scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-18 04:44:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

Merge remote-tracking branch 'origin/main' into litellm_ws_spend_log_per_turn

Browse source 
# Conflicts:
#	litellm/llms/custom_httpx/llm_http_handler.py
#	litellm/proxy/response_api_endpoints/endpoints.py
#	litellm/responses/main.py
#	litellm/responses/streaming_iterator.py
This commit is contained in:
mateo-berri 2026-10-17 12:44:44 -07:00
commit 423df37faa
398 changed files with 34016 additions and 2961 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/workflows/auto_update_price_and_context_window.yml vendored
View file

@ -24,12 +24,9 @@ jobs:
- name: Update JSON Data
run: |
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
- name: Regenerate JSON Schema
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create Pull Request
run: |
git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
git add model_prices_and_context_window.json
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
gh pr create --title "Update model_prices_and_context_window.json file" \
--body "Automated update for model_prices_and_context_window.json" \

7
.github/workflows/sync-together-ai-models.yml vendored
View file

@ -41,10 +41,6 @@ jobs:
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
env:
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
- name: Regenerate the JSON schema
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create a pull request when the registry changed
if: steps.existing.outputs.open_pr == ''
run: |
@ -57,8 +53,7 @@ jobs:
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b "$branch"
git add model_prices_and_context_window.json \
litellm/model_prices_and_context_window_backup.json \
model_prices_and_context_window.schema.json
litellm/model_prices_and_context_window_backup.json
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
gh auth setup-git
git push origin "$branch"

1
README.md
View file

@ -384,6 +384,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | |
| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | |
| [Nace (`nace`)](https://console.nace.ai/docs/drex) | | | | | | | | | | |
| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | |

32
backend/Dockerfile
View file

@ -4,6 +4,33 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
FROM $UV_IMAGE AS uvbin
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -54,7 +81,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -63,6 +90,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra saml \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

1
backend/routes/allowlist.py
View file

@ -165,6 +165,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
{
"/admin",
"/metrics",
"/swagger", # API documentation static assets belong to the backend
"/mcp", # lazily-mounted MCP sub-app serves on the backend component
}

45
ci_cd/cost_map_guard.py
View file

@ -1,11 +1,11 @@
"""Guard the cost map on pull requests.
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
files and may only add or update models.
Every pull request whose diff against its merge base touches one of the two cost map files gets the file
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
get the file checks and additionally may only touch those two files or the legacy schema
path and may only add or update models.
"""
from __future__ import annotations
@ -19,13 +19,12 @@ from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
CostMap = dict[str, object]
@ -35,7 +34,6 @@ CostMap = dict[str, object]
class Snapshot:
cost_map: str
backup: str
schema: str
def _parse_object(text: str, path: str) -> CostMap | str:
@ -46,13 +44,6 @@ def _parse_object(text: str, path: str) -> CostMap | str:
return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root"
def _rendered_schema(cost_map: CostMap) -> str:
try:
return render(build_schema(cost_map))
except SystemExit as error:
return str(error)
def _evaluation_output_price_failure(
key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
) -> str | None:
@ -78,29 +69,13 @@ def _file_failures(
*,
evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
) -> tuple[str, ...]:
schema_text: Final = _rendered_schema(head_map)
if not schema_text.startswith("{"):
return (schema_text,)
backup_failure: Final = (
()
if head.backup == head.cost_map
else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",)
)
schema_failure: Final = (
()
if head.schema == schema_text
else (
f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; "
"run `python ci_cd/generate_model_prices_schema.py` and commit the result",
)
)
return (
*backup_failure,
*schema_failure,
*(
f"{COST_MAP_PATH} does not validate against its schema: {error}"
for error in validation_errors(head_map, json.loads(schema_text))[:20]
),
*(
failure
for key, entry in head_map.items()
@ -135,7 +110,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
*(
f"bot PRs may only change the cost map files, not {path}"
for path in changed_files
if path not in GUARDED_PATHS
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
),
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),

418
ci_cd/generate_model_prices_schema.py
View file

@ -1,418 +0,0 @@
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Final, Optional
import jsonschema
REPO_ROOT = Path(__file__).parent.parent
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
JsonSchema = dict
NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
BOOLEAN: JsonSchema = {"type": "boolean"}
STRING: JsonSchema = {"type": "string"}
TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
WEEKDAY_PATTERN: Final = (
r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
)
EXTRA_BOOLEAN_KEYS = frozenset(
{
"gemini_native_audio",
"gemini_audio_only_live",
"uses_embed_content",
"use_openai_responses_path",
"bedrock_converse_supports_strict_tools",
"thinking_always_on",
}
)
HOURS_UTC: Final[JsonSchema] = {
"description": 'UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.',
"oneOf": [TIME_WINDOW, {"type": "array", "items": TIME_WINDOW, "minItems": 1}],
}
OFF_PEAK_WINDOW: Final[JsonSchema] = {
"type": "object",
"properties": {
"hours_utc": HOURS_UTC,
"weekdays": {
"type": "array",
"description": "ISO-8601 weekday numbers (1 = Monday .. 7 = Sunday) or English day names the window applies on.",
"items": {
"oneOf": [
{"type": "integer", "minimum": 1, "maximum": 7},
{"type": "string", "pattern": WEEKDAY_PATTERN},
]
},
"minItems": 1,
},
},
"required": ["hours_utc"],
"additionalProperties": False,
}
OBJECT_KEYS: dict[str, JsonSchema] = {
"off_peak_pricing": {
"type": "object",
"description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.",
"properties": {
"hours_utc": HOURS_UTC,
"windows": {"type": "array", "items": OFF_PEAK_WINDOW, "minItems": 1},
"weekday_timezone": {
"type": "string",
"description": "IANA zone the weekdays of each window are read on; defaults to UTC.",
},
"input_cost_per_token": NONNEG_NUMBER,
"output_cost_per_token": NONNEG_NUMBER,
"output_cost_per_reasoning_token": NONNEG_NUMBER,
"cache_read_input_token_cost": NONNEG_NUMBER,
"cache_creation_input_token_cost": NONNEG_NUMBER,
},
"anyOf": [{"required": ["hours_utc"]}, {"required": ["windows"]}],
"additionalProperties": False,
},
"search_context_cost_per_query": {
"type": "object",
"description": "USD cost per web search query, keyed by search context size.",
"properties": {
"search_context_size_low": NONNEG_NUMBER,
"search_context_size_medium": NONNEG_NUMBER,
"search_context_size_high": NONNEG_NUMBER,
},
"additionalProperties": False,
},
"guardrail_cost_per_unit": {
"type": "object",
"description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
"additionalProperties": NONNEG_NUMBER,
},
"metadata": {
"type": "object",
"description": "Free-form notes about the entry (e.g. pricing derivation).",
},
"provider_specific_entry": {
"type": "object",
"description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
},
}
ARRAY_KEYS: dict[str, JsonSchema] = {
"supported_audio_formats": {
"type": "array",
"description": "Audio container formats the model can return.",
"items": {"type": "string", "enum": ["mp3", "wav"]},
},
"supported_endpoints": {
"type": "array",
"description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
"items": STRING,
},
"supported_modalities": {
"type": "array",
"description": "Input modalities the model accepts.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
},
"supported_output_modalities": {
"type": "array",
"description": "Output modalities the model can produce.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
},
"reasoning_effort_levels": {
"type": "array",
"description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
"items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
},
"supported_regions": {
"type": "array",
"description": "Cloud regions the model is available in ('global' or region ids).",
"items": STRING,
},
"tiered_pricing": {
"type": "array",
"description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
"items": {
"type": "object",
"properties": {
"range": {
"type": "array",
"description": "[min, max] prompt-token span this tier applies to.",
"items": NONNEG_NUMBER,
"minItems": 2,
"maxItems": 2,
},
"max_results_range": {
"type": "array",
"description": "[min, max] result-count span this tier applies to (search models).",
"items": NONNEG_NUMBER,
"minItems": 2,
"maxItems": 2,
},
"input_cost_per_token": NONNEG_NUMBER,
"output_cost_per_token": NONNEG_NUMBER,
"output_cost_per_reasoning_token": NONNEG_NUMBER,
"cache_read_input_token_cost": NONNEG_NUMBER,
"cache_creation_input_token_cost": NONNEG_NUMBER,
"input_cost_per_query": NONNEG_NUMBER,
},
"additionalProperties": False,
},
},
}
INTEGER_KEYS: dict[str, JsonSchema] = {
"max_tokens": {
**NONNEG_INTEGER,
"description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
},
"max_input_tokens": {
**NONNEG_INTEGER,
"description": "Maximum prompt/context tokens the model accepts.",
},
"max_output_tokens": {
**NONNEG_INTEGER,
"description": "Maximum tokens the model can generate in one response.",
},
"output_vector_size": {
**NONNEG_INTEGER,
"description": "Embedding dimension for embedding models.",
},
"prompt_cache_min_tokens": {
**NONNEG_INTEGER,
"description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
},
"token_count_framing": {
**NONNEG_INTEGER,
"description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
},
"tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
"rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
}
NUMBER_KEYS: dict[str, JsonSchema] = {
"regional_processing_uplift_multiplier_eu": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
},
"regional_processing_uplift_multiplier_us": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
},
"regional_endpoint_uplift_multiplier": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).",
},
}
COST_DESCRIPTIONS: dict[str, str] = {
"input_cost_per_token": "USD per prompt token.",
"output_cost_per_token": "USD per generated token.",
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
"google_maps_grounding_cost_per_query": (
"USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
),
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
"output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
}
def cost_description(key: str) -> Optional[str]:
if key in COST_DESCRIPTIONS:
return COST_DESCRIPTIONS[key]
if key.endswith("_flex"):
return "Flex service-tier rate for the same-named base field."
if key.endswith("_priority"):
return "Priority service-tier rate for the same-named base field."
if "_above_" in key:
return "Rate applied once the prompt exceeds the token threshold in the field name."
return None
def cost_schema(key: str) -> JsonSchema:
description = cost_description(key)
return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
return {
"litellm_provider": {
"type": "string",
"description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
},
"mode": {
"type": "string",
"description": "Primary API surface / task type of the model.",
"enum": list(modes),
},
"source": {
"type": "string",
"description": "URL of the provider pricing/model page this entry was taken from.",
},
"deprecation_date": {
"type": "string",
"description": "Date the provider deprecates the model, YYYY-MM-DD.",
"format": "date",
"pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
},
"web_search_billing_unit": {
"type": "string",
"description": "Whether web search is billed per query or per prompt.",
"enum": ["per_query", "per_prompt"],
},
"bedrock_output_config_effort_ceiling": {
"type": "string",
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
"enum": ["low", "medium", "high", "max", "xhigh"],
},
"default_reasoning_effort": {
"type": "string",
"description": (
"Reasoning effort the provider applies when the request omits reasoning_effort. "
"Gates whether a non-default temperature or the top_p/logprobs sampling params are "
"accepted, which hold only when the effort resolves to 'none'."
),
"enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
},
"comment": STRING,
"token_count_fallback_model": {
"type": "string",
"minLength": 1,
"description": "Model that counts tokens for this one when the provider cannot count it.",
},
"audio_transcription_config": STRING,
"vertex_ai_audio_api": {
"type": "string",
"enum": ["lyria_predict", "lyria_interactions"],
},
}
def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
if key in curated:
return curated[key]
if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
return BOOLEAN
if "cost" in key:
return cost_schema(key)
return None
def build_schema(prices: dict) -> JsonSchema:
entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
if unclassified:
raise SystemExit(
f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
f"Add them to the key tables in {Path(__file__).name} and rerun it."
)
entry_properties = {key: classify(key, modes) for key in all_keys}
return {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "LiteLLM model_prices_and_context_window.json",
"description": (
"Schema for LiteLLM's model price and context window registry "
"(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
"Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
"optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
"All costs are USD per unit. New optional fields are added regularly, so consumers should "
"ignore unknown fields rather than reject them."
),
"type": "object",
"properties": {
"sample_spec": {
"type": "object",
"description": (
"Documentation placeholder illustrating the entry shape; not a real model and not "
"schema-conformant (several values are prose)."
),
},
"fallback_generalizations": {
"type": "object",
"description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
"properties": {
"rules": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": STRING,
"pattern": STRING,
"description": STRING,
},
"required": ["name", "pattern"],
"additionalProperties": True,
},
}
},
"additionalProperties": False,
},
},
"additionalProperties": {"$ref": "#/$defs/modelEntry"},
"$defs": {
"modelEntry": {
"type": "object",
"description": (
"Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
"are optional; boolean capability flags are simply omitted when unknown or false."
),
"required": ["litellm_provider"],
"properties": entry_properties,
"additionalProperties": True,
}
},
}
def render(schema: JsonSchema) -> str:
return json.dumps(schema, indent=2) + "\n"
def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
return tuple(
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
for error in validator.iter_errors(prices)
)
def main() -> int:
check = "--check" in sys.argv[1:]
prices = json.loads(PRICES_PATH.read_text())
rendered = render(build_schema(prices))
errors = validation_errors(prices, json.loads(rendered))
if errors:
print(f"{PRICES_PATH.name} does not validate against the generated schema:")
print("\n".join(errors[:20]))
return 1
if not check:
SCHEMA_PATH.write_text(rendered)
print(f"wrote {SCHEMA_PATH}")
return 0
if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
print(
f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
)
return 1
print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
return 0
if __name__ == "__main__":
sys.exit(main())

34
gateway/Dockerfile
View file

@ -20,6 +20,33 @@ RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${
make -j"$(nproc)" pgbouncer && \
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -66,11 +93,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
# Stage 2 — copy source and install the workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -79,6 +106,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
# compiled Rust extension must live next to the source or it is never imported.
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/

23
litellm-proxy-extras/litellm_proxy_extras/migrations/20261015120000_add_environment_variables_revision_trigger/migration.sql Normal file
View file

@ -0,0 +1,23 @@
SET LOCAL lock_timeout = '10s';
CREATE SEQUENCE IF NOT EXISTS "litellm_environment_variables_revision_seq";
GRANT USAGE ON SEQUENCE "litellm_environment_variables_revision_seq" TO PUBLIC;
CREATE OR REPLACE FUNCTION litellm_bump_environment_variables_revision() RETURNS TRIGGER AS $$
DECLARE
next_revision BIGINT := nextval(format('%I.litellm_environment_variables_revision_seq', TG_TABLE_SCHEMA)::regclass);
BEGIN
IF TG_OP = 'UPDATE' THEN
NEW."reload_revision" := GREATEST(next_revision, OLD."reload_revision" + 1);
ELSE
NEW."reload_revision" := next_revision;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS "litellm_environment_variables_revision" ON "LiteLLM_Config";
CREATE TRIGGER "litellm_environment_variables_revision"
BEFORE INSERT OR UPDATE ON "LiteLLM_Config"
FOR EACH ROW WHEN (NEW."param_name" = 'environment_variables')
EXECUTE PROCEDURE litellm_bump_environment_variables_revision();

71
litellm-rust/crates/inference-messages/tests/messages/request.rs
View file

@ -343,6 +343,77 @@ async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
assert_eq!(sent, expected);
}
#[rstest]
#[case::anthropic("anthropic", &[AnthropicBeta::DangerousToolUse20260903])]
#[case::deepseek("deepseek", &[])]
#[case::minimax("minimax", &[])]
#[tokio::test]
async fn safeguards_add_the_dangerous_tool_use_beta_only_where_the_beta_map_has_a_column(
call: MessagesCall,
#[case] provider: &str,
#[case] derived: &[AnthropicBeta],
) {
let upstream = upstream([message_response()]).await;
let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
..call
},
json!({"safeguards": safeguards}),
))
.await;
let request = only_request(&upstream).await;
let expected: BetaSet = derived
.iter()
.cloned()
.chain([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
.collect();
assert_eq!(sent_betas(&request), expected);
assert_eq!(
request.body_json::<Value>().unwrap()["safeguards"],
safeguards
);
}
#[rstest]
#[tokio::test]
async fn null_safeguards_are_dropped_instead_of_sent_without_their_beta(
call: MessagesCall,
#[values("anthropic", "azure_ai", "bedrock_mantle", "deepseek", "minimax")] provider: &str,
) {
let upstream = upstream([message_response()]).await;
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
..call
},
json!({"safeguards": null}),
))
.await;
let request = only_request(&upstream).await;
assert_eq!(
sent_betas(&request),
BetaSet::from_iter([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
);
assert!(
!request
.body_json::<Map<String, Value>>()
.unwrap()
.contains_key("safeguards")
);
}
#[rstest]
#[tokio::test]
async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: MessagesCall) {

91
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -109,7 +109,10 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
merge_beta_headers(
headers,
provider_feature_betas(request, BetaProvider::Anthropic),
)
}
}
@ -136,12 +139,20 @@ pub(crate) fn transform_messages_request(
messages: strip_encrypted_reasoning_blocks(messages),
params: MessagesOptionalParams {
context_management,
extra: without_null_safeguards(request.params.extra),
..request.params
},
..request
})
}
fn without_null_safeguards(extra: Map<String, Value>) -> Map<String, Value> {
extra
.into_iter()
.filter(|(key, value)| key != "safeguards" || !value.is_null())
.collect()
}
pub(crate) fn update_headers_with_anthropic_beta(
headers: Headers,
request: &MessagesRequest,
@ -149,12 +160,34 @@ pub(crate) fn update_headers_with_anthropic_beta(
merge_beta_headers(headers, feature_betas(request))
}
/// The betas a request needs on a host other than the first-party API: the Anthropic set
/// filtered and renamed through the host's `BetaProvider` policy.
pub(crate) fn update_headers_with_safeguards_beta(
headers: Headers,
request: &MessagesRequest,
provider: BetaProvider,
) -> Headers {
merge_beta_headers(
headers,
feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)),
)
}
/// The betas a request needs on a host with a column in the beta map, filtered and renamed
/// through that column.
pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet {
feature_betas(request)
.iter()
.filter_map(|beta| beta.on(provider))
on_provider(
feature_betas(request).union(mapped_host_betas(&request.params)),
provider,
)
}
fn on_provider(betas: BetaSet, provider: BetaProvider) -> BetaSet {
betas.iter().filter_map(|beta| beta.on(provider)).collect()
}
fn mapped_host_betas(params: &MessagesOptionalParams) -> BetaSet {
carries_safeguards(params)
.then_some(AnthropicBeta::DangerousToolUse20260903)
.into_iter()
.collect()
}
@ -206,6 +239,13 @@ fn uses_structured_output(params: &MessagesOptionalParams) -> bool {
.is_some_and(|config| config.format.is_some())
}
fn carries_safeguards(params: &MessagesOptionalParams) -> bool {
params
.extra
.get("safeguards")
.is_some_and(|value| !value.is_null())
}
fn messages_carry_output_config(messages: &[Message]) -> bool {
messages
.iter()
@ -1025,10 +1065,44 @@ mod tests {
json!({"messages": [{"role": "user", "content": "hi", "output_config": null}]}),
&["per-turn-control-2026-07-01"]
)]
#[case::safeguards(
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
&[]
)]
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
assert_eq!(feature_betas(&request(fields)), betas(expected));
}
#[rstest]
#[case::anthropic(BetaProvider::Anthropic, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
#[case::vertex_ai(BetaProvider::VertexAi, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
#[case::empty_list(BetaProvider::Anthropic, json!([]), &["dangerous-tool-use-2026-09-03"])]
#[case::not_a_list(BetaProvider::Anthropic, json!(1), &["dangerous-tool-use-2026-09-03"])]
#[case::null(BetaProvider::Anthropic, Value::Null, &[])]
#[case::bedrock_converse_has_a_null_cell(BetaProvider::BedrockConverse, json!([{"type": "dangerous_tool_use"}]), &[])]
fn safeguards_add_the_dangerous_tool_use_beta_where_the_beta_map_has_a_cell(
#[case] provider: BetaProvider,
#[case] safeguards: Value,
#[case] expected: &[&str],
) {
assert_eq!(
provider_feature_betas(&request(json!({"safeguards": safeguards})), provider),
betas(expected)
);
}
#[rstest]
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
assert_eq!(
update_headers_with_anthropic_beta(
client.clone(),
&request(json!({"safeguards": [{"type": "dangerous_tool_use"}]}))
),
client
);
}
#[rstest]
#[case::no_betas(&[("x-api-key", "k"), ("anthropic-version", "2023-06-01")], json!({}), &[("x-api-key", "k"), ("anthropic-version", "2023-06-01")])]
#[case::blank_beta_header(&[("Anthropic-Beta", " , "), ("x-api-key", "k")], json!({}), &[("Anthropic-Beta", " , "), ("x-api-key", "k")])]
@ -1067,6 +1141,11 @@ mod tests {
json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}),
&[("anthropic-beta", "claude-code-20250219,context-management-2025-06-27,effort-2025-11-24,interleaved-thinking-2025-05-14,per-turn-control-2026-07-01")],
)]
#[case::safeguards_add_the_dangerous_tool_use_beta_to_the_client_betas(
&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")],
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
&[("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")],
)]
fn request_headers_merge_the_feature_betas(
#[case] input: &[(&str, &str)],
#[case] fields: Value,

12
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -2,7 +2,7 @@ use litellm_auth::SecretValue;
use litellm_http::request::{has_bearer_auth, has_header};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
@ -12,7 +12,7 @@ use crate::{
common_utils::API_KEY_PLACEMENT,
messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
transformation::{transform_messages_request, update_headers_with_safeguards_beta},
},
},
azure_ai::common_utils::{
@ -104,7 +104,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
update_headers_with_safeguards_beta(headers, request, BetaProvider::AzureAi)
}
}
@ -544,6 +544,12 @@ mod tests {
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
)]
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
#[case::safeguards_need_the_dangerous_tool_use_beta(
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
&[("anthropic-beta", "claude-code-20250219")],
&[("x-api-key", "k"), ("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03")]
)]
#[case::null_safeguards_need_no_beta(json!({"safeguards": null}), &[], &[("x-api-key", "k")])]
fn request_headers_carry_the_anthropic_feature_betas(
#[case] fields: serde_json::Value,
#[case] forwarded: &[(&str, &str)],

21
litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs
View file

@ -6,7 +6,7 @@ use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{header_value, with_header, without_headers};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
use serde_json::Value;
@ -15,7 +15,7 @@ use crate::{
Error, ErrorDetail,
anthropic::messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
transformation::{transform_messages_request, update_headers_with_safeguards_beta},
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
@ -174,7 +174,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
None => headers,
};
update_headers_with_anthropic_beta(headers, request)
update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic)
}
fn wire_body(&self, body: Value) -> Value {
@ -356,6 +356,21 @@ mod tests {
);
}
#[rstest]
#[case::safeguards(json!([{"type": "dangerous_tool_use"}]), Some("dangerous-tool-use-2026-09-03"))]
#[case::null_safeguards(Value::Null, None)]
fn safeguards_need_the_dangerous_tool_use_beta(
#[case] safeguards: Value,
#[case] expected: Option<&str>,
) {
let request = serde_json::from_value(json!({"model": "claude-test", "max_tokens": 16, "messages": [], "safeguards": safeguards})).unwrap();
let headers = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.request_headers(vec![], &request);
assert_eq!(
header_value(&headers, "anthropic-beta").map(|value| value.to_string()),
expected.map(str::to_string)
);
}
#[rstest]
#[case::default(json!({}), false)]
#[case::override_empty(json!({"claude_platform_unsupported_params": []}), true)]

23
litellm-rust/crates/llms/src/deepseek/messages/transformation.rs
View file

@ -451,4 +451,27 @@ mod tests {
json!({"type": "enabled", "budget_tokens": 1024})
);
}
#[rstest]
#[case::without_client_betas(&[])]
#[case::with_client_betas(&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")])]
fn safeguards_and_client_betas_reach_deepseek_as_sent(#[case] client: &[(&str, &str)]) {
let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
let body = json!({
"model": "deepseek-v4-pro",
"max_tokens": 100,
"messages": [{"role": "user", "content": "hi"}],
"safeguards": safeguards
});
let headers: Headers = client
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect();
assert_eq!(
DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG
.request_headers(headers.clone(), &request_from(body.clone())),
headers
);
assert_eq!(transformed(body)["safeguards"], safeguards);
}
}

19
litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs
View file

@ -213,23 +213,14 @@ fn has_web_search_tool(tools: Option<&[Recognized<MessagesTool>]>) -> bool {
}
/// Python's Vertex `validate_anthropic_messages_environment` beta set: the shared feature
/// betas under Vertex's policy, plus the web search and safeguards betas Vertex needs spelled
/// out.
/// betas under Vertex's policy, plus the web search beta Vertex needs spelled out.
fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
let params = &request.params;
provider_feature_betas(request, BetaProvider::VertexAi).union(
[
has_web_search_tool(params.tools.as_deref())
.then_some(AnthropicBeta::WebSearch20250305),
params
.extra
.get("safeguards")
.is_some_and(|value| !value.is_null())
.then_some(AnthropicBeta::DangerousToolUse20260903),
]
.into_iter()
.flatten()
.collect(),
has_web_search_tool(params.tools.as_deref())
.then_some(AnthropicBeta::WebSearch20250305)
.into_iter()
.collect(),
)
}

1
litellm-rust/crates/model-catalog/AGENTS.md
View file

@ -2,5 +2,4 @@
For `model_prices_and_context_window.json` validation, we should eventually:
- Remove any schema file like `model_prices_and_context_window.schema.json`
- Stop skipping this crate's tests

32
litellm/__init__.py
View file

@ -1,34 +1,13 @@
### Hide pydantic namespace conflict warnings globally ###
from __future__ import annotations
import warnings
from litellm._process_defaults import apply_process_defaults as _apply_process_defaults
_apply_process_defaults()
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
# This warning can accumulate during streaming and cause memory leaks
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
### INIT VARIABLES #########################
import threading
import os
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
import dotenv as _dotenv
def _dev_env_hot_reload_enabled() -> bool:
"""The proxy exports this flag when started with ``--reload``. A reloaded
worker is a fresh process that inherits the reloader's environment, so an
edited ``.env`` value stays masked by the stale inherited one unless we
let the file win; overriding makes the edit take effect on reload."""
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
_dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
from collections.abc import Mapping, Sequence
from typing import (
Any,
@ -763,7 +742,7 @@ def is_openai_finetune_model(key: str) -> bool:
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
for key, value in model_cost_map.items():
for key, value in tuple(model_cost_map.items()):
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
open_ai_chat_completion_models.add(key)
elif value.get("litellm_provider") == "text-completion-openai":
@ -1694,6 +1673,9 @@ if TYPE_CHECKING:
from .llms.perplexity.decisions.transformation import (
PerplexityDecisionsConfig as PerplexityDecisionsConfig,
)
from .llms.nace.decisions.transformation import (
NaceDecisionsConfig as NaceDecisionsConfig,
)
from .llms.typesafe.decisions.transformation import (
TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
)

2
litellm/_lazy_imports_registry.py
View file

@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = (
"HostedVLLMRerankConfig",
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
"NaceDecisionsConfig",
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
"CloudflareDecisionsConfig",
@ -729,6 +730,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"HostedVLLMRerankConfig",
),
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
"NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),

24
litellm/_process_defaults.py Normal file
View file

@ -0,0 +1,24 @@
import os
import warnings
import dotenv
def _dev_env_hot_reload_enabled() -> bool:
"""The proxy exports this flag when started with ``--reload``. A reloaded
worker is a fresh process that inherits the reloader's environment, so an
edited ``.env`` value stays masked by the stale inherited one unless we
let the file win; overriding makes the edit take effect on reload."""
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
def apply_process_defaults() -> None:
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
# This warning can accumulate during streaming and cause memory leaks
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())

45
litellm/anthropic_beta_headers_manager.py
View file

@ -22,17 +22,28 @@ Environment Variables:
- LITELLM_ANTHROPIC_BETA_HEADERS_URL: Custom URL for remote config (optional)
"""
import asyncio
import json
import os
import threading
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from functools import cache
from importlib.resources import files
from typing import Final
from typing import Final, TypeAlias
import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
class GetAnthropicBetaHeadersConfig:
@ -174,13 +185,27 @@ def _load_beta_headers_config() -> dict:
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
# Get the URL from environment or use default
from litellm import anthropic_beta_headers_url
with _BETA_HEADERS_CONFIG_LOCK:
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
_BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
verbose_logger.debug("Loaded and cached beta headers config")
# Get the URL from environment or use default
from litellm import anthropic_beta_headers_url
return _BETA_HEADERS_CONFIG
_BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
verbose_logger.debug("Loaded and cached beta headers config")
return _BETA_HEADERS_CONFIG
async def aload_beta_headers_config() -> None:
if _BETA_HEADERS_CONFIG is None:
await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
@cache
def _bundled_beta_headers_config() -> BetaHeadersConfig:
return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())
def reload_beta_headers_config() -> dict:
@ -288,6 +313,14 @@ def is_beta_header_supported(
return beta_header in provider_mapping and provider_mapping[beta_header] is not None
def is_beta_header_supported_by_bundled_config(beta_header: str, provider: str) -> bool:
config: Final = _bundled_beta_headers_config()
aliases: Final = config.get("provider_aliases", {})
canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
provider_mapping: Final = config.get(canonical_provider, {})
return isinstance(provider_mapping, Mapping) and provider_mapping.get(beta_header) is not None
def get_provider_beta_header(
anthropic_beta_header: str,
provider: str,

10
litellm/caching/caching_handler.py
View file

@ -346,6 +346,7 @@ class LLMCachingHandler:
if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
verbose_logger.debug("Checking Async Cache")
logging_obj.reset_caching_details()
cached_result = await self._retrieve_from_cache(
call_type=call_type,
kwargs=kwargs,
@ -357,7 +358,7 @@ class LLMCachingHandler:
verbose_logger.debug("Cache Hit!")
cache_hit: Final = True
end_time: Final = datetime.datetime.now()
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model,
custom_llm_provider=kwargs.get("custom_llm_provider", None),
api_base=kwargs.get("api_base", None),
@ -366,7 +367,7 @@ class LLMCachingHandler:
cache_duration_ms: Final = (cache_check_end_time - cache_check_start_time) * 1000
self._update_litellm_logging_obj_environment(
logging_obj=logging_obj,
model=model,
model=provider_model,
kwargs=kwargs,
cached_result=cached_result,
is_async=True,
@ -381,7 +382,7 @@ class LLMCachingHandler:
call_type=call_type,
kwargs=kwargs,
logging_obj=logging_obj,
model=model,
model=provider_model,
custom_llm_provider=kwargs.get("custom_llm_provider", None),
args=args,
)
@ -467,6 +468,7 @@ class LLMCachingHandler:
new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
print_verbose("Checking Sync Cache")
logging_obj.reset_caching_details()
with response_cache_phase("get"):
cached_result = litellm.cache.get_cache(**new_kwargs)
if is_response_without_output(cached_result):
@ -514,7 +516,7 @@ class LLMCachingHandler:
if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
cached_result=cached_result
):
logging_obj.handle_sync_success_callbacks_for_async_calls(
logging_obj.handle_sync_cache_hit_success_callbacks(
result=cached_result,
start_time=start_time,
end_time=end_time,

7
litellm/constants.py
View file

@ -1,4 +1,5 @@
import os
import re
import sys
from enum import Enum
from types import MappingProxyType
@ -18,6 +19,9 @@ MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_PROFILE: Final = "jwt_bearer_obo"
MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_SCOPE: Final = "https://graph.microsoft.com/.default"
# Replies depend on the caller's delegated identity, which response-cache keys do not include.
RESPONSE_CACHE_EXCLUDED_PROVIDERS: Final = frozenset({"microsoft_365_copilot", "databricks_agent"})
CLAUDE_CODE_PICKER_PATTERN: Final = re.compile(r"claude|anthropic", re.IGNORECASE)
GATEWAY_CLIENT_HEADER: Final = "x-gateway-client"
CLAUDE_CODE_CLIENT: Final = "claude-code"
MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC"
GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth"
GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type"
@ -391,7 +395,7 @@ DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
@ -1615,6 +1619,7 @@ CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = get_env_int("CLOUDZERO_EXPORT_INTERVA
MCP_TOOL_NAME_PREFIX: Final = "mcp_tool"
MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = get_env_int("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)
PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096
INTERNAL_CALLBACK_NAMES: Final = frozenset({"cache", "vector_store_pre_call_hook"})
# Headers to control callbacks
X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"

119
litellm/cost_calculator.py
View file

@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from httpx import Response
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@ -150,6 +150,7 @@ from litellm.utils import (
TranscriptionResponse,
cached_get_model_info_helper,
token_counter,
unmapped_model_generalization,
)
if TYPE_CHECKING:
@ -875,6 +876,16 @@ def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | Non
return own_model_info["litellm_provider"]
def _pricing_provider_for_cost_candidate(
candidate_provider: str | None,
request_provider: str | None,
current_provider: object,
) -> str | None:
if candidate_provider != request_provider:
return candidate_provider
return current_provider if isinstance(current_provider, str) else None
def select_model_name_for_cost_calc(
model: str | None,
completion_response: object | None,
@ -1438,32 +1449,15 @@ def completion_cost(
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
) -> float:
"""
Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
Return the USD cost of a LiteLLM call.
Parameters:
completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request.
Pass `completion_response` to price a response LiteLLM returned. Without one, pass `model` plus
`messages` / `prompt` and `completion`, and token counts are computed from the text. Prices come from
`litellm.model_cost` unless `custom_cost_per_token` or `custom_cost_per_second` is given. `total_time`
is in milliseconds and only matters for per-second pricing such as Replicate and SageMaker.
[OPTIONAL PARAMS]
model (str): Optional. The name of the language model used in the completion calls
prompt (str): Optional. The input prompt passed to the llm
completion (str): Optional. The output completion text from the llm
total_time (float, int): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds
custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call.
custom_cost_per_second: Optional[float]: the cost per second for the llm api call.
Returns:
float: The cost in USD dollars for the completion based on the provided parameters.
Exceptions:
Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json
Note:
- If completion_response is provided, the function extracts token information and the model name from it.
- If completion_response is not provided, the function calculates token counts based on the model and input text.
- The cost is calculated based on the model, prompt tokens, and completion tokens.
- For certain models containing "togethercomputer" in the name, prices are based on the model size.
- For un-mapped Replicate models, the cost is calculated based on the total time used for the request.
Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model`
or add it to model_prices_and_context_window.json.
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
@ -1561,14 +1555,20 @@ def completion_cost(
)
requested_model: Final = model
potential_model_names: Final = [
selected_model,
_get_response_model(completion_response),
]
if model is not None:
potential_model_names.append(model)
request_pricing_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
raw_candidates: Final = (
(
selected_model,
base_model_provider if base_model_provider is not None else request_pricing_provider,
),
(_get_response_model(completion_response), request_pricing_provider),
*(((model, request_pricing_provider),) if model is not None else ()),
)
potential_model_names: Final = (
raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates)
)
for idx, model in enumerate(potential_model_names):
for idx, (model, pricing_provider) in enumerate(potential_model_names):
try:
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug("selected model name for cost calculation: %s", model)
@ -1913,13 +1913,6 @@ def completion_cost(
if litellm_logging_obj is not None:
request_model_for_cost = litellm_logging_obj.model
pricing_provider: str | None = (
base_model_provider
if base_model_provider is not None and idx == 0
else custom_llm_provider
if isinstance(custom_llm_provider, str)
else None
)
(
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
@ -1927,7 +1920,11 @@ def completion_cost(
model=model,
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=pricing_provider,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
response_time_ms=response_time_ms,
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
@ -2026,7 +2023,11 @@ def completion_cost(
if cost_per_token_usage_object is not None and model:
_token_type_breakdown = get_token_type_cost_breakdown(
model=model,
custom_llm_provider=pricing_provider,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
usage=cost_per_token_usage_object,
service_tier=service_tier,
data_residency=data_residency,
@ -2066,7 +2067,7 @@ def completion_cost(
)
if idx == len(potential_model_names) - 1:
raise e
raise Exception(f"Unable to calculat cost for received potential model names - {potential_model_names}")
raise Exception(f"Unable to calculate cost for received potential model names - {potential_model_names}")
except Exception as e:
raise e
@ -2081,13 +2082,20 @@ def get_response_cost_from_hidden_params(
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
response_cost: Final = additional_headers["llm_provider-x-litellm-response-cost"]
if response_cost is None:
return None
return float(additional_headers["llm_provider-x-litellm-response-cost"])
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
return None
def _billable_header_cost(value: object) -> float | None:
if not isinstance(value, (str, int, float)):
return None
try:
cost: Final = float(value)
except ValueError:
return None
return cost if math.isfinite(cost) and cost >= 0 else None
def response_cost_calculator(
response_object: ModelResponse
| EmbeddingResponse
@ -2281,6 +2289,25 @@ def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelIn
return None
_CostCandidate: TypeAlias = tuple[str | None, str | None]
def _is_unpriced_generalized_candidate(candidate: _CostCandidate) -> bool:
model, pricing_provider = candidate
if model is None:
return False
rule_info: Final = unmapped_model_generalization(model, pricing_provider)
return rule_info is not None and not _cost_map_entry_prices_anything(rule_info)
def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, ...]) -> tuple[_CostCandidate, ...]:
unpriced: Final = tuple(_is_unpriced_generalized_candidate(candidate) for candidate in candidates)
return (
*(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if not is_unpriced),
*(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if is_unpriced),
)
def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None

7
litellm/decisions/call.py
View file

@ -57,7 +57,7 @@ from litellm.types.decisions import (
UnsupportedDecisionsRequest,
)
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
from litellm.utils import ProviderConfigManager
from litellm.utils import ProviderConfigManager, get_model_cost_key
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
@ -667,7 +667,10 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": f"{call.custom_llm_provider}/{call.model}",
"provider_response_model": (
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
or f"{call.custom_llm_provider}/{call.model}"
),
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
}
)

14
litellm/experimental_mcp_client/client.py
View file

@ -498,6 +498,7 @@ class MCPClient:
elicitation_callback: "ElicitationCallback | None" = None,
logging_callback: Callable | None = None,
protocol_version: MCPUpstreamProtocol = "auto",
upstream_transport: httpx2.AsyncBaseTransport | None = None,
):
self.protocol_version: MCPUpstreamProtocol = TypeAdapter[MCPUpstreamProtocol](
MCPUpstreamProtocol
@ -524,6 +525,7 @@ class MCPClient:
self._sampling_callback: Callable | None = sampling_callback
self._elicitation_callback: ElicitationCallback | None = elicitation_callback
self._logging_callback: Callable | None = logging_callback
self._upstream_transport: httpx2.AsyncBaseTransport | None = upstream_transport
self._session_callbacks_override: _SessionCallbackMap | None = None
# handle the basic auth value if provided
if auth_value:
@ -610,7 +612,11 @@ class MCPClient:
return stdio_client(server_params), None
if self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
return (
sse_client(
url=self.server_url,
@ -622,7 +628,11 @@ class MCPClient:
)
# HTTP transport (default)
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)
http_client = httpx_client_factory(
headers=headers,

18
litellm/integrations/opentelemetry.py
View file

@ -6,9 +6,10 @@ from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
from typing import TYPE_CHECKING, Any, Final, cast
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -106,6 +107,14 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
# Cap on credential-scoped providers held at once; each one owns an exporter thread.
_MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
class _CacheHitView(TypedDict, total=False):
cache_hit: ReadOnly[object]
saved_cache_cost: ReadOnly[object]
_CACHE_HIT_VIEW: Final = TypeAdapter(_CacheHitView)
# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
@ -2434,6 +2443,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
litellm_params=litellm_params,
)
# Cost breakdown tracking
cache_view: Final = _CACHE_HIT_VIEW.validate_python(standard_logging_payload)
cache_hit: Final = cache_view.get("cache_hit") is True
cost_breakdown: Final[CostBreakdown | None] = standard_logging_payload.get("cost_breakdown")
if cost_breakdown:
for key, value in cost_breakdown.items():
@ -2443,6 +2454,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
key=f"gen_ai.cost.{key}",
value=value,
)
self.safe_set_attribute(span=span, key=LiteLLM.CACHE_HIT, value=cache_hit)
if cache_hit:
saved_cache_cost: Final = cache_view.get("saved_cache_cost")
if isinstance(saved_cache_cost, (int, float)):
self.safe_set_attribute(span=span, key=LiteLLM.SAVED_CACHE_COST, value=saved_cache_cost)
#############################################
########## LLM Request Attributes ###########
#############################################

32
litellm/integrations/otel/emitter.py
View file

@ -100,23 +100,19 @@ def _carried_keys(
return frozenset(key for key, _value in attributes)
def attribute_budget(span: Span, reserved: int, overwrites: frozenset[str] = frozenset()) -> int | None:
"""How many mapped attributes fit on ``span`` next to what it already carries and ``reserved`` more.
def carried_keys(span: Span) -> frozenset[str]:
"""The attribute keys ``span`` already holds, empty for a span that exposes none."""
if not isinstance(span, ReadableSpan):
return frozenset()
return _carried_keys(span.attributes or ())
``overwrites`` are the mapped keys already present on ``span``: setting one
replaces the value in place and consumes no slot against the limit, so only
the genuinely new pre-existing keys reduce the budget. Counting the
overwritten ones too reserves slots the fit can never spend and sheds
indexed message attributes for nothing.
"""
def attribute_budget(span: Span, reserved: int) -> int | None:
"""How many new attribute keys fit on ``span`` next to what it already carries and ``reserved`` more."""
limit: Final = span_attribute_limit(span)
if limit is None:
return None
if not isinstance(span, ReadableSpan):
return limit - reserved
existing_keys: Final = _carried_keys(span.attributes or ())
fresh: Final = len(existing_keys - overwrites) if overwrites else len(existing_keys)
return limit - fresh - reserved
return limit - len(carried_keys(span)) - reserved
def stamp_error(
@ -303,8 +299,14 @@ class SpanEmitter:
)
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
reserved: Final = len(stamped_later.keys() - mapped.keys())
for key, value in fit_indexed_messages(mapped, attribute_budget(span, reserved, frozenset(mapped))).items():
span.set_attribute(key, value)
overwrites: Final = carried_keys(span) & mapped.keys()
fitted: Final = fit_indexed_messages(
MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
attribute_budget(span, reserved),
)
for key, value in mapped.items():
if key in overwrites or key in fitted:
span.set_attribute(key, value)
if error:
stamped: Final = stamp_error(span, error)
if stamped is not None and self._event_recorder is not None and role is SpanRole.LLM_CALL:

8
litellm/integrations/otel/logger.py
View file

@ -538,8 +538,12 @@ class OpenTelemetryV2(CustomLogger):
# logger is a success/failure callback only, so ``pre_call`` never reaches it
# and no carrier exists. The payload plus the request-level provider-handoff
# stamp (``upstream_started``) is the affirmative signal of a real call; a
# gate rejection carries ``is_no_upstream_call`` and gets no span.
if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
# upstream stamp, since the payload proves the call completed.
if carrier is None and (
call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
):
return None
try:
return self._finish_carrier(carrier, call, start_time, end_time)

2
litellm/integrations/otel/mappers/genai.py
View file

@ -91,6 +91,8 @@ class GenAIMapper:
f"{LiteLLM.COST_PREFIX}margin_fixed_amount": lambda d: d.cost.margin_fixed_amount,
f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent,
f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount,
LiteLLM.CACHE_HIT: lambda d: d.cache_hit,
LiteLLM.SAVED_CACHE_COST: lambda d: d.saved_cache_cost,
LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
LiteLLM.REQUEST_ROUTE: lambda d: d.request_route,
LiteLLM.REQUEST_PURPOSE: lambda d: d.request_purpose,

5
litellm/integrations/otel/model/metadata.py
View file

@ -249,6 +249,10 @@ class LLMCallEvent:
# ``InternalCallOrigin`` such as ``autorouter_classifier``), ``None`` for the
# caller's own provider attempt.
purpose: str | None
# True for a response served from the litellm response cache. A cache hit
# never hands off to a provider, so ``upstream_started`` stays False even
# though the call completed and is worth a span.
cache_hit: bool
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
@ -273,6 +277,7 @@ class LLMCallEvent:
upstream_started=kwargs.get("api_call_start_time") is not None,
upstream_start_seconds=_epoch_seconds(kwargs.get("api_call_start_time")),
purpose=internal_call_origin(payload, kwargs),
cache_hit=bool(payload.get("cache_hit")) if payload else False, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
provisional_span_name=f"{operation.value} {model}".strip(),
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
trace=trace,

7
litellm/integrations/otel/model/payloads.py
View file

@ -441,6 +441,8 @@ class LLMCallSpanData:
embedding_output: EmbeddingOutput | None = None
promoted_metadata: Mapping[str, str] = field(default_factory=lambda: _EMPTY_METADATA)
routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({}))
cache_hit: bool | None = None
saved_cache_cost: float | None = None
@classmethod
def from_standard_logging_payload(
@ -456,6 +458,7 @@ class LLMCallSpanData:
metadata_keys: tuple[str, ...] = (),
) -> LLMCallSpanData:
params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {})
cache_hit: Final = as_bool(payload.get("cache_hit")) is True # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
# The single parse of the request's metadata — the request-vs-provider
# model split, the response model, api base, and identity all come from
# here rather than being re-derived from the raw payload dicts.
@ -490,7 +493,7 @@ class LLMCallSpanData:
error=_parse_error(payload),
response_cost=as_float(payload.get("response_cost")),
cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))),
server=ServerInfo.from_api_base(context.api_base),
server=None if cache_hit else ServerInfo.from_api_base(context.api_base),
identity=context.identity,
promoted_metadata=allowlisted_metadata(context.identity.metadata, metadata_keys),
is_streaming=as_bool(payload.get("stream")),
@ -507,6 +510,8 @@ class LLMCallSpanData:
session_id=session_id or None,
embedding_output=embedding_output if capture_content else None,
routing_attributes=routing_decision_attributes(decision),
cache_hit=cache_hit,
saved_cache_cost=as_float(payload.get("saved_cache_cost")) if cache_hit else None, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
)

2
litellm/integrations/otel/model/semconv.py
View file

@ -340,6 +340,8 @@ class LiteLLM:
# semconv key for an MCP server's *name* (the convention uses ``server.address``
# for its network location), so it lives under the vendor namespace.
MCP_SERVER_NAME: Final = "litellm.mcp.server.name"
CACHE_HIT: Final = "litellm.cache_hit"
SAVED_CACHE_COST: Final = "litellm.saved_cache_cost"
class Metric:

25
litellm/integrations/prometheus.py
View file

@ -259,6 +259,7 @@ _POSITIVE_SERIES_CAP: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field
_POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(gt=0)])
_SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)])
_DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0
_ADDITIONAL_HEADERS: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
@ -270,6 +271,15 @@ def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _Series
return None
def _gauge_reading_or_none(value: object) -> float | None:
if not isinstance(value, str | int | float):
return None
try:
return float(value)
except (ValueError, OverflowError):
return None
def _positive_or_ignored(setting: str, value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
if value is None:
return None
@ -3350,12 +3360,11 @@ class PrometheusLogger(CustomLogger):
model_group=enum_values.model_group,
)
remaining_requests: int | None = None
remaining_tokens: int | None = None
if additional_headers := standard_logging_payload["hidden_params"]["additional_headers"]:
# OpenAI / OpenAI Compatible headers
remaining_requests = additional_headers.get("x_ratelimit_remaining_requests", None)
remaining_tokens = additional_headers.get("x_ratelimit_remaining_tokens", None)
additional_headers: Final = _ADDITIONAL_HEADERS.validate_python(
standard_logging_payload["hidden_params"]["additional_headers"] or {}
)
remaining_requests: Final = _gauge_reading_or_none(additional_headers.get("x_ratelimit_remaining_requests"))
remaining_tokens: Final = _gauge_reading_or_none(additional_headers.get("x_ratelimit_remaining_tokens"))
if litellm_overhead_time_ms := standard_logging_payload["hidden_params"].get("litellm_overhead_time_ms"):
_labels = prometheus_label_factory(
@ -3373,7 +3382,7 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
if remaining_requests:
if remaining_requests is not None:
"""
"model_group",
"api_provider",
@ -3387,7 +3396,7 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_remaining_requests_metric.labels(**_labels).set(remaining_requests)
if remaining_tokens:
if remaining_tokens is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_remaining_tokens_metric"),
enum_values=enum_values,

2
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -44,7 +44,7 @@ from litellm.litellm_core_utils.private_json import (
stage_private_json,
write_private_json,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
@dataclass(frozen=True, slots=True)

4
litellm/litellm_core_utils/core_helpers.py
View file

@ -318,7 +318,7 @@ def remove_items_at_indices(items: _PoppableByIndex | None, indices: Iterable[in
items.pop(index)
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: dict) -> dict:
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: Mapping[str, object]) -> dict:
"""
Helper to get litellm metadata for spend tracking
@ -326,7 +326,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
and user_api_key values are in 'metadata'.
"""
potential_spend_tracking_metadata_substring: Final = "user_api_key"
for key, value in metadata.items():
for key, value in dict(metadata).items():
if potential_spend_tracking_metadata_substring in key:
litellm_metadata[key] = value
return litellm_metadata

47
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -1196,6 +1196,9 @@ def _map_sagemaker_exception(
)
_VERTEX_403_STATUS_CODE_PATTERN: Final = re.compile(r"(?<![0-9A-Za-z])403(?![0-9A-Za-z])")
def _map_vertex_exception(
*,
model: str,
@ -1206,6 +1209,8 @@ def _map_vertex_exception(
exception_provider: str,
extra_information: str,
) -> None:
status_code: Final[object] = getattr(original_exception, "status_code", None)
body_code: Final[int | None] = _get_body_error_code(error_str)
if "Vertex AI API has not been used in project" in error_str or "Unable to find your project" in error_str:
raise BadRequestError(
message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
@ -1252,7 +1257,11 @@ def _map_vertex_exception(
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
elif "403" in error_str:
elif (
(not isinstance(status_code, int) or status_code in (403, 500))
and body_code in (None, 403)
and _VERTEX_403_STATUS_CODE_PATTERN.search(error_str)
):
raise BadRequestError(
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
@ -1306,7 +1315,7 @@ def _map_vertex_exception(
elif (
isinstance(getattr(original_exception, "status_code", None), int)
and 500 <= original_exception.status_code < 600
and _get_body_error_code(error_str) == 429
and body_code == 429
):
# upstream gateway wraps a 429 inside a 5xx envelope
# e.g. HTTP 500/503 with {"error":{"code":429,...}}.
@ -2413,6 +2422,35 @@ def _map_exception_by_status(
)
_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_URL_SCHEME: Final = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*")
_HOST_WITH_OPTIONAL_PORT: Final = re.compile(
r"(?:(?:[A-Za-z0-9-]+\.)+[A-Za-z0-9-]+|localhost|\[[0-9A-Fa-f:.]+\])(?::\d+)?"
)
def with_deployment_api_base(litellm_params: object) -> Mapping[str, object]:
params: Final = as_str_mapping(litellm_params) or _NO_LITELLM_PARAMS
metadata: Final = as_str_mapping(params.get("metadata"))
deployment_api_base: Final = None if metadata is None else metadata.get("api_base")
if not isinstance(deployment_api_base, str) or not deployment_api_base:
return params
return MappingProxyType({**params, "api_base": deployment_api_base})
def _api_base_without_credentials(api_base: str) -> str:
scheme, separator, _ = api_base.partition("://")
scheme_prefix: Final = scheme + separator if separator and _URL_SCHEME.fullmatch(scheme) else ""
authority_and_rest: Final = api_base[len(scheme_prefix) :]
authority: Final = re.split(r"[/?#]", authority_and_rest, maxsplit=1)[0]
after_authority: Final = authority_and_rest[len(authority) :]
host: Final = authority.rpartition("@")[2]
if "@" in after_authority and _HOST_WITH_OPTIONAL_PORT.fullmatch(host) is None:
return ""
path: Final = re.split(r"[?#]", after_authority, maxsplit=1)[0]
return f"{scheme_prefix}{host}{path}"
def _is_guardrail_block(original_exception: Exception) -> bool:
from litellm.integrations.custom_guardrail import is_guardrail_intervention
@ -2483,8 +2521,9 @@ def exception_type(
if isinstance(custom_llm_provider, str) and len(custom_llm_provider) > 0:
exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception"
if _api_base:
extra_information += f"\nAPI Base: `{_api_base}`"
shown_api_base: Final = _api_base_without_credentials(_api_base) if _api_base else ""
if shown_api_base:
extra_information += f"\nAPI Base: `{shown_api_base}`"
if messages and len(messages) > 0 and litellm.redact_messages_in_exceptions is False:
extra_information += f"\nMessages: `{messages}`"

58
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -324,10 +324,16 @@ def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _Fetc
return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable:
try:
return urllib.parse.urlsplit(url)
except ValueError as error:
return ModelCostMapReloadUnavailable(reason=f"invalid URL {url}: {error}")
def _attempt_fetch_s3(parts: urllib.parse.SplitResult, url: str, timeout: int) -> _FetchAttemptOutcome:
from botocore.exceptions import BotoCoreError, ClientError
parts: Final = urllib.parse.urlsplit(url)
bucket: Final = parts.netloc
key: Final = parts.path.lstrip("/")
if not bucket or not key:
@ -343,9 +349,11 @@ def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return await asyncio.to_thread(_attempt_fetch_s3, url, timeout)
return await asyncio.to_thread(_attempt_fetch_s3, parts, url, timeout)
try:
response: Final = await client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -354,9 +362,11 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return _attempt_fetch_s3(url, timeout)
return _attempt_fetch_s3(parts, url, timeout)
try:
response: Final = client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -547,22 +557,39 @@ def get_model_cost_map_loaded_at() -> "datetime | None":
return _cost_map_source_info.loaded_at
def _expand_model_aliases(model_cost: dict) -> dict:
class AliasedCostEntry(dict[str, object]):
__slots__ = ("names",)
def __init__(self, entry: Mapping[str, object], names: tuple[str, ...]) -> None:
super().__init__(entry)
self.names: tuple[str, ...] = names
def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:
return MappingProxyType(
{
owner: (owner, *(alias for alias, alias_owner in alias_owners.items() if alias_owner == owner))
for owner in dict.fromkeys(alias_owners.values())
}
)
def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]:
"""
Expand ``aliases`` lists in model cost entries into top-level entries.
Each alias gets a reference to the **same** dict object as the canonical
entry (zero memory overhead). The ``aliases`` key is removed from the
entry so downstream code never sees it.
Each alias gets a reference to the **same** ``AliasedCostEntry`` as the
canonical entry, which carries every name it is served under. The
``aliases`` key is removed from the entry so downstream code never sees it.
If an alias collides with an existing canonical entry the alias is
skipped and a warning is logged.
"""
aliases_to_add: Final[dict[str, dict]] = {}
alias_owners: Final[dict[str, str]] = {}
keys_with_aliases: Final[list[str]] = []
for model_name, model_info in model_cost.items():
aliases: list | None = model_info.get("aliases")
aliases: object = model_info.get("aliases")
if aliases is None:
continue
keys_with_aliases.append(model_name)
@ -584,7 +611,7 @@ def _expand_model_aliases(model_cost: dict) -> dict:
model_name,
)
continue
if alias in aliases_to_add:
if alias in alias_owners:
verbose_logger.warning(
"LiteLLM model alias conflict: alias '%s' (from '%s') "
"was already claimed by another entry — skipping.",
@ -592,13 +619,14 @@ def _expand_model_aliases(model_cost: dict) -> dict:
model_name,
)
continue
aliases_to_add[alias] = model_info # same dict reference
alias_owners[alias] = model_name
# Remove the ``aliases`` key from entries so it doesn't pollute model info
for key in keys_with_aliases:
model_cost[key].pop("aliases", None)
model_cost.update(aliases_to_add)
for owner, names in _alias_groups(alias_owners).items():
model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names)))
return model_cost

384
litellm/litellm_core_utils/litellm_logging.py
View file

@ -13,6 +13,7 @@ import sys
import time
import traceback
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime as dt_object
from functools import lru_cache
from types import MappingProxyType, TracebackType
@ -41,6 +42,7 @@ from litellm.constants import (
DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
EMPTY_MAPPING,
INTERNAL_CALLBACK_NAMES,
PROVIDER_REQUEST_ID_HEADERS,
REDACTED_BY_LITELLM,
)
@ -557,6 +559,12 @@ def mask_api_base_credentials(api_base: str) -> str:
return api_base[:key_end] + "*" * 5 + api_base[-4:]
@dataclass(frozen=True, slots=True)
class UncachedPrice:
cost: float | None
cost_breakdown: CostBreakdown | None
def _timestamp_seconds(moment: object) -> float | None:
if isinstance(moment, datetime.datetime):
return moment.timestamp()
@ -810,6 +818,10 @@ class Logging(LiteLLMLoggingBaseClass):
new_call.sync_streaming_chunks = []
return new_call
def reset_caching_details(self) -> None:
"""Forget an earlier cache lookup's result, so a call that looks up again reports only its latest."""
self.caching_details = None
def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
self.response_timing_metrics = dict(timing_metrics)
@ -1899,6 +1911,7 @@ class Logging(LiteLLMLoggingBaseClass):
cache_hit: bool | None = None,
litellm_model_name: str | None = None,
router_model_id: str | None = None,
ignore_stamped_cost: bool = False,
) -> float | None:
"""
Calculate response cost using result + logging object variables.
@ -1934,7 +1947,9 @@ class Logging(LiteLLMLoggingBaseClass):
):
hidden_params: Final = result_hidden_params
if (
"response_cost" in hidden_params and hidden_params["response_cost"] is not None
not ignore_stamped_cost
and "response_cost" in hidden_params
and hidden_params["response_cost"] is not None
): # use cost if already calculated
self._record_zero_cost_diagnostic(
priced_result,
@ -2060,6 +2075,20 @@ class Logging(LiteLLMLoggingBaseClass):
_response_cost_calculator = response_cost_calculator
def uncached_price(self, result: object) -> UncachedPrice:
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
debug_key: Final = "response_cost_failure_debug_information"
had_failure_debug_info: Final = debug_key in self.model_call_details
try:
cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown)
finally:
self.cost_breakdown = previous_cost_breakdown
self.billed_token_rates = previous_billed_token_rates
if not had_failure_debug_info:
self.model_call_details.pop(debug_key, None)
def _record_zero_cost_diagnostic(
self,
result: object,
@ -2303,6 +2332,7 @@ class Logging(LiteLLMLoggingBaseClass):
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
for_async_request=True,
**kwargs,
)
@ -2545,7 +2575,8 @@ class Logging(LiteLLMLoggingBaseClass):
start_time,
end_time,
build_logging_payload: bool = True,
):
publish_logging_payload: bool = True,
) -> StandardLoggingPayload | None:
"""Resolve hidden params, compute response cost, and emit the standard logging payload."""
self._surface_response_headers_from_result(logging_result)
hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, {})
@ -2572,17 +2603,17 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
if not build_logging_payload:
return
return None
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logging_result, start_time, end_time
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
standard_logging_payload: Final = self._build_standard_logging_payload(
logging_result, start_time, end_time, isolate_call_details=not publish_logging_payload
)
if not publish_logging_payload:
return standard_logging_payload
self.model_call_details["standard_logging_object"] = standard_logging_payload
if standard_logging_payload is not None:
emit_standard_logging_payload(standard_logging_payload)
return standard_logging_payload
async def _prepare_baseline_cache_estimate(self, response_obj: object) -> None:
if self.baseline_cache_context is None:
@ -2598,12 +2629,16 @@ class Logging(LiteLLMLoggingBaseClass):
await invalidate_baseline_cache(self, reason, completed=completed)
def _build_standard_logging_payload(
self, init_response_obj: object, start_time: dt_object, end_time: dt_object
self,
init_response_obj: object,
start_time: dt_object,
end_time: dt_object,
isolate_call_details: bool = False,
) -> StandardLoggingPayload | None:
"""Build StandardLoggingPayload and accumulate its construction time."""
_start: Final = time.time()
payload: Final = get_standard_logging_object_payload(
kwargs=self.model_call_details,
kwargs=dict(self.model_call_details) if isolate_call_details else self.model_call_details,
init_response_obj=init_response_obj,
start_time=start_time,
end_time=end_time,
@ -2620,11 +2655,16 @@ class Logging(LiteLLMLoggingBaseClass):
emit_standard_logging_payload(stored_payload)
def _transform_usage_objects(self, result: object) -> object:
return self._transform_usage_objects_for_payload(result, self.model_call_details.get("standard_logging_object"))
def _transform_usage_objects_for_payload(
self, result: object, standard_logging_payload: StandardLoggingPayload | None
) -> object:
if isinstance(result, ResponsesAPIResponse):
result = result.model_copy()
transformed_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(result.usage)
setattr(result, "usage", transformed_usage)
if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None:
if standard_logging_payload is not None:
response_dict: Final = result.model_dump() if hasattr(result, "model_dump") else dict(result)
# Ensure usage is properly included with transformed chat format
if transformed_usage is not None:
@ -2633,7 +2673,7 @@ class Logging(LiteLLMLoggingBaseClass):
if hasattr(transformed_usage, "model_dump")
else dict(transformed_usage)
)
standard_logging_payload["response"] = response_dict
standard_logging_payload["response"] = response_dict # rebind-ok: mirrors the transformed usage
elif isinstance(result, TranscriptionResponse):
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
TranscriptionUsageObjectTransformation,
@ -2673,6 +2713,27 @@ class Logging(LiteLLMLoggingBaseClass):
standard_logging_object: StandardLoggingPayload | None = None,
build_logging_payload: bool = True,
) -> tuple[datetime.datetime, datetime.datetime, object]:
prepared: Final = self._prepare_success_logging(
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
standard_logging_object=standard_logging_object,
build_logging_payload=build_logging_payload,
publish_logging_payload=True,
)
return prepared[0], prepared[1], prepared[2]
def _prepare_success_logging(
self,
result: object,
start_time: datetime.datetime | None,
end_time: datetime.datetime | None,
cache_hit: bool | None,
standard_logging_object: StandardLoggingPayload | None,
build_logging_payload: bool,
publish_logging_payload: bool,
) -> tuple[datetime.datetime, datetime.datetime, object, StandardLoggingPayload | None]:
try:
if start_time is None:
start_time = self.start_time
@ -2705,29 +2766,21 @@ class Logging(LiteLLMLoggingBaseClass):
):
result = logging_result
if standard_logging_object is None and result is not None and self.stream is not True:
if self._is_recognized_call_type_for_logging(logging_result=logging_result) or isinstance(
logging_result, (dict, list)
):
self._process_hidden_params_and_response_cost(
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
build_logging_payload=build_logging_payload,
)
elif standard_logging_object is not None:
self.model_call_details["standard_logging_object"] = standard_logging_object
else:
# Streaming reaches here before its cost is known, so the cost
# is seeded to None, but only when nothing has already
# established one. A stream that assembles into a response
# object recomputes the cost right after this; a pass-through
# stream cannot (its body is opaque) and carries the cost its
# upstream reported in the response headers, which an
# unconditional reset would discard.
self.model_call_details.setdefault("response_cost", None)
standard_logging_payload: Final = self._resolve_success_logging_payload(
result=result,
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
standard_logging_object=standard_logging_object,
build_logging_payload=build_logging_payload,
publish_logging_payload=publish_logging_payload,
)
result = self._transform_usage_objects(result=result)
result = (
self._transform_usage_objects(result=result)
if publish_logging_payload
else self._transform_usage_objects_for_payload(result, standard_logging_payload)
)
if (
litellm.max_budget
@ -2746,10 +2799,47 @@ class Logging(LiteLLMLoggingBaseClass):
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
return start_time, end_time, result
return start_time, end_time, result, standard_logging_payload
except Exception as e:
raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {e}")
def _resolve_success_logging_payload(
self,
result: object,
logging_result: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
standard_logging_object: StandardLoggingPayload | None,
build_logging_payload: bool,
publish_logging_payload: bool,
) -> StandardLoggingPayload | None:
if standard_logging_object is not None:
if not publish_logging_payload:
return standard_logging_object.copy()
self.model_call_details["standard_logging_object"] = standard_logging_object
return standard_logging_object
if result is None or self.stream is True:
# Streaming reaches here before its cost is known, so the cost
# is seeded to None, but only when nothing has already
# established one. A stream that assembles into a response
# object recomputes the cost right after this; a pass-through
# stream cannot (its body is opaque) and carries the cost its
# upstream reported in the response headers, which an
# unconditional reset would discard.
self.model_call_details.setdefault("response_cost", None)
return None
if not self._is_recognized_call_type_for_logging(logging_result=logging_result) and not isinstance(
logging_result, (dict, list)
):
return None
return self._process_hidden_params_and_response_cost(
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
build_logging_payload=build_logging_payload,
publish_logging_payload=publish_logging_payload,
)
def _is_recognized_call_type_for_logging(
self,
logging_result: object,
@ -2883,11 +2973,14 @@ class Logging(LiteLLMLoggingBaseClass):
if complete_streaming_response is not None:
self.success_handler(result=complete_streaming_response)
async def async_flush_passthrough_collected_chunks(self, collector: "PassthroughStreamCollector"):
async def async_flush_passthrough_collected_chunks(self, collector: "PassthroughStreamCollector") -> bool:
"""Log the response the collector built from the stream, and say whether there was one to log."""
complete_streaming_response: Final = collector.build_logged_response(litellm_logging_obj=self)
if complete_streaming_response is not None:
await self.async_success_handler(result=complete_streaming_response)
if complete_streaming_response is None:
return False
await self.async_success_handler(result=complete_streaming_response)
return True
def _restore_correlation_context(self) -> None:
"""Restore trace_id/session_id contextvars to their pre-call value.
@ -2931,6 +3024,8 @@ class Logging(LiteLLMLoggingBaseClass):
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
cache_hit: bool | None = None,
*,
for_async_request: bool = False,
**kwargs: object, # kwargs-ok: forwarded to _success_handler_body
) -> None:
"""Restores trace_id/session_id contextvars once this attempt's own success
@ -2938,7 +3033,12 @@ class Logging(LiteLLMLoggingBaseClass):
try:
with post_response_phase():
return self._success_handler_body(
result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
for_async_request=for_async_request,
**kwargs,
)
finally:
self._restore_correlation_context()
@ -2949,17 +3049,20 @@ class Logging(LiteLLMLoggingBaseClass):
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
cache_hit: bool | None = None,
for_async_request: bool = False,
**kwargs: Any, # kwargs-ok: forwarded from success_handler
) -> None:
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
return
start_time, end_time, result = self._success_handler_helper_fn(
start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
start_time=start_time,
end_time=end_time,
result=result,
cache_hit=cache_hit,
standard_logging_object=kwargs.get("standard_logging_object", None),
build_logging_payload=True,
publish_logging_payload=not for_async_request,
)
litellm_params = self.model_call_details.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
@ -2991,16 +3094,45 @@ class Logging(LiteLLMLoggingBaseClass):
)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
if not for_async_request:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
)
if standard_logging_payload is not None:
# Only emit for sync requests (async_success_handler handles async)
if is_sync_request:
emit_standard_logging_payload(standard_logging_payload)
callback_logging_payload: Final = (
self._build_standard_logging_payload(
self.model_call_details["complete_streaming_response"],
start_time,
end_time,
isolate_call_details=True,
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
if for_async_request and complete_streaming_response is not None
else request_logging_payload
)
shared_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
)
isolated_logging_payload: Final = (
callback_logging_payload
if callback_logging_payload is not None or shared_logging_payload is None
else shared_logging_payload.copy()
)
call_details: dict[str, Any] # mutable-ok: legacy loggers take a plain dict
call_details = ( # rebind-ok: logging hooks may return a new dict
self.model_call_details
if not for_async_request
else (
dict(self.model_call_details)
if isolated_logging_payload is None
else {**self.model_call_details, "standard_logging_object": isolated_logging_payload}
)
if standard_logging_payload is not None:
# Only emit for sync requests (async_success_handler handles async)
if is_sync_request:
emit_standard_logging_payload(standard_logging_payload)
)
callbacks: Final = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_success_callbacks,
global_callbacks=litellm.success_callback,
@ -3008,7 +3140,7 @@ class Logging(LiteLLMLoggingBaseClass):
## REDACT MESSAGES ##
result = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
model_call_details=call_details,
result=result,
)
## LOGGING HOOK ##
@ -3018,7 +3150,7 @@ class Logging(LiteLLMLoggingBaseClass):
if (
callback.should_run_guardrail(
data=self.model_call_details,
data=call_details,
event_type=GuardrailEventHooks.logging_only,
)
is not True
@ -3026,18 +3158,21 @@ class Logging(LiteLLMLoggingBaseClass):
continue
self.shadow_eval_request_snapshot = None
self.model_call_details, result = callback.logging_hook(
kwargs=self.model_call_details,
call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict
kwargs=call_details,
result=result,
call_type=self.call_type,
)
elif isinstance(callback, CustomLogger):
self.model_call_details, result = callback.logging_hook(
kwargs=self.model_call_details,
call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict
kwargs=call_details,
result=result,
call_type=self.call_type,
)
if not for_async_request:
self.model_call_details = call_details
call_details_view: Final = MappingProxyType[str, object](call_details)
self.mark_logging_complete(event_type="sync_success")
for callback in callbacks:
try:
@ -3051,7 +3186,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "promptlayer" and promptLayerLogger is not None:
print_verbose("reaches promptlayer for logging!")
promptLayerLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3059,7 +3194,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "supabase" and supabaseClient is not None:
print_verbose("reaches supabase for logging!")
kwargs = self.model_call_details
kwargs = call_details
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
if self.stream:
@ -3067,10 +3202,10 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches supabase for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
model = self._model_call_details_view["model"]
messages = self._model_call_details_view["messages"]
model = call_details_view["model"]
messages = call_details_view["messages"]
optional_params = kwargs.get("optional_params", {})
litellm_params = kwargs.get("litellm_params", {})
supabaseClient.log_event(
@ -3090,7 +3225,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "wandb" and weightsBiasesLogger is not None:
print_verbose("reaches wandb for logging!")
weightsBiasesLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3099,7 +3234,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "logfire" and logfireLogger is not None:
verbose_logger.debug("reaches logfire for success logging!")
kwargs = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine
kwargs[k] = v
@ -3109,10 +3244,10 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches logfire for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
logfireLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3123,7 +3258,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "lunary" and lunaryLogger is not None:
print_verbose("reaches lunary for logging!")
model = self.model
kwargs = self.model_call_details
kwargs = call_details
input = kwargs.get("messages", kwargs.get("input", None))
@ -3134,7 +3269,7 @@ class Logging(LiteLLMLoggingBaseClass):
if "complete_streaming_response" not in kwargs:
continue
else:
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
lunaryLogger.log_event(
type=type,
@ -3144,7 +3279,7 @@ class Logging(LiteLLMLoggingBaseClass):
input=input,
user_id=kwargs.get("user", None),
# user_props=self.model_call_details.get("user_props", None),
extra=self._model_call_details_view.get("optional_params", {}),
extra=call_details_view.get("optional_params", {}),
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3154,8 +3289,8 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "helicone" and heliconeLogger is not None:
print_verbose("reaches helicone for logging!")
model = self.model
messages = self._model_call_details_view["input"]
kwargs = self.model_call_details
messages = call_details_view["input"]
kwargs = call_details
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
if self.stream:
@ -3163,7 +3298,7 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches helicone for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
heliconeLogger.log_success(
model=model,
@ -3178,7 +3313,7 @@ class Logging(LiteLLMLoggingBaseClass):
global langFuseLogger
print_verbose("reaches langfuse for success logging!")
kwargs = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine
kwargs[k] = v
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
@ -3191,7 +3326,7 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches langfuse for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
globalLangfuseLogger=langFuseLogger,
@ -3216,7 +3351,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "greenscale" and greenscaleLogger is not None:
kwargs = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine
kwargs[k] = v
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
@ -3229,7 +3364,7 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches greenscale for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
greenscaleLogger.log_event(
kwargs=kwargs,
@ -3240,7 +3375,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "athina" and athinaLogger is not None:
deep_copy = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
deep_copy[k] = v
athinaLogger.log_event(
kwargs=deep_copy,
@ -3251,7 +3386,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "traceloop":
deep_copy = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response":
deep_copy[k] = v
traceloopLogger.log_event(
@ -3267,11 +3402,11 @@ class Logging(LiteLLMLoggingBaseClass):
if s3Logger is None:
s3Logger = S3Logger()
if self.stream:
if "complete_streaming_response" in self.model_call_details:
if "complete_streaming_response" in call_details:
print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response")
s3Logger.log_event(
kwargs=self.model_call_details,
response_obj=self.model_call_details["complete_streaming_response"],
kwargs=call_details,
response_obj=call_details["complete_streaming_response"],
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
@ -3280,7 +3415,7 @@ class Logging(LiteLLMLoggingBaseClass):
print_verbose("S3Logger Logger: Got Stream Event - No complete stream response as yet")
else:
s3Logger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3294,19 +3429,17 @@ class Logging(LiteLLMLoggingBaseClass):
openMeterLogger = OpenMeterLogger()
if self.stream and complete_streaming_response is None:
openMeterLogger.log_stream_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self._model_call_details_view["complete_response"]
call_details["complete_response"] = call_details.get("complete_streaming_response", {})
result = call_details_view["complete_response"]
openMeterLogger.log_success_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3319,20 +3452,18 @@ class Logging(LiteLLMLoggingBaseClass):
): # custom logger class
if self.stream and complete_streaming_response is None:
callback.log_stream_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self._model_call_details_view["complete_response"]
call_details["complete_response"] = call_details.get("complete_streaming_response", {})
result = call_details_view["complete_response"]
callback.log_success_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3343,7 +3474,7 @@ class Logging(LiteLLMLoggingBaseClass):
print_verbose(f"success callbacks: Running Custom Callback Function - {callback}")
customLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -4184,8 +4315,18 @@ class Logging(LiteLLMLoggingBaseClass):
start_time,
end_time,
cache_hit,
for_async_request=True,
)
def handle_sync_cache_hit_success_callbacks(
self,
result: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
cache_hit: bool | None = None,
) -> None:
executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
def _should_run_sync_callbacks_for_async_calls(self) -> bool:
"""
Returns:
@ -4197,7 +4338,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
_filtered_success_callbacks = self._remove_internal_custom_logger_callbacks(_combined_sync_callbacks)
_filtered_success_callbacks = self._remove_internal_litellm_callbacks(_filtered_success_callbacks)
return len(_filtered_success_callbacks) > 0
return any(
not (isinstance(callback, str) and callback in INTERNAL_CALLBACK_NAMES)
for callback in _filtered_success_callbacks # pyright: ignore[reportUnknownVariableType] # callback lists are untyped
)
def _should_run_sync_failure_callbacks_for_async_calls(self) -> bool:
"""
@ -6805,6 +6949,36 @@ def _autorouter_savings_for_payload(
return None
_CACHE_HIT_ZEROED_AMOUNTS: Final = MappingProxyType(
{
"cache_read_cost": CostBreakdown(cache_read_cost=0.0),
"cache_creation_cost": CostBreakdown(cache_creation_cost=0.0),
"reasoning_cost": CostBreakdown(reasoning_cost=0.0),
"original_cost": CostBreakdown(original_cost=0.0),
"discount_amount": CostBreakdown(discount_amount=0.0),
"margin_fixed_amount": CostBreakdown(margin_fixed_amount=0.0),
"margin_total_amount": CostBreakdown(margin_total_amount=0.0),
}
)
def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
"""Every amount at 0, keeping the pricing basis and rates a fresh call would have had, which savings reports read."""
zero_costs: Final = CostBreakdown(input_cost=0.0, output_cost=0.0, total_cost=0.0, tool_usage_cost=0.0)
if served is None:
return zero_costs
zeroed_additional_costs: Final = (
CostBreakdown(additional_costs=dict.fromkeys(served["additional_costs"], 0.0))
if "additional_costs" in served
else CostBreakdown()
)
return functools.reduce(
lambda breakdown, zeroed: breakdown | zeroed,
(_CACHE_HIT_ZEROED_AMOUNTS[key] for key in served if key in _CACHE_HIT_ZEROED_AMOUNTS),
served | zero_costs | zeroed_additional_costs,
)
def get_standard_logging_object_payload(
kwargs: dict | None,
init_response_obj: object,
@ -6888,17 +7062,10 @@ def get_standard_logging_object_payload(
"user", None
) # maintain backwards compatibility with old request body check
saved_cache_cost: float = 0.0
if cache_hit is True or served_from_cache(init_response_obj):
id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id
if cache_hit is True:
saved_cache_cost = (
logging_obj.response_cost_calculator(
result=init_response_obj,
cache_hit=False,
)
or 0.0
)
uncached_price: Final = logging_obj.uncached_price(result=init_response_obj) if cache_hit is True else None
saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0
## Get model cost information ##
base_model = get_base_model_from_metadata(model_call_details=kwargs)
@ -6915,7 +7082,7 @@ def get_standard_logging_object_payload(
# clean up litellm hidden params
clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
if clean_hidden_params["response_cost"] is None and raw_response_cost is not None:
if cache_hit is True or (clean_hidden_params["response_cost"] is None and raw_response_cost is not None):
clean_hidden_params["response_cost"] = llm_response_cost
if clean_hidden_params["litellm_overhead_time_ms"] is None and status == "success":
# /v1/messages dict results and the bridge stream wrappers keep it on the logging object;
@ -6977,7 +7144,14 @@ def get_standard_logging_object_payload(
):
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
request_cost_breakdown: Final = cost_breakdown_with_guardrail(
(
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
if uncached_price is not None
else logging_obj.cost_breakdown
),
guardrail_cost,
)
captured_baseline: Final = logging_obj.baseline_observation
autorouter_savings: Final = (
None

72
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -251,14 +251,7 @@ async def convert_to_streaming_response_async(
setattr(
model_response_object,
"usage",
Usage(
completion_tokens=response_object["usage"].get("completion_tokens", 0),
prompt_tokens=response_object["usage"].get("prompt_tokens", 0),
total_tokens=response_object["usage"].get("total_tokens", 0),
completion_tokens_details=_cached_completion_tokens_details(
_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])
),
),
_replayed_usage(_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])),
)
if "id" in response_object:
@ -345,12 +338,10 @@ def convert_to_streaming_response(
model_response_object.choices = choice_list
if "usage" in response_object and response_object["usage"] is not None:
setattr(model_response_object, "usage", Usage())
model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0)
model_response_object.usage.completion_tokens_details = _cached_completion_tokens_details(
_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])
setattr(
model_response_object,
"usage",
_replayed_usage(_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])),
)
if "id" in response_object:
@ -414,13 +405,28 @@ class _CachedMessageProviderFields(TypedDict, total=False):
class _CachedUsageDetails(TypedDict, total=False):
completion_tokens: ReadOnly[object]
prompt_tokens: ReadOnly[object]
total_tokens: ReadOnly[object]
completion_tokens_details: ReadOnly[object]
prompt_tokens_details: ReadOnly[object]
cache_read_input_tokens: ReadOnly[object]
cache_creation_input_tokens: ReadOnly[object]
class _PromptCacheTokenCounts(TypedDict, total=False):
cache_read_input_tokens: ReadOnly[int]
cache_creation_input_tokens: ReadOnly[int]
_CACHED_MESSAGE_PROVIDER_FIELDS: Final = TypeAdapter(_CachedMessageProviderFields)
_CACHED_USAGE_DETAILS: Final = TypeAdapter(_CachedUsageDetails)
_FIELD_MAP: Final = TypeAdapter(dict[str, object])
_COMPLETION_TOKENS_DETAILS: Final = TypeAdapter(CompletionTokensDetailsWrapper)
_PROMPT_TOKENS_DETAILS: Final = TypeAdapter(PromptTokensDetailsWrapper)
_PROMPT_CACHE_TOKEN_COUNTS: Final = TypeAdapter(_PromptCacheTokenCounts)
_TOKEN_COUNT: Final = TypeAdapter(int)
_PROMPT_CACHE_TOKEN_FIELDS: Final = ("cache_read_input_tokens", "cache_creation_input_tokens")
def _cached_provider_specific_fields(message: _CachedMessageProviderFields) -> Mapping[str, object] | None:
@ -438,6 +444,44 @@ def _cached_completion_tokens_details(usage: _CachedUsageDetails) -> CompletionT
return None
def _cached_prompt_tokens_details(usage: _CachedUsageDetails) -> PromptTokensDetailsWrapper | None:
details: Final = usage.get("prompt_tokens_details")
if not isinstance(details, dict):
return None
try:
return _PROMPT_TOKENS_DETAILS.validate_python(details)
except ValidationError:
return None
def _cached_token_count(count: object) -> int | None:
try:
return _TOKEN_COUNT.validate_python(count)
except ValidationError:
return None
def _cached_prompt_cache_token_counts(usage: _CachedUsageDetails) -> _PromptCacheTokenCounts:
return _PROMPT_CACHE_TOKEN_COUNTS.validate_python(
{
field: count
for field in _PROMPT_CACHE_TOKEN_FIELDS
if (count := _cached_token_count(usage.get(field))) is not None
}
)
def _replayed_usage(usage: _CachedUsageDetails) -> Usage:
return Usage(
completion_tokens=_cached_token_count(usage.get("completion_tokens")) or 0,
prompt_tokens=_cached_token_count(usage.get("prompt_tokens")) or 0,
total_tokens=_cached_token_count(usage.get("total_tokens")) or 0,
completion_tokens_details=_cached_completion_tokens_details(usage),
prompt_tokens_details=_cached_prompt_tokens_details(usage),
**_cached_prompt_cache_token_counts(usage),
)
def handle_invalid_parallel_tool_calls(
tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall],
) -> list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None:

120
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -9,7 +9,8 @@ import json
import mimetypes
import re
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from itertools import groupby, islice
from graphlib import CycleError, TopologicalSorter
from itertools import chain, groupby, islice
from os import PathLike
from pathlib import Path
from types import MappingProxyType
@ -1426,6 +1427,123 @@ def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str,
frontier = tuple(child for node in frontier for child in _subschemas(node))
def schema_fragment_ref(ref: str) -> str:
return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref
def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool:
definitions: Final = MappingProxyType(dict(_root_definitions(schema)))
if not _refs_in(schema) <= definitions.keys():
return True
try:
tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order())
except CycleError:
return True
return False
def schema_union_property_count(schema: Mapping[str, object]) -> int:
return sum(1 for _, node in _property_schemas(schema) if _is_union(node))
def schema_optional_property_count(schema: Mapping[str, object]) -> int:
return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema))
def schema_property_count(schema: Mapping[str, object]) -> int:
return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties")))
def schema_grammar_branch_count(schema: Mapping[str, object]) -> int:
return sum(_grammar_branches(node) for node in _schema_nodes(schema))
def schema_has_open_object(schema: Mapping[str, object]) -> bool:
return any(_is_open_object(node) for node in _schema_nodes(schema))
def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool:
return any(
isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern)
for node in _schema_nodes(schema)
)
def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str)
)
def _may_be_string(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type is None or node_type == "string" or "string" in _list_at(node, "type")
def is_object_schema(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type == "object" or (isinstance(node_type, list) and "object" in node_type)
def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))
_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...])
def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]:
value: Final = container.get(key)
return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else ()
def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]:
if container is None:
return iter(())
return ((key, value) for key in container if (value := _schema_at(container, key)) is not None)
def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
for defs_key in ("$defs", "definitions"):
yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key)))
def _refs_in(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str)
)
def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema))
def _is_union(node: Mapping[str, object]) -> bool:
return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1
def _grammar_branches(node: Mapping[str, object]) -> int:
properties: Final = _schema_at(node, "properties")
if properties is None:
return 0
optional: Final = len(_optional_property_names(node))
return 2**optional * (len(properties) - optional + 1)
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
properties: Final = _schema_at(node, "properties")
if properties is None:
return frozenset()
return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str))
def _is_open_object(node: Mapping[str, object]) -> bool:
additional_properties: Final = node.get("additionalProperties")
if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node:
return True
return is_object_schema(node) and "properties" not in node and additional_properties is not False
def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
for key, value in node.items():
if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict):

4
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -5504,14 +5504,14 @@ def _bedrock_tools_pt(
description = name
defs = parameters.pop("$defs", {})
defs_copy = copy.deepcopy(defs)
defs_copy = copy.deepcopy(defs) if isinstance(defs, dict) else {}
# Expand $ref references in parameters using the definitions
# Note: We don't pre-flatten defs as that causes exponential memory growth
# with circular references (see issue #19098). unpack_defs handles nested
# refs recursively and correctly detects/skips circular references.
unpack_defs(parameters, defs_copy)
normalize_json_schema_custom_types_to_object(parameters)
if parameters.get("type") not in _valid_json_schema_root_types:
if not (isinstance(root_type := parameters.get("type"), str) and root_type in _valid_json_schema_root_types):
parameters["type"] = "object"
tool_block = cast(
BedrockToolBlock,

23
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1,11 +1,11 @@
import asyncio
import json
import traceback
from collections.abc import Callable, Coroutine, Mapping, Sequence
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable
from pydantic import TypeAdapter
from typing_extensions import ReadOnly
@ -14,6 +14,7 @@ import litellm
from litellm._logging import redact_internal_details_from_client_message, verbose_logger
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
REALTIME_METERED_USAGE_KEY,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
@ -100,6 +101,16 @@ class ScopedWebSocket(Protocol):
def scope(self) -> _ASGIScope: ...
@runtime_checkable
class _ReplayScopeWebSocket(Protocol):
@property
def scope(self) -> MutableMapping[str, object]: ...
def client_websocket_scope(websocket: object) -> MutableMapping[str, object] | None:
return websocket.scope if isinstance(websocket, _ReplayScopeWebSocket) else None
class _ClientWebSocket(ScopedWebSocket, Protocol):
async def send_text(self, data: str) -> None: ...
async def receive_text(self) -> str: ...
@ -233,6 +244,7 @@ class RealTimeStreaming:
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
):
self.websocket: _ClientWebSocket = websocket
self._replay_scope: Final = client_websocket_scope(websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
self._logging_worker = logging_worker
@ -1451,6 +1463,13 @@ class RealTimeStreaming:
return item
async def _receive_client_message(self) -> str | None:
refused_opening_message: Final = (
None
if self._replay_scope is None
else self._replay_scope.pop(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, None)
)
if isinstance(refused_opening_message, str):
return refused_opening_message
try:
return await self.websocket.receive_text()
except Exception as e: # noqa: BLE001 # whatever the client socket raises, the client is gone

24
litellm/litellm_core_utils/streaming_handler.py
View file

@ -47,7 +47,7 @@ from litellm.types.utils import GenericStreamingChunk as GChunk
from ..exceptions import OpenAIError
from .core_helpers import map_finish_reason, process_response_headers
from .exception_mapping_utils import exception_type
from .exception_mapping_utils import exception_type, with_deployment_api_base
from .llm_response_utils.get_api_base import get_api_base
from .rules import Rules
@ -1579,11 +1579,7 @@ class CustomStreamWrapper:
except Exception as e:
traceback.format_exc()
setattr(e, "message", str(e))
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=e,
)
raise self._map_stream_exception(e)
def set_logging_event_loop(self, loop):
"""
@ -2370,6 +2366,14 @@ class CustomStreamWrapper:
if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta):
self._emitted_disqualifying_content = True
def _map_stream_exception(self, original_exception: Exception) -> Exception:
return exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=original_exception,
extra_kwargs=with_deployment_api_base(self.logging_obj.model_call_details.get("litellm_params")),
)
def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
"""
Common error handling for both __next__ and __anext__.
@ -2414,13 +2418,7 @@ class CustomStreamWrapper:
mapped_exception: Exception = e
else:
try:
mapped_exception = exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
mapped_exception = self._map_stream_exception(e)
except Exception as mapping_error:
mapped_exception = mapping_error
self._restore_consumer_correlation_context()

62
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -30,6 +30,7 @@ from litellm.llms.anthropic.pass_through.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
is_provider_native_tool_dict,
)
from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.pass_through.stream_assembly import (
build_complete_streaming_response,
build_usage_only_response_from_chunks,
@ -1456,6 +1457,9 @@ class AnthropicMessagesHandler(BaseTranslation):
return _SSEFieldRewrite("delta", "text", next(replacements))
cls._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta)
cls._redact_ended_stream_message_delta(
responses_so_far, kept_tool_use_ids=frozenset(cls._streamed_tool_use_ids(responses_so_far).values())
)
@classmethod
def _write_ended_stream_tool_call_rewrites(
@ -1523,6 +1527,64 @@ class AnthropicMessagesHandler(BaseTranslation):
return None
cls._rewrite_ended_stream_events(responses_so_far, rewrite_tool_use)
streamed_tool_use_ids: Final = cls._streamed_tool_use_ids(responses_so_far)
cls._redact_ended_stream_message_delta(
responses_so_far,
kept_tool_use_ids=frozenset(
tool_use_id for index, tool_use_id in streamed_tool_use_ids.items() if index not in rewrites_by_block
),
)
@classmethod
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
"""The id of each buffered ``tool_use`` block, by its content block index."""
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
tool_use_blocks: Final = (
(event.get("index"), block)
for event in events
if event.get("type") == "content_block_start"
and (block := as_str_mapping(event.get("content_block"))) is not None
and block.get("type") == "tool_use"
)
return MappingProxyType(
{
index: tool_use_id
for index, block in tool_use_blocks
if isinstance(index, int) and isinstance(tool_use_id := block.get("id"), str)
}
)
@classmethod
def _redact_ended_stream_message_delta(
cls,
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
*,
kept_tool_use_ids: frozenset[str],
) -> None:
"""Hold the buffered ``message_delta`` to the rules a re-emitted rewrite follows: a verdict survives
only for a tool call delivered unchanged, and no free-text ``explanation`` describes the text the
guardrail rewrote."""
def rewrite_stop_details(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
delta: Final = as_str_mapping(event.get("delta")) if event.get("type") == "message_delta" else None
if delta is None or "stop_details" not in delta:
return None
return _SSEFieldRewrite(
"delta", "stop_details", redacted_stop_details(delta["stop_details"], keep_explanations=False)
)
def rewrite_safeguard_results(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
delta: Final = as_str_mapping(event.get("delta")) if event.get("type") == "message_delta" else None
if delta is None or "safeguard_results" not in delta:
return None
return _SSEFieldRewrite(
"delta",
"safeguard_results",
filtered_safeguard_results(delta["safeguard_results"], kept_tool_use_ids, keep_explanations=False),
)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_stop_details)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_safeguard_results)
@staticmethod
def _rewrite_ended_stream_events(

114
litellm/llms/anthropic/chat/transformation.py
View file

@ -1,3 +1,4 @@
import copy
import json
import re
import time
@ -24,7 +25,18 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
is_object_schema,
sanitize_input_schema_for_anthropic,
schema_fragment_ref,
schema_grammar_branch_count,
schema_has_lookaround_pattern,
schema_has_open_object,
schema_has_unresolvable_ref,
schema_optional_property_count,
schema_string_formats,
schema_union_property_count,
unpack_defs,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
RemoteMedia,
@ -43,7 +55,7 @@ from litellm.llms.base_llm.base_utils import (
type_to_response_format_param,
unwrap_response_format_envelope,
)
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
@ -150,6 +162,12 @@ _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128
_DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
NATIVE_OUTPUT_MAX_UNION_PARAMETERS: Final = 16
NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24
NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048
def _tool_type(tool: object) -> str | None:
@ -753,7 +771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
grammar Anthropic compiles the schema into, so the API rejects them with a
400 ``invalid_request_error`` (e.g. "output_format.schema: For 'array' type,
property 'uniqueItems' is not supported"):
- maxItems/minItems/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints
- maxItems/minItems above 1/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints
- minimum/maximum/exclusiveMinimum/exclusiveMaximum/multipleOf: numeric constraints
- minLength/maxLength: string constraints
- minProperties/maxProperties/patternProperties/propertyNames: object constraints
@ -810,7 +828,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"else": "conditional (else): {}",
"not": "must not match: {}",
}
unsupported_fields: Final = set(constraint_labels)
min_items: Final = schema.get("minItems")
enforced_min_items: Final = (
isinstance(min_items, int) and not isinstance(min_items, bool) and min_items in (0, 1)
)
unsupported_fields: Final = set(constraint_labels) - ({"minItems"} if enforced_min_items else set())
# Build description additions from removed constraints. Iterating
# constraint_labels (not the set) keeps the note order deterministic across
@ -818,7 +840,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# PYTHONHASHSEED and stay cache-friendly.
constraint_descriptions: Final[list] = []
for field, label in constraint_labels.items():
if field not in schema:
if field not in schema or field not in unsupported_fields:
continue
value = schema[field]
# A falsy boolean constraint (e.g. ``uniqueItems: false``) imposes no
@ -853,7 +875,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Already handled above
continue
if key in ("properties", "$defs") and isinstance(value, dict):
if key in ("properties", "$defs", "definitions") and isinstance(value, dict):
nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict
result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()}
elif key == "items" and isinstance(value, dict):
@ -865,6 +887,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
]
elif key == "allOf" and isinstance(value, list):
result[key] = [AnthropicConfig.filter_anthropic_output_schema(item) for item in value]
elif key == "$ref" and isinstance(value, str):
result[key] = schema_fragment_ref(value)
elif key == "oneOf" and isinstance(value, list):
# Anthropic rejects oneOf ("Schema type 'oneOf' is not supported");
# the Anthropic SDK rewrites it to anyOf, so do the same.
@ -876,7 +900,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Anthropic requires additionalProperties=false for object schemas
# See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs
if result.get("type") == "object":
if is_object_schema(result):
result["additionalProperties"] = False
return result
@ -1628,32 +1652,55 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return json_schema
@staticmethod
def _defs_unpacked(json_schema: Mapping[str, object]) -> Mapping[str, object]:
unpacked: Final = copy.deepcopy(dict(json_schema))
defs: Final = unpacked.pop("$defs", unpacked.pop("definitions", None))
if isinstance(defs, dict) and defs:
unpack_defs(unpacked, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES)
return unpacked
@staticmethod
def native_output_schema(json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
if (
schema_has_unresolvable_ref(json_schema)
or schema_has_open_object(json_schema)
or schema_has_lookaround_pattern(json_schema)
or not schema_string_formats(json_schema) <= _NATIVE_OUTPUT_STRING_FORMATS
or schema_union_property_count(json_schema) > NATIVE_OUTPUT_MAX_UNION_PARAMETERS
or schema_optional_property_count(json_schema) > NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS
or schema_grammar_branch_count(json_schema) > NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES
):
return None
return AnthropicConfig.filter_anthropic_output_schema(dict(json_schema))
def uses_native_structured_output_for_schema(
self,
model: str,
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
open_object_takes_forced_json_tool: Final = (
json_schema is not None
and schema_has_open_object(json_schema)
and not self.is_thinking_requested(request_params)
and not self.is_thinking_enabled(dict(request_params))
and not request_params.get("tools")
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
)
return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema)))
def map_response_format_to_anthropic_output_format(self, value: dict | None) -> AnthropicOutputSchema | None:
json_schema: dict | None = self._extract_json_schema_from_response_format(value)
json_schema: Final = self._extract_json_schema_from_response_format(value)
if json_schema is None:
return None
# Resolve $ref/$defs before filtering — Anthropic doesn't support
# external schema references (e.g., /$defs/CalendarEvent).
import copy
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
unpack_defs,
)
json_schema = copy.deepcopy(json_schema)
defs: Final = json_schema.pop("$defs", json_schema.pop("definitions", {}))
if defs:
unpack_defs(json_schema, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES)
# Filter out unsupported fields for Anthropic's output_format API
filtered_schema: Final = self.filter_anthropic_output_schema(json_schema)
return AnthropicOutputSchema(
type="json_schema",
schema=filtered_schema,
)
native_schema: Final = self.native_output_format_schema(json_schema)
if native_schema is None:
return None
return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema))
def map_response_format_to_anthropic_tool(
self, value: dict | None, optional_params: dict, is_thinking_enabled: bool
@ -1820,7 +1867,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
output_key=param,
)
elif param == "response_format" and isinstance(value, dict):
if self.uses_native_structured_output(model):
if self.uses_native_structured_output_for_schema(
model,
self._extract_json_schema_from_response_format(value),
REQUEST_PARAMS.validate_python(non_default_params),
):
_output_format = self.map_response_format_to_anthropic_output_format(value)
if _output_format is not None:
optional_params["output_format"] = _output_format
@ -2979,6 +3030,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
completion_response["stop_reason"] = "stop"
if refusal_fields:
json_mode_message.provider_specific_fields = dict(refusal_fields)
if thinking_blocks:
json_mode_message.thinking_blocks = thinking_blocks
json_mode_message.reasoning_content = reasoning_content
_message = json_mode_message
model_response.choices[0].message = _message

15
litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py
View file

@ -12,6 +12,7 @@ import json
from collections.abc import Mapping
from typing import Any, Final, cast
from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -34,8 +35,13 @@ class FakeAnthropicMessagesStreamIterator:
- message_stop
"""
def __init__(self, response: AnthropicMessagesResponse):
def __init__(
self,
response: AnthropicMessagesResponse,
message_delta_fields: Mapping[str, object] | None = None,
):
self.response = response
self.message_delta_fields = message_delta_fields
self.chunks = self._create_streaming_chunks()
self.current_index = 0
@ -175,14 +181,9 @@ class FakeAnthropicMessagesStreamIterator:
cache_read_input_tokens: Final = usage.get("cache_read_input_tokens")
if cache_read_input_tokens is not None:
delta_usage["cache_read_input_tokens"] = cache_read_input_tokens
safeguard_results: Final = self.response.get("safeguard_results")
message_delta: Final = {
"type": "message_delta",
"delta": {
"stop_reason": self.response.get("stop_reason"),
"stop_sequence": self.response.get("stop_sequence"),
**({"safeguard_results": safeguard_results} if safeguard_results is not None else {}),
},
"delta": anthropic_message_delta(response_dict, self.message_delta_fields),
"usage": delta_usage,
}
chunks.append(f"event: message_delta\ndata: {json.dumps(message_delta)}\n\n".encode())

26
litellm/llms/anthropic/pass_through/messages/streaming_iterator.py
View file

@ -18,7 +18,11 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
from litellm.llms.anthropic.pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE
from litellm.llms.anthropic.pass_through.messages.utils import (
INCOMPLETE_STREAM_ERROR_MESSAGE,
MESSAGE_DELTA_OPTIONAL_KEYS,
anthropic_message_delta,
)
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
from litellm.proxy.pass_through_endpoints.success_handler import (
@ -280,30 +284,32 @@ def anthropic_messages_response_as_sse_events(response: AnthropicMessagesRespons
content_events: Final = (
event for index, block in enumerate(content_blocks) for event in _anthropic_content_block_events(index, block)
)
# A real message_start always carries a null stop_reason/stop_sequence and
# a zero output_tokens - those are only known once generation finishes, so
# copying the completed response's final values here would let a client
# treat the message as already finished, or double-count output tokens.
# A real message_start always carries a null stop_reason/stop_sequence/stop_details,
# a zero output_tokens and no safeguard_results - those are only known once
# generation finishes, so copying the completed response's final values here
# would let a client treat the message as already finished, or double-count
# output tokens.
message_start_usage: Final = {
**(response.get("usage") or {}),
"output_tokens": 0,
}
message_start_fields: Final = {
key: value for key, value in response.items() if key not in MESSAGE_DELTA_OPTIONAL_KEYS
}
message_start_payload: Final = {
"type": "message_start",
"message": {
**response,
**message_start_fields,
"content": [],
"stop_reason": None,
"stop_sequence": None,
"stop_details": None,
"usage": message_start_usage,
},
}
message_delta_payload: Final = {
"type": "message_delta",
"delta": {
"stop_reason": response.get("stop_reason"),
"stop_sequence": response.get("stop_sequence"),
},
"delta": anthropic_message_delta(response),
"usage": response.get("usage") or {},
}
return (

11
litellm/llms/anthropic/pass_through/messages/transformation.py
View file

@ -4,6 +4,7 @@ from typing import ClassVar, Final
import httpx
from litellm._logging import verbose_logger
from litellm.anthropic_beta_headers_manager import is_beta_header_supported_by_bundled_config
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -708,6 +709,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params: dict,
custom_llm_provider: str = "anthropic",
messages: Sequence[object] = (),
beta_map_provider: str | None = None,
) -> dict:
"""
Auto-inject anthropic-beta headers based on features used.
@ -724,8 +726,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params: Optional parameters including tools, context_management, output_format, speed
custom_llm_provider: Provider name for looking up correct tool search header
messages: Request messages, scanned for per-message output_config
beta_map_provider: Provider whose beta map column gates safeguards, defaults to custom_llm_provider
"""
beta_values: Final[set] = set()
beta_values: Final[set[str]] = set()
existing_beta: Final = tuple(
piece.strip()
@ -775,6 +778,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if optional_params.get("speed") == "fast":
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value)
if optional_params.get("safeguards") is not None and is_beta_header_supported_by_bundled_config(
ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value,
beta_map_provider or custom_llm_provider,
):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)

91
litellm/llms/anthropic/pass_through/messages/utils.py
View file

@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Final, cast, get_type_hints
from pydantic import JsonValue
from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.core_helpers import as_str_mapping
@ -49,6 +49,95 @@ def refusal_stop_details(explanation: str | None) -> AnthropicStopDetails:
return AnthropicStopDetails(type="refusal", category=None, explanation=explanation)
MESSAGE_DELTA_OPTIONAL_KEYS: Final = ("stop_details", "safeguard_results")
def anthropic_message_delta(
response: Mapping[str, object], optional_fields: Mapping[str, object] | None = None
) -> Mapping[str, object]:
"""
The ``delta`` of the ``message_delta`` event synthesized from a complete
Anthropic Messages response. A real stream carries ``stop_details`` and
``safeguard_results`` there, so a client that asked to stream reads them
from this event and nowhere else. ``optional_fields`` replaces the
response's own when it was rebuilt from a stream that carried them.
"""
optional_source: Final = response if optional_fields is None else optional_fields
present_optional: Final = {
key: value for key in MESSAGE_DELTA_OPTIONAL_KEYS if (value := optional_source.get(key)) is not None
}
return {
"stop_reason": response.get("stop_reason"),
"stop_sequence": response.get("stop_sequence"),
**present_optional,
}
_SAFEGUARD_RESULT_ITEMS: Final = TypeAdapter(tuple[object, ...])
def redacted_stop_details(raw: object, keep_explanations: bool) -> object:
"""``stop_details`` as a rewritten stream may deliver it: its free-text ``explanation`` can quote
the text a guardrail rewrote, so it is nulled unless the rewrite changed no text."""
stop_details: Final = as_str_mapping(raw)
if keep_explanations or stop_details is None or "explanation" not in stop_details:
return raw
return {**stop_details, "explanation": None}
def _without_explanation(raw: object) -> object:
mapping: Final = as_str_mapping(raw)
if mapping is None:
return raw
return {key: value for key, value in mapping.items() if key != "explanation"}
def _status_without_explanations(raw: object) -> object:
status: Final = as_str_mapping(_without_explanation(raw))
if status is None:
return raw
tool_uses: Final = as_str_mapping(status.get("tool_uses"))
if tool_uses is None:
return status
return {
**status,
"tool_uses": {tool_use_id: _without_explanation(verdict) for tool_use_id, verdict in tool_uses.items()},
}
def _result_without_explanations(raw: object) -> object:
result: Final = as_str_mapping(_without_explanation(raw))
if result is None:
return raw
if "status" not in result:
return result
return {**result, "status": _status_without_explanations(result["status"])}
def _kept_tool_use_verdicts(raw: object, kept_tool_use_ids: frozenset[str]) -> object:
result: Final = as_str_mapping(raw)
status: Final = as_str_mapping(result.get("status")) if result is not None else None
tool_uses: Final = as_str_mapping(status.get("tool_uses")) if status is not None else None
if result is None or status is None or tool_uses is None:
return raw
kept_tool_uses: Final = {
tool_use_id: verdict for tool_use_id, verdict in tool_uses.items() if tool_use_id in kept_tool_use_ids
}
return {**result, "status": {**status, "tool_uses": kept_tool_uses}}
def filtered_safeguard_results(raw: object, kept_tool_use_ids: frozenset[str], keep_explanations: bool) -> object:
"""``safeguard_results`` as a rewritten stream may deliver them: a ``tool_uses`` verdict survives only
for a tool call the client gets with the input the classifier judged, and the free-text ``explanation`` of a
result, its status and each verdict is dropped unless the rewrite changed no text."""
try:
results: Final = _SAFEGUARD_RESULT_ITEMS.validate_python(raw)
except ValidationError:
return raw if keep_explanations else None
kept: Final = tuple(_kept_tool_use_verdicts(result, kept_tool_use_ids) for result in results)
return kept if keep_explanations else tuple(_result_without_explanations(result) for result in kept)
def _mapping_field(container: object, key: str) -> object | None:
"""One key of a raw provider payload, or None when the payload is not a mapping."""
if not isinstance(container, Mapping):

25
litellm/llms/anthropic/pass_through/stream_assembly.py
View file

@ -1,16 +1,18 @@
import codecs
import json
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import as_str_mapping, map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.chat.handler import (
ModelResponseIterator as AnthropicModelResponseIterator,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.pass_through.messages.utils import MESSAGE_DELTA_OPTIONAL_KEYS
from litellm.types.utils import (
Choices,
Message,
@ -447,6 +449,27 @@ def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None:
)
def anthropic_message_delta_fields(all_chunks: Sequence[object]) -> Mapping[str, object]:
"""The ``stop_details`` and ``safeguard_results`` of the stream's ``message_delta``.
The assembled OpenAI shape has no slot for them, and a client that asked to stream reads them
from that event and nowhere else (a Claude Code auto-mode session takes its classifier verdict
from ``safeguard_results``), so a caller re-emitting a rewritten response merges them back in.
"""
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return MappingProxyType({})
message_delta: Final = next(
(event for event in reversed(_parsed_sse_events(sse_stream)) if event.get("type") == "message_delta"), None
)
delta: Final = as_str_mapping(message_delta.get("delta")) if message_delta is not None else None
if delta is None:
return MappingProxyType({})
return MappingProxyType(
{key: value for key in MESSAGE_DELTA_OPTIONAL_KEYS if (value := delta.get(key)) is not None}
)
def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
"""Whether raw SSE frames are Anthropic Messages events.

11
litellm/llms/azure_ai/chat/transformation.py
View file

@ -46,14 +46,10 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
)
def _is_grok_model(model: str) -> bool:
return "grok" in model
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
usage: Final = chunk.get("usage")
if _is_grok_model(str(chunk.get("model") or "")) and isinstance(usage, dict):
if isinstance(usage, dict):
XAIChatConfig.fold_reasoning_tokens_into_completion(usage)
return super().chunk_parser(chunk)
@ -127,7 +123,7 @@ class AzureAIStudioConfig(OpenAIConfig):
"""
Check if the model supports stop tokens.
"""
if _is_grok_model(model):
if "grok" in model:
# Reuse Xai method for Grok model
xai_config: Final = XAIChatConfig()
return xai_config.supports_stop_reason(model)
@ -345,8 +341,7 @@ class AzureAIStudioConfig(OpenAIConfig):
api_key=api_key,
json_mode=json_mode,
)
if _is_grok_model(model):
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
return response
def get_model_response_iterator(

69
litellm/llms/azure_ai/decisions/transformation.py
View file

@ -1,11 +1,21 @@
import asyncio
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import httpx
from pydantic import ValidationError
import litellm
from litellm.llms.azure_ai.common_utils import (
AZURE_ENTRA_LITELLM_PARAM_KEYS,
get_azure_ai_auth_headers,
)
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
_FOUNDRY_ROUTE_SUFFIX: Final = re.compile(r"(/api/projects/[^/]+)?(/openai/v1|/openai|/models|/v1)?$")
_AZURE_AI_CREDENTIAL_PARAM_KEYS: Final = AZURE_ENTRA_LITELLM_PARAM_KEYS | frozenset({"azure_ad_token_provider"})
class AzureAIDecisionsConfig(BaseDecisionsConfig):
@ -14,10 +24,69 @@ class AzureAIDecisionsConfig(BaseDecisionsConfig):
path = "/providers/microsoft/v1/systemone"
api_key_env = ("AZURE_AI_API_KEY",)
api_base_env = ("AZURE_AI_API_BASE",)
api_key_required = False
def missing_api_base_message(self, custom_llm_provider: str) -> str:
return "Missing AZURE_AI_API_BASE - set AZURE_AI_API_BASE or pass api_base, e.g. https://<resource>.services.ai.azure.com"
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
credential_params: Final = MappingProxyType(
{key: litellm_params[key] for key in _AZURE_AI_CREDENTIAL_PARAM_KEYS if key in litellm_params}
)
try:
auth_headers: Final = get_azure_ai_auth_headers(api_key, credential_params)
except ValidationError as error:
field_names: Final = tuple(
dict.fromkeys(
".".join(str(field) for field in detail["loc"])
for detail in error.errors(include_input=False, include_context=False, include_url=False)
)
)
raise litellm.BadRequestError(
message=f"Invalid Azure AI credential parameter(s): {', '.join(f'`{name}`' for name in field_names)}",
model=model,
llm_provider="azure_ai",
) from error
except ValueError as error:
raise litellm.AuthenticationError(message=str(error), model=model, llm_provider="azure_ai") from error
return {
**super().validate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
),
**auth_headers,
}
async def avalidate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
if api_key:
return self.validate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
return await asyncio.to_thread(
self.validate_environment,
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
def get_complete_url(self, api_base: str, model: str) -> str:
url: Final = httpx.URL(api_base)
resource_path: Final = _FOUNDRY_ROUTE_SUFFIX.sub("", url.path.rstrip("/"))

16
litellm/llms/base_llm/chat/transformation.py
View file

@ -8,7 +8,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Union
import httpx
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
from litellm.constants import DEFAULT_MAX_TOKENS, RESPONSE_FORMAT_TOOL_NAME
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -38,6 +38,9 @@ else:
LiteLLMLoggingObj = Any
REQUEST_PARAMS: Final = TypeAdapter(Mapping[str, object])
class BaseLLMException(Exception):
def __init__(
self,
@ -126,6 +129,17 @@ class BaseConfig(ABC):
thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
return thinking is True or thinking_type == "enabled" or non_default_params.get("reasoning_effort") is not None
def is_thinking_requested(self, non_default_params: Mapping[str, object]) -> bool:
match non_default_params:
case {"thinking": True | {"type": "enabled" | "adaptive"}}:
return True
case {"reasoning_effort": None | "none"}:
return False
case {"reasoning_effort": object()}:
return True
case _:
return False
def is_max_tokens_in_request(self, non_default_params: dict) -> bool:
"""
OpenAI spec allows max_tokens or max_completion_tokens to be specified.

40
litellm/llms/base_llm/decisions/transformation.py
View file

@ -390,8 +390,24 @@ def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRe
)
_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_LABELS: Final[TypeAdapter[tuple[str, ...]]] = TypeAdapter(tuple[str, ...])
def _normalize_answer(value: object) -> Mapping[str, object]:
answer: Final = _MAPPING.validate_python(value)
legend: Final = answer.get("legend")
if answer.get("type") != "score" or not isinstance(legend, list):
return answer
labels: Final = _LABELS.validate_python(legend)
return {**answer, "legend": {str(index): label for index, label in enumerate(labels)}}
def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(payload), request)
response: Final = _MAPPING.validate_python(payload)
answers: Final = _MAPPING.validate_python(response.get("answers"))
normalized: Final = {**response, "answers": {name: _normalize_answer(answer) for name, answer in answers.items()}}
return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(normalized), request)
def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None:
@ -510,13 +526,33 @@ class BaseDecisionsConfig(ABC):
def request_model(self, model: str) -> str:
return model
def validate_environment(self, headers: Mapping[str, str], model: str, api_key: str | None) -> dict[str, str]:
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
return {
**{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS},
**({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}),
"Content-Type": "application/json",
}
async def avalidate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
return self.validate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
def get_complete_url(self, api_base: str, model: str) -> str:
return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}"

5
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -203,6 +203,11 @@ class BaseTranslation(ABC):
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
return None
def needs_ended_stream_scan(self, guardrail: "CustomGuardrail", responses_so_far: Sequence[object]) -> bool:
"""Whether the end-of-stream round scans ``guardrail`` even when a sampled round already covered every
text, because this translation hands that guardrail marks only the ended round carries."""
return False
def released_stream_as_ended(self, responses_so_far: Sequence[object]) -> tuple[object, ...]:
"""The chunks a client left the stream with, closed the way this endpoint ends a stream, so the
end-of-stream scan also inspects tool calls the stream never finished"""

198
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -29,6 +29,7 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_lookaround_regex_patterns,
is_object_schema,
native_redacted_thinking_data,
parse_content_for_reasoning,
tool_with_sanitized_parameters,
@ -53,12 +54,21 @@ from litellm.llms.anthropic.chat.transformation import (
tools_without_unsupported_tool_search,
)
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images
from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround, bedrock_supports_tool_search
from litellm.llms.bedrock.common_utils import (
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_supports_tool_search,
deployment_model_info,
)
from litellm.llms.bedrock.request_metadata import (
bedrock_request_metadata_headers,
bedrock_request_metadata_is_owned,
bedrock_request_metadata_is_rewritten,
bedrock_request_metadata_violation,
caller_request_metadata_violations_are_dropped,
merge_bedrock_invoke_headers,
resolve_bedrock_request_metadata,
)
@ -346,77 +356,15 @@ class AmazonConverseConfig(BaseConfig):
and v is not None
}
def _validate_request_metadata(self, metadata: dict) -> None:
"""
Validate requestMetadata according to AWS Bedrock Converse API constraints.
Constraints:
- Maximum of 16 items
- Keys: 1-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{1,256}
- Values: 0-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{0,256}
"""
import re
if not isinstance(metadata, dict):
def _validate_request_metadata(self, metadata: object) -> None:
violation: Final = bedrock_request_metadata_violation(metadata)
if violation is not None:
raise litellm.exceptions.BadRequestError(
message="requestMetadata must be a dictionary",
message=violation.message,
model="bedrock",
llm_provider="bedrock",
)
if len(metadata) > 16:
raise litellm.exceptions.BadRequestError(
message="requestMetadata can contain a maximum of 16 items",
model="bedrock",
llm_provider="bedrock",
)
key_pattern: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
value_pattern: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
for key, value in metadata.items():
if not isinstance(key, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata keys must be strings",
model="bedrock",
llm_provider="bedrock",
)
if not isinstance(value, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata values must be strings",
model="bedrock",
llm_provider="bedrock",
)
if len(key) == 0 or len(key) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata key length must be 1-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if len(value) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata value length must be 0-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if not key_pattern.match(key):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
if not value_pattern.match(value):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
@staticmethod
def _is_openai_gpt_reasoning_model(model: str) -> bool:
return re.search(r"openai\.gpt-\d", model) is not None
@ -939,17 +887,22 @@ class AmazonConverseConfig(BaseConfig):
return _tool
@staticmethod
def _supports_native_structured_outputs(model: str, custom_llm_provider: str | None = None) -> bool:
"""Check if the Bedrock model supports native structured outputs (outputConfig.textFormat).
def _supports_native_structured_outputs(
model: str,
custom_llm_provider: str | None,
thinking_requested: bool,
model_info: Mapping[str, object] | None,
) -> bool:
"""Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request.
Delegates to the standard ``supports_native_structured_output`` utility
which looks up the flag in ``litellm.model_cost`` via
``_get_model_info_helper``.
``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and
``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) only when the
request asks for thinking, which the json tool path would drop.
Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
"""
from litellm.utils import supports_native_structured_output
return supports_native_structured_output(model=model, custom_llm_provider=custom_llm_provider)
return bedrock_chat_uses_native_structured_output(
model, thinking_requested, custom_llm_provider or "bedrock_converse", model_info
)
@staticmethod
def _add_additional_properties_to_schema(schema: dict) -> dict:
@ -964,9 +917,9 @@ class AmazonConverseConfig(BaseConfig):
if not isinstance(schema, dict):
return schema
result: Final = dict(schema)
result: Final[dict[str, object]] = dict(schema)
if result.get("type") == "object" and "additionalProperties" not in result:
if is_object_schema(result) and "additionalProperties" not in result:
result["additionalProperties"] = False
# Recurse into nested schemas
@ -977,16 +930,46 @@ class AmazonConverseConfig(BaseConfig):
if "items" in result and isinstance(result["items"], dict):
result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(result["items"])
for defs_key in ("$defs", "definitions"):
if defs_key in result and isinstance(result[defs_key], dict):
if isinstance(definitions := result.get(defs_key), dict):
result[defs_key] = {
k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result[defs_key].items()
k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in definitions.items()
}
for key in ("anyOf", "allOf", "oneOf"):
if key in result and isinstance(result[key], list):
result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in result[key]]
if isinstance(variants := result.get(key), list):
result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in variants]
return result
def _native_output_schema(
self,
model: str,
json_schema: Mapping[str, object],
thinking_requested: bool,
model_info: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
if not self._supports_native_structured_outputs(
model, self.custom_llm_provider, thinking_requested, model_info
):
return None
if BedrockModelInfo.get_base_model(model).startswith("anthropic"):
return bedrock_cleaned_output_schema(model, json_schema, self.custom_llm_provider or "bedrock_converse")
return json_schema
def _output_config_for_anthropic_format(
self, model: str, output_config_format: Mapping[str, object], model_info: Mapping[str, object] | None
) -> OutputConfigBlock | None:
native_format: Final = bedrock_native_output_format(
model, output_config_format, self.custom_llm_provider or "bedrock_converse", model_info
)
if native_format is None:
return None
schema, name, description = (native_format.get(key) for key in ("schema", "name", "description"))
return AmazonConverseConfig._create_output_config_for_response_format(
json_schema={**schema} if isinstance(schema, Mapping) else None,
name=name if isinstance(name, str) else None,
description=description if isinstance(description, str) else None,
)
@staticmethod
def _create_output_config_for_response_format(
json_schema: dict | None = None,
@ -1046,6 +1029,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params: dict,
model: str,
drop_params: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params)
base_model: Final = BedrockModelInfo.get_base_model(model)
@ -1062,6 +1046,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params=optional_params,
non_default_params=non_default_params,
is_thinking_enabled=is_thinking_enabled,
model_info=model_info,
)
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["maxTokens"] = (
@ -1199,7 +1184,7 @@ class AmazonConverseConfig(BaseConfig):
elif param == "context_management" and isinstance(value, (dict, list)):
self._map_context_management_param(value, optional_params)
if param == "requestMetadata":
self._map_request_metadata_param(value, optional_params)
self._map_request_metadata_param(value, optional_params, drop_params)
if param == "service_tier" and isinstance(value, str):
self._map_service_tier_param(value, optional_params)
@ -1232,9 +1217,10 @@ class AmazonConverseConfig(BaseConfig):
return optional_params
def _map_request_metadata_param(self, value: object, optional_params: dict) -> None:
def _map_request_metadata_param(self, value: object, optional_params: dict, drop_params: bool) -> None:
if value is not None and isinstance(value, dict):
self._validate_request_metadata(value)
if not caller_request_metadata_violations_are_dropped(drop_params):
self._validate_request_metadata(value)
optional_params["requestMetadata"] = value
def _map_context_management_param(self, value: dict | list, optional_params: dict) -> None:
@ -1273,6 +1259,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params: dict,
non_default_params: dict,
is_thinking_enabled: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
"""
Handles translation of response_format parameter to Bedrock format.
@ -1296,13 +1283,23 @@ class AmazonConverseConfig(BaseConfig):
if "type" in value and value["type"] == "text":
return optional_params
if self._supports_native_structured_outputs(model, self.custom_llm_provider) and json_schema is not None:
native_schema: Final = (
self._native_output_schema(
model,
json_schema,
self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)),
model_info,
)
if json_schema is not None
else None
)
if native_schema is not None:
# Use Bedrock's native structured outputs API (outputConfig.textFormat)
# No synthetic tool injection, no fake_stream needed.
# Requires an explicit schema — json_object with no schema falls through
# to the tool-call path below.
output_config: Final = self._create_output_config_for_response_format(
json_schema=json_schema,
json_schema=dict(native_schema),
name=name,
description=description,
)
@ -1644,7 +1641,11 @@ class AmazonConverseConfig(BaseConfig):
tool_choice.pop("type", None)
def _prepare_request_params(
self, optional_params: dict, model: str, drop_params: bool = False
self,
optional_params: dict,
model: str,
drop_params: bool = False,
model_info: Mapping[str, object] | None = None,
) -> tuple[dict, dict, dict, OutputConfigBlock | None]:
"""Prepare and separate request parameters."""
# Consume the internal ``_output_config_normalized`` marker set by
@ -1677,7 +1678,7 @@ class AmazonConverseConfig(BaseConfig):
# Extract requestMetadata before processing other parameters
request_metadata: Final = inference_params.pop("requestMetadata", None)
if request_metadata is not None:
if request_metadata is not None and not caller_request_metadata_violations_are_dropped(drop_params):
self._validate_request_metadata(request_metadata)
output_config: OutputConfigBlock | None = inference_params.pop("outputConfig", None)
@ -1687,17 +1688,12 @@ class AmazonConverseConfig(BaseConfig):
and output_config_format is not None
and output_config_format.get("type") == "json_schema"
and base_model.startswith("anthropic")
and self._supports_native_structured_outputs(model, self.custom_llm_provider)
):
output_config = self._create_output_config_for_response_format(
json_schema=output_config_format.get("schema"),
name=output_config_format.get("name"),
description=output_config_format.get("description"),
)
elif output_config is None and output_config_format is not None:
output_config = self._output_config_for_anthropic_format(model, output_config_format, model_info)
if output_config is None and output_config_format is not None:
litellm.verbose_logger.warning(
"Bedrock Converse: dropping `output_config.format` for model=%s — "
"model does not advertise `supports_native_structured_output` in "
"model does not take native structured output for this schema per "
"model_prices_and_context_window.json. The schema will not be "
"enforced; pass `response_format` to use the synthetic tool-call "
"fallback.",
@ -1987,7 +1983,7 @@ class AmazonConverseConfig(BaseConfig):
additional_request_params,
request_metadata,
output_config,
) = self._prepare_request_params(optional_params, model, drop_params)
) = self._prepare_request_params(optional_params, model, drop_params, deployment_model_info(litellm_params))
original_tools: Final = inference_params.pop("tools", [])
@ -2109,7 +2105,7 @@ class AmazonConverseConfig(BaseConfig):
request_metadata: Final = resolve_bedrock_request_metadata(
litellm_params=litellm_params, caller_metadata=_data.get("requestMetadata")
)
if bedrock_request_metadata_is_owned():
if bedrock_request_metadata_is_rewritten(litellm_params.get("drop_params") is True):
_data.pop("requestMetadata", None)
if request_metadata is not None:
_data["requestMetadata"] = request_metadata
@ -2189,7 +2185,7 @@ class AmazonConverseConfig(BaseConfig):
request_metadata: Final = resolve_bedrock_request_metadata(
litellm_params=litellm_params, caller_metadata=_data.get("requestMetadata")
)
if bedrock_request_metadata_is_owned():
if bedrock_request_metadata_is_rewritten(litellm_params.get("drop_params") is True):
_data.pop("requestMetadata", None)
if request_metadata is not None:
_data["requestMetadata"] = request_metadata

14
litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py
View file

@ -92,8 +92,16 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
optional_params: dict,
model: str,
drop_params: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
return AmazonConverseConfig.map_openai_params(self, non_default_params, optional_params, model, drop_params)
return AmazonConverseConfig.map_openai_params(
self,
{key: value for key, value in non_default_params.items() if key != "requestMetadata"},
optional_params,
model,
drop_params,
model_info,
)
def transform_request(
self,
@ -107,7 +115,9 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
self,
model=model,
messages=messages,
optional_params=_without_tool_config_injection_points(optional_params),
optional_params=_without_tool_config_injection_points(
{key: value for key, value in optional_params.items() if key != "requestMetadata"}
),
litellm_params=litellm_params,
headers=headers,
)

23
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -23,7 +23,10 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
)
from litellm.llms.bedrock.common_utils import (
apply_bedrock_invoke_structured_output,
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_supports_tool_search,
deployment_model_info,
get_anthropic_beta_from_headers,
normalize_bedrock_opus_output_config_effort,
normalize_custom_field_on_tools,
@ -49,6 +52,7 @@ else:
LiteLLMLoggingObj = Any
_NON_DEFAULT_PARAMS: Final = TypeAdapter(Mapping[str, object])
_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
@ -71,6 +75,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
anthropic_version: str = "bedrock-2023-05-31"
def __init__(self, model_info: Mapping[str, object] | None = None) -> None:
super().__init__()
self._deployment_model_info: Final = model_info
@property
def custom_llm_provider(self) -> str | None:
return "bedrock"
@ -81,10 +89,18 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
def get_supported_openai_params(self, model: str) -> list[str]:
return AnthropicConfig.get_supported_openai_params(self, model)
def uses_native_structured_output(self, model: str) -> bool:
from litellm.utils import supports_native_structured_output
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
return AnthropicConfig.native_output_schema(json_schema)
return supports_native_structured_output(model=model, custom_llm_provider="bedrock")
def uses_native_structured_output_for_schema(
self,
model: str,
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
return bedrock_chat_uses_native_structured_output(
model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
def map_openai_params(
self,
@ -240,6 +256,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
apply_bedrock_invoke_structured_output(
model=model,
request_body=anthropic_request,
model_info=deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)),
)
strip_unsupported_bedrock_invoke_output_config_keys(
model=model,

104
litellm/llms/bedrock/common_utils.py
View file

@ -29,6 +29,7 @@ import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -249,15 +250,99 @@ def convert_bedrock_invoke_output_format_to_inline_schema(
request_body["messages"] = new_messages
def _bedrock_model_supports(model: str, key: str) -> bool:
def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "bedrock") -> bool:
from litellm.utils import supports_factory
return supports_factory(model=model, custom_llm_provider="bedrock", key=key)
return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key=key)
_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
def deployment_model_info(litellm_params: Mapping[str, object] | None) -> Mapping[str, object]:
model_info: Final = (litellm_params or {}).get("model_info")
return _DEPLOYMENT_MODEL_INFO.validate_python(model_info) if isinstance(model_info, Mapping) else {}
def _bedrock_native_output_opt_in(
model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None
) -> bool:
deployment_flag: Final = (model_info or {}).get("supports_bedrock_native_structured_output")
if isinstance(deployment_flag, bool):
return deployment_flag
return _bedrock_model_supports(model, "supports_bedrock_native_structured_output", custom_llm_provider)
def bedrock_supports_native_structured_output(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return _bedrock_model_supports(
model, "supports_native_structured_output", custom_llm_provider
) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
def bedrock_chat_uses_native_structured_output(
model: str,
thinking_requested: bool,
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> bool:
return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) or (
thinking_requested and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
)
BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64
def bedrock_cleaned_output_schema(
model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock"
) -> Mapping[str, object] | None:
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
if (
not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider)
and schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
):
return None
return AnthropicConfig.native_output_schema(schema)
def bedrock_native_output_schema(
model: str,
schema: Mapping[str, object],
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> Mapping[str, object] | None:
native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider)
if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider):
return schema if native_schema is None else native_schema
if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info):
return native_schema
return None
def bedrock_native_output_format(
model: str,
output_format: Mapping[str, object],
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> Mapping[str, object] | None:
schema: Final = output_format.get("schema")
if not isinstance(schema, Mapping):
return (
output_format if bedrock_supports_native_structured_output(model, custom_llm_provider, model_info) else None
)
native_schema: Final = bedrock_native_output_schema(model, schema, custom_llm_provider, model_info)
if native_schema is None:
return None
return {**output_format, "schema": native_schema}
def apply_bedrock_invoke_structured_output(
model: str,
request_body: dict[str, object], # mutable-ok: edited in place like siblings
model_info: Mapping[str, object] | None = None,
) -> None:
"""
Route Anthropic structured-output params to what the Bedrock model supports.
@ -277,17 +362,18 @@ def apply_bedrock_invoke_structured_output(
if schema_format is None:
return
if _bedrock_model_supports(model, "supports_native_structured_output"):
native_format: Final = bedrock_native_output_format(model, schema_format, model_info=model_info)
if native_format is not None:
existing_output_config: Final = request_body.get("output_config")
if isinstance(existing_output_config, dict):
existing_output_config["format"] = schema_format
existing_output_config["format"] = native_format
else:
request_body["output_config"] = {"format": schema_format} # rebind-ok: out-param
request_body["output_config"] = {"format": native_format} # rebind-ok: out-param
return
verbose_logger.warning(
"Bedrock Invoke: model=%s does not advertise `supports_native_structured_output` "
"in model_prices_and_context_window.json, so the JSON schema was inlined into "
"Bedrock Invoke: model=%s does not take native structured output for this schema "
"per model_prices_and_context_window.json, so the JSON schema was inlined into "
"the last user message and is NOT enforced by the model.",
model,
)
@ -1296,7 +1382,6 @@ def is_claude_4_5_on_bedrock(model: str) -> bool:
_BEDROCK_MODEL_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
def bedrock_converse_supports_strict_tools(model: str) -> bool:
@ -1327,8 +1412,7 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
"""
params: Final = litellm_params or {}
model_info: Final = _DEPLOYMENT_MODEL_INFO.validate_python(params.get("model_info") or {})
deployment_flag: Final = model_info.get("supports_regex_lookaround")
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
if isinstance(deployment_flag, bool):
return deployment_flag
base_model: Final = params.get("base_model")

2
litellm/llms/bedrock/files/transformation.py
View file

@ -1021,7 +1021,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
converse_config: Final = AmazonConverseConfig()
mapped_params = converse_config.map_openai_params(
non_default_params=optional_params,
non_default_params={k: v for k, v in optional_params.items() if k != "requestMetadata"},
optional_params={},
model=model,
drop_params=False,

5
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -3,6 +3,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -69,6 +70,9 @@ else:
LiteLLMLoggingObj = Any
_MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
class AmazonAnthropicClaudeMessagesConfig(
AnthropicMessagesConfig,
AmazonInvokeConfig,
@ -751,6 +755,7 @@ class AmazonAnthropicClaudeMessagesConfig(
apply_bedrock_invoke_structured_output(
model=model,
request_body=anthropic_messages_request,
model_info=_MODEL_INFO.validate_python(getattr(litellm_params, "model_info", None)),
)
normalize_bedrock_opus_output_config_effort(
model=model,

878
litellm/llms/bedrock/passthrough/guardrail_translation/handler.py
View file

@ -1,18 +1,32 @@
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias
import base64
import copy
import json
import struct
from binascii import crc32
from collections.abc import AsyncGenerator, AsyncIterable, Callable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias, TypeVar
from pydantic import TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
StreamTransformSink,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.pass_through.guardrail_translation.handler import (
PassThroughEndpointHandler,
@ -21,6 +35,7 @@ if TYPE_CHECKING:
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
_CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"})
_INVOKE_STREAM_ACTION: Final = "invoke-with-response-stream"
_EVENT_STREAM_CONTENT_TYPE: Final = "vnd.amazon.eventstream"
_EVENT_STREAM_MEDIA_TYPE: Final = "application/vnd.amazon.eventstream"
@ -30,6 +45,10 @@ def _is_converse_endpoint(endpoint: str) -> bool:
return bool(parts) and parts[-1] in _CONVERSE_ACTIONS
def _is_invoke_stream_endpoint(endpoint: str) -> bool:
return endpoint.rstrip("/").split("/")[-1] == _INVOKE_STREAM_ACTION
def _generic_passthrough_handler() -> "PassThroughEndpointHandler":
"""
Fallback for non-Converse Bedrock routes (e.g. invoke). The generic
@ -217,8 +236,6 @@ class _StreamFrame(TypedDict):
def _unpack_uint32(buffer: bytes) -> int:
import struct
return struct.unpack("!I", buffer)[0]
@ -253,6 +270,578 @@ def _collect_stream_delta_text_holders(delta: object) -> list[_DeltaHolder]:
return holders
def _delta_frame_texts(payload: Mapping[str, object]) -> tuple[tuple[_StreamGroupKey, str], ...]:
"""The guardrailable texts of one ``contentBlockDelta`` payload, each keyed by its content block and kind."""
block_index: Final = payload.get("contentBlockIndex")
# Group per content block: a guardrail edit that changes length must not move text between blocks.
return tuple(
((block_index if isinstance(block_index, int) else None, group_key), container[key])
for group_key, container, key in _collect_stream_delta_text_holders(payload.get("delta"))
)
@dataclass(frozen=True, slots=True)
class _StreamTextGroups:
"""Stream texts joined per group, keeping where each fragment came from so a rewrite can be split back."""
groups: tuple[_StreamGroupKey, ...]
members: Mapping[_StreamGroupKey, Sequence[tuple[int, int]]]
fragments: Mapping[_StreamGroupKey, Sequence[str]]
def texts(self) -> tuple[str, ...]:
return tuple("".join(self.fragments[group]) for group in self.groups)
def _group_stream_texts(frame_texts: Sequence[Sequence[tuple[_StreamGroupKey, str]]]) -> _StreamTextGroups:
"""Group the per-frame texts by key in first-seen order; ``groups`` keeps only the groups with any text."""
order: Final[list[_StreamGroupKey]] = [] # mutable-ok: built in one pass over the frames
members: Final[dict[_StreamGroupKey, list[tuple[int, int]]]] = {} # mutable-ok: built in one pass over the frames
fragments: Final[dict[_StreamGroupKey, list[str]]] = {} # mutable-ok: built in one pass over the frames
for frame_idx, texts in enumerate(frame_texts):
for local_idx, (group_key, text) in enumerate(texts):
if group_key not in members:
members[group_key] = []
fragments[group_key] = []
order.append(group_key)
members[group_key].append((frame_idx, local_idx))
fragments[group_key].append(text)
return _StreamTextGroups(
groups=tuple(group for group in order if "".join(fragments[group])),
members=members,
fragments=fragments,
)
def _split_group_texts(groups: _StreamTextGroups, rewritten_texts: Sequence[str]) -> Mapping[tuple[int, int], str]:
"""Cut each group's rewritten text back into its fragments, sized like the original fragments."""
split: Final[dict[tuple[int, int], str]] = {} # mutable-ok: built in one pass over the groups
for group_key, rewritten in zip(groups.groups, rewritten_texts):
members = groups.members[group_key]
originals = groups.fragments[group_key]
total_orig = sum(len(t) for t in originals) or 1
pos = 0
for k, member in enumerate(members):
if k == len(members) - 1:
split[member] = rewritten[pos:]
else:
end = pos + round(len(rewritten) * len(originals[k]) / total_orig)
split[member] = rewritten[pos:end]
pos = end
return split
def _event_stream_message(headers_bytes: bytes, payload: bytes) -> bytes:
"""One AWS event-stream message: prelude, prelude CRC, headers, payload and message CRC."""
prelude: Final = struct.pack("!II", 12 + len(headers_bytes) + len(payload) + 4, len(headers_bytes))
prelude_crc: Final = crc32(prelude) & 0xFFFFFFFF
message: Final = prelude + struct.pack("!I", prelude_crc) + headers_bytes + payload
return message + struct.pack("!I", crc32(message) & 0xFFFFFFFF)
def _string_header(name: str, value: str) -> bytes:
name_bytes: Final = name.encode()
value_bytes: Final = value.encode()
return bytes((len(name_bytes),)) + name_bytes + b"\x07" + struct.pack("!H", len(value_bytes)) + value_bytes
def _string_headers(headers: Mapping[str, str]) -> bytes:
return b"".join(_string_header(name, value) for name, value in headers.items())
def _converse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
return _event_stream_message(
_string_headers({":event-type": event_type, ":content-type": "application/json", ":message-type": "event"}),
json.dumps(payload, separators=(",", ":")).encode(),
)
def _converse_exception(message: str, status_code: int) -> bytes:
"""An in-stream exception frame, which boto3 raises as ``EventStreamError`` carrying ``message``."""
return _event_stream_message(
_string_headers(
{
":exception-type": "validationException" if status_code < 500 else "internalServerException",
":content-type": "application/json",
":message-type": "exception",
}
),
json.dumps({"message": message}, separators=(",", ":")).encode(),
)
def _base64_json(event: Mapping[str, object]) -> str:
return base64.b64encode(json.dumps(event, separators=(",", ":")).encode()).decode()
def _invoke_chunk(event: Mapping[str, object]) -> bytes:
return _event_stream_message(
_string_headers({":event-type": "chunk", ":content-type": "application/json", ":message-type": "event"}),
json.dumps({"bytes": _base64_json(event)}, separators=(",", ":")).encode(),
)
@dataclass(slots=True)
class ConverseStreamFrame:
"""One frame of a Converse event stream as the proxy relays it.
``payload`` is the parsed JSON of an event frame; exception frames and bytes that do not decode keep it None and
are relayed as received. ``ends_stream`` marks the empty item the framer adds once the upstream has ended, so an
exception frame or a closed connection ends the stream as ``messageStop`` does. ``guardrailed_payload`` is set
when a guardrail rewrote the frame's text."""
raw: bytes
event_type: str | None = None
payload: Mapping[str, object] | None = None
guardrailed_payload: Mapping[str, object] | None = None
ends_stream: bool = False
@property
def block_index(self) -> int | None:
index: Final = self.payload.get("contentBlockIndex") if self.payload is not None else None
return index if isinstance(index, int) else None
def scanned_texts(self) -> tuple[tuple[_StreamGroupKey, str], ...]:
current: Final = self.guardrailed_payload if self.guardrailed_payload is not None else self.payload
return _delta_frame_texts(current) if self.event_type == "contentBlockDelta" and current is not None else ()
def received_texts(self) -> tuple[tuple[_StreamGroupKey, str], ...]:
return (
_delta_frame_texts(self.payload)
if self.event_type == "contentBlockDelta" and self.payload is not None
else ()
)
def encode(self) -> bytes:
if self.guardrailed_payload is None:
return self.raw
headers_length: Final = _unpack_uint32(self.raw[4:8])
return _event_stream_message(
self.raw[12 : 12 + headers_length], json.dumps(self.guardrailed_payload, separators=(",", ":")).encode()
)
def _anthropic_sse(event: Mapping[str, object] | None) -> bytes:
if event is None:
return b""
return f"event: {event['type']}\ndata: {json.dumps(event, separators=(',', ':'))}\n\n".encode()
@dataclass(slots=True)
class InvokeStreamFrame:
"""One frame of an invoke-with-response-stream as the proxy relays it.
``event`` is the Anthropic Messages stream event a ``chunk`` frame carries base64 encoded in the ``bytes`` of its
``payload``. Exception frames and bytes that do not decode keep it None and are relayed as received.
``ends_stream`` marks the empty item the framer adds once the upstream has ended, and ``guardrailed_event`` is set
when a guardrail rewrote the event."""
raw: bytes
payload: Mapping[str, object] | None = None
event: Mapping[str, object] | None = None
guardrailed_event: Mapping[str, object] | None = None
ends_stream: bool = False
def sse(self) -> bytes:
"""The event as a ``/v1/messages`` stream carries it, the shape its guardrail translation scans."""
return _anthropic_sse(self.guardrailed_event if self.guardrailed_event is not None else self.event)
def received_sse(self) -> bytes:
"""The event as received from the upstream, before any guardrail rewrote it."""
return _anthropic_sse(self.event)
def encode(self) -> bytes:
if self.guardrailed_event is None or self.payload is None:
return self.raw
headers_length: Final = _unpack_uint32(self.raw[4:8])
return _event_stream_message(
self.raw[12 : 12 + headers_length],
json.dumps({**self.payload, "bytes": _base64_json(self.guardrailed_event)}, separators=(",", ":")).encode(),
)
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_Frame: Final = TypeVar("_Frame", ConverseStreamFrame, InvokeStreamFrame)
def _event_message(raw: bytes) -> tuple[str, bytes] | None:
"""The event type and payload of one event-stream frame, or None for an exception frame or bytes that do not
decode."""
from botocore.eventstream import EventStreamBuffer, ParserError
buffer: Final = EventStreamBuffer()
buffer.add_data(raw)
try:
message: Final = next(iter(buffer))
except (ParserError, StopIteration) as e:
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: could not decode event-stream frame, relaying it unmodified: %s", e
)
return None
# botocore-stubs types EventStreamMessage.headers as int; at runtime it is the decoded header dict.
headers: Final = _JSON_OBJECT.validate_python(message.headers)
event_type: Final = headers.get(":event-type")
if headers.get(":message-type") != "event" or not isinstance(event_type, str):
return None
return event_type, message.payload
def _decode_converse_frame(raw: bytes) -> ConverseStreamFrame:
message: Final = _event_message(raw)
if message is None:
return ConverseStreamFrame(raw=raw)
event_type, payload_bytes = message
try:
payload: Final = _JSON_OBJECT.validate_python(json.loads(payload_bytes))
except (json.JSONDecodeError, UnicodeDecodeError, ValidationError):
return ConverseStreamFrame(raw=raw, event_type=event_type)
return ConverseStreamFrame(raw=raw, event_type=event_type, payload=payload)
def _decode_invoke_frame(raw: bytes) -> InvokeStreamFrame:
message: Final = _event_message(raw)
if message is None or message[0] != "chunk":
return InvokeStreamFrame(raw=raw)
try:
payload: Final = _JSON_OBJECT.validate_python(json.loads(message[1]))
encoded: Final = payload.get("bytes")
event: Final = (
_JSON_OBJECT.validate_python(json.loads(base64.b64decode(encoded))) if isinstance(encoded, str) else None
)
except (ValueError, ValidationError):
return InvokeStreamFrame(raw=raw)
if event is None or not isinstance(event.get("type"), str):
return InvokeStreamFrame(raw=raw)
return InvokeStreamFrame(raw=raw, payload=payload, event=event)
async def _event_stream_frames(
chunks: AsyncIterable[bytes], frame_type: type[_Frame], decode: Callable[[bytes], _Frame]
) -> AsyncGenerator[_Frame, None]:
"""Cut the upstream bytes into frames as soon as each frame has fully arrived, then add an empty item that ends
the stream once the upstream has ended.
Bytes that cannot be framed (a length below the 16-byte minimum, or a stream that ends inside a frame) are relayed
as received."""
pending = b"" # rebind-ok: bytes received but not yet framed
framing = True # rebind-ok: cleared once a frame length is unusable
async for chunk in chunks:
if not framing:
yield frame_type(raw=chunk)
continue
pending += chunk
while len(pending) >= 4:
total_length = _unpack_uint32(pending[:4])
if total_length < 16:
framing = False
break
if len(pending) < total_length:
break
yield decode(pending[:total_length])
pending = pending[total_length:]
if not framing and pending:
yield frame_type(raw=pending)
pending = b""
if pending:
yield frame_type(raw=pending)
yield frame_type(raw=b"", ends_stream=True)
def _converse_frames(items: Sequence[object]) -> tuple[ConverseStreamFrame, ...]:
"""The frames among ``items``, with the bytes an earlier guardrail's block message added read back into frames."""
return tuple(
item if isinstance(item, ConverseStreamFrame) else _decode_converse_frame(item)
for item in items
if isinstance(item, (ConverseStreamFrame, bytes))
)
def _reads_stream_marks(guardrail: "CustomGuardrail") -> bool:
return getattr(guardrail, "reads_stream_marks", False) is True
def _json_object_member(value: object, key: str) -> Mapping[str, object] | None:
"""``value[key]`` when both are JSON objects, else None."""
try:
return _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_python(value).get(key))
except ValidationError:
return None
def _reject_json_constant(name: str) -> object:
raise ValueError(f"{name} is not JSON")
def _finished_tool_input(text: str) -> object:
"""A tool call's streamed input as the JSON document Converse returns, or the text as received when it is not
JSON (cut short, rewritten by a guardrail, or holding NaN or nesting too deep to send on)."""
try:
return json.loads(text, parse_constant=_reject_json_constant) if text else {}
except (ValueError, RecursionError):
return text
def _finished_converse_content(
frames: Sequence[ConverseStreamFrame], groups: _StreamTextGroups
) -> list[dict[str, object]]: # mutable-ok: the content of the finished response guardrails receive
"""The finished reply's text, reasoning and tool-call blocks, shaped as a non-streaming Converse response
holds them."""
texts: Final = dict(zip(groups.groups, groups.texts()))
tool_starts: Final[dict[int | None, Mapping[str, object]]] = {} # mutable-ok: built in one pass over the frames
keys: Final[dict[_StreamGroupKey, None]] = {} # mutable-ok: blocks in stream order, built in one pass
for frame in frames:
tool_use = (
_json_object_member(_json_object_member(frame.payload, "start"), "toolUse")
if frame.event_type == "contentBlockStart"
else None
)
if tool_use is not None:
tool_starts[frame.block_index] = tool_use
keys[(frame.block_index, "tool")] = None
keys.update((key, None) for key, _ in frame.scanned_texts())
content: Final[list[dict[str, object]]] = [] # mutable-ok: built in one pass over the blocks
for block_index, kind in keys:
text = texts.get((block_index, kind), "")
if kind == "tool":
tool_start = tool_starts.get(block_index, {})
named = {name: tool_start[name] for name in ("toolUseId", "name", "type") if name in tool_start}
content.append({"toolUse": {**named, "input": _finished_tool_input(text)}})
elif text and kind == "reasoning":
content.append({"reasoningContent": {"reasoningText": {"text": text}}})
elif text:
content.append({"text": text})
return content
def _stream_marked_request_data(
request_data: Mapping[str, object], frames: Sequence[ConverseStreamFrame], groups: _StreamTextGroups, ended: bool
) -> dict[str, object]: # mutable-ok: the request dict guardrails receive
"""``request_data`` with the marks chat completions gives a streamed answer: ``stream`` on every scan and, once
the stream has ended, the finished ``response``."""
if not ended:
return {**request_data, "stream": True}
stop_reason: Final = next(
(
frame.payload.get("stopReason")
for frame in frames
if frame.event_type == "messageStop" and frame.payload is not None
),
None,
)
message: Final = {"role": "assistant", "content": _finished_converse_content(frames, groups)}
response: Final = {"output": {"message": message}, **({"stopReason": stop_reason} if stop_reason else {})}
return {**request_data, "stream": True, "response": response}
def _invoke_frames(items: Sequence[object]) -> tuple[InvokeStreamFrame, ...]:
return tuple(item for item in items if isinstance(item, InvokeStreamFrame))
def _sse_events(sse: bytes) -> tuple[Mapping[str, object], ...]:
return tuple(
_JSON_OBJECT.validate_python(json.loads(line[len("data:") :]))
for line in sse.decode().splitlines()
if line.startswith("data:")
)
def _invoke_message_stopped(frames: Sequence[InvokeStreamFrame]) -> bool:
"""Whether a ``message_delta`` with a ``stop_reason`` came, as a ``/v1/messages`` stream ends."""
return any(
frame.event is not None
and frame.event.get("type") == "message_delta"
and isinstance(delta := frame.event.get("delta"), Mapping)
and _JSON_OBJECT.validate_python(delta).get("stop_reason") is not None
for frame in frames
)
def _invoke_stream_has_ended(frames: Sequence[InvokeStreamFrame]) -> bool:
"""Ended on a ``message_delta`` with a ``stop_reason`` or once the upstream has ended."""
return any(frame.ends_stream for frame in frames) or _invoke_message_stopped(frames)
_UPSTREAM_ENDED_SSE: Final = _anthropic_sse(
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": 0},
}
)
def _invoke_scan_sse(frames: Sequence[InvokeStreamFrame], events: Sequence[bytes]) -> tuple[bytes, ...]:
"""The ``/v1/messages`` events to scan for these frames, closed with a ``message_delta`` when the upstream ended
without one so the translation scans them as a whole stream, tool calls included."""
if any(frame.ends_stream for frame in frames) and not _invoke_message_stopped(frames):
return (*events, _UPSTREAM_ENDED_SSE)
return tuple(events)
async def _scan_invoke_stream(
frames: Sequence[InvokeStreamFrame],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"],
user_api_key_dict: Optional["UserAPIKeyAuth"],
request_data: dict[str, object] | None, # mutable-ok: the request dict guardrails receive
) -> None:
"""Scan the Anthropic events relayed so far as ``/v1/messages`` scans its stream, with the request marked as
streaming and those events as its ``responses``. Once the stream has ended, the rewrites are written back into the
frames, which reach the caller only when the stream is held until moderated."""
received: Final = tuple(frame.sse() for frame in frames)
scan_events: Final = _invoke_scan_sse(frames, received)
scanned: Final[list[object]] = list(scan_events) # mutable-ok: the /v1/messages translation rewrites it in place
if request_data is not None:
request_data["stream"] = True # rebind-ok: marks the request guardrails receive as a stream
request_data["responses"] = scanned # rebind-ok: each scan sees every event relayed so far
await AnthropicMessagesHandler().process_output_streaming_response( # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict
responses_so_far=scanned,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=request_data,
deliver_ended_stream_rewrites=_invoke_stream_has_ended(frames),
)
for frame, before, after in zip(frames, received, scanned):
if isinstance(after, bytes) and after != before and (rewritten := _sse_events(after)):
frame.guardrailed_event = rewritten[0]
def _invoke_block_frames(
exc: "ModifyResponseException", stream_started: bool, frames: Sequence[InvokeStreamFrame]
) -> tuple[bytes, ...]:
"""The ``/v1/messages`` block events for the stream relayed so far, each in its own ``chunk`` frame."""
sse_chunks: Final = AnthropicMessagesHandler().build_block_sse_chunks(
exc, stream_started=stream_started, responses_so_far=[frame.sse() for frame in frames]
)
return tuple(_invoke_chunk(event) for event in _sse_events(b"".join(sse_chunks)))
def _write_back_stream_texts(
frames: Sequence[ConverseStreamFrame], groups: _StreamTextGroups, rewritten_texts: Sequence[str]
) -> None:
split: Final = _split_group_texts(groups, rewritten_texts)
for frame_idx, frame in enumerate(frames):
current = frame.guardrailed_payload if frame.guardrailed_payload is not None else frame.payload
if frame.event_type != "contentBlockDelta" or current is None:
continue
rewritten = copy.deepcopy(current)
changed = False
for local_idx, (_, container, key) in enumerate(_collect_stream_delta_text_holders(rewritten.get("delta"))):
new_text = split.get((frame_idx, local_idx))
if new_text is not None and new_text != container[key]:
container[key] = new_text
changed = True
if changed:
frame.guardrailed_payload = rewritten
_INVOKE_DELTA_TEXT_FIELDS: Final = MappingProxyType(
{
"text_delta": ("text", "text"),
"input_json_delta": ("tool", "partial_json"),
"thinking_delta": ("reasoning", "thinking"),
}
)
@dataclass(frozen=True, slots=True)
class _InvokeDeltaText:
"""The guardrailable text one Anthropic event's delta carries, keyed by its content block and kind like the
Converse holders: text, tool input and thinking. Signatures and citations are never rewritten."""
group: _StreamGroupKey
delta: Mapping[str, object]
field: str
text: str
def _invoke_delta_text(event: Mapping[str, object] | None) -> _InvokeDeltaText | None:
if event is None:
return None
try:
delta: Final = _JSON_OBJECT.validate_python(event.get("delta"))
except ValidationError:
return None
kind: Final = delta.get("type")
field: Final = _INVOKE_DELTA_TEXT_FIELDS.get(kind) if isinstance(kind, str) else None
text: Final = delta.get(field[1]) if field is not None else None
if field is None or not isinstance(text, str):
return None
index: Final = event.get("index")
return _InvokeDeltaText((index if isinstance(index, int) else None, field[0]), delta, field[1], text)
async def _post_call_rewritten_texts(
groups: _StreamTextGroups,
proxy_logging_obj: "ProxyLogging",
user_api_key_dict: "UserAPIKeyAuth",
data: dict[str, object], # mutable-ok: the request dict post_call hooks receive
) -> Sequence[str] | None:
"""Run the post_call guardrails on a buffered stream's texts as one Converse response with a content block per
group, and return each group's text as the guardrails left it, or None when the response came back in another
shape."""
synthetic_response: Final[dict] = {
"output": {
"message": {
"role": "assistant",
"content": [{"text": text} for text in groups.texts()],
}
},
"stopReason": "end_turn",
}
processed: Final = await proxy_logging_obj.post_call_success_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=synthetic_response,
)
if not isinstance(processed, dict):
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: post_call_success_hook returned %s, leaving event stream unmodified",
type(processed).__name__,
)
return None
try:
processed_blocks: Final = processed["output"]["message"]["content"]
return [processed_blocks[i]["text"] for i in range(len(groups.groups))]
except (KeyError, IndexError, TypeError):
return None
async def _guardrail_buffered_invoke_stream(
body_bytes: bytes,
proxy_logging_obj: "ProxyLogging",
user_api_key_dict: "UserAPIKeyAuth",
data: dict[str, object], # mutable-ok: the request dict post_call hooks receive
) -> bytes:
"""Scan a whole Claude invoke stream the way the buffered Converse path scans its stream, and write the guardrail
rewrites back into the ``chunk`` frames that carried each text."""
async def received() -> AsyncGenerator[bytes, None]:
yield body_bytes
frames: Final = [frame async for frame in _event_stream_frames(received(), InvokeStreamFrame, _decode_invoke_frame)]
texts: Final = [_invoke_delta_text(frame.event) for frame in frames]
groups: Final = _group_stream_texts([[] if text is None else [(text.group, text.text)] for text in texts])
if not groups.groups:
return body_bytes
rewritten_texts: Final = await _post_call_rewritten_texts(groups, proxy_logging_obj, user_api_key_dict, data)
if rewritten_texts is None:
return body_bytes
split: Final = _split_group_texts(groups, rewritten_texts)
for frame_idx, (frame, text) in enumerate(zip(frames, texts)):
if (
text is not None
and frame.event is not None
and (new_text := split.get((frame_idx, 0))) is not None
and new_text != text.text
):
frame.guardrailed_event = {**frame.event, "delta": {**text.delta, text.field: new_text}}
return b"".join(frame.encode() for frame in frames)
class BedrockPassthroughGuardrailHandler(BaseTranslation):
@staticmethod
def is_event_stream_content_type(content_type: str) -> bool:
@ -263,19 +852,32 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
return _EVENT_STREAM_MEDIA_TYPE
@staticmethod
def event_stream_endpoint_is_de_anonymizable(endpoint: str) -> bool:
return _is_converse_endpoint(endpoint)
def event_stream_endpoint_is_de_anonymizable(endpoint: str, model: str, body: object) -> bool:
"""The streams the buffered path can scan whole and rewrite, the same ones the relay scans."""
return BedrockPassthroughGuardrailHandler.event_stream_endpoint_is_relayable(endpoint, model, body)
@staticmethod
def event_stream_endpoint_is_relayable(endpoint: str, model: str, body: object) -> bool:
"""Converse streams of every model, and invoke streams of Claude, whose chunks carry Anthropic Messages stream
events. Claude is told by its model id or, for ids that do not name it such as application inference profile
ARNs, by the ``anthropic_version`` Bedrock requires in every Claude Messages request body."""
return _is_converse_endpoint(endpoint) or (
_is_invoke_stream_endpoint(endpoint)
and (
BaseAWSLLM.get_bedrock_invoke_provider(model) == "anthropic"
or (isinstance(body, Mapping) and "anthropic_version" in body)
)
)
@staticmethod
async def de_anonymize_event_stream(
body_bytes: bytes,
proxy_logging_obj: "ProxyLogging",
user_api_key_dict: "UserAPIKeyAuth",
data: dict,
data: dict[str, object], # mutable-ok: the request dict post_call hooks receive
) -> bytes:
import json as _json
import struct
from binascii import crc32 as esm_crc32
if _is_invoke_stream_endpoint(str(data.get("endpoint") or "")):
return await _guardrail_buffered_invoke_stream(body_bytes, proxy_logging_obj, user_api_key_dict, data)
from botocore.eventstream import EventStreamBuffer
@ -304,16 +906,11 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
frames.append({"raw": frame_raw, "texts": []})
continue
texts: list[tuple[_StreamGroupKey, str]] = []
texts: Sequence[tuple[_StreamGroupKey, str]] = ()
if event_type == "contentBlockDelta":
try:
payload_dict: dict[str, object] = _json.loads(payload_bytes)
block_index = payload_dict.get("contentBlockIndex")
# Group per content block: a guardrail edit that changes length must not move text between blocks.
texts = [
((block_index if isinstance(block_index, int) else None, group_key), container[key])
for group_key, container, key in _collect_stream_delta_text_holders(payload_dict.get("delta"))
]
payload_dict: dict[str, object] = json.loads(payload_bytes)
texts = _delta_frame_texts(payload_dict)
except Exception as e:
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: could not parse "
@ -325,66 +922,17 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
trailing_bytes: Final = body_bytes[offset:]
group_order: Final[list[_StreamGroupKey]] = []
group_members: Final[dict[_StreamGroupKey, list[tuple[int, int]]]] = {}
group_texts: Final[dict[_StreamGroupKey, list[str]]] = {}
for frame_idx, frame in enumerate(frames):
for local_idx, (group_key, text) in enumerate(frame["texts"]):
if group_key not in group_members:
group_members[group_key] = []
group_texts[group_key] = []
group_order.append(group_key)
group_members[group_key].append((frame_idx, local_idx))
group_texts[group_key].append(text)
active_groups: Final = [gk for gk in group_order if "".join(group_texts[gk])]
if not active_groups:
groups: Final = _group_stream_texts([frame["texts"] for frame in frames])
if not groups.groups:
return body_bytes
synthetic_response: Final[dict] = {
"output": {
"message": {
"role": "assistant",
"content": [{"text": "".join(group_texts[gk])} for gk in active_groups],
}
},
"stopReason": "end_turn",
}
processed: Final = await proxy_logging_obj.post_call_success_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=synthetic_response,
de_anonymized_texts: Final = await _post_call_rewritten_texts(
groups, proxy_logging_obj, user_api_key_dict, data
)
if not isinstance(processed, dict):
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: post_call_success_hook returned %s, "
"leaving event stream unmodified",
type(processed).__name__,
)
if de_anonymized_texts is None:
return body_bytes
try:
processed_blocks: Final = processed["output"]["message"]["content"]
de_anonymized_texts: Final = [processed_blocks[i]["text"] for i in range(len(active_groups))]
except (KeyError, IndexError, TypeError):
return body_bytes
new_text_map: Final[dict[tuple[int, int], str]] = {}
for group_key, de_anonymized_text in zip(active_groups, de_anonymized_texts):
members = group_members[group_key]
orig_texts = group_texts[group_key]
total_orig = sum(len(t) for t in orig_texts) or 1
de_anon_len = len(de_anonymized_text)
pos = 0
for k, member in enumerate(members):
if k == len(members) - 1:
new_text_map[member] = de_anonymized_text[pos:]
else:
end = pos + round(de_anon_len * len(orig_texts[k]) / total_orig)
new_text_map[member] = de_anonymized_text[pos:end]
pos = end
new_text_map: Final = _split_group_texts(groups, de_anonymized_texts)
result_parts: Final[list[bytes]] = []
@ -399,27 +947,19 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
headers_bytes = frame_raw[12 : 12 + orig_hdrs_len]
try:
payload_dict = _json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
for local_idx, (_, container, key) in enumerate(
_collect_stream_delta_text_holders(payload_dict.get("delta"))
):
new_text = new_text_map.get((frame_idx, local_idx))
if new_text is not None:
container[key] = new_text
new_payload = _json.dumps(payload_dict, separators=(",", ":")).encode()
new_payload = json.dumps(payload_dict, separators=(",", ":")).encode()
except Exception:
result_parts.append(frame_raw)
continue
new_total = 12 + orig_hdrs_len + len(new_payload) + 4
prelude = struct.pack("!II", new_total, orig_hdrs_len)
prelude_crc_val = esm_crc32(prelude) & 0xFFFFFFFF
prelude_crc_b = struct.pack("!I", prelude_crc_val)
part_for_msg_crc = prelude_crc_b + headers_bytes + new_payload
msg_crc_val = esm_crc32(part_for_msg_crc, prelude_crc_val) & 0xFFFFFFFF
msg_crc_b = struct.pack("!I", msg_crc_val)
result_parts.append(prelude + prelude_crc_b + headers_bytes + new_payload + msg_crc_b)
result_parts.append(_event_stream_message(headers_bytes, new_payload))
result_parts.append(trailing_bytes)
return b"".join(result_parts)
@ -477,8 +1017,13 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
request_data: dict | None = None,
) -> object:
endpoint: Final = (request_data or {}).get("endpoint", "")
if endpoint and not _is_converse_endpoint(endpoint):
endpoint: Final[object] = (request_data or {}).get("endpoint", "")
if (
isinstance(endpoint, str)
and endpoint
and not _is_converse_endpoint(endpoint)
and not _is_invoke_stream_endpoint(endpoint)
):
return await _generic_passthrough_handler().process_output_response(
response=response,
guardrail_to_apply=guardrail_to_apply,
@ -503,6 +1048,20 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
if not texts:
return response
await self._scan_output_texts(
texts, holders, guardrail_to_apply, litellm_logging_obj, user_api_key_dict, request_data
)
return response
async def _scan_output_texts(
self,
texts: list[str],
holders: list[_StringHolder],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"],
user_api_key_dict: Optional["UserAPIKeyAuth"],
request_data: dict[str, object] | None, # mutable-ok: the request dict guardrails receive
) -> None:
effective_request_data = request_data or {}
if "litellm_metadata" not in effective_request_data and user_api_key_dict is not None:
user_metadata: Final = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
@ -513,8 +1072,8 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
}
inputs: Final = GenericGuardrailAPIInputs(texts=texts)
model: Final = effective_request_data.get("model") if effective_request_data else None
if model:
model: Final = effective_request_data.get("model")
if isinstance(model, str) and model:
inputs["model"] = model
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
@ -528,4 +1087,141 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
if guardrailed_texts:
_write_back_texts(guardrailed_texts, holders)
return response
async def process_output_streaming_response(
self,
responses_so_far: Sequence[object],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
request_data: dict[str, object] | None = None, # mutable-ok: the request dict guardrails receive
stream_transform_sink: StreamTransformSink | None = None,
deliver_ended_stream_rewrites: bool = False,
) -> Sequence[object]:
"""
Scan the Converse frames relayed so far as the buffered path scans a whole response: one text per content
block and kind. Once the stream has ended, guardrail rewrites are written back into the frames, which reach
the caller only when the stream is held until moderated. A guardrail that sets ``reads_stream_marks`` gets
a copy of the request marked as chat completions marks a streamed answer; the others get it unchanged.
Such a guardrail also gets the ended scan of a reply with no text, such as a call to a tool that takes no
input, with no texts to scan. Invoke frames are scanned as Anthropic Messages events.
"""
invoke_frames: Final = _invoke_frames(responses_so_far)
if invoke_frames:
await _scan_invoke_stream(
invoke_frames, guardrail_to_apply, litellm_logging_obj, user_api_key_dict, request_data
)
return responses_so_far
frames: Final = _converse_frames(responses_so_far)
groups: Final = _group_stream_texts([frame.scanned_texts() for frame in frames])
ended: Final = self._check_streaming_has_ended(frames)
reads_marks: Final = _reads_stream_marks(guardrail_to_apply)
if not groups.groups:
if request_data is not None and reads_marks and ended and _finished_converse_content(frames, groups):
await self._scan_output_texts(
[],
[],
guardrail_to_apply,
litellm_logging_obj,
user_api_key_dict,
_stream_marked_request_data(request_data, frames, groups, ended),
)
return responses_so_far
content: Final = [{"text": text} for text in groups.texts()]
await self.process_output_response( # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict
response={"output": {"message": {"role": "assistant", "content": content}}},
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=(
_stream_marked_request_data(request_data, frames, groups, ended)
if request_data is not None and reads_marks
else request_data
),
)
if ended:
_write_back_stream_texts(frames, groups, [block["text"] for block in content])
return responses_so_far
def needs_ended_stream_scan(self, guardrail: "CustomGuardrail", responses_so_far: Sequence[object]) -> bool:
return _reads_stream_marks(guardrail)
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
invoke_frames: Final = _invoke_frames(responses_so_far)
if invoke_frames:
return AnthropicMessagesHandler().get_streaming_scan_key(
list(_invoke_scan_sse(invoke_frames, [frame.received_sse() for frame in invoke_frames]))
)
frames: Final = _converse_frames(responses_so_far)
return StreamingScanKey(
texts=_group_stream_texts([frame.received_texts() for frame in frames]).texts(),
stream_ended=self._check_streaming_has_ended(frames),
)
def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool:
return _invoke_stream_has_ended(_invoke_frames(responses_so_far)) or any(
frame.event_type == "messageStop" or frame.ends_stream for frame in _converse_frames(responses_so_far)
)
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes]:
"""
End the message with the guardrail's message as a new text block and ``stopReason`` ``guardrail_intervened``,
closing any content block left open and starting the message when nothing was relayed yet. An invoke stream
gets the ``/v1/messages`` block events instead.
"""
invoke_frames: Final = _invoke_frames(responses_so_far or ())
if invoke_frames or _is_invoke_stream_endpoint(str(exc.request_data.get("endpoint") or "")):
return _invoke_block_frames(exc, stream_started, invoke_frames)
frames: Final = _converse_frames(responses_so_far or ())
started: Final = frozenset(
index
for index in (
frame.block_index for frame in frames if frame.event_type in ("contentBlockStart", "contentBlockDelta")
)
if index is not None
)
stopped: Final = frozenset(
index
for index in (frame.block_index for frame in frames if frame.event_type == "contentBlockStop")
if index is not None
)
block_index: Final = max(started | stopped, default=-1) + 1
return (
*(
()
if any(frame.event_type == "messageStart" for frame in frames)
else (_converse_event("messageStart", {"role": "assistant"}),)
),
*(_converse_event("contentBlockStop", {"contentBlockIndex": index}) for index in sorted(started - stopped)),
_converse_event("contentBlockDelta", {"contentBlockIndex": block_index, "delta": {"text": exc.message}}),
_converse_event("contentBlockStop", {"contentBlockIndex": block_index}),
_converse_event("messageStop", {"stopReason": "guardrail_intervened"}),
)
@staticmethod
def owns_event_stream_item(item: object) -> bool:
return isinstance(item, (ConverseStreamFrame, InvokeStreamFrame))
@staticmethod
def event_stream_items(
chunks: AsyncIterable[bytes], endpoint: str
) -> AsyncGenerator[ConverseStreamFrame | InvokeStreamFrame, None]:
if _is_invoke_stream_endpoint(endpoint):
return _event_stream_frames(chunks, InvokeStreamFrame, _decode_invoke_frame)
return _event_stream_frames(chunks, ConverseStreamFrame, _decode_converse_frame)
@staticmethod
def encode_event_stream_item(item: object) -> bytes:
if isinstance(item, (ConverseStreamFrame, InvokeStreamFrame)):
return item.encode()
if isinstance(item, bytes):
return item
raise TypeError(f"cannot relay {type(item).__name__} on a Bedrock event stream")
@staticmethod
def event_stream_error_item(message: str, status_code: int) -> bytes:
return _converse_exception(message, status_code)

171
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -1,18 +1,33 @@
import json
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from functools import partial
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Optional, cast
import httpx
from httpx import Response
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import AWS_ERROR_TYPE_HEADER
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
from litellm.types.utils import ModelResponseStream
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError, BedrockEventStreamDecoderBase, BedrockModelInfo
from ..request_metadata import (
BEDROCK_REQUEST_METADATA_HEADER,
BedrockRequestMetadataViolation,
bedrock_request_metadata_headers,
bedrock_request_metadata_is_owned,
bedrock_request_metadata_violation,
caller_request_metadata_violations_are_dropped,
merge_bedrock_invoke_headers,
resolve_bedrock_request_metadata,
)
if TYPE_CHECKING:
from botocore.eventstream import EventStreamMessage
@ -24,10 +39,87 @@ if TYPE_CHECKING:
BEDROCK_STREAMING_ACTIONS: Final = frozenset({"invoke-with-response-stream", "converse-stream"})
_BEDROCK_CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"})
_BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stream"})
def _bedrock_endpoint_action(endpoint: str) -> str:
return endpoint.partition("?")[0].rstrip("/").rsplit("/", 1)[-1]
def is_bedrock_streaming_endpoint(endpoint: str) -> bool:
return endpoint.partition("?")[0].rstrip("/").rsplit("/", 1)[-1] in BEDROCK_STREAMING_ACTIONS
return _bedrock_endpoint_action(endpoint) in BEDROCK_STREAMING_ACTIONS
def _caller_request_metadata_violation(
action: str, body: Mapping[str, object]
) -> BedrockRequestMetadataViolation | None:
if action in _BEDROCK_INVOKE_ACTIONS and "requestMetadata" in body and body["requestMetadata"] is None:
return BedrockRequestMetadataViolation("type", "requestMetadata must be a dictionary")
return bedrock_request_metadata_violation(body.get("requestMetadata"))
def _aws_error_type(action: str, violation: BedrockRequestMetadataViolation) -> str:
if action in _BEDROCK_CONVERSE_ACTIONS and violation.kind == "type":
return "SerializationException"
return "ValidationException"
def _caller_request_metadata(action: str, body: Mapping[str, object], litellm_params: Mapping[str, object]) -> object:
violation: Final = (
None
if caller_request_metadata_violations_are_dropped(litellm_params.get("drop_params") is True)
else _caller_request_metadata_violation(action, body)
)
if violation is not None:
raise BedrockError(
status_code=400,
message=violation.message,
headers={AWS_ERROR_TYPE_HEADER: _aws_error_type(action, violation)},
)
return body.get("requestMetadata")
def _with_body_request_metadata(
action: str, body: Mapping[str, object], litellm_params: Mapping[str, object]
) -> Mapping[str, object]:
resolved: Final = resolve_bedrock_request_metadata(
litellm_params, caller_metadata=_caller_request_metadata(action, body, litellm_params)
)
without_caller: Final = {key: value for key, value in body.items() if key != "requestMetadata"}
return without_caller if resolved is None else {**without_caller, "requestMetadata": resolved}
def _without_owned_request_metadata_header(headers: Mapping[str, str]) -> Mapping[str, str]:
if not bedrock_request_metadata_is_owned():
return headers
return {name: value for name, value in headers.items() if name.lower() != BEDROCK_REQUEST_METADATA_HEADER.lower()}
def _with_request_metadata(
action: str | None,
headers: Mapping[str, str],
body: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[Mapping[str, str], Mapping[str, object]]:
if action in _BEDROCK_CONVERSE_ACTIONS:
return headers, _with_body_request_metadata(action, body, litellm_params)
if action in _BEDROCK_INVOKE_ACTIONS and "requestMetadata" in body:
return _without_owned_request_metadata_header(headers), _with_body_request_metadata(
action, body, litellm_params
)
if action in _BEDROCK_INVOKE_ACTIONS:
return _with_invoke_request_metadata(headers, body, litellm_params)
return headers, body
def _with_invoke_request_metadata(
headers: Mapping[str, str],
body: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[Mapping[str, str], Mapping[str, object]]:
owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
return merge_bedrock_invoke_headers(dict(headers), (), metadata_headers, owned_names), body
_TEXT_ONLY_DELTA_FIELDS: Final = frozenset({"content", "role"})
@ -91,6 +183,57 @@ def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelR
return None
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_JSON_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def _leaf_texts(block: Mapping[str, object]) -> Iterator[str]:
"""The prompt text a content block carries itself: its text or JSON, or a tool call's input."""
text: Final = block.get("text")
if isinstance(text, str):
yield text
if "json" in block:
yield json.dumps(block["json"], default=str)
if "toolUse" in block:
yield json.dumps(_JSON_OBJECT.validate_python(block["toolUse"]).get("input"), default=str)
def _block_texts(block: Mapping[str, object]) -> Iterator[str]:
"""The prompt text a request content block carries, including the blocks of a tool result's content, which
Bedrock does not nest further."""
yield from _leaf_texts(block)
if "toolResult" not in block:
return
tool_result: Final = _JSON_OBJECT.validate_python(block["toolResult"])
yield from chain.from_iterable(map(_leaf_texts, _JSON_OBJECTS.validate_python(tool_result.get("content", ()))))
def _content_text(content: object) -> str:
if isinstance(content, str):
return content
return "\n".join(chain.from_iterable(map(_block_texts, _JSON_OBJECTS.validate_python(content))))
def _chat_message(turn: Mapping[str, object]) -> ChatCompletionUserMessage:
return ChatCompletionUserMessage(role="user", content=_content_text(turn.get("content", "")))
def _count_prompt_tokens(model: str, request_body: object) -> int:
"""Estimate a Bedrock request's prompt tokens from its system prompt and turns, the way a chat completions
stream that ended before the provider reported usage is estimated."""
from litellm.litellm_core_utils.token_counter import token_counter
body: Final = _JSON_OBJECT.validate_python(request_body)
if "messages" not in body:
return 0
system: Final = body.get("system")
messages: Final = (
*((ChatCompletionSystemMessage(role="system", content=_content_text(system)),) if system else ()),
*map(_chat_message, _JSON_OBJECTS.validate_python(body["messages"])),
)
return token_counter(model=model, messages=messages)
def _build_logged_response(
chunks: Sequence[ModelResponseStream], litellm_logging_obj: "LiteLLMLoggingObj"
) -> Optional["CostResponseTypes"]:
@ -98,7 +241,13 @@ def _build_logged_response(
if len(chunks) == 0:
return None
return stream_chunk_builder(chunks=list(chunks), logging_obj=litellm_logging_obj)
return stream_chunk_builder(
chunks=list(chunks),
logging_obj=litellm_logging_obj,
count_prompt_tokens=partial(
_count_prompt_tokens, litellm_logging_obj.model, litellm_logging_obj.model_call_details.get("input")
),
)
class BedrockEventStreamCollector:
@ -223,18 +372,24 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
def sign_request(
self,
headers: dict,
litellm_params: dict,
request_data: dict | None,
headers: dict[str, str],
litellm_params: dict[str, object],
request_data: dict[str, object] | None,
api_base: str,
model: str | None = None,
) -> tuple[dict, bytes | None]:
optional_params: Final = litellm_params.copy()
signed_headers, signed_body = _with_request_metadata(
action=_bedrock_endpoint_action(api_base) if bedrock_request_metadata_is_owned() else None,
headers=headers,
body=request_data or {},
litellm_params=litellm_params,
)
return self._sign_request(
service_name="bedrock",
headers=headers,
headers=dict(signed_headers),
optional_params=optional_params,
request_data=request_data or {},
request_data=dict(signed_body),
api_base=api_base,
model=model,
api_key=self.get_bedrock_bearer_token(optional_params),

93
litellm/llms/bedrock/realtime/handler.py
View file

@ -8,7 +8,7 @@ import asyncio
import contextlib
import importlib.metadata
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, NoReturn, Protocol, runtime_checkable
@ -21,7 +21,7 @@ from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY,
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
BEDROCK_REALTIME_SDK_DISTRIBUTION,
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
@ -37,9 +37,8 @@ from litellm.types.realtime import RealtimeResponseTransformInput
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
from ..common_utils import BedrockError
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
_CLIENT_MODALITIES_ADAPTER: Final[TypeAdapter["list[str] | None"]] = TypeAdapter(list[str] | None)
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
_EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
@ -177,25 +176,35 @@ async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message:
return
def _pending_session_update(scope: Mapping[str, object]) -> str | None:
"""A fallback attempt on the same websocket replays the session.update the failed attempt never acked."""
if scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True:
committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
raise BedrockError(
status_code=400,
message=(
"Bedrock realtime session already committed to a provider stream; it cannot be replayed"
+ (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
),
)
pending: Final = scope.get(BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY)
def _is_session_committed(scope: Mapping[str, object]) -> bool:
return scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True
def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
"""A fallback attempt on a websocket whose Bedrock stream already took input would silently start over."""
if scope is None or not _is_session_committed(scope):
return
committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
raise BedrockError(
status_code=400,
message=(
"Bedrock realtime session already committed to a provider stream; it cannot be replayed"
+ (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
),
)
def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
"""A fallback attempt on the same websocket replays the opening message Bedrock never accepted a stream for."""
raise_if_session_committed(scope)
pending: Final = scope.get(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY)
return pending if isinstance(pending, str) else None
def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn:
error: Final = _as_bedrock_error(failure)
verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error)))
if scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True:
if _is_session_committed(scope):
scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error))
raise error from failure
@ -207,26 +216,36 @@ def _parse_client_message(message: str) -> Mapping[str, JsonValue]:
return _EMPTY_JSON_OBJECT
async def _ack_session_update(
async def _send_opening_message(
client_ws: RealtimeClientWebSocket,
bedrock_stream: BedrockBidirectionalStream,
message: str,
bedrock_messages: Sequence[str],
send_to_bedrock: Callable[[str], Awaitable[None]],
) -> None:
"""Keep the opening message for a fallback to replay until Bedrock accepts the stream it opened."""
client_ws.scope[BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY] = message # rebind-ok: scope outlives attempts
for bedrock_message in bedrock_messages:
await send_to_bedrock(bedrock_message)
await bedrock_stream.await_output()
client_ws.scope.pop(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, None)
client_ws.scope[BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY] = True # rebind-ok: scope outlives the attempt
async def _ack_session_update(
client_ws: RealtimeClientWebSocket,
transformation_config: BedrockRealtimeConfig,
model: str,
logging_obj: LiteLLMLogging | None,
parsed_client_message: Mapping[str, JsonValue],
) -> bool:
"""Ack the client's session.update once Bedrock accepted the stream; False means the client is gone."""
await bedrock_stream.await_output()
client_ws.scope.pop(BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY, None)
client_ws.scope[BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY] = True # rebind-ok: scope outlives the attempt
"""Ack the client's session.update; False means the client is gone."""
if logging_obj is None:
return True
requested_modalities: Final = _CLIENT_MODALITIES_ADAPTER.validate_python(
_json_dict(parsed_client_message.get("session")).get("modalities")
)
modalities: Final = requested_modalities(_json_dict(parsed_client_message.get("session")))
try:
await client_ws.send_text(
json.dumps(transformation_config.session_updated_event(model, logging_obj, requested_modalities))
json.dumps(transformation_config.session_updated_event(model, logging_obj, modalities))
)
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
@ -314,7 +333,7 @@ class BedrockRealtime(BaseAWSLLM):
except ImportError as e:
raise _sdk_import_error(self._sdk_version_lookup(), e) from e
pending_session_update: Final = _pending_session_update(websocket.scope)
pending_opening_message: Final = _pending_opening_message(websocket.scope)
# Get AWS region
if aws_region_name is None:
@ -376,7 +395,7 @@ class BedrockRealtime(BaseAWSLLM):
try:
await self._run_session(
websocket, open_bidirectional_stream, model, logging_obj, pending_session_update, drop_params
websocket, open_bidirectional_stream, model, logging_obj, pending_opening_message, drop_params
)
finally:
await _close_bedrock_client(bedrock_client)
@ -387,7 +406,7 @@ class BedrockRealtime(BaseAWSLLM):
open_bidirectional_stream: Callable[[], Awaitable[BedrockBidirectionalStream]],
model: str,
logging_obj: LiteLLMLogging,
pending_session_update: str | None,
pending_opening_message: str | None,
drop_params: bool = False,
) -> None:
transformation_config: Final = BedrockRealtimeConfig(drop_params=drop_params)
@ -396,7 +415,7 @@ class BedrockRealtime(BaseAWSLLM):
verbose_proxy_logger.debug("Bedrock Realtime: Bidirectional stream established")
if pending_session_update is None:
if pending_opening_message is None:
await websocket.send_text(json.dumps(transformation_config.session_created_event(model, logging_obj)))
verbose_proxy_logger.debug("Bedrock Realtime: sent session.created to client on connect")
@ -418,7 +437,7 @@ class BedrockRealtime(BaseAWSLLM):
model,
session_state,
logging_obj,
initial_message=pending_session_update,
initial_message=pending_opening_message,
)
logged_events: Final = (
@ -531,18 +550,22 @@ class BedrockRealtime(BaseAWSLLM):
if not await _reject_session_update(client_ws, parsed_client_message, rejection):
break
continue
client_ws.scope[BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY] = message
transformed_messages = transformation_config.transform_realtime_request(
message=message,
model=model,
session_configuration_request=session_state.get("session_configuration_request"),
)
for bedrock_message in transformed_messages:
await send_to_bedrock(bedrock_message)
if transformed_messages and not _is_session_committed(client_ws.scope):
await _send_opening_message(
client_ws, bedrock_stream, message, transformed_messages, send_to_bedrock
)
else:
for bedrock_message in transformed_messages:
await send_to_bedrock(bedrock_message)
if is_session_update and not await _ack_session_update(
client_ws, bedrock_stream, transformation_config, model, logging_obj, parsed_client_message
client_ws, transformation_config, model, logging_obj, parsed_client_message
):
break
finally:

123
litellm/llms/bedrock/realtime/transformation.py
View file

@ -123,26 +123,81 @@ def _integer_setting(value: JsonValue) -> int | None:
return None if number is None else int(number)
def _malformed_number_settings(session_config: Mapping[str, JsonValue]) -> tuple[str, ...]:
def requested_modalities(session_config: Mapping[str, JsonValue]) -> tuple[str, ...] | None:
modalities: Final = session_config.get("modalities")
if not isinstance(modalities, list):
return None
names: Final = tuple(modality for modality in modalities if isinstance(modality, str))
return names if len(names) == len(modalities) else None
_OPENAI_VOICES: Final = frozenset(
("alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse", "marin", "cedar")
)
def _unplayable_voice(session_config: Mapping[str, JsonValue]) -> str | None:
voice: Final = session_config.get("voice")
if not isinstance(voice, str):
return None
return voice if not voice.strip() or voice.lower() in _OPENAI_VOICES else None
def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[str, str], ...]:
instructions: Final = session_config.get("instructions")
voice: Final = session_config.get("voice")
input_audio_format: Final = session_config.get("input_audio_format")
output_audio_format: Final = session_config.get("output_audio_format")
modalities: Final = session_config.get("modalities")
temperature: Final = session_config.get("temperature")
max_output_tokens: Final = session_config.get("max_response_output_tokens")
input_rate_hertz: Final = session_config.get("input_sample_rate_hertz")
output_rate_hertz: Final = session_config.get("output_sample_rate_hertz")
tools: Final = session_config.get("tools")
malformed: Final = (
("temperature", temperature is not None and _number_setting(temperature) is None),
("instructions", "a string", instructions is not None and not isinstance(instructions, str)),
("voice", "a string", voice is not None and not isinstance(voice, str)),
("tools", "a list", tools is not None and not isinstance(tools, list)),
(
"input_audio_format",
"a string",
input_audio_format is not None and not isinstance(input_audio_format, str),
),
(
"output_audio_format",
"a string",
output_audio_format is not None and not isinstance(output_audio_format, str),
),
(
"modalities",
"a list of strings",
modalities is not None and requested_modalities(session_config) is None,
),
("temperature", "a number", temperature is not None and _number_setting(temperature) is None),
(
"max_response_output_tokens",
"a number or 'inf'",
max_output_tokens is not None
and max_output_tokens != "inf"
and _integer_setting(max_output_tokens) is None,
),
("output_sample_rate_hertz", output_rate_hertz is not None and _integer_setting(output_rate_hertz) is None),
(
"input_sample_rate_hertz",
"a number",
input_rate_hertz is not None and _integer_setting(input_rate_hertz) is None,
),
(
"output_sample_rate_hertz",
"a number",
output_rate_hertz is not None and _integer_setting(output_rate_hertz) is None,
),
)
return tuple(name for name, is_malformed in malformed if is_malformed)
return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockSessionUpdateRejection:
code: Literal["invalid_type", "session_update_not_applied"]
code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
message: str
param: str
@ -312,7 +367,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
return 8000 # G.711 typically uses 8kHz
return 24000 if is_output else 16000
def transform_session_update_event(self, json_message: dict) -> list[str]:
def transform_session_update_event(self, json_message: Mapping[str, JsonValue]) -> list[str]:
"""
Transform session.update event to Bedrock session configuration.
@ -325,7 +380,10 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
verbose_logger.debug("Handling session.update")
messages: Final[list[str]] = []
session_config: Final = json_message.get("session", {})
requested_session: Final = json_message.get("session")
session_config: Final[Mapping[str, JsonValue]] = (
requested_session if isinstance(requested_session, dict) else {}
)
locked_settings: Final = self._apply_locked_settings(session_config, self._started_settings)
# Settings Bedrock locks at prompt start may only change before it
@ -336,13 +394,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
self.output_sample_rate_hertz = locked_settings.output_sample_rate_hertz
# Update audio input configuration from session if provided
if "input_audio_format" in session_config:
input_format: Final = session_config["input_audio_format"]
input_format: Final = session_config.get("input_audio_format")
if isinstance(input_format, str):
self.input_sample_rate_hertz = self._map_audio_format_to_sample_rate(input_format, is_output=False)
# Allow direct override of the input sample rate if provided (custom extension)
if "input_sample_rate_hertz" in session_config:
self.input_sample_rate_hertz = session_config["input_sample_rate_hertz"]
requested_input_rate: Final = _integer_setting(session_config.get("input_sample_rate_hertz"))
if requested_input_rate is not None:
self.input_sample_rate_hertz = requested_input_rate
if self.prompt_started:
return messages
@ -352,7 +411,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
messages.append(json.dumps(session_start))
# Send prompt start
tools: Final[Sequence[JsonValue] | None] = session_config.get("tools")
tools: Final = session_config.get("tools")
prompt_start_config: Final[Mapping[str, object]] = {
"promptName": self.prompt_name,
"textOutputConfiguration": {"mediaType": self.text_media_type},
@ -370,7 +429,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
"toolUseOutputConfiguration": {"mediaType": "application/json"},
"toolConfiguration": {"tools": self._transform_tools_to_bedrock_format(tools)},
}
if tools
if isinstance(tools, list) and tools
else {}
),
}
@ -445,12 +504,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
if isinstance(requested_tools, list)
else base.tools_json
)
system_prompt: Final = base.system_prompt if instructions is None else str(instructions)
voice_id: Final = base.voice_id if voice is None else str(voice)
system_prompt: Final = instructions if isinstance(instructions, str) else base.system_prompt
voice_id: Final = (
voice if isinstance(voice, str) and _unplayable_voice(session_config) is None else base.voice_id
)
format_sample_rate: Final = (
base.output_sample_rate_hertz
if output_audio_format is None
else self._map_audio_format_to_sample_rate(str(output_audio_format), is_output=True)
self._map_audio_format_to_sample_rate(output_audio_format, is_output=True)
if isinstance(output_audio_format, str)
else base.output_sample_rate_hertz
)
requested_output_rate: Final = _integer_setting(session_config.get("output_sample_rate_hertz"))
output_sample_rate_hertz: Final = (
@ -476,16 +537,26 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
session_config: Final = json_message.get("session")
if not isinstance(session_config, Mapping):
return None
malformed: Final = () if self.drop_params else _malformed_number_settings(session_config)
malformed: Final = () if self.drop_params else _malformed_settings(session_config)
if malformed:
name, expected = malformed[0]
return BedrockSessionUpdateRejection(
code="invalid_type",
message=(
f"Invalid type for 'session.{malformed[0]}': Bedrock Nova Sonic needs a number for temperature "
"and output_sample_rate_hertz, and a number or 'inf' for max_response_output_tokens. "
f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
"Set drop_params to drop it instead"
),
param=f"session.{malformed[0]}",
param=f"session.{name}",
)
unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
if unplayable_voice is not None:
return BedrockSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
"Set drop_params to drop it instead"
),
param="session.voice",
)
if not self.prompt_started:
return None
@ -785,7 +856,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
try:
json_message: Final[dict[str, object]] = json.loads(message)
json_message: Final[dict[str, JsonValue]] = json.loads(message)
except json.JSONDecodeError:
verbose_logger.warning("Invalid JSON message: %s", message[:200])
return []
@ -816,11 +887,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
self,
model: str,
logging_obj: LiteLLMLoggingObj,
modalities: list[str] | None = None,
modalities: Sequence[str] | None = None,
) -> OpenAIRealtimeStreamSession:
session: Final = OpenAIRealtimeStreamSession(
id=logging_obj.litellm_trace_id,
modalities=modalities if modalities is not None else ["text", "audio"],
modalities=list(modalities) if modalities is not None else ["text", "audio"],
)
if model is not None and isinstance(model, str):
session["model"] = model
@ -842,7 +913,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
self,
model: str,
logging_obj: LiteLLMLoggingObj,
modalities: list[str] | None = None,
modalities: Sequence[str] | None = None,
) -> OpenAIRealtimeStreamSessionEvents:
"""Build the OpenAI session.updated ack reflecting the client's requested modalities."""
return OpenAIRealtimeStreamSessionEvents(

129
litellm/llms/bedrock/request_metadata.py
View file

@ -8,17 +8,22 @@ sets ``litellm.bedrock_request_metadata_fields`` (``litellm_settings`` on the pr
Two properties are load-bearing for that billing record and are asserted by the tests:
proxy identity is resolved first so it can never be evicted by caller-supplied pairs, and the
whole ``user_api_key_`` prefix is reserved so a caller cannot write a proxy-authoritative
looking key. Values that break Bedrock's constraints are dropped rather than sanitised or
rejected, because an operator flipping this setting on must not turn a working request into a
400 and a silently rewritten attribution key is worse than an absent one.
looking key. Identity and ``spend_logs_metadata`` values that break Bedrock's constraints are
dropped rather than sanitised or rejected, because an operator flipping this setting on must not
turn a working request into a 400 and a silently rewritten attribution key is worse than an
absent one. For the same reason an Invoke header that would pass Bedrock's 8,500-character limit
keeps only the leading pairs that fit. A caller's own malformed ``requestMetadata`` is rejected with
a 400 unless ``drop_params`` is set, in which case its bad pairs are dropped the same way, with or
without the setting.
"""
from __future__ import annotations
import json
import re
from collections.abc import Mapping
from typing import Final
from collections.abc import Mapping, Sized
from dataclasses import dataclass
from typing import Final, Literal
import litellm
@ -26,15 +31,27 @@ BEDROCK_REQUEST_METADATA_HEADER: Final = "X-Amzn-Bedrock-Request-Metadata"
BEDROCK_REQUEST_METADATA_MAX_PAIRS: Final = 16
BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX: Final = "user_api_key_"
BEDROCK_REQUEST_METADATA_CLIENT_FIELD: Final = "spend_logs_metadata"
BEDROCK_REQUEST_METADATA_HEADER_MAX_CHARS: Final = 8500
_METADATA_PARAM_NAMES: Final[tuple[str, ...]] = ("metadata", "litellm_metadata")
_KEY_PATTERN: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
_VALUE_PATTERN: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
_ALLOWED_CHARACTERS: Final = r"[a-zA-Z0-9\s:_@$#=/+,.-]"
_KEY_PATTERN: Final = re.compile(rf"{_ALLOWED_CHARACTERS}{{1,256}}", re.ASCII)
_VALUE_PATTERN: Final = re.compile(rf"{_ALLOWED_CHARACTERS}{{0,256}}", re.ASCII)
_OWNED_HEADER_NAMES: Final[frozenset[str]] = frozenset((BEDROCK_REQUEST_METADATA_HEADER.lower(),))
@dataclass(frozen=True, slots=True)
class BedrockRequestMetadataViolation:
kind: Literal["type", "constraint"]
message: str
def _is_forwardable(key: str, value: str) -> bool:
return _KEY_PATTERN.match(key) is not None and _VALUE_PATTERN.match(value) is not None
return _KEY_PATTERN.fullmatch(key) is not None and _VALUE_PATTERN.fullmatch(value) is not None
def _is_reserved(key: str) -> bool:
return key.strip().lower().startswith(BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX)
def _text_pairs(source: object) -> tuple[tuple[str, str], ...]:
@ -99,13 +116,16 @@ def _client_pairs(
candidates: Final = tuple(
(key, value)
for key, value in (*_text_pairs(caller_metadata), *spend_logs_pairs)
if not key.startswith(BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX) and _is_forwardable(key, value)
if not _is_reserved(key) and _is_forwardable(key, value)
)
return tuple(
pair
for index, pair in enumerate(candidates)
if pair[0] not in tuple(earlier for earlier, _ in candidates[:index])
)[:budget]
first_index: Final = {key: index for index, (key, _) in reversed(tuple(enumerate(candidates)))}
return tuple(pair for index, pair in enumerate(candidates) if first_index[pair[0]] == index)[:budget]
def _forwardable_caller_pairs(caller_metadata: object) -> tuple[tuple[str, str], ...]:
return tuple(pair for pair in _text_pairs(caller_metadata) if _is_forwardable(*pair))[
:BEDROCK_REQUEST_METADATA_MAX_PAIRS
]
def resolve_bedrock_request_metadata(
@ -113,17 +133,18 @@ def resolve_bedrock_request_metadata(
caller_metadata: object = None,
) -> dict[str, str] | None:
"""
Resolve the ``requestMetadata`` pairs to send to Bedrock, or ``None`` when the feature is
off or nothing survives Bedrock's constraints. The result is a plain dict because it is
written straight onto the Converse body, which Bedrock types as ``dict[str, str]``.
Resolve the ``requestMetadata`` pairs to send to Bedrock, or ``None`` when nothing survives
Bedrock's constraints. The result is a plain dict because it is written straight onto the
Converse body, which Bedrock types as ``dict[str, str]``.
``caller_metadata`` is any ``requestMetadata`` the caller passed explicitly. It has already
been validated (and rejected with a 400) by the Converse transformation, so it is only
filtered here for the reserved identity prefix and the remaining slot budget.
``caller_metadata`` is any ``requestMetadata`` the caller passed explicitly. Callers reject a
malformed one with a 400 unless ``drop_params`` is set, so here its pairs are only filtered for
Bedrock's rules, the reserved identity prefix, and the remaining slot budget. With the feature
off only Bedrock's rules apply, since no identity is written for a caller to forge.
"""
allowed_fields: Final = _allowed_fields()
if not allowed_fields:
return None
return dict(_forwardable_caller_pairs(caller_metadata)) or None
sources: Final = _metadata_sources(litellm_params)
identity: Final = _identity_pairs(sources, allowed_fields)
client: Final = _client_pairs(
@ -136,6 +157,36 @@ def resolve_bedrock_request_metadata(
return resolved or None
def _pair_violation(key: str, value: str) -> BedrockRequestMetadataViolation | None:
if _KEY_PATTERN.fullmatch(key) is None:
return BedrockRequestMetadataViolation(
"constraint", f"requestMetadata key {key!r} must be 1-256 characters of {_ALLOWED_CHARACTERS}"
)
if _VALUE_PATTERN.fullmatch(value) is None:
return BedrockRequestMetadataViolation(
"constraint", f"requestMetadata value for key {key!r} must be 0-256 characters of {_ALLOWED_CHARACTERS}"
)
return None
def bedrock_request_metadata_violation(caller_metadata: object) -> BedrockRequestMetadataViolation | None:
"""Why a caller's ``requestMetadata`` breaks Bedrock's rules, in the order Bedrock checks them: a
wrong JSON type before the pair count and character rules. ``None`` means absent or valid."""
text_pairs: Final = _text_pairs(caller_metadata)
entry_count: Final = len(caller_metadata) if isinstance(caller_metadata, Sized) else 0
if caller_metadata is None:
return None
if not isinstance(caller_metadata, Mapping):
return BedrockRequestMetadataViolation("type", "requestMetadata must be a dictionary")
if len(text_pairs) != entry_count:
return BedrockRequestMetadataViolation("type", "requestMetadata keys and values must be strings")
if len(text_pairs) > BEDROCK_REQUEST_METADATA_MAX_PAIRS:
return BedrockRequestMetadataViolation(
"constraint", f"requestMetadata can contain a maximum of {BEDROCK_REQUEST_METADATA_MAX_PAIRS} items"
)
return next((violation for key, value in text_pairs if (violation := _pair_violation(key, value))), None)
def bedrock_request_metadata_is_owned() -> bool:
"""
Whether the proxy OWNS the request-metadata field and header name for this request.
@ -149,25 +200,51 @@ def bedrock_request_metadata_is_owned() -> bool:
return bool(_allowed_fields())
def caller_request_metadata_violations_are_dropped(drop_params: bool) -> bool:
return drop_params or litellm.drop_params is True
def bedrock_request_metadata_is_rewritten(drop_params: bool) -> bool:
return bedrock_request_metadata_is_owned() or caller_request_metadata_violations_are_dropped(drop_params)
def _header_value(pairs: Mapping[str, str]) -> str:
return json.dumps(pairs, separators=(",", ":"))
def _fits_header(pairs: Mapping[str, str]) -> bool:
return len(_header_value(pairs)) <= BEDROCK_REQUEST_METADATA_HEADER_MAX_CHARS
def _longest_fitting_prefix(resolved: Mapping[str, str]) -> dict[str, str]:
pairs: Final = tuple(resolved.items())
return next(
(dict(pairs[:count]) for count in range(len(pairs), 0, -1) if _fits_header(dict(pairs[:count]))),
{},
)
def bedrock_request_metadata_headers(
litellm_params: Mapping[str, object] | None,
) -> tuple[frozenset[str], tuple[tuple[str, str], ...]]:
"""
The signed ``X-Amzn-Bedrock-Request-Metadata`` header for the Invoke paths, which have no
body field for request metadata.
The signed ``X-Amzn-Bedrock-Request-Metadata`` header for an Invoke request whose body carries
no ``requestMetadata`` of its own.
Returns the header names the proxy OWNS and, separately, the pairs to send. Ownership is
reported whenever forwarding is enabled, including when nothing resolves, because a caller
can suppress the resolver (omit the allow-listed fields, or send values that all fail
Bedrock's rules) and an owned-but-empty result must still evict the caller's header rather
than fall back to it.
than fall back to it. Only the leading pairs that fit Bedrock's header limit are sent, so
identity always survives and long ``spend_logs_metadata`` pairs drop off the end.
"""
if not bedrock_request_metadata_is_owned():
return frozenset(), ()
resolved: Final = resolve_bedrock_request_metadata(litellm_params)
if resolved is None:
fitting: Final = {} if resolved is None else _longest_fitting_prefix(resolved)
if not fitting:
return _OWNED_HEADER_NAMES, ()
return _OWNED_HEADER_NAMES, ((BEDROCK_REQUEST_METADATA_HEADER, json.dumps(resolved, separators=(",", ":"))),)
return _OWNED_HEADER_NAMES, ((BEDROCK_REQUEST_METADATA_HEADER, _header_value(fitting)),)
def merge_bedrock_invoke_headers(

167
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -4,7 +4,7 @@ import json
import ssl
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
from contextlib import asynccontextmanager
from functools import lru_cache, partial
from functools import lru_cache, partial, reduce
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -35,7 +35,7 @@ import litellm.litellm_core_utils
import litellm.types
import litellm.types.utils
from litellm._logging import redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, update_headers_with_filtered_beta
from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@ -49,6 +49,7 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
synthesize_subtitle_document,
)
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
from litellm.litellm_core_utils.realtime_errors import (
@ -215,6 +216,7 @@ if TYPE_CHECKING:
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -1515,8 +1517,65 @@ class BaseLLMHTTPHandler:
api_base: str,
api_key: str | None,
headers: Mapping[str, str],
) -> tuple[str, dict[str, str], dict[str, object]]:
outbound_headers: Final = provider_config.validate_environment(headers=headers, model=model, api_key=api_key)
litellm_params: Mapping[str, object],
) -> tuple[str, Mapping[str, str], Mapping[str, object]]:
outbound_headers: Final = provider_config.validate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
return self._prepare_decisions_request_with_headers(
model=model,
logging_obj=logging_obj,
provider_config=provider_config,
body=body,
logged_body=logged_body,
api_base=api_base,
api_key=api_key,
outbound_headers=outbound_headers,
)
async def _aprepare_decisions_request(
self,
model: str,
logging_obj: LiteLLMLoggingObj | None,
provider_config: BaseDecisionsConfig,
body: Mapping[str, object],
logged_body: Mapping[str, object],
api_base: str,
api_key: str | None,
headers: Mapping[str, str],
litellm_params: Mapping[str, object],
) -> tuple[str, Mapping[str, str], Mapping[str, object]]:
outbound_headers: Final = await provider_config.avalidate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
return self._prepare_decisions_request_with_headers(
model=model,
logging_obj=logging_obj,
provider_config=provider_config,
body=body,
logged_body=logged_body,
api_base=api_base,
api_key=api_key,
outbound_headers=outbound_headers,
)
def _prepare_decisions_request_with_headers(
self,
model: str,
logging_obj: LiteLLMLoggingObj | None,
provider_config: BaseDecisionsConfig,
body: Mapping[str, object],
logged_body: Mapping[str, object],
api_base: str,
api_key: str | None,
outbound_headers: Mapping[str, str],
) -> tuple[str, Mapping[str, str], Mapping[str, object]]:
url: Final = provider_config.get_complete_url(api_base=api_base, model=model)
data: Final = dict(body)
if logging_obj is not None:
@ -1553,6 +1612,7 @@ class BaseLLMHTTPHandler:
api_base=api_base,
api_key=api_key,
headers=headers,
litellm_params=litellm_params,
)
signed_headers, signed_body = provider_config.sign_request(
headers=outbound_headers,
@ -1567,7 +1627,7 @@ class BaseLLMHTTPHandler:
response: Final = sync_httpx_client.post(
url,
content=signed_body,
json=data if signed_body is None else None,
json=dict(data) if signed_body is None else None,
headers=dict(signed_headers),
timeout=timeout,
logging_obj=logging_obj,
@ -1594,7 +1654,7 @@ class BaseLLMHTTPHandler:
litellm_params: Mapping[str, object],
client: AsyncHTTPHandler | None = None,
) -> DecisionsIRResponse:
url, outbound_headers, data = self._prepare_decisions_request(
url, outbound_headers, data = await self._aprepare_decisions_request(
model=model,
logging_obj=logging_obj,
provider_config=provider_config,
@ -1603,6 +1663,7 @@ class BaseLLMHTTPHandler:
api_base=api_base,
api_key=api_key,
headers=headers,
litellm_params=litellm_params,
)
sign: Final = partial(
provider_config.sign_request,
@ -1625,7 +1686,7 @@ class BaseLLMHTTPHandler:
response: Final = await async_httpx_client.post(
url,
content=signed_body,
json=data if signed_body is None else None,
json=dict(data) if signed_body is None else None,
headers=dict(signed_headers),
timeout=timeout,
logging_obj=logging_obj,
@ -2245,6 +2306,16 @@ class BaseLLMHTTPHandler:
merged_headers.update(extra_headers_from_kwargs)
if provider_specific_headers:
merged_headers.update(provider_specific_headers)
request_params: Final = reduce(
delete_nested_value,
litellm_params.get("additional_drop_params") or (),
anthropic_messages_optional_request_params,
)
if (
request_params.get("safeguards") is not None
and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
):
await aload_beta_headers_config()
(
headers,
api_base,
@ -2252,7 +2323,7 @@ class BaseLLMHTTPHandler:
headers=merged_headers or {},
model=model,
messages=messages,
optional_params=anthropic_messages_optional_request_params,
optional_params=request_params,
litellm_params=dict(litellm_params),
api_key=api_key,
api_base=api_base,
@ -2283,20 +2354,11 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
)
additional_drop_params: Final[list[str]] = litellm_params.get("additional_drop_params") or []
if additional_drop_params:
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
for path in additional_drop_params:
anthropic_messages_optional_request_params = delete_nested_value(
anthropic_messages_optional_request_params, path
)
# Prepare request body
request_body: Final = anthropic_messages_provider_config.transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
anthropic_messages_optional_request_params=request_params,
litellm_params=litellm_params,
headers=headers,
)
@ -2403,7 +2465,7 @@ class BaseLLMHTTPHandler:
held_back_tool_names: Final = self._server_fulfilled_tools_in_request(
logging_obj=logging_obj,
tools=anthropic_messages_optional_request_params.get("tools"),
tools=request_params.get("tools"),
)
initial_response = AgenticAnthropicStreamingIterator(
completion_stream=completion_stream,
@ -2411,7 +2473,7 @@ class BaseLLMHTTPHandler:
model=model,
messages=messages,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
anthropic_messages_optional_request_params=request_params,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs_for_agentic,
@ -2434,7 +2496,7 @@ class BaseLLMHTTPHandler:
model=model,
messages=messages,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
anthropic_messages_optional_request_params=request_params,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs_for_agentic,
@ -6807,6 +6869,7 @@ class BaseLLMHTTPHandler:
first_message: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs: object,
) -> Exception | None:
"""
@ -6822,6 +6885,35 @@ class BaseLLMHTTPHandler:
"""
_ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
_ws_guardrail_callbacks: list = []
_ws_output_guardrail_callbacks: list = []
try:
import litellm as _litellm
# Use duck-typing so any guardrail that exposes the PII
# masking interface works, not just _OPTIONAL_PresidioPIIMasking.
# This avoids a layering violation (SDK importing from proxy).
_ws_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and callable(getattr(cb, "_unmask_pii_text", None))
and getattr(cb, "output_parse_pii", False)
]
_ws_output_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and getattr(cb, "apply_to_output", False)
]
except Exception as _guardrail_exc:
verbose_logger.warning(
"Responses WebSocket: failed to collect guardrail callbacks — PII masking will be skipped. Error: %s",
_guardrail_exc,
)
if responses_api_provider_config is None or not responses_api_provider_config.supports_native_websocket():
from litellm.responses.streaming_iterator import (
ManagedResponsesWebSocketHandler,
@ -6840,6 +6932,8 @@ class BaseLLMHTTPHandler:
first_message=first_message,
quota_callbacks=_ws_quota_callbacks,
turn_admission=turn_admission,
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
frame_guard=frame_guard,
**kwargs,
)
return await handler.run()
@ -6920,36 +7014,6 @@ class BaseLLMHTTPHandler:
if litellm_metadata:
_request_data["litellm_metadata"] = litellm_metadata
_ws_guardrail_callbacks: list = []
_ws_output_guardrail_callbacks: list = []
try:
import litellm as _litellm
# Use duck-typing so any guardrail that exposes the PII
# masking interface works, not just _OPTIONAL_PresidioPIIMasking.
# This avoids a layering violation (SDK importing from proxy).
_ws_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and callable(getattr(cb, "_unmask_pii_text", None))
and getattr(cb, "output_parse_pii", False)
]
_ws_output_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and getattr(cb, "apply_to_output", False)
]
except Exception as _guardrail_exc:
verbose_logger.warning(
"Responses WebSocket: failed to collect guardrail "
"callbacks — PII masking will be skipped. Error: %s",
_guardrail_exc,
)
streaming: Final = ResponsesWebSocketStreaming(
websocket=websocket,
backend_ws=cast(ClientConnection, backend_ws),
@ -6964,6 +7028,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
request_defaults=request_defaults,
turn_admission=turn_admission,
frame_guard=frame_guard,
)
return await streaming.bidirectional_forward()

2
litellm/llms/fal_ai/cost_calculator.py
View file

@ -87,7 +87,7 @@ def _keyed_rows(model: str, quality: str) -> tuple[tuple[int, int, float], ...]:
suffix: Final = f"/{model}"
return tuple(
(width, height, float(raw_cost))
for key in litellm.model_cost
for key in (*litellm.model_cost,)
if isinstance(key, str) and key.startswith(prefix) and key.endswith(suffix)
for size in (key[len(prefix) : -len(suffix)],)
for dimensions in (_parse_keyed_dimensions(size),)

2
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -544,7 +544,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return cached[2]
index: Final[list[tuple[str, dict]]] = []
for key, model_info in model_cost.items():
for key, model_info in (*model_cost.items(),):
if not key.startswith("fireworks_ai/"):
continue
if not isinstance(model_info, dict):

9
litellm/llms/nace/decisions/transformation.py Normal file
View file

@ -0,0 +1,9 @@
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class NaceDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("NACE_API_KEY",)
api_base_env = ("NACE_API_BASE", "NACE_BASE_URL")
def get_default_api_base(self) -> str | None:
return "https://console.nace.ai"

2
litellm/llms/ollama/common_utils.py
View file

@ -197,7 +197,7 @@ class OllamaModelInfo(BaseLLMModelInfo):
"ollama/" + stripped_model,
"ollama_chat/" + stripped_model,
}
model_cost_keys: Final = {key.lower() for key in model_cost}
model_cost_keys: Final = {key.lower() for key in (*model_cost,)}
return any(name.lower() in model_cost_keys for name in potential_model_names)
@staticmethod

4
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -28,7 +28,9 @@ def _catalogue_declares_default_effort() -> bool:
the one litellm gave before it existed. Scanning costs ~80us on the largest published map and
only on the fallback path, which is noise beside the request it precedes.
"""
return any(isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in litellm.model_cost.values())
return any(
isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in (*litellm.model_cost.values(),)
)
def _normalize_reasoning_effort_for_chat_completion(

40
litellm/llms/openai/openai.py
View file

@ -11,13 +11,14 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
import openai
from openai import AsyncOpenAI, OpenAI
from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
from openai._base_client import make_request_options
from openai._constants import RAW_RESPONSE_HEADER
from openai._legacy_response import LegacyAPIResponse
from openai._types import RequestOptions
from openai.types import CreateEmbeddingResponse
from openai.types.beta.assistant_deleted import AssistantDeleted
from openai.types.chat import ChatCompletion, ChatCompletionChunk
from openai.types.file_deleted import FileDeleted
from pydantic import BaseModel, TypeAdapter
from typing_extensions import overload
@ -354,7 +355,14 @@ _NO_EXTRA_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({})
_SDK_OPTION_KEYS: Final = frozenset(("extra_headers", "extra_query", "extra_body"))
def _embedding_request_without_sdk_defaults(
def _raw_chat_response(
result: ChatCompletion | Stream[ChatCompletionChunk] | AsyncStream[ChatCompletionChunk],
) -> LegacyAPIResponse[ChatCompletion]:
assert isinstance(result, LegacyAPIResponse)
return result
def _request_without_sdk_transform(
data: Mapping[str, object], timeout: float | httpx.Timeout
) -> tuple[Mapping[str, object], RequestOptions]:
body: Final = {k: v for k, v in data.items() if k not in _SDK_OPTION_KEYS}
@ -517,7 +525,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
"""
start_time: Final = time.time()
try:
raw_response = await openai_aclient.chat.completions.with_raw_response.create(**data, timeout=timeout)
body, options = _request_without_sdk_transform(data, timeout)
raw_response = _raw_chat_response(
await openai_aclient.post(
"/chat/completions",
body=body,
cast_to=ChatCompletion,
options=options,
stream=bool(data.get("stream")),
stream_cls=AsyncStream[ChatCompletionChunk],
)
)
end_time = time.time()
if hasattr(raw_response, "headers"):
@ -558,7 +576,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
"""
raw_response = None
try:
raw_response = openai_client.chat.completions.with_raw_response.create(**data, timeout=timeout)
body, options = _request_without_sdk_transform(data, timeout)
raw_response = _raw_chat_response(
openai_client.post(
"/chat/completions",
body=body,
cast_to=ChatCompletion,
options=options,
stream=bool(data.get("stream")),
stream_cls=Stream[ChatCompletionChunk],
)
)
if hasattr(raw_response, "headers"):
headers = dict(raw_response.headers)
@ -1257,7 +1285,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
logging_obj: LiteLLMLoggingObj,
) -> LegacyAPIResponse[CreateEmbeddingResponse]:
if "encoding_format" not in data:
body, options = _embedding_request_without_sdk_defaults(data, timeout)
body, options = _request_without_sdk_transform(data, timeout)
bypass_response: Final = await openai_aclient.post(
"/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse
)
@ -1274,7 +1302,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
logging_obj: LiteLLMLoggingObj,
) -> LegacyAPIResponse[CreateEmbeddingResponse]:
if "encoding_format" not in data:
body, options = _embedding_request_without_sdk_defaults(data, timeout)
body, options = _request_without_sdk_transform(data, timeout)
bypass_response: Final = openai_client.post(
"/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse
)

2
litellm/llms/openai_like/chat/transformation.py
View file

@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage
from litellm.types.utils import ModelResponse
@ -119,6 +120,7 @@ class OpenAILikeChatConfig(OpenAIGPTConfig):
choice["message"] = message
returned_response: Final = ModelResponse(**response_json)
set_provider_response_headers_in_hidden_params(returned_response, response.headers)
if custom_llm_provider is not None:
returned_response.model = custom_llm_provider + "/" + (returned_response.model or "")

2
litellm/llms/openai_like/messages/transformation.py
View file

@ -8,6 +8,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.openai_like.json_loader import SimpleProviderConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
@ -57,6 +58,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
headers=normalized,
optional_params=optional_params,
messages=messages,
beta_map_provider=LlmProviders.OPENAI_LIKE.value,
)
return merged, api_base

136
litellm/llms/pass_through/guardrail_translation/handler.py
View file

@ -6,17 +6,21 @@ It uses the field targeting configuration from litellm_logging_obj
to extract specific fields for guardrail processing.
"""
from collections.abc import Mapping
from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Final, Optional, Protocol, runtime_checkable
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import without_server_streaming_classification
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
StreamTransformSink,
)
from litellm.types.passthrough_endpoints.pass_through_endpoints import PassThroughGuardrailSettings
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.utils import ProxyLogging
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
@ -208,7 +212,7 @@ class _EventStreamMediaTypeProvider(Protocol):
@runtime_checkable
class _EventStreamEndpointChecker(Protocol):
def event_stream_endpoint_is_de_anonymizable(self, endpoint: str) -> bool: ...
def event_stream_endpoint_is_de_anonymizable(self, endpoint: str, model: str, body: object) -> bool: ...
_PROVIDER_HANDLERS: dict[str, type[BaseTranslation]] = {}
@ -225,6 +229,15 @@ def _get_provider_handlers() -> dict[str, type[BaseTranslation]]:
return _PROVIDER_HANDLERS
def _provider_handler(provider: object) -> type[BaseTranslation] | None:
return _get_provider_handlers().get(provider) if isinstance(provider, str) else None
def _owns_event_stream_item(handler_cls: type[BaseTranslation], item: object) -> bool:
owns: Final[Callable[[object], bool] | None] = getattr(handler_cls, "owns_event_stream_item", None)
return owns is not None and owns(item)
class LlmPassthroughRouteHandler(BaseTranslation):
"""
Dispatcher for allm_passthrough_route guardrail translation.
@ -277,6 +290,100 @@ class LlmPassthroughRouteHandler(BaseTranslation):
request_data=request_data,
)
@staticmethod
def _event_stream_translation(items: Sequence[object], provider: object = None) -> BaseTranslation | None:
"""The provider handler for a relayed event stream: the one that owns its frames, else ``provider``'s."""
handlers: Final = _get_provider_handlers()
owner: Final = next(
(
handler_cls
for handler_cls in handlers.values()
if items and _owns_event_stream_item(handler_cls, items[0])
),
handlers.get(provider) if isinstance(provider, str) else None,
)
return owner() if owner is not None else None
async def process_output_streaming_response(
self,
responses_so_far: Sequence[object],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
request_data: dict[str, object] | None = None, # mutable-ok: the request dict guardrails receive
stream_transform_sink: StreamTransformSink | None = None,
deliver_ended_stream_rewrites: bool = False,
) -> Sequence[object]:
translation: Final = self._event_stream_translation(
responses_so_far, (request_data or {}).get("custom_llm_provider")
)
if translation is not None:
await translation.process_output_streaming_response( # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict
responses_so_far=list(responses_so_far),
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=request_data,
stream_transform_sink=stream_transform_sink,
deliver_ended_stream_rewrites=deliver_ended_stream_rewrites,
)
return responses_so_far
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
translation: Final = self._event_stream_translation(responses_so_far)
return translation.get_streaming_scan_key(responses_so_far) if translation is not None else None
def needs_ended_stream_scan(self, guardrail: "CustomGuardrail", responses_so_far: Sequence[object]) -> bool:
translation: Final = self._event_stream_translation(responses_so_far)
return translation is not None and translation.needs_ended_stream_scan(guardrail, responses_so_far)
def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool:
has_ended: Final[Callable[[Sequence[object]], bool] | None] = getattr(
self._event_stream_translation(responses_so_far), "_check_streaming_has_ended", None
)
return has_ended is not None and has_ended(responses_so_far)
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes] | None:
translation: Final = self._event_stream_translation(
responses_so_far or (), exc.request_data.get("custom_llm_provider")
)
if translation is None:
return None
return translation.build_block_sse_chunks(exc, stream_started=stream_started, responses_so_far=responses_so_far)
@staticmethod
def event_stream_items(
provider: object, endpoint: object, chunks: AsyncIterable[bytes]
) -> AsyncIterator[object] | None:
"""The upstream event stream of ``endpoint`` cut into the items the provider's guardrail translation scans, or
None when the provider has no streaming guardrail translation."""
framer: Final[Callable[[AsyncIterable[bytes], str], AsyncIterator[object]] | None] = getattr(
_provider_handler(provider), "event_stream_items", None
)
return framer(chunks, endpoint if isinstance(endpoint, str) else "") if framer is not None else None
@staticmethod
def encode_event_stream_item(provider: object, item: object) -> bytes:
encoder: Final[Callable[[object], bytes] | None] = getattr(
_provider_handler(provider), "encode_event_stream_item", None
)
if encoder is None:
raise TypeError(f"provider {provider!r} has no event-stream encoder")
return encoder(item)
@staticmethod
def event_stream_error_item(provider: object, message: str, status_code: int) -> bytes | None:
"""An in-stream error frame in the provider's event-stream format, or None when the format has none."""
builder: Final[Callable[[str, int], bytes] | None] = getattr(
_provider_handler(provider), "event_stream_error_item", None
)
return builder(message, status_code) if builder is not None else None
@staticmethod
def is_event_stream_response(provider: str | None, content_type: str) -> bool:
handler_cls: Final = _get_provider_handlers().get(provider or "")
@ -297,11 +404,26 @@ class LlmPassthroughRouteHandler(BaseTranslation):
return getattr(handler_cls, "de_anonymize_event_stream", None)
@staticmethod
def supports_event_stream_de_anonymization(provider: str | None, endpoint: str | None) -> bool:
handler_cls: Final = _get_provider_handlers().get(provider or "")
def supports_event_stream_relay(provider: object, endpoint: object, model: object, body: object) -> bool:
"""Whether the provider's guardrail translation can scan this endpoint's event stream for ``model`` and the
request ``body`` frame by frame as it is relayed."""
relayable: Final[Callable[[str, str, object], bool] | None] = getattr(
_provider_handler(provider), "event_stream_endpoint_is_relayable", None
)
return relayable is not None and relayable(
endpoint if isinstance(endpoint, str) else "", model if isinstance(model, str) else "", body
)
@staticmethod
def supports_event_stream_de_anonymization(provider: object, endpoint: object, model: object, body: object) -> bool:
"""Whether the provider's guardrail translation can scan this endpoint's whole event stream for ``model`` and
the request ``body`` and rewrite its frames."""
handler_cls: Final = _provider_handler(provider)
if not isinstance(handler_cls, _EventStreamEndpointChecker):
return False
return handler_cls.event_stream_endpoint_is_de_anonymizable(endpoint or "")
return handler_cls.event_stream_endpoint_is_de_anonymizable(
endpoint if isinstance(endpoint, str) else "", model if isinstance(model, str) else "", body
)
@staticmethod
async def de_anonymize_event_stream(

5
litellm/main.py
View file

@ -4132,6 +4132,7 @@ def _complete_sagemaker(ctx: CompletionDispatchContext) -> _CompletionDispatchRe
_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
_BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
@ -5695,7 +5696,9 @@ def completion(
and provider_config.accepts_reasoning_summary_in_reasoning_effort,
)
optional_params = get_optional_params(
**{**optional_param_args, "reasoning_effort": provider_reasoning_effort}, **non_default_params
**{**optional_param_args, "reasoning_effort": provider_reasoning_effort},
model_info=_MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None,
**non_default_params,
)
processed_non_default_params: Final = pre_process_non_default_params(
model=model,

271
litellm/model_prices_and_context_window_backup.json
View file

@ -1546,7 +1546,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1623,7 +1623,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1700,7 +1700,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1777,7 +1777,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -12877,6 +12877,7 @@
"supports_prompt_caching": true
},
"azure_ai/ministral-3b": {
"deprecation_date": "2026-12-09",
"input_cost_per_token": 4e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 128000,
@ -12940,6 +12941,7 @@
"supports_vision": true
},
"azure_ai/mistral-medium-2505": {
"deprecation_date": "2026-12-09",
"input_cost_per_token": 4e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12977,6 +12979,7 @@
"supports_tool_choice": true
},
"azure_ai/mistral-small-2503": {
"deprecation_date": "2026-12-09",
"input_cost_per_token": 1e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 128000,
@ -29699,6 +29702,7 @@
},
"vertex_ai/gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"deprecation_date": "2027-09-02",
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -29806,6 +29810,7 @@
"supports_native_streaming": true
},
"vertex_ai/gemini-3.8-live": {
"deprecation_date": "2027-09-25",
"input_cost_per_audio_token": 3e-06,
"input_cost_per_image_token": 1e-06,
"input_cost_per_token": 7.5e-07,
@ -31960,6 +31965,7 @@
},
"gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"deprecation_date": "2027-09-02",
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -32409,11 +32415,16 @@
]
},
"github_copilot/claude-fable-5": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_read_input_token_cost": 1e-06,
"input_cost_per_token": 1e-05,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/messages"
@ -32427,11 +32438,16 @@
"thinking_always_on": true
},
"github_copilot/claude-fable-5.1": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 1e-05,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/messages"
@ -32490,11 +32506,16 @@
"thinking_always_on": false
},
"github_copilot/claude-opus-4.8": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/messages"
@ -32524,13 +32545,17 @@
"supports_vision": true
},
"github_copilot/claude-opus-5": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"deprecation_date": "2026-11-16",
"input_cost_per_token": 5e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"source": "https://github.blog/changelog/2026-10-15-upcoming-deprecation-of-selected-github-copilot-models-in-mid-november",
"output_cost_per_token": 2.5e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/messages"
@ -32543,11 +32568,16 @@
"supports_vision": true
},
"github_copilot/claude-opus-5.5": {
"cache_creation_input_token_cost": 5e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 4e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/messages"
@ -32561,11 +32591,16 @@
"thinking_always_on": true
},
"github_copilot/claude-sonnet-5": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/messages"
@ -32578,11 +32613,16 @@
"supports_vision": true
},
"github_copilot/claude-sonnet-5.5": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/messages"
@ -32596,11 +32636,15 @@
"thinking_always_on": true
},
"github_copilot/gemini-3.7-flash": {
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_token": 7.5e-07,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 3.75e-06,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions"
],
@ -32610,11 +32654,15 @@
"supports_vision": true
},
"github_copilot/gemini-3.8-flash": {
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_token": 7.5e-07,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 3.75e-06,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions"
],
@ -32780,11 +32828,18 @@
"supports_vision": true
},
"github_copilot/gpt-5.4": {
"cache_read_input_token_cost": 2.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -32796,11 +32851,15 @@
"supports_vision": true
},
"github_copilot/gpt-5.4-mini": {
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_token": 7.5e-07,
"litellm_provider": "github_copilot",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 4.5e-06,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -32811,11 +32870,18 @@
"supports_vision": true
},
"github_copilot/gpt-5.5": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"litellm_provider": "github_copilot",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -32826,11 +32892,20 @@
"supports_vision": true
},
"github_copilot/gpt-5.6-luna": {
"cache_creation_input_token_cost": 2.5e-07,
"cache_creation_input_token_cost_above_200k_tokens": 5e-07,
"cache_read_input_token_cost": 2e-08,
"cache_read_input_token_cost_above_200k_tokens": 4e-08,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_200k_tokens": 4e-07,
"litellm_provider": "github_copilot",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 1.2e-06,
"output_cost_per_token_above_200k_tokens": 1.8e-06,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -32841,11 +32916,20 @@
"supports_vision": true
},
"github_copilot/gpt-5.6-sol": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_read_input_token_cost": 4e-07,
"cache_read_input_token_cost_above_272k_tokens": 8e-07,
"input_cost_per_token": 4e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 2e-05,
"output_cost_per_token_above_272k_tokens": 3e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -32856,13 +32940,21 @@
"supports_vision": true
},
"github_copilot/gpt-5.6-terra": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_272k_tokens": 4e-07,
"deprecation_date": "2026-11-16",
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"source": "https://github.blog/changelog/2026-10-15-upcoming-deprecation-of-selected-github-copilot-models-in-mid-november",
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_above_272k_tokens": 1.8e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -32873,11 +32965,20 @@
"supports_vision": true
},
"github_copilot/gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
"cache_read_input_token_cost": 1e-06,
"cache_read_input_token_cost_above_272k_tokens": 2e-06,
"input_cost_per_token": 1e-05,
"input_cost_per_token_above_272k_tokens": 2e-05,
"litellm_provider": "github_copilot",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 5e-05,
"output_cost_per_token_above_272k_tokens": 7.5e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -32888,11 +32989,20 @@
"supports_vision": true
},
"github_copilot/gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_above_272k_tokens": 2e-08,
"input_cost_per_token": 1e-07,
"input_cost_per_token_above_272k_tokens": 2e-07,
"litellm_provider": "github_copilot",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 5e-07,
"output_cost_per_token_above_272k_tokens": 7.5e-07,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -32903,11 +33013,20 @@
"supports_vision": true
},
"github_copilot/gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_272k_tokens": 4e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 1e-05,
"output_cost_per_token_above_272k_tokens": 1.5e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -32918,11 +33037,20 @@
"supports_vision": true
},
"github_copilot/gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_above_272k_tokens": 2e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 1e-05,
"output_cost_per_token_above_272k_tokens": 1.5e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -32933,11 +33061,15 @@
"supports_vision": true
},
"github_copilot/kimi-k3": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "github_copilot",
"max_input_tokens": 917504,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/chat/completions"
],
@ -32947,11 +33079,15 @@
"supports_vision": true
},
"github_copilot/mai-code-1.1-flash": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 2e-07,
"litellm_provider": "github_copilot",
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 1.2e-06,
"source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml",
"supported_endpoints": [
"/v1/responses"
],
@ -45295,6 +45431,38 @@
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/getting-started/pricing"
},
"perplexity/pplx-decider-v1.2": {
"input_cost_per_token": 2e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
"perplexity/pplx-decider-latest": {
"input_cost_per_token": 2e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
"perplexity/pplx-decider-flash-v1.2": {
"input_cost_per_token": 1e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
"perplexity/pplx-decider-flash-latest": {
"input_cost_per_token": 1e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
"perplexity/sonar": {
"input_cost_per_token": 1e-06,
"litellm_provider": "perplexity",
@ -65431,6 +65599,40 @@
"supports_vision": false,
"source": "https://deepinfra.com/pricing"
},
"deepinfra/deepseek-ai/DeepSeek-V4.1-Flash": {
"max_tokens": 131072,
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 4.2e-07,
"cache_read_input_token_cost": 4.2e-09,
"supports_prompt_caching": true,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash"
},
"deepinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"max_tokens": 131072,
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 4.2e-07,
"cache_read_input_token_cost": 4.2e-09,
"supports_prompt_caching": true,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash-1017"
},
"deepinfra/openai/gpt-oss-120b-Ultra": {
"max_tokens": 131072,
"max_input_tokens": 131072,
@ -71184,6 +71386,38 @@
"notes": "Self-hosted decision model; infrastructure costs are paid separately"
}
},
"nace/drex-v1.0": {
"input_cost_per_token": 4e-08,
"litellm_provider": "nace",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://console.nace.ai/docs/drex/reference/models"
},
"nace/drex-v1.5": {
"input_cost_per_token": 5e-08,
"litellm_provider": "nace",
"max_input_tokens": 131072,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://console.nace.ai/docs/drex/reference/models"
},
"nace/drex-v2.0": {
"input_cost_per_token": 6e-08,
"litellm_provider": "nace",
"max_input_tokens": 131072,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://console.nace.ai/docs/drex/reference/models"
},
"nace/drex-latest": {
"input_cost_per_token": 6e-08,
"litellm_provider": "nace",
"max_input_tokens": 131072,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://console.nace.ai/docs/drex/reference/models"
},
"typesafe/jev-1.13.0": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "typesafe",
@ -72466,7 +72700,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -72499,7 +72733,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -72856,6 +73090,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -72887,6 +73122,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -72933,6 +73169,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -73003,6 +73240,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -73039,6 +73277,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -73082,6 +73321,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -73123,6 +73363,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -73164,6 +73405,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -73286,6 +73528,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -73329,6 +73572,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -74301,6 +74545,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -74337,6 +74582,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -74388,6 +74634,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -74468,6 +74715,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -74509,6 +74757,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -74557,6 +74806,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -74603,6 +74853,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -74649,6 +74900,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -74699,6 +74951,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -74747,6 +75000,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -75141,6 +75395,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-4-argon": {
"deprecation_date": "2027-10-13",
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,

14
litellm/passthrough/main.py
View file

@ -99,7 +99,8 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]):
self._litellm_logging_obj = litellm_logging_obj
self._spend = _SpendCollection(provider_config, litellm_logging_obj)
self._flush_scheduled = False
self._background_tasks: set[asyncio.Task] = set() # mutable-ok: instance set for background task tracking
self._background_tasks: set[asyncio.Task[bool]] = set() # mutable-ok: instance set for background task tracking
self._flush: asyncio.Task[bool] | None = None
self._hidden_params: dict[str, object] = {} # mutable-ok: router attaches response headers here in place
@property
@ -157,6 +158,7 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]):
)
self._background_tasks.add(task)
self._flush = task
task.add_done_callback(self._background_tasks.discard)
except Exception as e: # noqa: BLE001 # Safe catch-all for verbose logging
@ -166,6 +168,16 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]):
e,
)
async def wait_for_spend_flush(self) -> bool:
"""Wait until the spend-tracking flush scheduled when the stream ended or was closed has finished, and say
whether it logged the stream, which it does not when the stream returned nothing it could read. Cancelling the
wait leaves the flush running."""
flush: Final = self._flush
if flush is None:
return False
outcome: Final = await asyncio.shield(asyncio.gather(flush, return_exceptions=True))
return outcome[0] is True
def __aiter__(self) -> AsyncPassthroughStreamingResponse:
return self

75
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -862,12 +862,17 @@ def _raise_unless_oauth2_discovery_server(
def _dcr_bridge_relays_client_registration(mcp_server: MCPServer) -> bool:
"""True when a DCR-bridge server relays client registration to the upstream authorization
server instead of short-circuiting to an admin-configured OAuth client. In the relay arm the
server instead of short-circuiting to the server's own OAuth client. In the relay arm the
upstream holds each client's own registration, so the authorize and token relays pass the
client's ``client_id`` and ``redirect_uri`` through verbatim and the authorization code
returns directly to the client's redirect URI without transiting the gateway. Gateway-side
redirect trust and the ``/callback`` state relay therefore only apply to the short-circuit
arm, where the upstream only knows the gateway's own callback."""
arm, where the upstream only knows the gateway's own callback.
Authorize and token decide per request from the server's current client: once the gateway
holds its shared client (registered at ``/register`` under ``PROXY_BASE_URL``, or entered by an
admin) every caller runs the gateway-callback arm, while a server that still has none keeps
relaying, so clients registered before the shared client existed keep their own binding."""
return bool(
mcp_server.is_dcr_bridge
and mcp_server.effective_registration_url
@ -880,6 +885,27 @@ def _dcr_bridge_relays_client_registration(mcp_server: MCPServer) -> bool:
)
def _register_relays_bridge_client(mcp_server: MCPServer) -> bool:
"""``/register`` relays the caller's own registration only when the gateway cannot keep a shared
client: with ``PROXY_BASE_URL`` set it registers one client bound to its own ``/callback`` and
keeps it as the server's client, so an upstream that restricts redirect domains (Atlassian)
never sees a laptop loopback redirect. Without a database the shared client would live only in
this process, so a database-free gateway keeps relaying as well."""
return _dcr_bridge_relays_client_registration(mcp_server) and (
configured_proxy_base_url() is None or not _shared_client_store_available()
)
def _shared_client_store_available() -> bool:
from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 # avoids circular import
try:
get_prisma_client_or_throw("Database not connected. Cannot keep the shared MCP OAuth client.")
except HTTPException:
return False
return True
def _require_s256_pkce(
code_challenge: str | None,
code_challenge_method: str | None,
@ -1028,6 +1054,17 @@ async def _managed_authorization_scope(
return selected_scope, scope_user_id
async def _hydrate_bridge_shared_client(mcp_server: MCPServer, request: Request) -> None:
"""Whichever worker served ``/register`` persisted the bridge's shared gateway client; a worker
whose catalog snapshot predates that write still holds no ``client_id``, so the gateway-callback
arms load the persisted client before presenting one upstream. Callers that carry their own
client (a gateway-minted ephemeral client sealed into the state or the code) skip this, since the
upstream bound their code to that client."""
if not mcp_server.is_dcr_bridge or mcp_server.client_id:
return
await _reuse_persisted_dcr_client_if_available(mcp_server, f"{get_request_base_url(request)}/callback")
async def authorize_with_server(
request: Request,
mcp_server: MCPServer,
@ -1069,6 +1106,8 @@ async def authorize_with_server(
# calling this for its enforcement side effect, then falls through to the gateway
# /callback flow below, which reads the original code_challenge names.
bridge_challenge, bridge_method = _require_s256_pkce(code_challenge, code_challenge_method)
if ephemeral_dcr_client is None:
await _hydrate_bridge_shared_client(resolved_server, request)
# A gateway-minted ephemeral client is registered against {base}/callback, so its
# flow must run the short-circuit arm; the relay arm is only for clients that
# registered themselves through the front door and hold their own redirect binding.
@ -1233,6 +1272,7 @@ async def exchange_token_with_server(
refresh_token: str | None = None,
scope: str | None = None,
client_token_endpoint_auth_method: MCPTokenEndpointAuthMethod | None = None,
sealed_client: bool = False,
):
raise_if_not_oauth2(mcp_server)
if grant_type not in ("authorization_code", "refresh_token"):
@ -1282,11 +1322,14 @@ async def exchange_token_with_server(
# recovered from a sealed code) must authenticate the way its own registration was granted,
# not the way the server row is configured; callers that carry no method keep the row's method
# as before.
resolved_client_id: Final = resolved_server.client_id if resolved_server.client_id else client_id
resolved_client_secret: Final = resolved_server.client_secret if resolved_server.client_id else client_secret
if not sealed_client:
await _hydrate_bridge_shared_client(resolved_server, request)
server_client_wins: Final = bool(resolved_server.client_id) and not sealed_client
resolved_client_id: Final = resolved_server.client_id if server_client_wins else client_id
resolved_client_secret: Final = resolved_server.client_secret if server_client_wins else client_secret
resolved_auth_method: Final = (
resolved_server.token_endpoint_auth_method
if resolved_server.client_id
if server_client_wins
else (client_token_endpoint_auth_method or resolved_server.token_endpoint_auth_method)
)
try:
@ -1830,10 +1873,13 @@ async def _persist_dcr_client_registration(
empty and otherwise adopts whatever a concurrent registration persisted first.
The client-forwarded token modes (``true_passthrough`` / ``oauth_delegate``) are skipped
unconditionally: the caller holds the upstream token and the gateway must hold no OAuth
client identity for these servers. Persisting here would stamp ``oauth2_flow`` and a
``client_id`` onto a server whose mode promises the gateway stores nothing, making a
fresh pass-through server read as gateway-authorized.
unless the server is a DCR bridge: the caller holds the upstream token and the gateway holds
no OAuth client identity for these servers, so persisting would stamp ``oauth2_flow`` and a
``client_id`` onto a server whose mode promises the gateway stores nothing. A DCR bridge is
the exception by design: its gateway-hosted sign-in registers one shared client bound to the
gateway ``/callback`` (the same app an admin may enter by hand), and that client must survive
for every later caller's authorize and token exchange. ``oauth2_flow`` is still never stamped
on a client-forwarded row.
``redirect_uris`` records what the client is bound to so a later origin change can be
detected as a positive mismatch and trigger re-registration instead of stranding the
@ -1842,7 +1888,7 @@ async def _persist_dcr_client_registration(
``update_mcp_server`` merges credential blobs: a re-registered public client must not
inherit the previous client's secret or auth method.
"""
if mcp_server.is_client_forwarded_token:
if mcp_server.is_client_forwarded_token and not mcp_server.is_dcr_bridge:
return "skipped"
try:
@ -1910,20 +1956,23 @@ async def _persist_dcr_client_registration(
):
await _refresh_persisted_dcr_server(stored)
return "reused"
stamped_flow: Final[Literal["authorization_code"] | None] = (
None if mcp_server.is_client_forwarded_token else "authorization_code"
)
updated_row: Final = await update_mcp_server(
prisma_client=prisma_client,
data=(
UpdateMCPServerRequest(
server_id=mcp_server.server_id,
credentials=credentials,
oauth2_flow="authorization_code",
oauth2_flow=stamped_flow,
token_url=mcp_server.token_url,
)
if mcp_server.token_url
else UpdateMCPServerRequest(
server_id=mcp_server.server_id,
credentials=credentials,
oauth2_flow="authorization_code",
oauth2_flow=stamped_flow,
)
),
touched_by="mcp_oauth_dcr",
@ -2211,7 +2260,7 @@ async def register_client_with_server(
)
return dummy_return
bridge_relay: Final = _dcr_bridge_relays_client_registration(resolved_server)
bridge_relay: Final = _register_relays_bridge_client(resolved_server)
if bridge_relay and not client_redirect_uris:
raise HTTPException(
status_code=400,

3
litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py
View file

@ -248,6 +248,7 @@ class SubjectTokenRefusal(LiteLLMBaseModel):
"invalid_grant",
"invalid_client",
"invalid_scope",
"invalid_target",
"temporarily_unavailable",
]
description: str = Field(min_length=1)
@ -1527,7 +1528,7 @@ def _subject_token_refusal_response(refusal: SubjectTokenRefusal) -> Response:
return _oauth_error(503, refusal.error, refusal.description)
case "invalid_client":
return _oauth_error(401, refusal.error, refusal.description)
case "unsupported_grant_type" | "invalid_request" | "invalid_grant" | "invalid_scope":
case "unsupported_grant_type" | "invalid_request" | "invalid_grant" | "invalid_scope" | "invalid_target":
return _oauth_error(400, refusal.error, refusal.description)
case _:
assert_never(refusal.error)

44
litellm/proxy/_experimental/mcp_server/id_jag_grant.py
View file

@ -23,6 +23,7 @@ from urllib.parse import unquote_plus
import jwt
from fastapi import Request
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.models.user import LiteLLM_UserTable
@ -48,6 +49,7 @@ _INACTIVE_SUBJECT: Final = frozenset(("no_active_key", "unresolvable"))
_REPLAY_KEY_PREFIX: Final = "mcp_gateway_dcr:id_jag:"
SCOPE_MCP: Final = "litellm:mcp"
SCOPE_LLM: Final = "litellm:llm"
_RESOURCE_CLAIM_ADAPTER: Final = TypeAdapter(list[str])
SCOPE_ROUTE_GROUPS: Final = MappingProxyType(
{SCOPE_MCP: ("mcp_inference_routes",), SCOPE_LLM: ("llm_api_routes_without_mcp",)}
)
@ -65,9 +67,9 @@ class GrantedScope:
def granted_scope(claim: object, *, mcp_resource: bool) -> GrantedScope | SubjectTokenRefusal:
"""The routes the assertion buys. With ``aud`` the bare origin the ``scope`` claim decides and
no claim is the full credential. With ``aud`` the MCP resource (``{base}/mcp``) the identity
provider named that resource, so the credential is ceilinged to the MCP routes whatever the
claim says: ``litellm:llm`` cannot be granted and is refused."""
no claim is the full credential. With the MCP resource (``{base}/mcp``) as ``aud`` or as every
``resource``, the identity provider named that resource, so the credential is ceilinged to the
MCP routes whatever the claim says: ``litellm:llm`` cannot be granted and is refused."""
if claim is not None and not isinstance(claim, str):
return SubjectTokenRefusal(error="invalid_scope", description=_SCOPE_DESCRIPTION)
requested: Final = frozenset(claim.split()) if claim else frozenset()
@ -90,6 +92,32 @@ def names_mcp_resource(aud: object, base_url: str | None) -> bool:
return base_url is not None and f"{base_url}/mcp" in audiences and base_url not in audiences
def _resource_claim_values(claim: object) -> tuple[str, ...] | None:
if isinstance(claim, str):
return (claim,)
if not isinstance(claim, list):
return None
try:
values: Final = _RESOURCE_CLAIM_ADAPTER.validate_python(claim, strict=True)
except ValidationError:
return None
return tuple(values) if values else None
def resource_names_mcp(claims: Mapping[str, object], base_url: str | None) -> bool | SubjectTokenRefusal:
"""The signed RFC 8707 ``resource`` claim (a URI or an array of URIs) restricts the credential like an
``aud`` of ``{base}/mcp`` does."""
if "resource" not in claims:
return False
resources: Final = _resource_claim_values(claims["resource"])
if resources is None:
return SubjectTokenRefusal(error="invalid_target", description=_RESOURCE_DESCRIPTION)
served_resources: Final = frozenset(gateway_audiences(base_url))
if any(resource not in served_resources for resource in resources):
return SubjectTokenRefusal(error="invalid_target", description=_RESOURCE_DESCRIPTION)
return frozenset(resources) == frozenset((f"{base_url}/mcp",))
def _route_groups(scopes: Sequence[str]) -> Iterator[str]:
for scope in scopes:
yield from SCOPE_ROUTE_GROUPS[scope]
@ -98,8 +126,10 @@ def _route_groups(scopes: Sequence[str]) -> Iterator[str]:
_SCOPE_DESCRIPTION: Final = (
f"scope must be {SCOPE_MCP} (MCP gateway routes), {SCOPE_LLM} (LLM API routes), or both, space separated"
)
_RESOURCE_DESCRIPTION: Final = "the assertion's resource claim must name this gateway's origin or its /mcp resource"
_MCP_RESOURCE_DESCRIPTION: Final = (
f"the /mcp audience only grants MCP routes ({SCOPE_MCP}); use the bare origin as audience for LLM routes"
f"the /mcp audience or resource only grants MCP routes ({SCOPE_MCP}); use the bare origin as audience and "
"resource for LLM routes"
)
@ -450,8 +480,12 @@ async def identity_from_id_jag(
if isinstance(marker, SubjectTokenRefusal):
return marker
replay_key, replay_ttl_seconds = marker
mcp_resource: Final = resource_names_mcp(verified, prerequisites.base_url)
if isinstance(mcp_resource, SubjectTokenRefusal):
return mcp_resource
scope: Final = granted_scope(
verified.get("scope"), mcp_resource=names_mcp_resource(verified.get("aud"), prerequisites.base_url)
verified.get("scope"),
mcp_resource=mcp_resource or names_mcp_resource(verified.get("aud"), prerequisites.base_url),
)
if isinstance(scope, SubjectTokenRefusal):
return scope

29
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -1970,6 +1970,7 @@ class MCPServerManager:
per_user_oauth_token_store: InvalidatableOAuthTokenStore | None = None,
per_user_token_cache: MCPPerUserTokenCache | None = None,
discovery_clock: Callable[[], float] = time.monotonic,
upstream_transport: httpx2.AsyncBaseTransport | None = None,
):
self._per_user_oauth_token_store = per_user_oauth_token_store or LazyPerUserOAuthTokenStore(
self.get_mcp_server_by_id
@ -1979,6 +1980,7 @@ class MCPServerManager:
oauth_token_store=self._per_user_oauth_token_store,
token_exchanger=build_token_exchanger(),
)
self._upstream_transport: httpx2.AsyncBaseTransport | None = upstream_transport
discovery_ttl: Final = _mcp_discovery_cache_ttl()
self._prompt_discovery_cache = _DiscoveryCache[ListPromptsResult](
discovery_ttl, discovery_clock, TypeAdapter(ListPromptsResult)
@ -4348,6 +4350,7 @@ class MCPServerManager:
if resolved_server.allow_elicitation
else None
),
upstream_transport=self._upstream_transport,
)
_create_mcp_client = create_mcp_client
@ -4528,7 +4531,7 @@ class MCPServerManager:
)
else:
page: Final = (
await client.list_tools_page(params)
await self._fetch_tools_page_with_timeout(client, server, params)
if params is not None
else ListToolsResult(tools=await self._fetch_tools_with_timeout(client, server.name))
)
@ -5618,6 +5621,19 @@ class MCPServerManager:
)
raise_classified_list_failure(e, server_name)
async def _fetch_tools_page_with_timeout(
self, client: MCPClient, server: MCPServer, params: PaginatedRequestParams
) -> ListToolsResult:
deadline: Final = (
MCP_TOOL_LISTING_TIMEOUT if server.timeout is None else max(server.timeout, MCP_TOOL_LISTING_TIMEOUT)
)
try:
with anyio.fail_after(deadline):
return await client.list_tools_page(params)
except TimeoutError as e:
verbose_logger.warning("Timeout while listing tools from %s", server.name)
raise MCPServerListError(ServerListFault(tag="timeout"), server.name) from e
_SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024
def assign_unique_short_prefix(
@ -7466,7 +7482,10 @@ class MCPServerManager:
return "ok"
# Add timeout wrapper to prevent hanging
await asyncio.wait_for(client.run_with_session(_noop), timeout=MCP_HEALTH_CHECK_TIMEOUT)
await asyncio.wait_for(
client.run_with_session(_noop, quiet_on_error=True),
timeout=MCP_HEALTH_CHECK_TIMEOUT,
)
self._remember_upstream_initialize_instructions(server, client)
status = "healthy"
except asyncio.TimeoutError:
@ -7478,6 +7497,12 @@ class MCPServerManager:
except Exception as e:
health_check_error = str(e)
status = "unhealthy"
verbose_logger.warning(
"MCP health check failed for %s: %s%s",
server.name,
type(e).__name__,
_upstream_failure_suffix(e),
)
return LiteLLM_MCPServerTable(
server_id=server.server_id,

56
litellm/proxy/_experimental/mcp_server/outbound_credentials/login_assertion_exchange.py
View file

@ -5,7 +5,9 @@ login assertion on ID-JAG leg 1. It presents a refresh token that the IdP issued
one RFC 8693 exchange (``requested_token_type`` refresh_token), which is how a SAML requesting app works at
an IdP whose ID-JAG exchange does not take a ``saml2`` subject directly. That exchange has to happen at
login: a SAML assertion is valid for minutes, the first MCP call that needs it can come hours later, and
the refresh token outlives the assertion by as long as the IdP decides.
the refresh token outlives the assertion by as long as the IdP decides. An OIDC login signed in through the very
IdP client the ID-JAG server uses already holds such a refresh token: it is bound to that client, so it is
presented as is and no further exchange is posted, which Okta refuses for an id_token subject anyway.
The servers consulted are the two configuration authorities, the pod-local config declaration and the DB
rows, like the retention gate, not the in-memory registry, which is a per-process snapshot that can lag a
@ -19,12 +21,17 @@ the ID-JAG call reports the missing refresh token as the precondition it is.
from __future__ import annotations
import asyncio
import os
from collections.abc import Iterable, Mapping, Sequence
from datetime import datetime, timezone
from typing import Final, Protocol
from litellm._logging import verbose_proxy_logger
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result
from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import (
SSOClientConfig,
sso_client_config,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import (
SSOIdentityAssertion,
with_login_refresh_token,
@ -123,12 +130,31 @@ async def exchange_login_assertion_for_refresh_token(
token_endpoint: TokenExchangeEndpoint,
now: datetime,
deadline_seconds: float = _LOGIN_EXCHANGE_DEADLINE_SECONDS,
sso_client: SSOClientConfig | None = None,
) -> SSOIdentityAssertion:
"""The assertion as captured when no server wants a refresh token, otherwise the assertion plus the
refresh token the first target's IdP client exchanged it for. An OIDC login's own SSO refresh token
does not count, it is bound to the SSO client and the ID-JAG client cannot redeem it."""
refresh token one target's IdP client can redeem. An OIDC login's own refresh token is that token
when the SSO client that received it is the target's client at the same token endpoint; otherwise it
is bound to a client the ID-JAG client is not, and the assertion is exchanged for one at the first
target's client."""
if not targets or assertion.exchanged_refresh_token is not None:
return assertion
reusable: Final = _target_redeeming_the_login_refresh_token(assertion, targets, sso_client)
if reusable is not None and assertion.refresh_token is not None:
verbose_proxy_logger.info(
"ID-JAG: the refresh token issued at OIDC login belongs to client_id=%s at %s, the IdP client of a server "
"with id_jag_subject_source: refresh_token, so it is kept as that subject and no further exchange is posted",
reusable.client_id,
reusable.org_token_endpoint,
)
return with_login_refresh_token(
assertion,
assertion.refresh_token.get_secret_value(),
None,
now,
endpoint=reusable.org_token_endpoint,
client_id=reusable.client_id,
)
config: Final = targets[0]
if len(targets) > 1:
verbose_proxy_logger.warning(
@ -185,6 +211,24 @@ async def exchange_login_assertion_for_refresh_token(
)
def _target_redeeming_the_login_refresh_token(
assertion: SSOIdentityAssertion, targets: tuple[IdJagConfig, ...], sso_client: SSOClientConfig | None
) -> IdJagConfig | None:
"""The target whose IdP client is the SSO client the login's refresh token was issued to. A refresh
token is redeemable only by the client that received it at the endpoint that issued it, so both have
to match exactly before the login token is presented anywhere."""
if assertion.refresh_token is None or sso_client is None:
return None
return next(
(
config
for config in targets
if config.org_token_endpoint == sso_client.token_endpoint and config.client_id == sso_client.client_id
),
None,
)
async def exchange_login_assertion_if_configured(
assertion: SSOIdentityAssertion | None,
) -> SSOIdentityAssertion | None:
@ -209,7 +253,11 @@ async def exchange_login_assertion_if_configured(
servers: Final = await login_exchange_servers(global_mcp_server_manager, db_rows)
targets: Final = refresh_token_exchange_targets(to_server_spec(server) for server in servers)
return await exchange_login_assertion_for_refresh_token(
assertion, targets, TokenEndpointClient(), datetime.now(timezone.utc)
assertion,
targets,
TokenEndpointClient(),
datetime.now(timezone.utc),
sso_client=sso_client_config(os.environ),
)
except Exception as exc: # noqa: BLE001 # the login itself must not fail on an egress-side exchange
verbose_proxy_logger.warning("ID-JAG: the login-time refresh token exchange failed unexpectedly: %s", exc)

9
litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py
View file

@ -429,10 +429,11 @@ def _login_refresh_token_subject(
assertion: SSOIdentityAssertion, config: IdJagConfig, now: datetime
) -> Result[_IdJagSubjectToken, CredError]:
"""The subject for a server configured with ``id_jag_subject_source: refresh_token``: the refresh
token the ID-JAG client exchanged the login assertion for when the user signed in. Never the SSO
refresh token an OIDC login holds, that one is bound to the SSO client. The assertion's own expiry
does not apply here, a SAML assertion is spent minutes after login while the refresh token it
bought lives as long as the IdP said."""
token the ID-JAG client holds from the user's sign-in, exchanged from the login assertion or, when
the SSO client is this server's IdP client, the one the OIDC login itself returned. The login-time
hook records which client and endpoint it is bound to, and a token bound elsewhere is never sent
here. The assertion's own expiry does not apply, a SAML assertion is spent minutes after login
while the refresh token it bought lives as long as the IdP said."""
if assertion.exchanged_refresh_token is None:
return Error(
CredError.of_precondition_required(

13
litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py
View file

@ -229,10 +229,19 @@ def carry_exchanged_refresh_token(
) -> SSOIdentityAssertion:
"""The renewed OIDC assertion still carrying the refresh token the ID-JAG client exchanged at login.
Renewal only replaces the id_token and the SSO refresh token, the exchanged one is unrelated to
them and stays valid for as long as the IdP said."""
them and stays valid for as long as the IdP said. When the login's own refresh token was kept as
that subject (the SSO client is the ID-JAG client), a rotation the renewal just received replaces
it too, since the IdP has retired the token the row held."""
login_token_reused: Final = (
previous.refresh_token is not None
and previous.exchanged_refresh_token is not None
and previous.exchanged_refresh_token.get_secret_value() == previous.refresh_token.get_secret_value()
)
return renewed.model_copy(
update={
"exchanged_refresh_token": previous.exchanged_refresh_token,
"exchanged_refresh_token": renewed.refresh_token
if login_token_reused
else previous.exchanged_refresh_token,
"exchanged_refresh_token_expires_at": previous.exchanged_refresh_token_expires_at,
"exchanged_refresh_token_endpoint": previous.exchanged_refresh_token_endpoint,
"exchanged_refresh_token_client_id": previous.exchanged_refresh_token_client_id,

3
litellm/proxy/_experimental/mcp_server/upstream.py
View file

@ -207,6 +207,7 @@ async def prepare_upstream_client(
protocol_version: MCPUpstreamProtocol,
sampling_callback: SamplingCallback | None = None,
elicitation_callback: ElicitationCallback | None = None,
upstream_transport: httpx2.AsyncBaseTransport | None = None,
) -> MCPClient:
transport: Final[MCPTransportType] = server.transport or MCPTransport.sse
server_spec: Final = None if transport == MCPTransport.stdio else to_server_spec_fail_closed(server)
@ -263,6 +264,7 @@ async def prepare_upstream_client(
server_url=server.url or "",
transport_type=transport,
protocol_version=protocol_version,
upstream_transport=upstream_transport,
auth_type=server.auth_type,
timeout=timeout,
extra_headers=resolved_headers,
@ -304,6 +306,7 @@ async def prepare_upstream_client(
server_url=server.url or "",
transport_type=transport,
protocol_version=protocol_version,
upstream_transport=upstream_transport,
auth_type=server.auth_type,
auth_value=auth_value,
auth_header_name=auth_header_name,

25
litellm/proxy/auth/route_checks.py
View file

@ -72,14 +72,22 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spe
_LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp"))
_AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS))
# Method-scoped because the generic credential CRUD handlers share these paths:
# the route check cannot admit PATCH/DELETE on a name like "user_connections"
# or "*/user_connection*" without also opening the admin CRUD handlers.
_INTERNAL_USER_CREDENTIAL_CONNECTION_ROUTES: Final = (
# Writes that only touch the caller's own credential row (BYOK connections and
# per-user MCP credentials). Method-scoped because the generic credential CRUD
# handlers share these paths: the route check cannot admit PATCH/DELETE on a
# name like "user_connections" without also opening the admin CRUD handlers.
_SELF_SCOPED_CREDENTIAL_ROUTES: Final = (
("GET", "/credentials/user_connections"),
("POST", "/credentials/{credential_name:path}/user_connection/start"),
("POST", "/credentials/{credential_name:path}/user_connection/poll"),
("DELETE", "/credentials/{credential_name:path}/user_connection"),
("POST", "/v1/mcp/server/oauth/{server_id}/register"),
("POST", "/v1/mcp/server/{server_id}/oauth-user-credential"),
("DELETE", "/v1/mcp/server/{server_id}/oauth-user-credential"),
("POST", "/v1/mcp/server/{server_id}/user-credential"),
("DELETE", "/v1/mcp/server/{server_id}/user-credential"),
("POST", "/v1/mcp/server/{server_id}/user-env-vars"),
("DELETE", "/v1/mcp/server/{server_id}/user-env-vars"),
)
@ -349,7 +357,7 @@ class RouteChecks:
_user_role == LitellmUserRoles.INTERNAL_USER.value
and (
RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.internal_user_routes.value)
or RouteChecks.is_internal_user_credential_connection_route(route=route, request=request)
or RouteChecks.is_self_scoped_credential_route(route=route, request=request)
)
or user_is_org_admin(request_data=request_data, user_object=user_obj)
and RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.org_admin_allowed_routes.value)
@ -702,13 +710,13 @@ class RouteChecks:
_get_request_method = get_request_method
@staticmethod
def is_internal_user_credential_connection_route(route: str, request: Request | None) -> bool:
def is_self_scoped_credential_route(route: str, request: Request | None) -> bool:
method: Final = RouteChecks.get_request_method(request)
if method is None:
return False
return any(
method == expected_method and RouteChecks.check_route_access(route=route, allowed_routes=(pattern,))
for expected_method, pattern in _INTERNAL_USER_CREDENTIAL_CONNECTION_ROUTES
for expected_method, pattern in _SELF_SCOPED_CREDENTIAL_ROUTES
)
@staticmethod
@ -1033,6 +1041,9 @@ class RouteChecks:
if route == "/session/logout":
return
if RouteChecks.is_self_scoped_credential_route(route=route, request=request):
return
# Hard-block known write routes regardless of HTTP method (defensive
# — these are POSTs in practice, but pinning them here protects
# against future GET-shaped writes).

2
litellm/proxy/client/cli/commands/agents.py
View file

@ -22,7 +22,7 @@ from litellm.proxy.common_utils.codex_model_catalog import (
CodexStockModel,
codex_models_response_json,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
from .auth import cli_context_obj, context_secret_vault, get_stored_api_key, login
from .claude_settings import ClaudeSettingsError, install_statusline_script

2
litellm/proxy/client/cli/commands/autoroute/config.py
View file

@ -3,7 +3,7 @@ from typing import Final, Literal
from pydantic import ConfigDict, Field, JsonValue, TypeAdapter
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
TIER_NAMES: Final[tuple[str, ...]] = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")
AUTOROUTER_MODEL_NAME: Final = "autorouter"

2
litellm/proxy/client/cli/commands/claude_settings.py
View file

@ -35,7 +35,7 @@ from litellm.litellm_core_utils.private_json import (
stage_private_json,
write_private_bytes,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
from . import statusline_script
from .cmd_quoting import quote_for_cmd

2
litellm/proxy/client/cli/commands/codex_settings.py
View file

@ -23,7 +23,7 @@ from litellm.litellm_core_utils.private_json import (
stage_private_bytes,
stage_private_json,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
from .agents import CODEX_PROXY_PROVIDER, codex_proxy_provider

2
litellm/proxy/client/cli/commands/configure.py
View file

@ -7,7 +7,7 @@ from typing import Final
import click
from InquirerPy import inquirer
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
from .auth import CliContextObj
from .claude_settings import (

2
litellm/proxy/client/cli/commands/configure_profiles.py
View file

@ -17,7 +17,7 @@ from litellm.litellm_core_utils.private_json import (
ensure_private_dir,
stage_private_json,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
from .agents import codex_config_path
from .claude_settings import claude_settings_path, configure_state_path

8
litellm/proxy/client/cli/commands/configure_setup.py
View file

@ -14,12 +14,8 @@ from InquirerPy import inquirer
from InquirerPy.base.control import Choice
from pydantic import TypeAdapter, ValidationError
from litellm.proxy.common_utils.model_listing_utils import (
CLAUDE_CODE_CLIENT,
CLAUDE_CODE_PICKER_PATTERN,
GATEWAY_CLIENT_HEADER,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.constants import CLAUDE_CODE_CLIENT, CLAUDE_CODE_PICKER_PATTERN, GATEWAY_CLIENT_HEADER
from litellm.types.llms.base_model import LiteLLMBaseModel
from .agents import codex_config_path
from .claude_settings import (

2
litellm/proxy/client/cli/commands/debug.py
View file

@ -20,7 +20,7 @@ import click
import requests
from pydantic import ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
from ...http_client import HTTPClient
from ._cli_context import cli_context_values

2
litellm/proxy/client/cli/commands/pi.py
View file

@ -19,7 +19,7 @@ import requests
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator
from pydantic.types import StringConstraints
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
PI_CONFIG_DIR_ENV: Final = "PI_CODING_AGENT_DIR"
PI_PROVIDER_NAME: Final = "litellm"

2
litellm/proxy/client/cli/commands/pkce_login.py
View file

@ -24,7 +24,7 @@ from pydantic import ConfigDict, Field, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.cli_token_utils import CLI_TOKEN_FRESHNESS_BUFFER_SECONDS
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
if TYPE_CHECKING:
from .auth import CliTokenData

5
litellm/proxy/client/cli/interface.py
View file

@ -8,6 +8,7 @@ from typing import Final, TypeAlias
import click
from litellm._logging import verbose_logger
from litellm.proxy.common_utils.banner import show_banner
from .commands.auth import cli_context_obj, context_secret_vault
@ -115,8 +116,6 @@ def show_commands():
def setup_shell(ctx: click.Context):
"""Set up the interactive shell with banner and initial info."""
from litellm.proxy.common_utils.banner import show_banner
show_banner()
# Show server connection info
@ -137,8 +136,6 @@ def handle_special_commands(user_input: str) -> bool:
return True
elif user_input.lower() == "clear":
click.clear()
from litellm.proxy.common_utils.banner import show_banner
show_banner()
show_commands()
return True

150
litellm/proxy/client/cli/main.py
View file

@ -1,42 +1,20 @@
# stdlib imports
# third party imports
import functools
import importlib
from collections.abc import Callable, Iterator, Mapping, MutableMapping
from typing import Final, Literal, Protocol
import click
from litellm._version import version as litellm_version
from litellm.litellm_core_utils.cli_token_utils import load_cli_token
from litellm.proxy.client.health import HealthManagementClient
from .commands.agents import agent_commands
from .commands.auth import (
auth_group,
cli_context_obj,
context_secret_vault,
get_stored_api_key,
keychain_unreadable_notice,
login,
logout,
whoami,
)
from .commands.autoroute.commands import autoroute_group
from .commands.chat import chat
from .commands.config import config_commands, get_config_value, hidden_command_names
from .commands.configure import configure_group, reconfigure_group, unconfigure_group
from .commands.credentials import credentials
from .commands.debug import debug
from .commands.encryption import encryption
from .commands.http import http
from .commands.keys import keys
from .commands.model_groups import model_groups
# local imports
from .commands.models import models
from .commands.teams import teams
from .commands.up import down, up
from .commands.users import users
from .interface import interactive_shell
from .commands.config import get_config_value, hidden_command_names
class _CliContextObjSlots(Protocol):
@ -62,6 +40,8 @@ def print_version(base_url: str, api_key: str | None):
click.echo(f"LiteLLM Proxy CLI Version: {litellm_version}")
if base_url:
click.echo(f"LiteLLM Proxy Server URL: {base_url}")
from litellm.proxy.client.health import HealthManagementClient
try:
health_client: Final = HealthManagementClient(base_url=base_url, api_key=api_key)
server_version: Final = health_client.get_server_version()
@ -87,7 +67,81 @@ class HideConfiguredCommandsGroup(click.Group):
return [name for name in super().list_commands(ctx) if name not in hidden]
@click.group(cls=HideConfiguredCommandsGroup, invoke_without_command=True)
class LazyCommands(MutableMapping[str, click.Command]):
"""Subcommands by name, each imported only when Click looks it up, so `lite <cmd>` loads just that command"""
def __init__(self, loaders: Mapping[str, Callable[[], click.Command]]) -> None:
self._loaders = dict(loaders)
def __getitem__(self, name: str) -> click.Command:
return self._loaders[name]()
def __setitem__(self, name: str, command: click.Command) -> None:
self._loaders[name] = lambda: command
def __delitem__(self, name: str) -> None:
del self._loaders[name]
def __iter__(self) -> Iterator[str]:
return iter(self._loaders)
def __len__(self) -> int:
return len(self._loaders)
def _command_in(module: str, attribute: str) -> Callable[[], click.Command]:
@functools.cache
def load() -> click.Command:
match getattr(importlib.import_module(module, __package__), attribute):
case click.Command() as command:
return command
case _:
raise TypeError(f"{module}.{attribute} is not a click command")
return load
def _agent_command(name: str) -> Callable[[], click.Command]:
@functools.cache
def load() -> click.Command:
from .commands.agents import agent_commands
return next(command for command in agent_commands() if command.name == name)
return load
AGENT_COMMAND_NAMES: Final = ("claude", "codex", "opencode", "pi")
LITE_COMMANDS: Final = LazyCommands(
{
"login": _command_in(".commands.auth", "login"),
"logout": _command_in(".commands.auth", "logout"),
"whoami": _command_in(".commands.auth", "whoami"),
"auth": _command_in(".commands.auth", "auth_group"),
"models": _command_in(".commands.models", "models"),
"credentials": _command_in(".commands.credentials", "credentials"),
"encryption": _command_in(".commands.encryption", "encryption"),
"chat": _command_in(".commands.chat", "chat"),
"http": _command_in(".commands.http", "http"),
"debug": _command_in(".commands.debug", "debug"),
"keys": _command_in(".commands.keys", "keys"),
"teams": _command_in(".commands.teams", "teams"),
"users": _command_in(".commands.users", "users"),
**{name: _agent_command(name) for name in AGENT_COMMAND_NAMES},
"up": _command_in(".commands.up", "up"),
"down": _command_in(".commands.up", "down"),
"model-groups": _command_in(".commands.model_groups", "model_groups"),
"autoroute": _command_in(".commands.autoroute.commands", "autoroute_group"),
"config": _command_in(".commands.config", "config_commands"),
"configure": _command_in(".commands.configure", "configure_group"),
"reconfigure": _command_in(".commands.configure", "reconfigure_group"),
"unconfigure": _command_in(".commands.configure", "unconfigure_group"),
}
)
@click.group(cls=HideConfiguredCommandsGroup, invoke_without_command=True, commands=LITE_COMMANDS)
@click.option(
"--version",
"-v",
@ -166,6 +220,8 @@ def cli(ctx: click.Context, show_version: bool, base_url: str | None, api_key: s
# If no subcommand was invoked, start interactive mode
if ctx.invoked_subcommand is None:
from .interface import interactive_shell
interactive_shell(ctx, cli)
@ -177,46 +233,6 @@ def version(ctx: click.Context):
print_version(ctx_obj.get("base_url"), ctx_obj.get("api_key"))
# Add authentication commands as top-level commands
cli.add_command(login)
cli.add_command(logout)
cli.add_command(whoami)
# Add the auth command group (e.g. `lite auth print-token`, used as Claude Code's apiKeyHelper)
cli.add_command(auth_group, name="auth")
# Add the models command group
cli.add_command(models)
# Add the credentials command group
cli.add_command(credentials)
# Add the encryption migration command group
cli.add_command(encryption)
# Add the chat command group
cli.add_command(chat)
# Add the http command group
cli.add_command(http)
cli.add_command(debug)
# Add the keys command group
cli.add_command(keys)
# Add the teams command group
cli.add_command(teams)
# Add the users command group
cli.add_command(users)
# Add a top-level command per coding agent (claude, codex, opencode, ...)
for agent_command in agent_commands():
cli.add_command(agent_command)
# Add the up/down commands (route Claude Code through the local LiteLLM proxy)
cli.add_command(up)
cli.add_command(down)
# Add the model-groups command group (discover models your key can access)
cli.add_command(model_groups)
# Add the autoroute command group (QA auto-routing against your real proxy)
cli.add_command(autoroute_group, name="autoroute")
cli.add_command(config_commands)
# Add configure/unconfigure (persistently wire a coding agent to the proxy with a virtual key)
cli.add_command(configure_group)
cli.add_command(reconfigure_group)
cli.add_command(unconfigure_group)
LITELLM_PROXY_DEPRECATION_NOTICE: Final = (
"The `litellm-proxy` command is deprecated and will be removed in a future release; "
"run `lite` instead, it takes the same commands and options."

Some files were not shown because too many files have changed in this diff Show more