scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-28 06:23:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 94

Merge origin/main into litellm_chat_prompt_cache_breakpoint_to_cache_control

Browse source 
Main's #47100 sends native structured output first on Bedrock models that opt
in, and retries with the json tool when Bedrock rejects it. That retry adds the
operator's tool_config cachePoint after the json tool, which the cache step did
not see on the first attempt, so a 1h breakpoint behind a 5m point got Bedrock's
ttl order 400 on the retry

Under drop_params the cache step now plans for both attempts, so that 1h
breakpoint goes out as 5m on each. Without drop_params it plans for the first
attempt only, since counting the retry's point there would keep a breakpoint
with no ttl at 5m ahead of a later 1h one on the native attempt

The Converse cache planning moves out of completion() into one helper, since
the merged function grew past what basedpyright analyzes
This commit is contained in:
mateo-berri 2026-10-18 02:57:48 +00:00
commit 342c18ced2
452 changed files with 38563 additions and 3805 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
.github/e2e-stack/assert_tests_ran.py vendored
View file

@ -1,69 +0,0 @@
import os
import re
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Final
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
from coverage_registry.management_cases import MANAGEMENT_CASES
def main() -> int:
selected: Final = tuple(sys.argv[2:])
try:
report: Final = ET.parse(Path(sys.argv[1])).getroot()
except (ET.ParseError, OSError):
_ = sys.stdout.write("::error::could not read the test execution report\n")
return 1
cases: Final = tuple(report.iter("testcase"))
expected_count: Final = os.environ.get("E2E_REQUIRED_TEST_COUNT")
passed: Final = frozenset(
case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
)
missing: Final = tuple(path for path in selected if path not in passed)
required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
passed_nodes: Final = frozenset(
prop.get("value")
for case in cases
if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
for prop in case.findall("./properties/property")
if prop.get("name") == "management_node"
)
missing_nodes: Final = required_nodes - passed_nodes
for node in sorted(missing_nodes):
_ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
for path in selected:
collected: Final = sum(case.get("file") == path for case in cases)
skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
_ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
for case in cases:
if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error", "skipped")):
continue
outcome = "skipped" if case.find("skipped") is not None else "failed"
_ = sys.stdout.write(f" {outcome}: {case.get('classname', '')}::{case.get('name', '')}\n")
for prop in case.findall("./properties/property"):
name = prop.get("name", "")
value = prop.get("value", "")
if name in ("oauth_failure_phase", "oauth_exception_type", "oauth_frame") and re.fullmatch(
r"[A-Za-z0-9_.:<>-]{1,240}", value
):
_ = sys.stdout.write(f" {name}: {value}\n")
if expected_count is not None and (
len(cases) != int(expected_count) or any(case.find("skipped") is not None for case in cases)
):
_ = sys.stdout.write("::error::required test count was not met or a required case was skipped\n")
return 1
if (
selected
and not missing
and not missing_nodes
and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
):
return 0
_ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
return 1
if __name__ == "__main__":
sys.exit(main())

4
.github/workflows/test-linting.yml vendored
View file

@ -539,8 +539,8 @@ jobs:
- name: test_unit_passed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
- name: test_e2e_idp_stack
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_idp_stack.py
- name: test_e2e_metadata
env:

180
.github/workflows/test-mcp-oauth-e2e.yml vendored
View file

@ -1,180 +0,0 @@
name: MCP OAuth happy path
on:
pull_request:
paths:
- '.github/workflows/test-mcp-oauth-e2e.yml'
- '.github/e2e-stack/**'
- 'tests/e2e/*.py'
- 'tests/e2e/pytest.ini'
- 'tests/e2e/idp_realm.json'
- 'tests/e2e/mcp/**'
- 'litellm/experimental_mcp_client/**'
- 'litellm/proxy/_experimental/mcp_server/**'
- 'litellm/proxy/auth/**'
- 'litellm/proxy/management_endpoints/mcp_management_endpoints.py'
- 'litellm/proxy/_types.py'
- 'litellm/types/mcp_server/mcp_server_manager.py'
- 'litellm/proxy/management_endpoints/*sso*.py'
- 'litellm/proxy/management_endpoints/sso/**'
- 'litellm/proxy/common_utils/encrypt_decrypt_utils.py'
- 'litellm/proxy/proxy_server.py'
- 'litellm/proxy/schema.prisma'
- 'ui/litellm-dashboard/src/app/connect/**'
- 'ui/litellm-dashboard/src/app/mcp/oauth/**'
- 'pyproject.toml'
- 'uv.lock'
workflow_dispatch:
permissions: {}
concurrency:
group: mcp-oauth-${{ github.ref }}
cancel-in-progress: true
jobs:
oauth:
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
environment: e2e-changed
timeout-minutes: 45
permissions:
contents: read
id-token: write
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: litellm
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U litellm"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_HOST: 127.0.0.1
DATABASE_PORT: '5432'
DATABASE_USER: litellm
DATABASE_PASSWORD: dbpassword9090
DATABASE_NAME: litellm
DATABASE_URL: postgresql://litellm:dbpassword9090@127.0.0.1:5432/litellm
E2E_KEYCLOAK_URL: http://127.0.0.1:8081
E2E_KEYCLOAK_ADMIN_USER: admin
E2E_KEYCLOAK_ADMIN_PASSWORD: e2e-ephemeral-idp-not-a-secret
E2E_FIXTURE_MODE: live
E2E_PROVIDER_CACHE: '0'
E2E_MCP_OAUTH_LIVE: '1'
E2E_REQUIRED_TEST_COUNT: '4'
steps:
- name: Checkout the tested source
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
persist-credentials: false
- name: Require and materialize the upstream login
env:
STORAGE_STATE: ${{ secrets.E2E_LINEAR_STORAGE_STATE_B64 }}
run: |
umask 077
python3 - <<'PY'
import base64
import json
import os
import secrets
from pathlib import Path
encoded = os.environ.get("STORAGE_STATE", "")
if not encoded:
raise SystemExit("E2E_LINEAR_STORAGE_STATE_B64 is required; capture and provision a test-account login")
state = json.loads(base64.b64decode(encoded, validate=True))
if not isinstance(state, dict) or not state.get("cookies"):
raise SystemExit("The captured login must contain browser cookies")
directory = Path(os.environ["RUNNER_TEMP"]) / "mcp-oauth-private"
directory.mkdir(mode=0o700)
path = directory / "linear-state.json"
path.write_text(json.dumps(state))
with open(os.environ["GITHUB_ENV"], "a") as output:
output.write(f"E2E_LINEAR_STORAGE_STATE={path}\n")
for name in ("LITELLM_MASTER_KEY", "LITELLM_SALT_KEY"):
value = "sk-e2e-" + secrets.token_hex(24)
print(f"::add-mask::{value}")
output.write(f"{name}={value}\n")
PY
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.13'
- uses: ./.github/actions/setup-uv-with-retries
with:
version: '0.10.9'
- uses: ./.github/actions/cache-cargo-build
- name: Install the frozen E2E environment
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra proxy-runtime --extra extra_proxy --group ci --group proxy-dev --group e2e-dev
uv run --no-sync python scripts/prisma_generate_if_needed.py
uv run --no-sync playwright install --with-deps chromium
- name: Configure license access
id: aws
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
with:
role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
aws-region: us-east-1
role-session-name: mcp-oauth-${{ github.run_id }}
role-duration-seconds: 900
output-env-credentials: false
output-credentials: true
- name: Load the E2E license
env:
AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
AWS_DEFAULT_REGION: us-east-1
run: |
license="$(aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license --query SecretString --output text)"
test -n "${license}"
echo "::add-mask::${license}"
echo "LITELLM_LICENSE=${license}" >> "${GITHUB_ENV}"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
- name: Build the gateway consent UI at the tested commit
run: |
cd ui/litellm-dashboard
../../scripts/with_dashboard_node.sh npm ci
../../scripts/with_dashboard_node.sh npm run build
mkdir -p ../../litellm/proxy/_experimental/out
cp -r out/. ../../litellm/proxy/_experimental/out/
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name index.html | while read -r page; do
mkdir -p "${page%.html}"
mv "${page}" "${page%.html}/index.html"
done
- name: Prepare the isolated database and IdP
run: |
umask 077
bash .github/e2e-stack/start-idp.sh
uv run --no-sync python migrations/run.py > "${RUNNER_TEMP}/mcp-oauth-private/migrations.log" 2>&1
- name: Run every required OAuth variant without retries
run: |
umask 077
uv run --no-sync pytest -c tests/e2e/pytest.ini tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
--rootdir=. --reruns 0 --tb=short -o junit_family=xunit1 \
--junitxml="${RUNNER_TEMP}/mcp-oauth-private/results.xml" \
> "${RUNNER_TEMP}/mcp-oauth-private/pytest.log" 2>&1
- name: Report JUnit results and reject skipped or missing cases
if: always()
run: |
uv run --no-sync python .github/e2e-stack/assert_tests_ran.py \
"${RUNNER_TEMP}/mcp-oauth-private/results.xml" tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py
- name: Remove private login and logs
if: always()
run: |
docker rm -f e2e-keycloak >/dev/null 2>&1 || true
rm -rf "${RUNNER_TEMP}/mcp-oauth-private"

1
.github/workflows/test-unit.yml vendored
View file

@ -658,6 +658,7 @@ jobs:
- shard: proxy-extras
test-path: >-
tests/unit/litellm_proxy_extras
tests/unit/db_scripts
python-version: "3.12"
workers: 2
reruns: 0

1
backend/routes/allowlist.py
View file

@ -165,6 +165,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
{
"/admin",
"/metrics",
"/swagger", # API documentation static assets belong to the backend
"/mcp", # lazily-mounted MCP sub-app serves on the backend component
}

201
db_scripts/strip_vector_store_internal_metadata.py Normal file
View file

@ -0,0 +1,201 @@
import argparse
import os
import sys
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from itertools import accumulate, repeat, takewhile
from typing import Final, Literal, TextIO
import httpx
from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
from litellm.vector_stores.utils import is_litellm_written_metadata_key
DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
LIST_PAGE_SIZE: Final = "100"
class VectorStore(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
id: str
metadata: dict[str, JsonValue] | None = None
class VectorStorePage(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
data: tuple[VectorStore, ...]
has_more: bool = False
class OpenAIErrorBody(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
message: str = ""
class OpenAIErrorResponse(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
error: OpenAIErrorBody
@dataclass(frozen=True, slots=True)
class Options:
api_key: str
api_base: str
vector_store_ids: tuple[str, ...]
extra_keys: frozenset[str]
apply: bool
@dataclass(frozen=True, slots=True)
class StoreResult:
store_id: str
matched_keys: tuple[str, ...]
status: Literal["dry_run", "updated", "failed"]
error: str = ""
def is_proxy_written_key(key: str, extra_keys: frozenset[str]) -> bool:
return is_litellm_written_metadata_key(key) or key in extra_keys
def matching_keys(store: VectorStore, extra_keys: frozenset[str]) -> tuple[str, ...]:
return tuple(sorted(key for key in (store.metadata or {}) if is_proxy_written_key(key, extra_keys)))
def fetch_page(client: httpx.Client, after: str | None) -> VectorStorePage:
params: Final = {"limit": LIST_PAGE_SIZE} if after is None else {"limit": LIST_PAGE_SIZE, "after": after}
return VectorStorePage.model_validate_json(client.get("/vector_stores", params=params).raise_for_status().content)
def fetch_next_page(client: httpx.Client, page: VectorStorePage | None) -> VectorStorePage | None:
if page is None or not page.has_more or not page.data:
return None
return fetch_page(client, after=page.data[-1].id)
def list_all_stores(client: httpx.Client) -> Iterator[VectorStore]:
pages: Final = accumulate(
repeat(None), lambda page, _: fetch_next_page(client, page), initial=fetch_page(client, after=None)
)
for page in takewhile(lambda page: page is not None, pages):
yield from page.data
def retrieve_stores(client: httpx.Client, store_ids: tuple[str, ...]) -> Iterator[VectorStore]:
for store_id in store_ids:
yield VectorStore.model_validate_json(client.get(f"/vector_stores/{store_id}").raise_for_status().content)
def describe_http_error(response: httpx.Response) -> str:
try:
message: Final = OpenAIErrorResponse.model_validate_json(response.content).error.message
except ValidationError:
return f"HTTP {response.status_code}"
return f"HTTP {response.status_code}: {message}"
def delete_keys(client: httpx.Client, store_id: str, keys: tuple[str, ...]) -> StoreResult:
try:
response: Final = client.post(f"/vector_stores/{store_id}", json={"metadata": dict.fromkeys(keys)})
except httpx.TransportError as error:
return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=type(error).__name__)
if response.is_success:
return StoreResult(store_id=store_id, matched_keys=keys, status="updated")
return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=describe_http_error(response))
def clean_store(client: httpx.Client, store: VectorStore, options: Options) -> StoreResult | None:
keys: Final = matching_keys(store, options.extra_keys)
if not keys:
return None
if not options.apply:
return StoreResult(store_id=store.id, matched_keys=keys, status="dry_run")
return delete_keys(client, store.id, keys)
def format_result(result: StoreResult) -> str:
key_list: Final = ", ".join(result.matched_keys)
count: Final = len(result.matched_keys)
match result.status:
case "dry_run":
return f"{result.store_id}: would delete {count} keys: {key_list}"
case "updated":
return f"{result.store_id}: deleted {count} keys: {key_list}"
case "failed":
return f"{result.store_id}: failed to delete {count} keys ({result.error}): {key_list}"
def clean_and_report(
client: httpx.Client, stores: Iterator[VectorStore], options: Options, out: TextIO
) -> Iterator[StoreResult | None]:
for store in stores:
result: Final = clean_store(client, store, options)
if result is not None:
print(format_result(result), file=out)
yield result
def run(options: Options, client: httpx.Client, out: TextIO) -> int:
stores: Final = (
retrieve_stores(client, options.vector_store_ids) if options.vector_store_ids else list_all_stores(client)
)
scanned: Final = tuple(clean_and_report(client, stores, options, out))
results: Final = tuple(result for result in scanned if result is not None)
updated: Final = sum(1 for result in results if result.status == "updated")
failed: Final = sum(1 for result in results if result.status == "failed")
mode: Final = "applied" if options.apply else "dry run, pass --apply to delete"
print(
f"scanned={len(scanned)} matched={len(results)} updated={updated} failed={failed} ({mode})",
file=out,
)
return 1 if failed else 0
def parse_options(argv: Sequence[str] | None) -> Options:
parser: Final = argparse.ArgumentParser(
description="Remove LiteLLM proxy-written keys from OpenAI vector store metadata. Dry run by default."
)
parser.add_argument("--api-key", default=os.environ.get("OPENAI_API_KEY"), help="defaults to $OPENAI_API_KEY")
parser.add_argument("--api-base", default=DEFAULT_API_BASE)
parser.add_argument("--vector-store-id", action="append", default=[], help="repeatable, defaults to every store")
parser.add_argument("--extra-key", action="append", default=[], help="repeatable, another key name to delete")
parser.add_argument("--apply", action="store_true", help="delete the matching keys instead of listing them")
args: Final = parser.parse_args(argv)
if not args.api_key:
parser.error("--api-key or OPENAI_API_KEY is required")
return Options(
api_key=args.api_key,
api_base=args.api_base.rstrip("/"),
vector_store_ids=tuple(args.vector_store_id),
extra_keys=frozenset(args.extra_key),
apply=args.apply,
)
def main(argv: Sequence[str] | None = None, transport: httpx.BaseTransport | None = None) -> int:
options: Final = parse_options(argv)
with httpx.Client(
base_url=options.api_base,
headers={"Authorization": f"Bearer {options.api_key}"},
timeout=60.0,
transport=transport,
) as client:
try:
return run(options, client, sys.stdout)
except httpx.HTTPStatusError as error:
sys.stderr.write(f"could not read vector stores: {describe_http_error(error.response)}\n")
return 2
except httpx.TransportError as error:
sys.stderr.write(f"could not reach {options.api_base}: {type(error).__name__}\n")
return 2
except ValidationError:
sys.stderr.write(f"unexpected vector store response shape from {options.api_base}\n")
return 2
if __name__ == "__main__":
sys.exit(main())

71
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -3,6 +3,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
"""
import asyncio
import json
from collections.abc import Mapping, Sequence
from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta, timezone
@ -227,7 +228,7 @@ class CheckBatchCost:
"so it will no longer be polled"
)
async def _claim_job_for_costing(self, job: "ManagedObjectRow") -> bool:
async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool:
"""
Atomically flip batch_processed from false to true, returning whether this pod won
the row. Every pod and uvicorn worker schedules its own poller against the shared
@ -241,10 +242,16 @@ class CheckBatchCost:
"""
if not self._has_batch_processed_column:
return True
from litellm_enterprise.proxy.hooks.managed_files import postgres_storable_json
try:
claimed: Final = await _managed_object_table(self.prisma_client).update_many(
where={"id": job.id, "batch_processed": False},
data={"batch_processed": True},
data={
"batch_processed": True,
"status": response.status if response.status != "completed" else "complete",
"file_object": postgres_storable_json(response),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
@ -253,7 +260,9 @@ class CheckBatchCost:
return False
return claimed > 0
async def _release_job_claim(self, job: "ManagedObjectRow") -> None:
async def _release_job_claim(
self, job: "ManagedObjectRow", original_status: str | None, original_file_object: object
) -> None:
"""Give a claimed row back once billing it failed, so a later poll cycle retries it.
Safe to match on batch_processed=True: while this poller is active the retrieve
@ -265,7 +274,11 @@ class CheckBatchCost:
try:
await _managed_object_table(self.prisma_client).update_many(
where={"id": job.id, "batch_processed": True},
data={"batch_processed": False},
data={
"batch_processed": False,
"status": original_status,
"file_object": json.dumps(original_file_object),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
@ -564,8 +577,6 @@ class CheckBatchCost:
@staticmethod
def _get_input_file_id(job: "ManagedObjectRow") -> Optional[str]:
import json
from litellm.types.utils import LiteLLMBatch
file_object: Final = job.file_object
@ -718,8 +729,7 @@ class CheckBatchCost:
)
except Exception as _e:
verbose_proxy_logger.warning(
f"CheckBatchCost: failed to create managed file ID for "
f"{_file_attr}={_raw_file_id!r}: {_e}"
f"CheckBatchCost: failed to create managed file ID for {_file_attr}={_raw_file_id!r}: {_e}"
)
# Pass the deployment's router-registered pricing (litellm_params custom
@ -755,8 +765,9 @@ class CheckBatchCost:
failed_requests=output_file_result.failed_requests + error_file_failed_requests,
)
)
logging_model: Final = batch_result.models[0] if batch_result.models else litellm_model_name or model_id
logging_obj = LiteLLMLogging(
model=batch_result.models[0],
model=logging_model,
messages=[{"role": "user", "content": "<retrieve_batch>"}],
stream=False,
call_type="aretrieve_batch",
@ -787,7 +798,9 @@ class CheckBatchCost:
custom_llm_provider=str(llm_provider) if llm_provider else None,
)
if not await self._claim_job_for_costing(job):
original_status: Final = job.status
original_file_object: Final = job.file_object
if not await self._claim_job_for_costing(job, response.model_copy(update={"usage": batch_result.usage})):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
"in this window, so its cost is already being tracked there"
@ -806,7 +819,7 @@ class CheckBatchCost:
batch_completion_cost=batch_result.completion_cost,
)
except Exception:
await self._release_job_claim(job)
await self._release_job_claim(job, original_status, original_file_object)
raise
# Record batch duration (completed_at - created_at)
@ -1099,10 +1112,7 @@ class CheckBatchCost:
continue
## RETRIEVE THE BATCH JOB OUTPUT FILE
if (
response.status in PROVIDER_TERMINAL_BATCH_STATUSES
and response.output_file_id is not None
):
if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None:
try:
tracked = await self._track_completed_batch_cost(
job=job,
@ -1133,24 +1143,19 @@ class CheckBatchCost:
# Track this job for the final metrics summary
processed_models.append(tracked)
# mark the job as complete
try:
update_data: dict = {
"status": response.status if response.status != "completed" else "complete",
"file_object": response.model_dump_json(),
}
if self._has_batch_processed_column:
update_data["batch_processed"] = True
await _managed_object_table(self.prisma_client).update(
where={"id": job.id},
data=update_data,
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
if not self._has_batch_processed_column:
# nothing claimed the row, so it stays in the poll page and its lines wait for that retry
if not self._has_batch_processed_column:
try:
await _managed_object_table(self.prisma_client).update(
where={"id": job.id},
data={
"status": response.status if response.status != "completed" else "complete",
"file_object": response.model_dump_json(),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
continue
self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)

8
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -416,7 +416,7 @@ def _storable_json_value(value: JsonValue) -> JsonValue:
return value
def _postgres_storable_json(file_object: BaseModel) -> str:
def postgres_storable_json(file_object: BaseModel) -> str:
dumped: Final = file_object.model_dump_json()
if "\\u0000" not in dumped:
return dumped
@ -650,7 +650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
# FIX: Update status and file_object on every operation to keep state in sync
update_columns: Final = {
"file_object": _postgres_storable_json(stored_object),
"file_object": postgres_storable_json(stored_object),
"status": file_object.status,
"updated_by": user_api_key_dict.user_id,
}
@ -666,7 +666,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
data={
"create": {
"unified_object_id": unified_object_id,
"file_object": _postgres_storable_json(stored_object),
"file_object": postgres_storable_json(stored_object),
"model_object_id": model_object_id,
"file_purpose": file_purpose,
"created_by": resolve_resource_owner_id(user_api_key_dict),
@ -702,7 +702,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"unified_object_id": response_id,
"model_object_id": decrypted_response_id(response_id),
"file_purpose": "response",
"file_object": _postgres_storable_json(_without_model_output(response)),
"file_object": postgres_storable_json(_without_model_output(response)),
"status": PENDING_RESPONSE_STATUSES[0],
"created_by": resolve_resource_owner_id(user_api_key_dict),
"team_id": user_api_key_dict.team_id,

1
litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs
View file

@ -94,6 +94,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
"docker_model_runner",
"ragflow",
"pinstripes",
"openinfer",
"darkbloom",
"meta",
"cognition",

71
litellm-rust/crates/inference-messages/tests/messages/request.rs
View file

@ -343,6 +343,77 @@ async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
assert_eq!(sent, expected);
}
#[rstest]
#[case::anthropic("anthropic", &[AnthropicBeta::DangerousToolUse20260903])]
#[case::deepseek("deepseek", &[])]
#[case::minimax("minimax", &[])]
#[tokio::test]
async fn safeguards_add_the_dangerous_tool_use_beta_only_where_the_beta_map_has_a_column(
call: MessagesCall,
#[case] provider: &str,
#[case] derived: &[AnthropicBeta],
) {
let upstream = upstream([message_response()]).await;
let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
..call
},
json!({"safeguards": safeguards}),
))
.await;
let request = only_request(&upstream).await;
let expected: BetaSet = derived
.iter()
.cloned()
.chain([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
.collect();
assert_eq!(sent_betas(&request), expected);
assert_eq!(
request.body_json::<Value>().unwrap()["safeguards"],
safeguards
);
}
#[rstest]
#[tokio::test]
async fn null_safeguards_are_dropped_instead_of_sent_without_their_beta(
call: MessagesCall,
#[values("anthropic", "azure_ai", "bedrock_mantle", "deepseek", "minimax")] provider: &str,
) {
let upstream = upstream([message_response()]).await;
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
..call
},
json!({"safeguards": null}),
))
.await;
let request = only_request(&upstream).await;
assert_eq!(
sent_betas(&request),
BetaSet::from_iter([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
);
assert!(
!request
.body_json::<Map<String, Value>>()
.unwrap()
.contains_key("safeguards")
);
}
#[rstest]
#[tokio::test]
async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: MessagesCall) {

91
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -109,7 +109,10 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
merge_beta_headers(
headers,
provider_feature_betas(request, BetaProvider::Anthropic),
)
}
}
@ -136,12 +139,20 @@ pub(crate) fn transform_messages_request(
messages: strip_encrypted_reasoning_blocks(messages),
params: MessagesOptionalParams {
context_management,
extra: without_null_safeguards(request.params.extra),
..request.params
},
..request
})
}
fn without_null_safeguards(extra: Map<String, Value>) -> Map<String, Value> {
extra
.into_iter()
.filter(|(key, value)| key != "safeguards" || !value.is_null())
.collect()
}
pub(crate) fn update_headers_with_anthropic_beta(
headers: Headers,
request: &MessagesRequest,
@ -149,12 +160,34 @@ pub(crate) fn update_headers_with_anthropic_beta(
merge_beta_headers(headers, feature_betas(request))
}
/// The betas a request needs on a host other than the first-party API: the Anthropic set
/// filtered and renamed through the host's `BetaProvider` policy.
pub(crate) fn update_headers_with_safeguards_beta(
headers: Headers,
request: &MessagesRequest,
provider: BetaProvider,
) -> Headers {
merge_beta_headers(
headers,
feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)),
)
}
/// The betas a request needs on a host with a column in the beta map, filtered and renamed
/// through that column.
pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet {
feature_betas(request)
.iter()
.filter_map(|beta| beta.on(provider))
on_provider(
feature_betas(request).union(mapped_host_betas(&request.params)),
provider,
)
}
fn on_provider(betas: BetaSet, provider: BetaProvider) -> BetaSet {
betas.iter().filter_map(|beta| beta.on(provider)).collect()
}
fn mapped_host_betas(params: &MessagesOptionalParams) -> BetaSet {
carries_safeguards(params)
.then_some(AnthropicBeta::DangerousToolUse20260903)
.into_iter()
.collect()
}
@ -206,6 +239,13 @@ fn uses_structured_output(params: &MessagesOptionalParams) -> bool {
.is_some_and(|config| config.format.is_some())
}
fn carries_safeguards(params: &MessagesOptionalParams) -> bool {
params
.extra
.get("safeguards")
.is_some_and(|value| !value.is_null())
}
fn messages_carry_output_config(messages: &[Message]) -> bool {
messages
.iter()
@ -1025,10 +1065,44 @@ mod tests {
json!({"messages": [{"role": "user", "content": "hi", "output_config": null}]}),
&["per-turn-control-2026-07-01"]
)]
#[case::safeguards(
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
&[]
)]
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
assert_eq!(feature_betas(&request(fields)), betas(expected));
}
#[rstest]
#[case::anthropic(BetaProvider::Anthropic, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
#[case::vertex_ai(BetaProvider::VertexAi, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
#[case::empty_list(BetaProvider::Anthropic, json!([]), &["dangerous-tool-use-2026-09-03"])]
#[case::not_a_list(BetaProvider::Anthropic, json!(1), &["dangerous-tool-use-2026-09-03"])]
#[case::null(BetaProvider::Anthropic, Value::Null, &[])]
#[case::bedrock_converse_has_a_null_cell(BetaProvider::BedrockConverse, json!([{"type": "dangerous_tool_use"}]), &[])]
fn safeguards_add_the_dangerous_tool_use_beta_where_the_beta_map_has_a_cell(
#[case] provider: BetaProvider,
#[case] safeguards: Value,
#[case] expected: &[&str],
) {
assert_eq!(
provider_feature_betas(&request(json!({"safeguards": safeguards})), provider),
betas(expected)
);
}
#[rstest]
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
assert_eq!(
update_headers_with_anthropic_beta(
client.clone(),
&request(json!({"safeguards": [{"type": "dangerous_tool_use"}]}))
),
client
);
}
#[rstest]
#[case::no_betas(&[("x-api-key", "k"), ("anthropic-version", "2023-06-01")], json!({}), &[("x-api-key", "k"), ("anthropic-version", "2023-06-01")])]
#[case::blank_beta_header(&[("Anthropic-Beta", " , "), ("x-api-key", "k")], json!({}), &[("Anthropic-Beta", " , "), ("x-api-key", "k")])]
@ -1067,6 +1141,11 @@ mod tests {
json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}),
&[("anthropic-beta", "claude-code-20250219,context-management-2025-06-27,effort-2025-11-24,interleaved-thinking-2025-05-14,per-turn-control-2026-07-01")],
)]
#[case::safeguards_add_the_dangerous_tool_use_beta_to_the_client_betas(
&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")],
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
&[("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")],
)]
fn request_headers_merge_the_feature_betas(
#[case] input: &[(&str, &str)],
#[case] fields: Value,

12
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -2,7 +2,7 @@ use litellm_auth::SecretValue;
use litellm_http::request::{has_bearer_auth, has_header};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
@ -12,7 +12,7 @@ use crate::{
common_utils::API_KEY_PLACEMENT,
messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
transformation::{transform_messages_request, update_headers_with_safeguards_beta},
},
},
azure_ai::common_utils::{
@ -104,7 +104,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
update_headers_with_safeguards_beta(headers, request, BetaProvider::AzureAi)
}
}
@ -544,6 +544,12 @@ mod tests {
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
)]
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
#[case::safeguards_need_the_dangerous_tool_use_beta(
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
&[("anthropic-beta", "claude-code-20250219")],
&[("x-api-key", "k"), ("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03")]
)]
#[case::null_safeguards_need_no_beta(json!({"safeguards": null}), &[], &[("x-api-key", "k")])]
fn request_headers_carry_the_anthropic_feature_betas(
#[case] fields: serde_json::Value,
#[case] forwarded: &[(&str, &str)],

21
litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs
View file

@ -6,7 +6,7 @@ use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{header_value, with_header, without_headers};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
use serde_json::Value;
@ -15,7 +15,7 @@ use crate::{
Error, ErrorDetail,
anthropic::messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
transformation::{transform_messages_request, update_headers_with_safeguards_beta},
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
@ -174,7 +174,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
None => headers,
};
update_headers_with_anthropic_beta(headers, request)
update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic)
}
fn wire_body(&self, body: Value) -> Value {
@ -356,6 +356,21 @@ mod tests {
);
}
#[rstest]
#[case::safeguards(json!([{"type": "dangerous_tool_use"}]), Some("dangerous-tool-use-2026-09-03"))]
#[case::null_safeguards(Value::Null, None)]
fn safeguards_need_the_dangerous_tool_use_beta(
#[case] safeguards: Value,
#[case] expected: Option<&str>,
) {
let request = serde_json::from_value(json!({"model": "claude-test", "max_tokens": 16, "messages": [], "safeguards": safeguards})).unwrap();
let headers = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.request_headers(vec![], &request);
assert_eq!(
header_value(&headers, "anthropic-beta").map(|value| value.to_string()),
expected.map(str::to_string)
);
}
#[rstest]
#[case::default(json!({}), false)]
#[case::override_empty(json!({"claude_platform_unsupported_params": []}), true)]

23
litellm-rust/crates/llms/src/deepseek/messages/transformation.rs
View file

@ -451,4 +451,27 @@ mod tests {
json!({"type": "enabled", "budget_tokens": 1024})
);
}
#[rstest]
#[case::without_client_betas(&[])]
#[case::with_client_betas(&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")])]
fn safeguards_and_client_betas_reach_deepseek_as_sent(#[case] client: &[(&str, &str)]) {
let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
let body = json!({
"model": "deepseek-v4-pro",
"max_tokens": 100,
"messages": [{"role": "user", "content": "hi"}],
"safeguards": safeguards
});
let headers: Headers = client
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect();
assert_eq!(
DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG
.request_headers(headers.clone(), &request_from(body.clone())),
headers
);
assert_eq!(transformed(body)["safeguards"], safeguards);
}
}

19
litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs
View file

@ -213,23 +213,14 @@ fn has_web_search_tool(tools: Option<&[Recognized<MessagesTool>]>) -> bool {
}
/// Python's Vertex `validate_anthropic_messages_environment` beta set: the shared feature
/// betas under Vertex's policy, plus the web search and safeguards betas Vertex needs spelled
/// out.
/// betas under Vertex's policy, plus the web search beta Vertex needs spelled out.
fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
let params = &request.params;
provider_feature_betas(request, BetaProvider::VertexAi).union(
[
has_web_search_tool(params.tools.as_deref())
.then_some(AnthropicBeta::WebSearch20250305),
params
.extra
.get("safeguards")
.is_some_and(|value| !value.is_null())
.then_some(AnthropicBeta::DangerousToolUse20260903),
]
.into_iter()
.flatten()
.collect(),
has_web_search_tool(params.tools.as_deref())
.then_some(AnthropicBeta::WebSearch20250305)
.into_iter()
.collect(),
)
}

7
litellm/__init__.py
View file

@ -676,6 +676,7 @@ qwen_ai_platform_models: Set = set()
moonshot_models: Set = set()
publicai_models: Set = set()
darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
@ -742,7 +743,7 @@ def is_openai_finetune_model(key: str) -> bool:
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
for key, value in model_cost_map.items():
for key, value in tuple(model_cost_map.items()):
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
open_ai_chat_completion_models.add(key)
elif value.get("litellm_provider") == "text-completion-openai":
@ -945,6 +946,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
publicai_models.add(key)
elif value.get("litellm_provider") == "darkbloom":
darkbloom_models.add(key)
elif value.get("litellm_provider") == "openinfer":
openinfer_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
@ -1113,6 +1116,7 @@ model_list = list(
| moonshot_models
| publicai_models
| darkbloom_models
| openinfer_models
| v0_models
| morph_models
| lambda_ai_models
@ -1229,6 +1233,7 @@ def _build_models_by_provider() -> dict:
"moonshot": moonshot_models,
"publicai": publicai_models,
"darkbloom": darkbloom_models,
"openinfer": openinfer_models,
"v0": v0_models,
"morph": morph_models,
"lambda_ai": lambda_ai_models,

2
litellm/_logging.py
View file

@ -1000,6 +1000,8 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
lg.handlers.clear() # remove any existing handlers
lg.addHandler(handler) # add JSON formatter handler
lg.propagate = False # prevent bubbling to parent/root
if (configured_level := os.getenv("LITELLM_LOG")) is not None:
logging.getLogger().setLevel(resolve_log_level(configured_level))
def _get_uvicorn_json_log_config():

45
litellm/anthropic_beta_headers_manager.py
View file

@ -22,17 +22,28 @@ Environment Variables:
- LITELLM_ANTHROPIC_BETA_HEADERS_URL: Custom URL for remote config (optional)
"""
import asyncio
import json
import os
import threading
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from functools import cache
from importlib.resources import files
from typing import Final
from typing import Final, TypeAlias
import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
class GetAnthropicBetaHeadersConfig:
@ -174,13 +185,27 @@ def _load_beta_headers_config() -> dict:
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
# Get the URL from environment or use default
from litellm import anthropic_beta_headers_url
with _BETA_HEADERS_CONFIG_LOCK:
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
_BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
verbose_logger.debug("Loaded and cached beta headers config")
# Get the URL from environment or use default
from litellm import anthropic_beta_headers_url
return _BETA_HEADERS_CONFIG
_BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
verbose_logger.debug("Loaded and cached beta headers config")
return _BETA_HEADERS_CONFIG
async def aload_beta_headers_config() -> None:
if _BETA_HEADERS_CONFIG is None:
await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
@cache
def _bundled_beta_headers_config() -> BetaHeadersConfig:
return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())
def reload_beta_headers_config() -> dict:
@ -288,6 +313,14 @@ def is_beta_header_supported(
return beta_header in provider_mapping and provider_mapping[beta_header] is not None
def is_beta_header_supported_by_bundled_config(beta_header: str, provider: str) -> bool:
config: Final = _bundled_beta_headers_config()
aliases: Final = config.get("provider_aliases", {})
canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
provider_mapping: Final = config.get(canonical_provider, {})
return isinstance(provider_mapping, Mapping) and provider_mapping.get(beta_header) is not None
def get_provider_beta_header(
anthropic_beta_header: str,
provider: str,

16
litellm/batches/batch_utils.py
View file

@ -794,11 +794,17 @@ def _batch_response_was_successful(
message batch results lines report ``result.type == "succeeded"``; Bedrock
batch output lines report ``modelOutput`` (and no ``error``).
"""
if custom_llm_provider == "anthropic":
return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
if custom_llm_provider == "bedrock":
return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
_response: Final[dict[str, object]] = cast( # cast-ok: batch response bodies come from provider JSON
dict[str, object], batch_job_output_file.get("response", None) or {}
envelope_key: Final = "result" if custom_llm_provider == "anthropic" else "response"
envelope: Final = batch_job_output_file.get(envelope_key)
if not isinstance(envelope, Mapping):
if envelope is not None:
verbose_logger.warning("batch output %s must be an object, treating the line as unsuccessful", envelope_key)
return False
response: Final = cast(Mapping[object, object], envelope) # cast-ok: mapping shape validated above
return (
response.get("type") == "succeeded"
if custom_llm_provider == "anthropic"
else response.get("status_code") == 200
)
return _response.get("status_code", None) == 200

8
litellm/caching/caching_handler.py
View file

@ -346,6 +346,7 @@ class LLMCachingHandler:
if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
verbose_logger.debug("Checking Async Cache")
logging_obj.reset_caching_details()
cached_result = await self._retrieve_from_cache(
call_type=call_type,
kwargs=kwargs,
@ -357,7 +358,7 @@ class LLMCachingHandler:
verbose_logger.debug("Cache Hit!")
cache_hit: Final = True
end_time: Final = datetime.datetime.now()
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model,
custom_llm_provider=kwargs.get("custom_llm_provider", None),
api_base=kwargs.get("api_base", None),
@ -366,7 +367,7 @@ class LLMCachingHandler:
cache_duration_ms: Final = (cache_check_end_time - cache_check_start_time) * 1000
self._update_litellm_logging_obj_environment(
logging_obj=logging_obj,
model=model,
model=provider_model,
kwargs=kwargs,
cached_result=cached_result,
is_async=True,
@ -381,7 +382,7 @@ class LLMCachingHandler:
call_type=call_type,
kwargs=kwargs,
logging_obj=logging_obj,
model=model,
model=provider_model,
custom_llm_provider=kwargs.get("custom_llm_provider", None),
args=args,
)
@ -467,6 +468,7 @@ class LLMCachingHandler:
new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
print_verbose("Checking Sync Cache")
logging_obj.reset_caching_details()
with response_cache_phase("get"):
cached_result = litellm.cache.get_cache(**new_kwargs)
if is_response_without_output(cached_result):

39
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -41,7 +41,11 @@ from litellm.responses.sse_output_recovery import (
record_output_item_chunk,
record_output_text_chunk,
)
from litellm.responses.utils import ResponsesAPIRequestUtils, normalize_responses_api_stream_options
from litellm.responses.utils import (
ResponsesAPIRequestUtils,
auto_reasoning_summary,
normalize_responses_api_stream_options,
)
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionReasoningItem,
@ -176,6 +180,16 @@ def _without_audio_input_parts(input_items: list[object]) -> list[object]:
return [_without_audio_input_parts_in_item(item) for item in input_items]
def _effort_with_requested_summary(nested_effort: object, top_level_effort: object) -> object:
match top_level_effort, nested_effort:
case {"summary": str() as summary}, str() as effort:
return {"effort": effort, "summary": summary}
case {"summary": str() as summary}, Mapping() as reasoning if "summary" not in reasoning:
return {**reasoning, "summary": summary}
case _:
return nested_effort
def _served_provider(litellm_params: Mapping[str, object]) -> str | None:
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
return custom_llm_provider if isinstance(custom_llm_provider, str) else None
@ -953,6 +967,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
optional_params: dict,
responses_api_request: "ResponsesAPIOptionalRequestParams",
drop_params: bool = False,
model: str | None = None,
custom_llm_provider: str | None = None,
) -> None:
"""Map optional_params into responses_api_request (mutates in place)."""
for key, value in optional_params.items():
@ -984,7 +1000,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif key == "previous_response_id":
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
responses_api_request["reasoning"] = self.map_reasoning_effort(value)
responses_api_request["reasoning"] = self.map_reasoning_effort(value, model, custom_llm_provider)
elif key == "web_search_options":
self._add_web_search_tool(responses_api_request, value)
@ -1100,6 +1116,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
optional_params,
responses_api_request,
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
model=model,
custom_llm_provider=_served_provider(litellm_params),
)
stream: Final = optional_params.get("stream") or litellm_params.get("stream", False)
@ -1705,14 +1723,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Extract supported params from extra_body and merge into optional_params
extra_body_copy: Final = extra_body.copy()
top_level_effort: Final = cast(object, optional_params.get("reasoning_effort")) # cast-ok: upcast
for key, value in extra_body_copy.items():
if key in supported_responses_api_params:
# Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort)
optional_params[key] = extra_body.pop(key)
extra_body.pop(key)
optional_params[key] = (
_effort_with_requested_summary(cast(object, value), top_level_effort) # cast-ok: upcast
if key == "reasoning_effort"
else value
)
return optional_params
def map_reasoning_effort(self, reasoning_effort: object) -> Reasoning:
def map_reasoning_effort(
self, reasoning_effort: object, model: str | None = None, custom_llm_provider: str | None = None
) -> Reasoning:
# If dict is passed, convert it directly to Reasoning object
if isinstance(reasoning_effort, dict):
return Reasoning(
@ -1726,7 +1752,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
return (
Reasoning(effort=reasoning_effort, summary="detailed")
Reasoning(
effort=reasoning_effort,
summary="detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider),
)
if auto_summary_enabled
else Reasoning(effort=reasoning_effort)
)

3
litellm/constants.py
View file

@ -1034,6 +1034,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"https://api.prisminference.com/v1",
"https://api.reka.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
"https://api.openinfer.ai/v1",
]
@ -1102,6 +1103,7 @@ openai_compatible_providers: Final[list] = [
"docker_model_runner",
"ragflow",
"pinstripes", # Pinstripes - JSON-configured provider
"openinfer",
"darkbloom",
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
"cognition",
@ -1635,6 +1637,7 @@ OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_to
CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"

18
litellm/integrations/opentelemetry.py
View file

@ -6,9 +6,10 @@ from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
from typing import TYPE_CHECKING, Any, Final, cast
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -106,6 +107,14 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
# Cap on credential-scoped providers held at once; each one owns an exporter thread.
_MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
class _CacheHitView(TypedDict, total=False):
cache_hit: ReadOnly[object]
saved_cache_cost: ReadOnly[object]
_CACHE_HIT_VIEW: Final = TypeAdapter(_CacheHitView)
# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
@ -2434,6 +2443,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
litellm_params=litellm_params,
)
# Cost breakdown tracking
cache_view: Final = _CACHE_HIT_VIEW.validate_python(standard_logging_payload)
cache_hit: Final = cache_view.get("cache_hit") is True
cost_breakdown: Final[CostBreakdown | None] = standard_logging_payload.get("cost_breakdown")
if cost_breakdown:
for key, value in cost_breakdown.items():
@ -2443,6 +2454,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
key=f"gen_ai.cost.{key}",
value=value,
)
self.safe_set_attribute(span=span, key=LiteLLM.CACHE_HIT, value=cache_hit)
if cache_hit:
saved_cache_cost: Final = cache_view.get("saved_cache_cost")
if isinstance(saved_cache_cost, (int, float)):
self.safe_set_attribute(span=span, key=LiteLLM.SAVED_CACHE_COST, value=saved_cache_cost)
#############################################
########## LLM Request Attributes ###########
#############################################

32
litellm/integrations/otel/emitter.py
View file

@ -100,23 +100,19 @@ def _carried_keys(
return frozenset(key for key, _value in attributes)
def attribute_budget(span: Span, reserved: int, overwrites: frozenset[str] = frozenset()) -> int | None:
"""How many mapped attributes fit on ``span`` next to what it already carries and ``reserved`` more.
def carried_keys(span: Span) -> frozenset[str]:
"""The attribute keys ``span`` already holds, empty for a span that exposes none."""
if not isinstance(span, ReadableSpan):
return frozenset()
return _carried_keys(span.attributes or ())
``overwrites`` are the mapped keys already present on ``span``: setting one
replaces the value in place and consumes no slot against the limit, so only
the genuinely new pre-existing keys reduce the budget. Counting the
overwritten ones too reserves slots the fit can never spend and sheds
indexed message attributes for nothing.
"""
def attribute_budget(span: Span, reserved: int) -> int | None:
"""How many new attribute keys fit on ``span`` next to what it already carries and ``reserved`` more."""
limit: Final = span_attribute_limit(span)
if limit is None:
return None
if not isinstance(span, ReadableSpan):
return limit - reserved
existing_keys: Final = _carried_keys(span.attributes or ())
fresh: Final = len(existing_keys - overwrites) if overwrites else len(existing_keys)
return limit - fresh - reserved
return limit - len(carried_keys(span)) - reserved
def stamp_error(
@ -303,8 +299,14 @@ class SpanEmitter:
)
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
reserved: Final = len(stamped_later.keys() - mapped.keys())
for key, value in fit_indexed_messages(mapped, attribute_budget(span, reserved, frozenset(mapped))).items():
span.set_attribute(key, value)
overwrites: Final = carried_keys(span) & mapped.keys()
fitted: Final = fit_indexed_messages(
MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
attribute_budget(span, reserved),
)
for key, value in mapped.items():
if key in overwrites or key in fitted:
span.set_attribute(key, value)
if error:
stamped: Final = stamp_error(span, error)
if stamped is not None and self._event_recorder is not None and role is SpanRole.LLM_CALL:

8
litellm/integrations/otel/logger.py
View file

@ -538,8 +538,12 @@ class OpenTelemetryV2(CustomLogger):
# logger is a success/failure callback only, so ``pre_call`` never reaches it
# and no carrier exists. The payload plus the request-level provider-handoff
# stamp (``upstream_started``) is the affirmative signal of a real call; a
# gate rejection carries ``is_no_upstream_call`` and gets no span.
if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
# upstream stamp, since the payload proves the call completed.
if carrier is None and (
call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
):
return None
try:
return self._finish_carrier(carrier, call, start_time, end_time)

2
litellm/integrations/otel/mappers/genai.py
View file

@ -91,6 +91,8 @@ class GenAIMapper:
f"{LiteLLM.COST_PREFIX}margin_fixed_amount": lambda d: d.cost.margin_fixed_amount,
f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent,
f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount,
LiteLLM.CACHE_HIT: lambda d: d.cache_hit,
LiteLLM.SAVED_CACHE_COST: lambda d: d.saved_cache_cost,
LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
LiteLLM.REQUEST_ROUTE: lambda d: d.request_route,
LiteLLM.REQUEST_PURPOSE: lambda d: d.request_purpose,

5
litellm/integrations/otel/model/metadata.py
View file

@ -249,6 +249,10 @@ class LLMCallEvent:
# ``InternalCallOrigin`` such as ``autorouter_classifier``), ``None`` for the
# caller's own provider attempt.
purpose: str | None
# True for a response served from the litellm response cache. A cache hit
# never hands off to a provider, so ``upstream_started`` stays False even
# though the call completed and is worth a span.
cache_hit: bool
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
@ -273,6 +277,7 @@ class LLMCallEvent:
upstream_started=kwargs.get("api_call_start_time") is not None,
upstream_start_seconds=_epoch_seconds(kwargs.get("api_call_start_time")),
purpose=internal_call_origin(payload, kwargs),
cache_hit=bool(payload.get("cache_hit")) if payload else False, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
provisional_span_name=f"{operation.value} {model}".strip(),
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
trace=trace,

7
litellm/integrations/otel/model/payloads.py
View file

@ -441,6 +441,8 @@ class LLMCallSpanData:
embedding_output: EmbeddingOutput | None = None
promoted_metadata: Mapping[str, str] = field(default_factory=lambda: _EMPTY_METADATA)
routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({}))
cache_hit: bool | None = None
saved_cache_cost: float | None = None
@classmethod
def from_standard_logging_payload(
@ -456,6 +458,7 @@ class LLMCallSpanData:
metadata_keys: tuple[str, ...] = (),
) -> LLMCallSpanData:
params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {})
cache_hit: Final = as_bool(payload.get("cache_hit")) is True # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
# The single parse of the request's metadata — the request-vs-provider
# model split, the response model, api base, and identity all come from
# here rather than being re-derived from the raw payload dicts.
@ -490,7 +493,7 @@ class LLMCallSpanData:
error=_parse_error(payload),
response_cost=as_float(payload.get("response_cost")),
cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))),
server=ServerInfo.from_api_base(context.api_base),
server=None if cache_hit else ServerInfo.from_api_base(context.api_base),
identity=context.identity,
promoted_metadata=allowlisted_metadata(context.identity.metadata, metadata_keys),
is_streaming=as_bool(payload.get("stream")),
@ -507,6 +510,8 @@ class LLMCallSpanData:
session_id=session_id or None,
embedding_output=embedding_output if capture_content else None,
routing_attributes=routing_decision_attributes(decision),
cache_hit=cache_hit,
saved_cache_cost=as_float(payload.get("saved_cache_cost")) if cache_hit else None, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
)

2
litellm/integrations/otel/model/semconv.py
View file

@ -340,6 +340,8 @@ class LiteLLM:
# semconv key for an MCP server's *name* (the convention uses ``server.address``
# for its network location), so it lives under the vendor namespace.
MCP_SERVER_NAME: Final = "litellm.mcp.server.name"
CACHE_HIT: Final = "litellm.cache_hit"
SAVED_CACHE_COST: Final = "litellm.saved_cache_cost"
class Metric:

124
litellm/litellm_core_utils/completion_timeout.py
View file

@ -2,15 +2,25 @@
from __future__ import annotations
from collections.abc import Callable
from collections.abc import Callable, Mapping
from enum import Enum
from functools import partial
from types import MappingProxyType
from typing import Final
import httpx
from openai import Timeout as SDKTimeout
from typing_extensions import assert_never
from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS
class StreamKind(Enum):
NOT_STREAMED = "not_streamed"
LIVE = "live"
FAKED = "faked"
class CompletionTimeout:
"""Resolves HTTP timeout for ``completion()`` from model vs global settings."""
@ -36,17 +46,91 @@ class CompletionTimeout:
return COMPLETION_HTTP_FALLBACK_SECONDS
return float(global_timeout)
@staticmethod
def _longer_limit(first: float | None, second: float | None) -> float | None:
return None if first is None or second is None else max(first, second)
@staticmethod
def _longer(first: float | httpx.Timeout, second: float | httpx.Timeout) -> float | httpx.Timeout:
if not isinstance(first, httpx.Timeout) and not isinstance(second, httpx.Timeout):
return max(first, second)
first_limits: Final = httpx.Timeout(first)
second_limits: Final = httpx.Timeout(second)
longer: Final = CompletionTimeout._longer_limit
return httpx.Timeout(
connect=longer(first_limits.connect, second_limits.connect),
read=longer(first_limits.read, second_limits.read),
write=longer(first_limits.write, second_limits.write),
pool=longer(first_limits.pool, second_limits.pool),
)
@staticmethod
def for_stream(
stream_kind: StreamKind, request_timeout: float | httpx.Timeout, stream_timeout: float | httpx.Timeout | None
) -> float | httpx.Timeout:
"""
``stream_timeout`` bounds each read of a live stream, so it replaces the request timeout there. A fake
stream is one non-streaming request whose first byte arrives when generation ends, so it takes the longer
of each limit and never fails sooner than the same call would with either one alone.
"""
if stream_timeout is None:
return request_timeout
match stream_kind:
case StreamKind.NOT_STREAMED:
return request_timeout
case StreamKind.LIVE:
return stream_timeout
case StreamKind.FAKED:
return CompletionTimeout._longer(request_timeout, stream_timeout)
case _:
assert_never(stream_kind)
@staticmethod
def as_limit(timeout: object) -> float | httpx.Timeout:
if isinstance(timeout, (httpx.Timeout, SDKTimeout)):
return CompletionTimeout.normalize(timeout)
if isinstance(timeout, (float, int, str)):
return float(timeout)
raise TypeError(f"timeout must be a number or an httpx.Timeout, got {type(timeout).__name__}")
@staticmethod
def stream_request_kwargs(non_stream_timeout: object, stream_timeout: object) -> Mapping[str, object]:
if stream_timeout is None:
return MappingProxyType({"timeout": non_stream_timeout})
return MappingProxyType(
{
"timeout": None
if non_stream_timeout is None
else CompletionTimeout.for_stream(
StreamKind.FAKED,
CompletionTimeout.as_limit(non_stream_timeout),
CompletionTimeout.as_limit(stream_timeout),
),
"stream_timeout": stream_timeout,
}
)
@staticmethod
def _coerce(
timeout: object, custom_llm_provider: str, supports_httpx_timeout: Callable[[str], bool]
) -> float | httpx.Timeout:
limit: Final = CompletionTimeout.as_limit(timeout)
if not isinstance(limit, httpx.Timeout) or supports_httpx_timeout(custom_llm_provider):
return limit
return float(limit.read) if limit.read is not None else COMPLETION_HTTP_FALLBACK_SECONDS
@staticmethod
def resolve(
model_timeout: float | str | httpx.Timeout | SDKTimeout | None,
kwargs: dict,
kwargs: Mapping[str, object],
custom_llm_provider: str,
*,
global_timeout: float | str | None,
supports_httpx_timeout: Callable[[str], bool],
stream_kind: StreamKind,
) -> float | httpx.Timeout:
"""
Resolution order (first non-None wins):
Request timeout resolution order (first non-None wins):
1. ``model_timeout`` (call argument / merged ``litellm_params``)
2. ``kwargs["timeout"]``
@ -54,24 +138,20 @@ class CompletionTimeout:
4. ``global_timeout`` (the explicitly-configured ``litellm.request_timeout``),
or 600 when nothing was configured.
A stream then applies ``kwargs["stream_timeout"]`` per :meth:`for_stream`.
Coerce :class:`httpx.Timeout` when the provider does not support it.
"""
resolved: float | str | httpx.Timeout | SDKTimeout
if model_timeout is not None:
resolved = model_timeout
elif kwargs.get("timeout") is not None:
resolved = kwargs["timeout"]
elif kwargs.get("request_timeout") is not None:
resolved = kwargs["request_timeout"]
else:
resolved = CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
if isinstance(resolved, (httpx.Timeout, SDKTimeout)):
normalized: Final = CompletionTimeout.normalize(resolved)
if supports_httpx_timeout(custom_llm_provider):
return normalized
read_timeout: Final = normalized.read
return (
float(read_timeout) if read_timeout is not None else COMPLETION_HTTP_FALLBACK_SECONDS
) # default 10 min timeout
return float(resolved)
coerce: Final = partial(
CompletionTimeout._coerce,
custom_llm_provider=custom_llm_provider,
supports_httpx_timeout=supports_httpx_timeout,
)
non_stream_candidates: Final = (model_timeout, kwargs.get("timeout"), kwargs.get("request_timeout"))
explicit: Final = next((candidate for candidate in non_stream_candidates if candidate is not None), None)
request_timeout: Final = coerce(
explicit if explicit is not None else CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
)
stream_timeout: Final = None if stream_kind is StreamKind.NOT_STREAMED else kwargs.get("stream_timeout")
return CompletionTimeout.for_stream(
stream_kind, request_timeout, None if stream_timeout is None else coerce(stream_timeout)
)

47
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -1196,6 +1196,9 @@ def _map_sagemaker_exception(
)
_VERTEX_403_STATUS_CODE_PATTERN: Final = re.compile(r"(?<![0-9A-Za-z])403(?![0-9A-Za-z])")
def _map_vertex_exception(
*,
model: str,
@ -1206,6 +1209,8 @@ def _map_vertex_exception(
exception_provider: str,
extra_information: str,
) -> None:
status_code: Final[object] = getattr(original_exception, "status_code", None)
body_code: Final[int | None] = _get_body_error_code(error_str)
if "Vertex AI API has not been used in project" in error_str or "Unable to find your project" in error_str:
raise BadRequestError(
message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
@ -1252,7 +1257,11 @@ def _map_vertex_exception(
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
elif "403" in error_str:
elif (
(not isinstance(status_code, int) or status_code in (403, 500))
and body_code in (None, 403)
and _VERTEX_403_STATUS_CODE_PATTERN.search(error_str)
):
raise BadRequestError(
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
@ -1306,7 +1315,7 @@ def _map_vertex_exception(
elif (
isinstance(getattr(original_exception, "status_code", None), int)
and 500 <= original_exception.status_code < 600
and _get_body_error_code(error_str) == 429
and body_code == 429
):
# upstream gateway wraps a 429 inside a 5xx envelope
# e.g. HTTP 500/503 with {"error":{"code":429,...}}.
@ -2413,6 +2422,35 @@ def _map_exception_by_status(
)
_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_URL_SCHEME: Final = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*")
_HOST_WITH_OPTIONAL_PORT: Final = re.compile(
r"(?:(?:[A-Za-z0-9-]+\.)+[A-Za-z0-9-]+|localhost|\[[0-9A-Fa-f:.]+\])(?::\d+)?"
)
def with_deployment_api_base(litellm_params: object) -> Mapping[str, object]:
params: Final = as_str_mapping(litellm_params) or _NO_LITELLM_PARAMS
metadata: Final = as_str_mapping(params.get("metadata"))
deployment_api_base: Final = None if metadata is None else metadata.get("api_base")
if not isinstance(deployment_api_base, str) or not deployment_api_base:
return params
return MappingProxyType({**params, "api_base": deployment_api_base})
def _api_base_without_credentials(api_base: str) -> str:
scheme, separator, _ = api_base.partition("://")
scheme_prefix: Final = scheme + separator if separator and _URL_SCHEME.fullmatch(scheme) else ""
authority_and_rest: Final = api_base[len(scheme_prefix) :]
authority: Final = re.split(r"[/?#]", authority_and_rest, maxsplit=1)[0]
after_authority: Final = authority_and_rest[len(authority) :]
host: Final = authority.rpartition("@")[2]
if "@" in after_authority and _HOST_WITH_OPTIONAL_PORT.fullmatch(host) is None:
return ""
path: Final = re.split(r"[?#]", after_authority, maxsplit=1)[0]
return f"{scheme_prefix}{host}{path}"
def _is_guardrail_block(original_exception: Exception) -> bool:
from litellm.integrations.custom_guardrail import is_guardrail_intervention
@ -2483,8 +2521,9 @@ def exception_type(
if isinstance(custom_llm_provider, str) and len(custom_llm_provider) > 0:
exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception"
if _api_base:
extra_information += f"\nAPI Base: `{_api_base}`"
shown_api_base: Final = _api_base_without_credentials(_api_base) if _api_base else ""
if shown_api_base:
extra_information += f"\nAPI Base: `{shown_api_base}`"
if messages and len(messages) > 0 and litellm.redact_messages_in_exceptions is False:
extra_information += f"\nMessages: `{messages}`"

58
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -324,10 +324,16 @@ def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _Fetc
return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable:
try:
return urllib.parse.urlsplit(url)
except ValueError as error:
return ModelCostMapReloadUnavailable(reason=f"invalid URL {url}: {error}")
def _attempt_fetch_s3(parts: urllib.parse.SplitResult, url: str, timeout: int) -> _FetchAttemptOutcome:
from botocore.exceptions import BotoCoreError, ClientError
parts: Final = urllib.parse.urlsplit(url)
bucket: Final = parts.netloc
key: Final = parts.path.lstrip("/")
if not bucket or not key:
@ -343,9 +349,11 @@ def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return await asyncio.to_thread(_attempt_fetch_s3, url, timeout)
return await asyncio.to_thread(_attempt_fetch_s3, parts, url, timeout)
try:
response: Final = await client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -354,9 +362,11 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return _attempt_fetch_s3(url, timeout)
return _attempt_fetch_s3(parts, url, timeout)
try:
response: Final = client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -547,22 +557,39 @@ def get_model_cost_map_loaded_at() -> "datetime | None":
return _cost_map_source_info.loaded_at
def _expand_model_aliases(model_cost: dict) -> dict:
class AliasedCostEntry(dict[str, object]):
__slots__ = ("names",)
def __init__(self, entry: Mapping[str, object], names: tuple[str, ...]) -> None:
super().__init__(entry)
self.names: tuple[str, ...] = names
def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:
return MappingProxyType(
{
owner: (owner, *(alias for alias, alias_owner in alias_owners.items() if alias_owner == owner))
for owner in dict.fromkeys(alias_owners.values())
}
)
def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]:
"""
Expand ``aliases`` lists in model cost entries into top-level entries.
Each alias gets a reference to the **same** dict object as the canonical
entry (zero memory overhead). The ``aliases`` key is removed from the
entry so downstream code never sees it.
Each alias gets a reference to the **same** ``AliasedCostEntry`` as the
canonical entry, which carries every name it is served under. The
``aliases`` key is removed from the entry so downstream code never sees it.
If an alias collides with an existing canonical entry the alias is
skipped and a warning is logged.
"""
aliases_to_add: Final[dict[str, dict]] = {}
alias_owners: Final[dict[str, str]] = {}
keys_with_aliases: Final[list[str]] = []
for model_name, model_info in model_cost.items():
aliases: list | None = model_info.get("aliases")
aliases: object = model_info.get("aliases")
if aliases is None:
continue
keys_with_aliases.append(model_name)
@ -584,7 +611,7 @@ def _expand_model_aliases(model_cost: dict) -> dict:
model_name,
)
continue
if alias in aliases_to_add:
if alias in alias_owners:
verbose_logger.warning(
"LiteLLM model alias conflict: alias '%s' (from '%s') "
"was already claimed by another entry — skipping.",
@ -592,13 +619,14 @@ def _expand_model_aliases(model_cost: dict) -> dict:
model_name,
)
continue
aliases_to_add[alias] = model_info # same dict reference
alias_owners[alias] = model_name
# Remove the ``aliases`` key from entries so it doesn't pollute model info
for key in keys_with_aliases:
model_cost[key].pop("aliases", None)
model_cost.update(aliases_to_add)
for owner, names in _alias_groups(alias_owners).items():
model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names)))
return model_cost

92
litellm/litellm_core_utils/litellm_logging.py
View file

@ -13,12 +13,13 @@ import sys
import time
import traceback
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime as dt_object
from functools import lru_cache
from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
from httpx import Response
from httpx import Response, Timeout
from pydantic import BaseModel, JsonValue, TypeAdapter
import litellm
@ -558,6 +559,12 @@ def mask_api_base_credentials(api_base: str) -> str:
return api_base[:key_end] + "*" * 5 + api_base[-4:]
@dataclass(frozen=True, slots=True)
class UncachedPrice:
cost: float | None
cost_breakdown: CostBreakdown | None
def _timestamp_seconds(moment: object) -> float | None:
if isinstance(moment, datetime.datetime):
return moment.timestamp()
@ -664,6 +671,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.message_copy_duration_ms: float = (time.time() - _copy_start) * 1000
self.callback_duration_ms: float = 0.0
self.stream = stream
self.applied_timeout: float | Timeout | None = None
self.start_time = start_time # log the call start time
self.call_type = call_type
self.relays_llm_passthrough: Final = call_type in (
@ -811,6 +819,10 @@ class Logging(LiteLLMLoggingBaseClass):
new_call.sync_streaming_chunks = []
return new_call
def reset_caching_details(self) -> None:
"""Forget an earlier cache lookup's result, so a call that looks up again reports only its latest."""
self.caching_details = None
def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
self.response_timing_metrics = dict(timing_metrics)
@ -1045,6 +1057,11 @@ class Logging(LiteLLMLoggingBaseClass):
if "custom_llm_provider" in self.model_call_details:
self.custom_llm_provider = self.model_call_details["custom_llm_provider"]
def record_applied_timeout(self, timeout: float | Timeout) -> None:
self.applied_timeout = timeout
self.litellm_params["timeout"] = timeout
self.model_call_details["litellm_params"] = self.litellm_params
def update_from_kwargs(
self,
kwargs: dict,
@ -1890,6 +1907,7 @@ class Logging(LiteLLMLoggingBaseClass):
cache_hit: bool | None = None,
litellm_model_name: str | None = None,
router_model_id: str | None = None,
ignore_stamped_cost: bool = False,
) -> float | None:
"""
Calculate response cost using result + logging object variables.
@ -1925,7 +1943,9 @@ class Logging(LiteLLMLoggingBaseClass):
):
hidden_params: Final = result_hidden_params
if (
"response_cost" in hidden_params and hidden_params["response_cost"] is not None
not ignore_stamped_cost
and "response_cost" in hidden_params
and hidden_params["response_cost"] is not None
): # use cost if already calculated
self._record_zero_cost_diagnostic(
priced_result,
@ -2051,6 +2071,20 @@ class Logging(LiteLLMLoggingBaseClass):
_response_cost_calculator = response_cost_calculator
def uncached_price(self, result: object) -> UncachedPrice:
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
debug_key: Final = "response_cost_failure_debug_information"
had_failure_debug_info: Final = debug_key in self.model_call_details
try:
cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown)
finally:
self.cost_breakdown = previous_cost_breakdown
self.billed_token_rates = previous_billed_token_rates
if not had_failure_debug_info:
self.model_call_details.pop(debug_key, None)
def _record_zero_cost_diagnostic(
self,
result: object,
@ -3031,7 +3065,7 @@ class Logging(LiteLLMLoggingBaseClass):
complete_streaming_response: (
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
) = None
if "complete_streaming_response" in self.model_call_details:
if "complete_streaming_response" in self.model_call_details and not for_async_request:
return # break out of this.
complete_streaming_response = self._get_assembled_streaming_response(
result=result,
@ -6909,6 +6943,36 @@ def _autorouter_savings_for_payload(
return None
_CACHE_HIT_ZEROED_AMOUNTS: Final = MappingProxyType(
{
"cache_read_cost": CostBreakdown(cache_read_cost=0.0),
"cache_creation_cost": CostBreakdown(cache_creation_cost=0.0),
"reasoning_cost": CostBreakdown(reasoning_cost=0.0),
"original_cost": CostBreakdown(original_cost=0.0),
"discount_amount": CostBreakdown(discount_amount=0.0),
"margin_fixed_amount": CostBreakdown(margin_fixed_amount=0.0),
"margin_total_amount": CostBreakdown(margin_total_amount=0.0),
}
)
def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
"""Every amount at 0, keeping the pricing basis and rates a fresh call would have had, which savings reports read."""
zero_costs: Final = CostBreakdown(input_cost=0.0, output_cost=0.0, total_cost=0.0, tool_usage_cost=0.0)
if served is None:
return zero_costs
zeroed_additional_costs: Final = (
CostBreakdown(additional_costs=dict.fromkeys(served["additional_costs"], 0.0))
if "additional_costs" in served
else CostBreakdown()
)
return functools.reduce(
lambda breakdown, zeroed: breakdown | zeroed,
(_CACHE_HIT_ZEROED_AMOUNTS[key] for key in served if key in _CACHE_HIT_ZEROED_AMOUNTS),
served | zero_costs | zeroed_additional_costs,
)
def get_standard_logging_object_payload(
kwargs: dict | None,
init_response_obj: object,
@ -6992,17 +7056,10 @@ def get_standard_logging_object_payload(
"user", None
) # maintain backwards compatibility with old request body check
saved_cache_cost: float = 0.0
if cache_hit is True or served_from_cache(init_response_obj):
id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id
if cache_hit is True:
saved_cache_cost = (
logging_obj.response_cost_calculator(
result=init_response_obj,
cache_hit=False,
)
or 0.0
)
uncached_price: Final = logging_obj.uncached_price(result=init_response_obj) if cache_hit is True else None
saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0
## Get model cost information ##
base_model = get_base_model_from_metadata(model_call_details=kwargs)
@ -7019,7 +7076,7 @@ def get_standard_logging_object_payload(
# clean up litellm hidden params
clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
if clean_hidden_params["response_cost"] is None and raw_response_cost is not None:
if cache_hit is True or (clean_hidden_params["response_cost"] is None and raw_response_cost is not None):
clean_hidden_params["response_cost"] = llm_response_cost
if clean_hidden_params["litellm_overhead_time_ms"] is None and status == "success":
# /v1/messages dict results and the bridge stream wrappers keep it on the logging object;
@ -7081,7 +7138,14 @@ def get_standard_logging_object_payload(
):
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
request_cost_breakdown: Final = cost_breakdown_with_guardrail(
(
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
if uncached_price is not None
else logging_obj.cost_breakdown
),
guardrail_cost,
)
captured_baseline: Final = logging_obj.baseline_observation
autorouter_savings: Final = (
None

22
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -6166,3 +6166,25 @@ def resolve_structured_messages(
if structured:
return [msg if isinstance(msg, dict) else msg.model_dump() for msg in structured]
return None
def is_flat_responses_tool(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type")
return tool_type in ("function", "custom") and tool_type not in tool and "input_schema" not in tool
def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
if is_flat_responses_tool(tool):
chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
[dict(tool)] # pyright: ignore[reportArgumentType] # a JSON dict in a flat Responses tool shape
)
return tuple(chat_tools)
anthropic_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(
[dict(tool)] # pyright: ignore[reportArgumentType] # a JSON dict in an Anthropic or chat tool shape
)
return tuple(anthropic_tools)

4
litellm/litellm_core_utils/served_output_texts.py
View file

@ -126,13 +126,13 @@ def _chat_stream_choice_text(chunks: Sequence[ModelResponseStream], index: int)
content
for chunk in chunks
for choice in chunk.choices
if choice.index == index and isinstance(content := choice.delta.content, str)
if (choice.index or 0) == index and isinstance(content := choice.delta.content, str)
)
return "".join(contents) if contents else None
def _chat_stream_texts(chunks: Sequence[ModelResponseStream]) -> ServedTexts | None:
choice_count: Final = max((choice.index + 1 for chunk in chunks for choice in chunk.choices), default=0)
choice_count: Final = max(((choice.index or 0) + 1 for chunk in chunks for choice in chunk.choices), default=0)
texts: Final = tuple(_chat_stream_choice_text(chunks, index) for index in range(choice_count))
return texts if any(text is not None for text in texts) else None

24
litellm/litellm_core_utils/streaming_handler.py
View file

@ -47,7 +47,7 @@ from litellm.types.utils import GenericStreamingChunk as GChunk
from ..exceptions import OpenAIError
from .core_helpers import map_finish_reason, process_response_headers
from .exception_mapping_utils import exception_type
from .exception_mapping_utils import exception_type, with_deployment_api_base
from .llm_response_utils.get_api_base import get_api_base
from .rules import Rules
@ -1579,11 +1579,7 @@ class CustomStreamWrapper:
except Exception as e:
traceback.format_exc()
setattr(e, "message", str(e))
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=e,
)
raise self._map_stream_exception(e)
def set_logging_event_loop(self, loop):
"""
@ -2370,6 +2366,14 @@ class CustomStreamWrapper:
if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta):
self._emitted_disqualifying_content = True
def _map_stream_exception(self, original_exception: Exception) -> Exception:
return exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=original_exception,
extra_kwargs=with_deployment_api_base(self.logging_obj.model_call_details.get("litellm_params")),
)
def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
"""
Common error handling for both __next__ and __anext__.
@ -2414,13 +2418,7 @@ class CustomStreamWrapper:
mapped_exception: Exception = e
else:
try:
mapped_exception = exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
mapped_exception = self._map_stream_exception(e)
except Exception as mapping_error:
mapped_exception = mapping_error
self._restore_consumer_correlation_context()

62
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -30,6 +30,7 @@ from litellm.llms.anthropic.pass_through.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
is_provider_native_tool_dict,
)
from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.pass_through.stream_assembly import (
build_complete_streaming_response,
build_usage_only_response_from_chunks,
@ -1456,6 +1457,9 @@ class AnthropicMessagesHandler(BaseTranslation):
return _SSEFieldRewrite("delta", "text", next(replacements))
cls._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta)
cls._redact_ended_stream_message_delta(
responses_so_far, kept_tool_use_ids=frozenset(cls._streamed_tool_use_ids(responses_so_far).values())
)
@classmethod
def _write_ended_stream_tool_call_rewrites(
@ -1523,6 +1527,64 @@ class AnthropicMessagesHandler(BaseTranslation):
return None
cls._rewrite_ended_stream_events(responses_so_far, rewrite_tool_use)
streamed_tool_use_ids: Final = cls._streamed_tool_use_ids(responses_so_far)
cls._redact_ended_stream_message_delta(
responses_so_far,
kept_tool_use_ids=frozenset(
tool_use_id for index, tool_use_id in streamed_tool_use_ids.items() if index not in rewrites_by_block
),
)
@classmethod
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
"""The id of each buffered ``tool_use`` block, by its content block index."""
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
tool_use_blocks: Final = (
(event.get("index"), block)
for event in events
if event.get("type") == "content_block_start"
and (block := as_str_mapping(event.get("content_block"))) is not None
and block.get("type") == "tool_use"
)
return MappingProxyType(
{
index: tool_use_id
for index, block in tool_use_blocks
if isinstance(index, int) and isinstance(tool_use_id := block.get("id"), str)
}
)
@classmethod
def _redact_ended_stream_message_delta(
cls,
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
*,
kept_tool_use_ids: frozenset[str],
) -> None:
"""Hold the buffered ``message_delta`` to the rules a re-emitted rewrite follows: a verdict survives
only for a tool call delivered unchanged, and no free-text ``explanation`` describes the text the
guardrail rewrote."""
def rewrite_stop_details(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
delta: Final = as_str_mapping(event.get("delta")) if event.get("type") == "message_delta" else None
if delta is None or "stop_details" not in delta:
return None
return _SSEFieldRewrite(
"delta", "stop_details", redacted_stop_details(delta["stop_details"], keep_explanations=False)
)
def rewrite_safeguard_results(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
delta: Final = as_str_mapping(event.get("delta")) if event.get("type") == "message_delta" else None
if delta is None or "safeguard_results" not in delta:
return None
return _SSEFieldRewrite(
"delta",
"safeguard_results",
filtered_safeguard_results(delta["safeguard_results"], kept_tool_use_ids, keep_explanations=False),
)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_stop_details)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_safeguard_results)
@staticmethod
def _rewrite_ended_stream_events(

10
litellm/llms/anthropic/pass_through/adapters/handler.py
View file

@ -501,7 +501,11 @@ class LiteLLMMessagesToCompletionTransformationHandler:
return None
if resolved_provider == "litellm_proxy":
return None
from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary, responses_api_bridge_check
from litellm.responses.bridge_check import (
chat_target_accepts_reasoning_summary,
responses_api_bridge_check,
str_keyed_mapping,
)
if requested_summary is not None and chat_target_accepts_reasoning_summary(local_model, resolved_provider):
return _ChatTargetReasoning(effort=effort, summary=requested_summary)
@ -510,6 +514,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
web_search_options: Final = completion_kwargs.get("web_search_options")
tools: Final = completion_kwargs.get("tools")
client_summary: Final = thinking.get("summary")
model_info, _ = responses_api_bridge_check(
model=local_model,
custom_llm_provider=resolved_provider,
@ -517,8 +522,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
cast(OpenAIWebSearchOptions, web_search_options) if isinstance(web_search_options, dict) else None
),
tools=cast("list[dict[str, object]]", tools) if isinstance(tools, list) else None,
reasoning_effort=reasoning_effort,
reasoning_effort=effort if client_summary is None else {"effort": effort, "summary": client_summary},
api_base=resolved_api_base,
deployment_model_info=str_keyed_mapping(completion_kwargs.get("model_info")),
)
return None if model_info.get("mode") == "responses" else _ChatTargetReasoning(effort=effort)

11
litellm/llms/anthropic/pass_through/adapters/transformation.py
View file

@ -14,6 +14,7 @@ from litellm.llms.anthropic.pass_through.utils import (
json_copy_or_original,
prompt_cache_key_from_user_id,
)
from litellm.responses.utils import auto_reasoning_summary
# OpenAI has a 64-character limit for function/tool names
# Anthropic does not have this limit, so we need to truncate long names
@ -736,7 +737,7 @@ class LiteLLMAnthropicMessagesAdapter:
if reasoning_effort:
return {
"reasoning_effort": LiteLLMAnthropicMessagesAdapter._apply_reasoning_summary_wrapping(
reasoning_effort, thinking
reasoning_effort, thinking, model, None
)
}
return {}
@ -745,6 +746,8 @@ class LiteLLMAnthropicMessagesAdapter:
def _apply_reasoning_summary_wrapping(
reasoning_effort: str,
thinking: Mapping[str, object],
model: str,
custom_llm_provider: str | None,
) -> Any:
"""
Apply the reasoning_effort/summary wrapping rules shared by every
@ -762,7 +765,7 @@ class LiteLLMAnthropicMessagesAdapter:
if summary:
return {"effort": reasoning_effort, "summary": summary}
if is_reasoning_auto_summary_enabled():
return {"effort": reasoning_effort, "summary": "detailed"}
return {"effort": reasoning_effort, "summary": auto_reasoning_summary(model, custom_llm_provider)}
return reasoning_effort
def translate_anthropic_tool_choice_to_openai(
@ -1174,7 +1177,9 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs["reasoning_effort"] = (
reasoning_effort
if is_claude_target
else self._apply_reasoning_summary_wrapping(reasoning_effort, cast(dict[str, object], thinking))
else self._apply_reasoning_summary_wrapping(
reasoning_effort, cast(dict[str, object], thinking), model, custom_llm_provider
)
)
def _translate_output_format_to_openai(

15
litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py
View file

@ -12,6 +12,7 @@ import json
from collections.abc import Mapping
from typing import Any, Final, cast
from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -34,8 +35,13 @@ class FakeAnthropicMessagesStreamIterator:
- message_stop
"""
def __init__(self, response: AnthropicMessagesResponse):
def __init__(
self,
response: AnthropicMessagesResponse,
message_delta_fields: Mapping[str, object] | None = None,
):
self.response = response
self.message_delta_fields = message_delta_fields
self.chunks = self._create_streaming_chunks()
self.current_index = 0
@ -175,14 +181,9 @@ class FakeAnthropicMessagesStreamIterator:
cache_read_input_tokens: Final = usage.get("cache_read_input_tokens")
if cache_read_input_tokens is not None:
delta_usage["cache_read_input_tokens"] = cache_read_input_tokens
safeguard_results: Final = self.response.get("safeguard_results")
message_delta: Final = {
"type": "message_delta",
"delta": {
"stop_reason": self.response.get("stop_reason"),
"stop_sequence": self.response.get("stop_sequence"),
**({"safeguard_results": safeguard_results} if safeguard_results is not None else {}),
},
"delta": anthropic_message_delta(response_dict, self.message_delta_fields),
"usage": delta_usage,
}
chunks.append(f"event: message_delta\ndata: {json.dumps(message_delta)}\n\n".encode())

25
litellm/llms/anthropic/pass_through/messages/handler.py
View file

@ -55,15 +55,19 @@ _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object])
_SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]])
def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool:
from litellm.responses.bridge_check import responses_api_bridge_check
def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool:
from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping
model_info, _ = responses_api_bridge_check(model=model, custom_llm_provider=custom_llm_provider)
model_info, _ = responses_api_bridge_check(
model=model,
custom_llm_provider=custom_llm_provider,
deployment_model_info=str_keyed_mapping(deployment_model_info),
)
return model_info.get("mode") == "responses"
def _responses_mode_is_lost_by_prefix_strip(
requested_model: str, resolved_model: str, custom_llm_provider: str
requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object
) -> bool:
"""Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
@ -73,15 +77,16 @@ def _responses_mode_is_lost_by_prefix_strip(
"""
if requested_model == resolved_model:
return False
return _bridges_to_responses_api(requested_model, custom_llm_provider) and not _bridges_to_responses_api(
resolved_model, custom_llm_provider
)
return _bridges_to_responses_api(
requested_model, custom_llm_provider, deployment_model_info
) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info)
def _should_route_to_responses_api(
custom_llm_provider: str | None,
requested_model: str | None = None,
resolved_model: str | None = None,
deployment_model_info: object = None,
) -> bool:
"""Return True when the request should use the Responses API path.
@ -94,7 +99,9 @@ def _should_route_to_responses_api(
return True
if custom_llm_provider is None or requested_model is None or resolved_model is None:
return False
return _responses_mode_is_lost_by_prefix_strip(requested_model, resolved_model, custom_llm_provider)
return _responses_mode_is_lost_by_prefix_strip(
requested_model, resolved_model, custom_llm_provider, deployment_model_info
)
async def _with_native_safeguard_results(
@ -606,7 +613,7 @@ def anthropic_messages_handler(
if anthropic_messages_provider_config is None:
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
if kwargs.get("compaction") is None and _should_route_to_responses_api(
custom_llm_provider, original_model, model
custom_llm_provider, original_model, model, kwargs.get("model_info")
):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=max_tokens,

26
litellm/llms/anthropic/pass_through/messages/streaming_iterator.py
View file

@ -18,7 +18,11 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
from litellm.llms.anthropic.pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE
from litellm.llms.anthropic.pass_through.messages.utils import (
INCOMPLETE_STREAM_ERROR_MESSAGE,
MESSAGE_DELTA_OPTIONAL_KEYS,
anthropic_message_delta,
)
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
from litellm.proxy.pass_through_endpoints.success_handler import (
@ -280,30 +284,32 @@ def anthropic_messages_response_as_sse_events(response: AnthropicMessagesRespons
content_events: Final = (
event for index, block in enumerate(content_blocks) for event in _anthropic_content_block_events(index, block)
)
# A real message_start always carries a null stop_reason/stop_sequence and
# a zero output_tokens - those are only known once generation finishes, so
# copying the completed response's final values here would let a client
# treat the message as already finished, or double-count output tokens.
# A real message_start always carries a null stop_reason/stop_sequence/stop_details,
# a zero output_tokens and no safeguard_results - those are only known once
# generation finishes, so copying the completed response's final values here
# would let a client treat the message as already finished, or double-count
# output tokens.
message_start_usage: Final = {
**(response.get("usage") or {}),
"output_tokens": 0,
}
message_start_fields: Final = {
key: value for key, value in response.items() if key not in MESSAGE_DELTA_OPTIONAL_KEYS
}
message_start_payload: Final = {
"type": "message_start",
"message": {
**response,
**message_start_fields,
"content": [],
"stop_reason": None,
"stop_sequence": None,
"stop_details": None,
"usage": message_start_usage,
},
}
message_delta_payload: Final = {
"type": "message_delta",
"delta": {
"stop_reason": response.get("stop_reason"),
"stop_sequence": response.get("stop_sequence"),
},
"delta": anthropic_message_delta(response),
"usage": response.get("usage") or {},
}
return (

11
litellm/llms/anthropic/pass_through/messages/transformation.py
View file

@ -4,6 +4,7 @@ from typing import ClassVar, Final
import httpx
from litellm._logging import verbose_logger
from litellm.anthropic_beta_headers_manager import is_beta_header_supported_by_bundled_config
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -708,6 +709,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params: dict,
custom_llm_provider: str = "anthropic",
messages: Sequence[object] = (),
beta_map_provider: str | None = None,
) -> dict:
"""
Auto-inject anthropic-beta headers based on features used.
@ -724,8 +726,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params: Optional parameters including tools, context_management, output_format, speed
custom_llm_provider: Provider name for looking up correct tool search header
messages: Request messages, scanned for per-message output_config
beta_map_provider: Provider whose beta map column gates safeguards, defaults to custom_llm_provider
"""
beta_values: Final[set] = set()
beta_values: Final[set[str]] = set()
existing_beta: Final = tuple(
piece.strip()
@ -775,6 +778,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if optional_params.get("speed") == "fast":
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value)
if optional_params.get("safeguards") is not None and is_beta_header_supported_by_bundled_config(
ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value,
beta_map_provider or custom_llm_provider,
):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)

91
litellm/llms/anthropic/pass_through/messages/utils.py
View file

@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Final, cast, get_type_hints
from pydantic import JsonValue
from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.core_helpers import as_str_mapping
@ -49,6 +49,95 @@ def refusal_stop_details(explanation: str | None) -> AnthropicStopDetails:
return AnthropicStopDetails(type="refusal", category=None, explanation=explanation)
MESSAGE_DELTA_OPTIONAL_KEYS: Final = ("stop_details", "safeguard_results")
def anthropic_message_delta(
response: Mapping[str, object], optional_fields: Mapping[str, object] | None = None
) -> Mapping[str, object]:
"""
The ``delta`` of the ``message_delta`` event synthesized from a complete
Anthropic Messages response. A real stream carries ``stop_details`` and
``safeguard_results`` there, so a client that asked to stream reads them
from this event and nowhere else. ``optional_fields`` replaces the
response's own when it was rebuilt from a stream that carried them.
"""
optional_source: Final = response if optional_fields is None else optional_fields
present_optional: Final = {
key: value for key in MESSAGE_DELTA_OPTIONAL_KEYS if (value := optional_source.get(key)) is not None
}
return {
"stop_reason": response.get("stop_reason"),
"stop_sequence": response.get("stop_sequence"),
**present_optional,
}
_SAFEGUARD_RESULT_ITEMS: Final = TypeAdapter(tuple[object, ...])
def redacted_stop_details(raw: object, keep_explanations: bool) -> object:
"""``stop_details`` as a rewritten stream may deliver it: its free-text ``explanation`` can quote
the text a guardrail rewrote, so it is nulled unless the rewrite changed no text."""
stop_details: Final = as_str_mapping(raw)
if keep_explanations or stop_details is None or "explanation" not in stop_details:
return raw
return {**stop_details, "explanation": None}
def _without_explanation(raw: object) -> object:
mapping: Final = as_str_mapping(raw)
if mapping is None:
return raw
return {key: value for key, value in mapping.items() if key != "explanation"}
def _status_without_explanations(raw: object) -> object:
status: Final = as_str_mapping(_without_explanation(raw))
if status is None:
return raw
tool_uses: Final = as_str_mapping(status.get("tool_uses"))
if tool_uses is None:
return status
return {
**status,
"tool_uses": {tool_use_id: _without_explanation(verdict) for tool_use_id, verdict in tool_uses.items()},
}
def _result_without_explanations(raw: object) -> object:
result: Final = as_str_mapping(_without_explanation(raw))
if result is None:
return raw
if "status" not in result:
return result
return {**result, "status": _status_without_explanations(result["status"])}
def _kept_tool_use_verdicts(raw: object, kept_tool_use_ids: frozenset[str]) -> object:
result: Final = as_str_mapping(raw)
status: Final = as_str_mapping(result.get("status")) if result is not None else None
tool_uses: Final = as_str_mapping(status.get("tool_uses")) if status is not None else None
if result is None or status is None or tool_uses is None:
return raw
kept_tool_uses: Final = {
tool_use_id: verdict for tool_use_id, verdict in tool_uses.items() if tool_use_id in kept_tool_use_ids
}
return {**result, "status": {**status, "tool_uses": kept_tool_uses}}
def filtered_safeguard_results(raw: object, kept_tool_use_ids: frozenset[str], keep_explanations: bool) -> object:
"""``safeguard_results`` as a rewritten stream may deliver them: a ``tool_uses`` verdict survives only
for a tool call the client gets with the input the classifier judged, and the free-text ``explanation`` of a
result, its status and each verdict is dropped unless the rewrite changed no text."""
try:
results: Final = _SAFEGUARD_RESULT_ITEMS.validate_python(raw)
except ValidationError:
return raw if keep_explanations else None
kept: Final = tuple(_kept_tool_use_verdicts(result, kept_tool_use_ids) for result in results)
return kept if keep_explanations else tuple(_result_without_explanations(result) for result in kept)
def _mapping_field(container: object, key: str) -> object | None:
"""One key of a raw provider payload, or None when the payload is not a mapping."""
if not isinstance(container, Mapping):

2
litellm/llms/anthropic/pass_through/responses_adapters/handler.py
View file

@ -16,6 +16,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
)
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
from litellm.responses.utils import auto_reasoning_summary
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
AllAnthropicMessageValues,
@ -188,6 +189,7 @@ def _build_responses_kwargs(
include_encrypted_reasoning=_provider_returns_encrypted_reasoning(
model, forwarded_kwargs.get("custom_llm_provider")
),
auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")),
)
# Normalize reasoning effort based on model capabilities

8
litellm/llms/anthropic/pass_through/responses_adapters/transformation.py
View file

@ -544,6 +544,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
def translate_thinking_to_reasoning(
thinking: dict[str, object],
output_config: dict[str, object] | None = None,
auto_summary_level: str = "detailed",
) -> dict[str, object] | None:
"""
Convert Anthropic thinking param to Responses API reasoning param.
@ -576,7 +577,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if summary:
result["summary"] = summary
elif auto_summary:
result["summary"] = "detailed"
result["summary"] = auto_summary_level
return result
@classmethod
@ -591,6 +592,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
self,
anthropic_request: AnthropicMessagesRequest,
include_encrypted_reasoning: bool = True,
auto_summary_level: str = "detailed",
) -> dict[str, Any]:
"""
Translate a full Anthropic /v1/messages request dict to
@ -599,7 +601,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
``include_encrypted_reasoning`` asks the provider for ``reasoning.encrypted_content``
on every call, so a reasoning model's items can be replayed intact next turn even
when the client sent no ``thinking`` block; pass False for a provider whose
Responses API rejects ``include``.
Responses API rejects ``include``. ``auto_summary_level`` is the ``reasoning.summary`` the
opt-in auto summary injects when the client sent none.
"""
model: Final[str] = anthropic_request["model"]
messages_list: Final = cast(
@ -670,6 +673,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
reasoning: Final = self.translate_thinking_to_reasoning(
thinking,
output_config=cast(dict[str, Any] | None, output_config),
auto_summary_level=auto_summary_level,
)
if reasoning:
responses_kwargs["reasoning"] = reasoning

25
litellm/llms/anthropic/pass_through/stream_assembly.py
View file

@ -1,16 +1,18 @@
import codecs
import json
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import as_str_mapping, map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.chat.handler import (
ModelResponseIterator as AnthropicModelResponseIterator,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.pass_through.messages.utils import MESSAGE_DELTA_OPTIONAL_KEYS
from litellm.types.utils import (
Choices,
Message,
@ -447,6 +449,27 @@ def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None:
)
def anthropic_message_delta_fields(all_chunks: Sequence[object]) -> Mapping[str, object]:
"""The ``stop_details`` and ``safeguard_results`` of the stream's ``message_delta``.
The assembled OpenAI shape has no slot for them, and a client that asked to stream reads them
from that event and nowhere else (a Claude Code auto-mode session takes its classifier verdict
from ``safeguard_results``), so a caller re-emitting a rewritten response merges them back in.
"""
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return MappingProxyType({})
message_delta: Final = next(
(event for event in reversed(_parsed_sse_events(sse_stream)) if event.get("type") == "message_delta"), None
)
delta: Final = as_str_mapping(message_delta.get("delta")) if message_delta is not None else None
if delta is None:
return MappingProxyType({})
return MappingProxyType(
{key: value for key in MESSAGE_DELTA_OPTIONAL_KEYS if (value := delta.get(key)) is not None}
)
def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
"""Whether raw SSE frames are Anthropic Messages events.

21
litellm/llms/base_llm/anthropic_messages/transformation.py
View file

@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -180,7 +180,7 @@ class BaseAnthropicMessagesConfig(ABC):
return 2
def should_retry_anthropic_messages_on_http_error(
self, e: httpx.HTTPStatusError, litellm_params: dict, request_body: object = None
self, e: httpx.HTTPStatusError, litellm_params: Mapping[str, object], request_body: object = None
) -> bool:
"""
When True, async_anthropic_messages_handler will transform the request body
@ -202,7 +202,9 @@ class BaseAnthropicMessagesConfig(ABC):
) and is_anthropic_thinking_binding_mismatch_error(error_text)
return not caller_asked_for_this_error and is_anthropic_invalid_thinking_block_error(error_text)
def transform_anthropic_messages_request_on_http_error(self, e: httpx.HTTPStatusError, request_data: dict) -> dict:
def transform_anthropic_messages_request_on_http_error(
self, e: httpx.HTTPStatusError, request_data: dict[str, object]
) -> dict[str, object]:
"""
Mutates request_data in place when retrying after a recoverable HTTP error.
"""
@ -214,3 +216,16 @@ class BaseAnthropicMessagesConfig(ABC):
if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
return request_data
def native_output_fallback_litellm_params(
self,
model: str,
anthropic_messages_optional_request_params: Mapping[str, object],
litellm_params: GenericLiteLLMParams,
) -> GenericLiteLLMParams | None:
return None
def should_retry_anthropic_messages_without_native_output(
self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str
) -> bool:
return False

8
litellm/llms/base_llm/responses/transformation.py
View file

@ -1,7 +1,7 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
import httpx
@ -26,6 +26,8 @@ else:
LiteLLMLoggingObj = Any
BaseLLMException = Any
ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
class BaseResponsesAPIConfig(ABC):
def __init__(self):
@ -91,6 +93,10 @@ class BaseResponsesAPIConfig(ABC):
def get_supported_openai_params(self, model: str) -> list:
pass
def auto_reasoning_summary(self) -> ReasoningSummaryLevel:
"""The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
return "detailed"
@abstractmethod
def map_openai_params(
self,

1
litellm/llms/baseten/chat.py
View file

@ -54,6 +54,7 @@ class BasetenConfig(OpenAIGPTConfig):
return [
"max_tokens",
"max_completion_tokens",
"reasoning_effort",
"response_format",
"seed",
"stop",

24
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -42,14 +42,22 @@ def make_sync_call(
if client is None:
client = get_httpx_client() # Create a new client if none provided
response: Final = client.post(
api_base,
headers=headers,
data=data,
stream=not fake_stream,
logging_obj=logging_obj,
timeout=timeout,
)
try:
response: Final = client.post(
api_base,
headers=headers,
data=data,
stream=not fake_stream,
logging_obj=logging_obj,
timeout=timeout,
)
except httpx.HTTPStatusError as err:
raise BedrockError(
status_code=err.response.status_code,
message=error_response_text(err.response),
headers=err.response.headers,
response=err.response,
) from err
if response.status_code != 200:
raise BedrockError(

119
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -12,7 +12,7 @@ from itertools import chain, takewhile
from typing import TYPE_CHECKING, Final, Literal, cast, overload
import httpx
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
@ -62,12 +62,16 @@ from litellm.llms.bedrock.common_utils import (
bedrock_cleaned_output_schema,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_native_output_is_opt_in,
bedrock_supports_tool_search,
deployment_model_info,
requests_output_config_format,
)
from litellm.llms.bedrock.request_metadata import (
bedrock_request_metadata_headers,
bedrock_request_metadata_is_owned,
bedrock_request_metadata_is_rewritten,
bedrock_request_metadata_violation,
caller_request_metadata_violations_are_dropped,
merge_bedrock_invoke_headers,
resolve_bedrock_request_metadata,
)
@ -162,6 +166,18 @@ def _tools_the_model_accepts(
return [tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) for tool in tools]
_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
def _has_native_text_format(output_config: object) -> bool:
if not isinstance(output_config, Mapping):
return False
try:
return _OUTPUT_CONFIG_BLOCK.validate_python(output_config).get("textFormat") is not None
except ValidationError:
return False
class AmazonConverseConfig(BaseConfig):
"""
Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
@ -426,77 +442,15 @@ class AmazonConverseConfig(BaseConfig):
and v is not None
}
def _validate_request_metadata(self, metadata: dict) -> None:
"""
Validate requestMetadata according to AWS Bedrock Converse API constraints.
Constraints:
- Maximum of 16 items
- Keys: 1-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{1,256}
- Values: 0-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{0,256}
"""
import re
if not isinstance(metadata, dict):
def _validate_request_metadata(self, metadata: object) -> None:
violation: Final = bedrock_request_metadata_violation(metadata)
if violation is not None:
raise litellm.exceptions.BadRequestError(
message="requestMetadata must be a dictionary",
message=violation.message,
model="bedrock",
llm_provider="bedrock",
)
if len(metadata) > 16:
raise litellm.exceptions.BadRequestError(
message="requestMetadata can contain a maximum of 16 items",
model="bedrock",
llm_provider="bedrock",
)
key_pattern: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
value_pattern: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
for key, value in metadata.items():
if not isinstance(key, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata keys must be strings",
model="bedrock",
llm_provider="bedrock",
)
if not isinstance(value, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata values must be strings",
model="bedrock",
llm_provider="bedrock",
)
if len(key) == 0 or len(key) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata key length must be 1-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if len(value) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata value length must be 0-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if not key_pattern.match(key):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
if not value_pattern.match(value):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
@staticmethod
def _is_openai_gpt_reasoning_model(model: str) -> bool:
return re.search(r"openai\.gpt-\d", model) is not None
@ -1316,7 +1270,7 @@ class AmazonConverseConfig(BaseConfig):
elif param == "context_management" and isinstance(value, (dict, list)):
self._map_context_management_param(value, optional_params)
if param == "requestMetadata":
self._map_request_metadata_param(value, optional_params)
self._map_request_metadata_param(value, optional_params, drop_params)
if param == "service_tier" and isinstance(value, str):
self._map_service_tier_param(value, optional_params)
@ -1349,9 +1303,10 @@ class AmazonConverseConfig(BaseConfig):
return optional_params
def _map_request_metadata_param(self, value: object, optional_params: dict) -> None:
def _map_request_metadata_param(self, value: object, optional_params: dict, drop_params: bool) -> None:
if value is not None and isinstance(value, dict):
self._validate_request_metadata(value)
if not caller_request_metadata_violations_are_dropped(drop_params):
self._validate_request_metadata(value)
optional_params["requestMetadata"] = value
def _map_context_management_param(self, value: dict | list, optional_params: dict) -> None:
@ -1464,6 +1419,22 @@ class AmazonConverseConfig(BaseConfig):
optional_params["json_mode"] = True
return optional_params
@staticmethod
def sends_native_text_format(optional_params: Mapping[str, object]) -> bool:
return _has_native_text_format(optional_params.get("outputConfig"))
def uses_opt_in_native_output(
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> bool:
native_output_requested: Final = (
optional_params.get("json_mode") is True and self.sends_native_text_format(optional_params)
) or requests_output_config_format(optional_params)
return native_output_requested and bedrock_native_output_is_opt_in(
bedrock_capability_model(model, litellm_params.get("base_model")),
self.custom_llm_provider or "bedrock_converse",
deployment_model_info(litellm_params),
)
def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict):
"""
Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget.
@ -1818,7 +1789,7 @@ class AmazonConverseConfig(BaseConfig):
# Extract requestMetadata before processing other parameters
request_metadata: Final = inference_params.pop("requestMetadata", None)
if request_metadata is not None:
if request_metadata is not None and not caller_request_metadata_violations_are_dropped(drop_params):
self._validate_request_metadata(request_metadata)
output_config: OutputConfigBlock | None = inference_params.pop("outputConfig", None)
@ -2253,7 +2224,7 @@ class AmazonConverseConfig(BaseConfig):
request_metadata: Final = resolve_bedrock_request_metadata(
litellm_params=litellm_params, caller_metadata=_data.get("requestMetadata")
)
if bedrock_request_metadata_is_owned():
if bedrock_request_metadata_is_rewritten(litellm_params.get("drop_params") is True):
_data.pop("requestMetadata", None)
if request_metadata is not None:
_data["requestMetadata"] = request_metadata
@ -2333,7 +2304,7 @@ class AmazonConverseConfig(BaseConfig):
request_metadata: Final = resolve_bedrock_request_metadata(
litellm_params=litellm_params, caller_metadata=_data.get("requestMetadata")
)
if bedrock_request_metadata_is_owned():
if bedrock_request_metadata_is_rewritten(litellm_params.get("drop_params") is True):
_data.pop("requestMetadata", None)
if request_metadata is not None:
_data["requestMetadata"] = request_metadata

11
litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py
View file

@ -95,7 +95,12 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
model_info: Mapping[str, object] | None = None,
) -> dict:
return AmazonConverseConfig.map_openai_params(
self, non_default_params, optional_params, model, drop_params, model_info
self,
{key: value for key, value in non_default_params.items() if key != "requestMetadata"},
optional_params,
model,
drop_params,
model_info,
)
def transform_request(
@ -110,7 +115,9 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
self,
model=model,
messages=messages,
optional_params=_without_tool_config_injection_points(optional_params),
optional_params=_without_tool_config_injection_points(
{key: value for key, value in optional_params.items() if key != "requestMetadata"}
),
litellm_params=litellm_params,
headers=headers,
)

6
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -31,6 +31,7 @@ from litellm.llms.bedrock.common_utils import (
normalize_bedrock_opus_output_config_effort,
normalize_custom_field_on_tools,
normalize_tool_input_schema_types_for_bedrock_invoke,
requests_opt_in_native_output,
strip_unsupported_bedrock_invoke_output_config_keys,
tools_without_eager_input_streaming,
)
@ -102,6 +103,11 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
def uses_opt_in_native_output(
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> bool:
return requests_opt_in_native_output(model, optional_params, deployment_model_info(litellm_params))
def map_openai_params(
self,
non_default_params: dict,

62
litellm/llms/bedrock/chat/native_output_fallback.py Normal file
View file

@ -0,0 +1,62 @@
from collections.abc import Callable, Coroutine
from typing import Final, TypeAlias
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.common_utils import (
NativeOutputFallback,
is_bedrock_native_output_rejection,
warn_bedrock_native_output_rejected,
)
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
def _fallback_for(
error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
) -> NativeOutputFallback | None:
if not is_bedrock_native_output_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:
warn_bedrock_native_output_rejected(model, error.message)
return retry
async def _await_with_native_output_fallback(
first_attempt: Coroutine[object, object, NativeOutputResponse],
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory,
model: str,
) -> NativeOutputResponse:
try:
return await first_attempt
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
if retry is None:
raise
second_attempt: Final = attempt(retry)
return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt
def call_with_native_output_fallback(
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory | None,
model: str,
) -> NativeOutputAttemptResult:
if fallback is None:
return attempt(None)
try:
first_attempt: Final = attempt(None)
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
if retry is None:
raise
return attempt(retry)
if isinstance(first_attempt, Coroutine):
return _await_with_native_output_fallback(first_attempt, attempt, fallback, model)
return first_attempt

81
litellm/llms/bedrock/common_utils.py
View file

@ -11,6 +11,7 @@ import os
import re
import warnings
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
@ -184,7 +185,7 @@ def _get_local_model_cost_map() -> dict:
return GetModelCostMap.load_local_model_cost_map()
def pop_bedrock_invoke_output_config_format(request_body: dict) -> dict | None:
def pop_bedrock_invoke_output_config_format(request_body: dict[str, object]) -> dict[str, object] | None:
"""
Remove and return Anthropic's nested ``output_config.format`` field.
@ -205,8 +206,8 @@ def pop_bedrock_invoke_output_config_format(request_body: dict) -> dict | None:
def convert_bedrock_invoke_output_format_to_inline_schema(
output_format: dict,
request_body: dict,
output_format: Mapping[str, object],
request_body: dict[str, object],
) -> None:
"""
Embed an Anthropic structured-output schema into the last user message.
@ -292,6 +293,77 @@ def bedrock_chat_uses_native_structured_output(
)
def bedrock_native_output_is_opt_in(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return not _bedrock_model_supports(
model, "supports_native_structured_output", custom_llm_provider
) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return {
**litellm_params,
"model_info": {**deployment_model_info(litellm_params), "supports_bedrock_native_structured_output": False},
}
_BEDROCK_NATIVE_OUTPUT_UNSUPPORTED: Final = "output_config.format: Extra inputs are not permitted"
_BEDROCK_NATIVE_OUTPUT_REJECTIONS: Final = (
_BEDROCK_NATIVE_OUTPUT_UNSUPPORTED,
"compilation exceeded limits: Schema is too complex",
"output_config.format.schema:",
)
def is_bedrock_native_output_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and any(rejection in error_text for rejection in _BEDROCK_NATIVE_OUTPUT_REJECTIONS)
def warn_bedrock_native_output_rejected(model: str, error_text: str) -> None:
opt_out_advice: Final = (
" Set `supports_bedrock_native_structured_output: false` in this deployment's model_info to skip the "
"rejected call."
if _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED in error_text
else ""
)
verbose_logger.warning(
"Bedrock rejected native structured output for model=%s, so litellm sent the request again without it.%s",
model,
opt_out_advice,
)
@dataclass(frozen=True, slots=True)
class NativeOutputFallback:
optional_params: Mapping[str, object]
litellm_params: Mapping[str, object]
def requests_output_config_format(optional_params: Mapping[str, object]) -> bool:
match optional_params:
case {"output_config": {"format": object()}}:
return True
case _:
return False
def requests_opt_in_native_output(
model: str, anthropic_optional_params: Mapping[str, object], model_info: Mapping[str, object] | None
) -> bool:
native_output_requested: Final = isinstance(
anthropic_optional_params.get("output_format"), Mapping
) or requests_output_config_format(anthropic_optional_params)
return native_output_requested and bedrock_native_output_is_opt_in(model, model_info=model_info)
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
def json_object_or_none(value: object) -> Mapping[str, object] | None:
return _JSON_OBJECT.validate_python(value) if isinstance(value, Mapping) else None
BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64
@ -358,7 +430,8 @@ def apply_bedrock_invoke_structured_output(
"""
legacy_output_format: Final = request_body.pop("output_format", None)
output_config_format: Final = pop_bedrock_invoke_output_config_format(request_body)
schema_format: Final = legacy_output_format if isinstance(legacy_output_format, dict) else output_config_format
legacy_format: Final = json_object_or_none(legacy_output_format)
schema_format: Final = legacy_format if legacy_format is not None else output_config_format
if schema_format is None:
return

2
litellm/llms/bedrock/count_tokens/handler.py
View file

@ -48,7 +48,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model)
# Get AWS region using existing LiteLLM function
aws_region_name: Final = self._get_aws_region_name(
aws_region_name: Final = self.get_aws_region_name(
optional_params=litellm_params,
model=resolved_model,
model_id=None,

502
litellm/llms/bedrock/count_tokens/kimi_k3.py Normal file
View file

@ -0,0 +1,502 @@
"""
Exact Bedrock token counts for Kimi K3, which has no CountTokens support.
Renders the Converse body litellm sends through Moonshot's Kimi K3 chat template and counts it with the
official tokenizer, downloaded once from Hugging Face at a pinned revision and checked against its SHA-256.
"""
import asyncio
import base64
import copy
import hashlib
import json
import threading
import time
from collections.abc import Callable, Iterable, Mapping, Sequence
from concurrent.futures import Future
from dataclasses import dataclass
from enum import Enum
from itertools import chain, groupby
from pathlib import Path
from typing import Annotated, Final, Literal, TypeAlias
import regex
import tiktoken
from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, model_validator
from typing_extensions import Self
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.factory import (
chat_completion_tools_for,
is_flat_responses_tool,
)
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
KIMI_K3_REPO: Final = "moonshotai/Kimi-K3"
KIMI_K3_REVISION: Final = "f831ab66814297da540d832a5235f8e904f29d06"
KIMI_K3_RANKS_FILE: Final = "tiktoken.model"
KIMI_K3_RANKS_SHA256: Final = "b6c497a7469b33ced9c38afb1ad6e47f03f5e5dc05f15930799210ec050c5103"
_PATTERN: Final = "|".join(
(
r"[\p{Han}]+",
r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
r"\p{N}{1,3}",
r" ?[^\s\p{L}\p{N}]+[\r\n]*",
r"\s*[\r\n]+",
r"\s+(?!\S)",
r"\s+",
)
)
# Ends chunks only where the pattern above always splits, so their counts sum to the whole text's count while BPE
# stays linear. Only a run of over 1024 characters with no such split, like 2000 spaces, is cut mid-piece.
_CHUNK: Final = regex.compile(
r"(?s).{1,1024}(?:(?<=\S)(?= \S)|(?<=\p{N})(?=\P{N})|(?<=[^\s\p{N}])(?=\p{N})|(?<=\p{L})(?=[^\p{L}\p{M}'])"
r"|(?<=[\r\n])(?=\S)|\Z)|.{1024}\p{N}*"
)
@dataclass(frozen=True, slots=True)
class KimiK3TokenizerSource:
ranks_path: Callable[[], str]
sha256: str
def _hugging_face_ranks_path() -> str:
import huggingface_hub
return huggingface_hub.hf_hub_download(
repo_id=KIMI_K3_REPO,
filename=KIMI_K3_RANKS_FILE,
revision=KIMI_K3_REVISION,
local_files_only=litellm.disable_hf_tokenizer_download is True,
)
HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256)
def _loaded(source: KimiK3TokenizerSource) -> tiktoken.Encoding | str:
try:
data: Final = Path(source.ranks_path()).read_bytes()
except Exception as e: # noqa: BLE001 # a download or read failure of any kind falls back to the estimate
return type(e).__name__
if hashlib.sha256(data).hexdigest() != source.sha256:
return "SHA-256 mismatch"
rows: Final = (line.split() for line in data.splitlines() if line)
return tiktoken.Encoding(
name="kimi_k3",
pat_str=_PATTERN,
mergeable_ranks={base64.b64decode(token): int(rank) for token, rank in rows},
special_tokens={},
)
def load_kimi_k3_encoding(source: KimiK3TokenizerSource) -> tiktoken.Encoding | None:
loaded: Final = _loaded(source)
if isinstance(loaded, tiktoken.Encoding):
return loaded
verbose_logger.warning(
"Kimi K3 tokenizer unavailable (%s), so Kimi K3 token counts are estimated until a later count loads it. "
"Allow downloads of %s from Hugging Face repo %s at revision %s, or put it in the Hugging Face cache",
loaded,
KIMI_K3_RANKS_FILE,
KIMI_K3_REPO,
KIMI_K3_REVISION,
)
return None
def _started_load(source: KimiK3TokenizerSource) -> "Future[tiktoken.Encoding | None]":
encoding: Final = Future[tiktoken.Encoding | None]()
threading.Thread(
target=lambda: encoding.set_result(load_kimi_k3_encoding(source)), name="litellm-kimi-k3-tokenizer", daemon=True
).start()
return encoding
@dataclass(frozen=True, slots=True)
class _EncodingLoad:
encoding: "Future[tiktoken.Encoding | None]"
started_at: float
is_retry: bool
def retry_due(self, now: float, retry_after_seconds: float) -> bool:
return self.encoding.done() and self.encoding.result() is None and now - self.started_at >= retry_after_seconds
@dataclass(slots=True)
class KimiK3Encodings:
source: KimiK3TokenizerSource
retry_after_seconds: float
clock: Callable[[], float] = time.monotonic
current: _EncodingLoad | None = None
def _load(self) -> _EncodingLoad:
now: Final = self.clock()
current: Final = self.current
if current is not None and not current.retry_due(now, self.retry_after_seconds):
return current
load: Final = _EncodingLoad(encoding=_started_load(self.source), started_at=now, is_retry=current is not None)
self.current = load
return load
async def encoding(self, wait_seconds: float) -> tiktoken.Encoding | None:
load: Final = self._load()
remaining: Final = 0 if load.is_retry else load.started_at + wait_seconds - self.clock()
if load.encoding.done() or remaining <= 0:
return load.encoding.result() if load.encoding.done() else None
await asyncio.wait((asyncio.wrap_future(load.encoding),), timeout=remaining)
if load.encoding.done():
return load.encoding.result()
verbose_logger.warning(
"Kimi K3 tokenizer still loading after %s s, so Kimi K3 token counts are estimated until it loads. "
"Check that %s can be downloaded from Hugging Face repo %s at revision %s, or put it in the Hugging Face cache",
wait_seconds,
KIMI_K3_RANKS_FILE,
KIMI_K3_REPO,
KIMI_K3_REVISION,
)
return None
class _Converse(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
class _Text(_Converse):
text: str
class _CachePoint(_Converse):
cache_point: JsonValue = Field(alias="cachePoint")
class _ToolUse(_Converse):
tool_use_id: str = Field(alias="toolUseId")
name: str
arguments: Mapping[str, JsonValue] = Field(alias="input")
class _ToolUseBlock(_Converse):
tool_use: _ToolUse = Field(alias="toolUse")
class _ToolResult(_Converse):
tool_use_id: str = Field(alias="toolUseId")
content: tuple[_Text, ...]
status: str | None = None
class _ToolResultBlock(_Converse):
tool_result: _ToolResult = Field(alias="toolResult")
class _ReasoningText(_Converse):
text: str
signature: str | None = None
class _Reasoning(_Converse):
reasoning_text: _ReasoningText = Field(alias="reasoningText")
class _ReasoningBlock(_Converse):
reasoning_content: _Reasoning = Field(alias="reasoningContent")
class _UserMessage(_Converse):
role: Literal["user"]
content: tuple[_Text | _ToolResultBlock | _CachePoint, ...]
class _AssistantMessage(_Converse):
role: Literal["assistant"]
content: tuple[_Text | _ToolUseBlock | _ReasoningBlock | _CachePoint, ...]
def calls(self) -> tuple[_ToolUse, ...]:
return tuple(block.tool_use for block in self.content if isinstance(block, _ToolUseBlock))
class _InputSchema(_Converse):
definition: JsonValue = Field(alias="json")
class _ToolSpec(_Converse):
name: str
description: str | None = None
input_schema: _InputSchema = Field(alias="inputSchema")
def declared(self) -> Mapping[str, JsonValue]:
described: Final[Mapping[str, JsonValue]] = (
{} if self.description is None else {"description": self.description}
)
return {"name": self.name, **described, "parameters": self.input_schema.definition}
class _ToolSpecBlock(_Converse):
tool_spec: _ToolSpec = Field(alias="toolSpec")
class _ToolConfig(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
tools: tuple[_ToolSpecBlock | _CachePoint, ...]
_Message: TypeAlias = Annotated[_UserMessage | _AssistantMessage, Field(discriminator="role")]
def _result_ids(message: _Message) -> frozenset[str]:
if isinstance(message, _AssistantMessage):
return frozenset()
return frozenset(block.tool_result.tool_use_id for block in message.content if isinstance(block, _ToolResultBlock))
def _call_ids(message: _Message | None) -> frozenset[str]:
if not isinstance(message, _AssistantMessage):
return frozenset()
return frozenset(call.tool_use_id for call in message.calls())
class _ConverseRequest(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
messages: tuple[_Message, ...]
system: tuple[_Text | _CachePoint, ...] = ()
tool_config: _ToolConfig | None = Field(default=None, alias="toolConfig")
@model_validator(mode="after")
def _results_answer_the_previous_calls(self) -> Self:
pairs: Final = zip((None, *self.messages), self.messages, strict=False)
if not all(_result_ids(message) <= _call_ids(previous) for previous, message in pairs):
raise ValueError("a toolResult does not answer a toolUse of the previous assistant message")
return self
class _Control(Enum):
OPEN = "<|open|>"
CLOSE = "<|close|>"
SEP = "<|sep|>"
END_OF_MSG = "<|end_of_msg|>"
_Segment: TypeAlias = str | _Control
_Segments: TypeAlias = tuple[_Segment, ...]
def _open(tag: str, attrs: Iterable[tuple[str, str]] = ()) -> _Segments:
escaped: Final = (
(f" {key}", '="', value.replace("&", "&amp;").replace('"', "&quot;"), '"') for key, value in attrs
)
return (_Control.OPEN, tag, *chain.from_iterable(escaped), _Control.SEP)
def _close(tag: str) -> _Segments:
return (_Control.CLOSE, tag, _Control.SEP)
def _message(attrs: Iterable[tuple[str, str]], body: Iterable[_Segment]) -> _Segments:
return (*_open("message", attrs), *body, *_close("message"), _Control.END_OF_MSG)
def _tool_declare(specs: Sequence[_ToolSpec]) -> _Segments:
tools: Final = json.dumps(
[{"type": "function", "function": spec.declared()} for spec in specs],
ensure_ascii=False,
separators=(",", ":"),
sort_keys=True,
)
return _message(
(("role", "system"), ("type", "tool-declare")),
(f"# Tools\nHere are the available tools, described in JSONSchema.\n\n```json\n{tools}\n```",),
)
def _text(blocks: Iterable[_Converse]) -> str:
return "".join(block.text for block in blocks if isinstance(block, _Text))
def _json_type(value: JsonValue) -> str:
match value:
case bool():
return "boolean"
case None:
return "null"
case int() | float():
return "number"
case str():
return "string"
case dict():
return "object"
case _:
return "array"
def _argument(key: str, value: JsonValue) -> _Segments:
text: Final = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
return (*_open("argument", (("key", key), ("type", _json_type(value)))), text, *_close("argument"))
def _call(index: int, call: _ToolUse) -> _Segments:
arguments: Final = chain.from_iterable(_argument(key, value) for key, value in call.arguments.items())
return (*_open("call", (("tool", call.name), ("index", str(index)))), *arguments, *_close("call"))
def _tool_calls(calls: Sequence[_ToolUse]) -> _Segments:
if not calls:
return ()
rendered: Final = chain.from_iterable(_call(index, call) for index, call in enumerate(calls, start=1))
return (*_open("tools"), *rendered, *_close("tools"))
def _assistant_message(message: _AssistantMessage) -> _Segments:
reasoning: Final = "".join(
block.reasoning_content.reasoning_text.text for block in message.content if isinstance(block, _ReasoningBlock)
)
return _message(
(("role", "assistant"),),
(
*_open("think"),
reasoning,
*_close("think"),
*_open("response"),
_text(message.content),
*_close("response"),
*_tool_calls(message.calls()),
),
)
def _tool_messages(results: Sequence[_ToolResult], previous: _AssistantMessage) -> _Segments:
names: Final = {call.tool_use_id: call.name for call in previous.calls()}
return tuple(
chain.from_iterable(
_message(
(("role", "tool"), ("tool", names[result.tool_use_id]), ("index", str(index))),
(_text(result.content),),
)
for index, result in enumerate(results, start=1)
)
)
def _user_run(blocks: Sequence[_Text | _ToolResultBlock], previous: _Message | None) -> _Segments:
results: Final = tuple(block.tool_result for block in blocks if isinstance(block, _ToolResultBlock))
if results and isinstance(previous, _AssistantMessage):
return _tool_messages(results, previous)
return _message((("role", "user"),), (_text(blocks),))
def _user_messages(message: _UserMessage, previous: _Message | None) -> _Segments:
blocks: Final = tuple(block for block in message.content if not isinstance(block, _CachePoint))
runs: Final = groupby(blocks, key=lambda block: isinstance(block, _ToolResultBlock))
return tuple(chain.from_iterable(_user_run(tuple(run), previous) for _, run in runs))
def _turn(previous: _Message | None, message: _Message) -> _Segments:
if isinstance(message, _AssistantMessage):
return _assistant_message(message)
return _user_messages(message, previous)
def _rendered(request: _ConverseRequest) -> _Segments:
specs: Final = (
()
if request.tool_config is None
else tuple(block.tool_spec for block in request.tool_config.tools if isinstance(block, _ToolSpecBlock))
)
system: Final = _text(request.system)
turns: Final = chain.from_iterable(
_turn(previous, message) for previous, message in zip((None, *request.messages), request.messages, strict=False)
)
return (
*(_tool_declare(specs) if specs else ()),
*(_message((("role", "system"),), (system,)) if system else ()),
*turns,
*_open("message", (("role", "assistant"),)),
*_open("think"),
)
def _text_tokens(encoding: tiktoken.Encoding, text: str) -> int:
spans: Final = (match.span() for match in _CHUNK.finditer(text))
return sum(len(encoding.encode_ordinary(text[start:end])) for start, end in spans)
class _TextPart(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
type: Literal["text"]
class _TextOnlyMessage(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
content: str | tuple[_TextPart, ...] | None = None
def _text_only(message: AllMessageValues) -> bool:
try:
_TextOnlyMessage.model_validate(message)
except ValidationError:
return False
return True
def _as_converse_input(tool: ChatCompletionToolParam) -> tuple[Mapping[str, object], ...]:
return chat_completion_tools_for(tool) if is_flat_responses_tool(tool) else (tool,)
def _converse_request(
model: str, messages: Sequence[AllMessageValues], tools: Sequence[ChatCompletionToolParam] | None
) -> _ConverseRequest | None:
if not all(_text_only(message) for message in messages):
return None
try:
chat_tools: Final = tuple(
chat_tool for tool in tools or () for chat_tool in _as_converse_input(tool)
) # comprehension-ok: flattens each tool's translation
return _ConverseRequest.model_validate(
litellm.AmazonConverseConfig().transform_request(
model=model,
messages=copy.deepcopy(list(messages)),
optional_params={"tools": copy.deepcopy(list(chat_tools))} if chat_tools else {},
litellm_params={},
headers={},
)
)
except Exception: # noqa: BLE001 # a body litellm cannot send to Bedrock, or cannot render, falls back to the estimate
return None
def kimi_k3_tokens(
encoding: tiktoken.Encoding,
model: str,
messages: Sequence[AllMessageValues],
tools: Sequence[ChatCompletionToolParam] | None,
) -> int | None:
request: Final = _converse_request(model, messages, tools)
if request is None:
return None
return sum(
1 if isinstance(segment, _Control) else _text_tokens(encoding, segment) for segment in _rendered(request)
)
def kimi_k3_anthropic_tokens(encoding: tiktoken.Encoding, model: str, fields: Mapping[str, object]) -> int | None:
body: Final = {**fields, "model": model, "max_tokens": 1}
try:
translated, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
copy.deepcopy(body), # pyright: ignore[reportArgumentType] # raw /v1/messages fields
custom_llm_provider="bedrock",
)
except Exception: # noqa: BLE001 # a request the adapter cannot translate falls back to the estimate
return None
match translated:
case {"tools": tools}:
return kimi_k3_tokens(encoding, model, translated["messages"], tools)
case _:
return kimi_k3_tokens(encoding, model, translated["messages"], None)
def kimi_k3_template_tokens(message_count: int, has_tools: bool) -> int:
return 14 + 13 * max(message_count - 1, 0) + (40 if has_tools else 0)

2
litellm/llms/bedrock/count_tokens/mantle_handler.py
View file

@ -52,7 +52,7 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
) -> dict[str, JsonValue]:
try:
request: Final = _COUNT_REQUEST.validate_python(request_data)
aws_region_name: Final = self._get_aws_region_name(
aws_region_name: Final = self.get_aws_region_name(
optional_params=litellm_params, model=resolved_model, model_id=None
)
endpoint_url: Final = self.get_mantle_count_tokens_endpoint(aws_region_name)

8
litellm/llms/bedrock/files/transformation.py
View file

@ -66,8 +66,10 @@ from litellm.utils import get_llm_provider, get_optional_params
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import (
BedrockError,
deployment_model_info,
merge_bedrock_aws_request_params,
resolve_s3_encryption_key_id,
without_bedrock_native_output_opt_in,
)
S3_SIGNED_REQUEST_HEADERS_PARAM: Final = "_s3_signed_request_headers"
@ -989,10 +991,12 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
)
config: Final = AmazonAnthropicClaudeConfig()
litellm_params: Final = without_bedrock_native_output_opt_in({})
mapped_params = get_optional_params(
model=_invoke_route_model(model),
custom_llm_provider="bedrock",
messages=messages,
model_info=deployment_model_info(litellm_params),
**MappingProxyType(
{
k: v
@ -1007,7 +1011,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
optional_params={
k: v for k, v in mapped_params.items() if k not in _MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS
},
litellm_params={},
litellm_params=dict(litellm_params),
headers={},
)
@ -1021,7 +1025,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
converse_config: Final = AmazonConverseConfig()
mapped_params = converse_config.map_openai_params(
non_default_params=optional_params,
non_default_params={k: v for k, v in optional_params.items() if k != "requestMetadata"},
optional_params={},
model=model,
drop_params=False,

27
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -40,12 +40,16 @@ from litellm.llms.bedrock.common_utils import (
bedrock_supports_tool_search,
ensure_bedrock_anthropic_messages_tool_names,
get_anthropic_beta_from_headers,
is_bedrock_native_output_rejection,
normalize_bedrock_opus_output_config_effort,
normalize_custom_field_on_tools,
normalize_tool_input_schema_types_for_bedrock_invoke,
requests_opt_in_native_output,
requests_output_config_format,
strip_unsupported_bedrock_invoke_output_config_keys,
supports_bedrock_prompt_cache_ttl,
tools_without_eager_input_streaming,
warn_bedrock_native_output_rejected,
)
from litellm.llms.bedrock.request_metadata import (
bedrock_request_metadata_headers,
@ -111,6 +115,29 @@ class AmazonAnthropicClaudeMessagesConfig(
BaseAnthropicMessagesConfig.__init__(self, **kwargs)
AmazonInvokeConfig.__init__(self, **kwargs)
def native_output_fallback_litellm_params(
self,
model: str,
anthropic_messages_optional_request_params: Mapping[str, object],
litellm_params: GenericLiteLLMParams,
) -> GenericLiteLLMParams | None:
model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info)
if not requests_opt_in_native_output(model, anthropic_messages_optional_request_params, model_info):
return None
return litellm_params.model_copy(
update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}}
)
def should_retry_anthropic_messages_without_native_output(
self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str
) -> bool:
if not requests_output_config_format(request_body) or not is_bedrock_native_output_rejection(
e.response.status_code, e.response.text
):
return False
warn_bedrock_native_output_rejected(model, e.response.text)
return True
def validate_anthropic_messages_environment(
self,
headers: dict,

106
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -10,6 +10,7 @@ from httpx import Response
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import AWS_ERROR_TYPE_HEADER
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
@ -17,6 +18,16 @@ from litellm.types.utils import ModelResponseStream
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError, BedrockEventStreamDecoderBase, BedrockModelInfo
from ..request_metadata import (
BEDROCK_REQUEST_METADATA_HEADER,
BedrockRequestMetadataViolation,
bedrock_request_metadata_headers,
bedrock_request_metadata_is_owned,
bedrock_request_metadata_violation,
caller_request_metadata_violations_are_dropped,
merge_bedrock_invoke_headers,
resolve_bedrock_request_metadata,
)
if TYPE_CHECKING:
from botocore.eventstream import EventStreamMessage
@ -28,10 +39,87 @@ if TYPE_CHECKING:
BEDROCK_STREAMING_ACTIONS: Final = frozenset({"invoke-with-response-stream", "converse-stream"})
_BEDROCK_CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"})
_BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stream"})
def _bedrock_endpoint_action(endpoint: str) -> str:
return endpoint.partition("?")[0].rstrip("/").rsplit("/", 1)[-1]
def is_bedrock_streaming_endpoint(endpoint: str) -> bool:
return endpoint.partition("?")[0].rstrip("/").rsplit("/", 1)[-1] in BEDROCK_STREAMING_ACTIONS
return _bedrock_endpoint_action(endpoint) in BEDROCK_STREAMING_ACTIONS
def _caller_request_metadata_violation(
action: str, body: Mapping[str, object]
) -> BedrockRequestMetadataViolation | None:
if action in _BEDROCK_INVOKE_ACTIONS and "requestMetadata" in body and body["requestMetadata"] is None:
return BedrockRequestMetadataViolation("type", "requestMetadata must be a dictionary")
return bedrock_request_metadata_violation(body.get("requestMetadata"))
def _aws_error_type(action: str, violation: BedrockRequestMetadataViolation) -> str:
if action in _BEDROCK_CONVERSE_ACTIONS and violation.kind == "type":
return "SerializationException"
return "ValidationException"
def _caller_request_metadata(action: str, body: Mapping[str, object], litellm_params: Mapping[str, object]) -> object:
violation: Final = (
None
if caller_request_metadata_violations_are_dropped(litellm_params.get("drop_params") is True)
else _caller_request_metadata_violation(action, body)
)
if violation is not None:
raise BedrockError(
status_code=400,
message=violation.message,
headers={AWS_ERROR_TYPE_HEADER: _aws_error_type(action, violation)},
)
return body.get("requestMetadata")
def _with_body_request_metadata(
action: str, body: Mapping[str, object], litellm_params: Mapping[str, object]
) -> Mapping[str, object]:
resolved: Final = resolve_bedrock_request_metadata(
litellm_params, caller_metadata=_caller_request_metadata(action, body, litellm_params)
)
without_caller: Final = {key: value for key, value in body.items() if key != "requestMetadata"}
return without_caller if resolved is None else {**without_caller, "requestMetadata": resolved}
def _without_owned_request_metadata_header(headers: Mapping[str, str]) -> Mapping[str, str]:
if not bedrock_request_metadata_is_owned():
return headers
return {name: value for name, value in headers.items() if name.lower() != BEDROCK_REQUEST_METADATA_HEADER.lower()}
def _with_request_metadata(
action: str | None,
headers: Mapping[str, str],
body: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[Mapping[str, str], Mapping[str, object]]:
if action in _BEDROCK_CONVERSE_ACTIONS:
return headers, _with_body_request_metadata(action, body, litellm_params)
if action in _BEDROCK_INVOKE_ACTIONS and "requestMetadata" in body:
return _without_owned_request_metadata_header(headers), _with_body_request_metadata(
action, body, litellm_params
)
if action in _BEDROCK_INVOKE_ACTIONS:
return _with_invoke_request_metadata(headers, body, litellm_params)
return headers, body
def _with_invoke_request_metadata(
headers: Mapping[str, str],
body: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[Mapping[str, str], Mapping[str, object]]:
owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
return merge_bedrock_invoke_headers(dict(headers), (), metadata_headers, owned_names), body
_TEXT_ONLY_DELTA_FIELDS: Final = frozenset({"content", "role"})
@ -284,18 +372,24 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
def sign_request(
self,
headers: dict,
litellm_params: dict,
request_data: dict | None,
headers: dict[str, str],
litellm_params: dict[str, object],
request_data: dict[str, object] | None,
api_base: str,
model: str | None = None,
) -> tuple[dict, bytes | None]:
optional_params: Final = litellm_params.copy()
signed_headers, signed_body = _with_request_metadata(
action=_bedrock_endpoint_action(api_base) if bedrock_request_metadata_is_owned() else None,
headers=headers,
body=request_data or {},
litellm_params=litellm_params,
)
return self._sign_request(
service_name="bedrock",
headers=headers,
headers=dict(signed_headers),
optional_params=optional_params,
request_data=request_data or {},
request_data=dict(signed_body),
api_base=api_base,
model=model,
api_key=self.get_bedrock_bearer_token(optional_params),

129
litellm/llms/bedrock/request_metadata.py
View file

@ -8,17 +8,22 @@ sets ``litellm.bedrock_request_metadata_fields`` (``litellm_settings`` on the pr
Two properties are load-bearing for that billing record and are asserted by the tests:
proxy identity is resolved first so it can never be evicted by caller-supplied pairs, and the
whole ``user_api_key_`` prefix is reserved so a caller cannot write a proxy-authoritative
looking key. Values that break Bedrock's constraints are dropped rather than sanitised or
rejected, because an operator flipping this setting on must not turn a working request into a
400 and a silently rewritten attribution key is worse than an absent one.
looking key. Identity and ``spend_logs_metadata`` values that break Bedrock's constraints are
dropped rather than sanitised or rejected, because an operator flipping this setting on must not
turn a working request into a 400 and a silently rewritten attribution key is worse than an
absent one. For the same reason an Invoke header that would pass Bedrock's 8,500-character limit
keeps only the leading pairs that fit. A caller's own malformed ``requestMetadata`` is rejected with
a 400 unless ``drop_params`` is set, in which case its bad pairs are dropped the same way, with or
without the setting.
"""
from __future__ import annotations
import json
import re
from collections.abc import Mapping
from typing import Final
from collections.abc import Mapping, Sized
from dataclasses import dataclass
from typing import Final, Literal
import litellm
@ -26,15 +31,27 @@ BEDROCK_REQUEST_METADATA_HEADER: Final = "X-Amzn-Bedrock-Request-Metadata"
BEDROCK_REQUEST_METADATA_MAX_PAIRS: Final = 16
BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX: Final = "user_api_key_"
BEDROCK_REQUEST_METADATA_CLIENT_FIELD: Final = "spend_logs_metadata"
BEDROCK_REQUEST_METADATA_HEADER_MAX_CHARS: Final = 8500
_METADATA_PARAM_NAMES: Final[tuple[str, ...]] = ("metadata", "litellm_metadata")
_KEY_PATTERN: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
_VALUE_PATTERN: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
_ALLOWED_CHARACTERS: Final = r"[a-zA-Z0-9\s:_@$#=/+,.-]"
_KEY_PATTERN: Final = re.compile(rf"{_ALLOWED_CHARACTERS}{{1,256}}", re.ASCII)
_VALUE_PATTERN: Final = re.compile(rf"{_ALLOWED_CHARACTERS}{{0,256}}", re.ASCII)
_OWNED_HEADER_NAMES: Final[frozenset[str]] = frozenset((BEDROCK_REQUEST_METADATA_HEADER.lower(),))
@dataclass(frozen=True, slots=True)
class BedrockRequestMetadataViolation:
kind: Literal["type", "constraint"]
message: str
def _is_forwardable(key: str, value: str) -> bool:
return _KEY_PATTERN.match(key) is not None and _VALUE_PATTERN.match(value) is not None
return _KEY_PATTERN.fullmatch(key) is not None and _VALUE_PATTERN.fullmatch(value) is not None
def _is_reserved(key: str) -> bool:
return key.strip().lower().startswith(BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX)
def _text_pairs(source: object) -> tuple[tuple[str, str], ...]:
@ -99,13 +116,16 @@ def _client_pairs(
candidates: Final = tuple(
(key, value)
for key, value in (*_text_pairs(caller_metadata), *spend_logs_pairs)
if not key.startswith(BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX) and _is_forwardable(key, value)
if not _is_reserved(key) and _is_forwardable(key, value)
)
return tuple(
pair
for index, pair in enumerate(candidates)
if pair[0] not in tuple(earlier for earlier, _ in candidates[:index])
)[:budget]
first_index: Final = {key: index for index, (key, _) in reversed(tuple(enumerate(candidates)))}
return tuple(pair for index, pair in enumerate(candidates) if first_index[pair[0]] == index)[:budget]
def _forwardable_caller_pairs(caller_metadata: object) -> tuple[tuple[str, str], ...]:
return tuple(pair for pair in _text_pairs(caller_metadata) if _is_forwardable(*pair))[
:BEDROCK_REQUEST_METADATA_MAX_PAIRS
]
def resolve_bedrock_request_metadata(
@ -113,17 +133,18 @@ def resolve_bedrock_request_metadata(
caller_metadata: object = None,
) -> dict[str, str] | None:
"""
Resolve the ``requestMetadata`` pairs to send to Bedrock, or ``None`` when the feature is
off or nothing survives Bedrock's constraints. The result is a plain dict because it is
written straight onto the Converse body, which Bedrock types as ``dict[str, str]``.
Resolve the ``requestMetadata`` pairs to send to Bedrock, or ``None`` when nothing survives
Bedrock's constraints. The result is a plain dict because it is written straight onto the
Converse body, which Bedrock types as ``dict[str, str]``.
``caller_metadata`` is any ``requestMetadata`` the caller passed explicitly. It has already
been validated (and rejected with a 400) by the Converse transformation, so it is only
filtered here for the reserved identity prefix and the remaining slot budget.
``caller_metadata`` is any ``requestMetadata`` the caller passed explicitly. Callers reject a
malformed one with a 400 unless ``drop_params`` is set, so here its pairs are only filtered for
Bedrock's rules, the reserved identity prefix, and the remaining slot budget. With the feature
off only Bedrock's rules apply, since no identity is written for a caller to forge.
"""
allowed_fields: Final = _allowed_fields()
if not allowed_fields:
return None
return dict(_forwardable_caller_pairs(caller_metadata)) or None
sources: Final = _metadata_sources(litellm_params)
identity: Final = _identity_pairs(sources, allowed_fields)
client: Final = _client_pairs(
@ -136,6 +157,36 @@ def resolve_bedrock_request_metadata(
return resolved or None
def _pair_violation(key: str, value: str) -> BedrockRequestMetadataViolation | None:
if _KEY_PATTERN.fullmatch(key) is None:
return BedrockRequestMetadataViolation(
"constraint", f"requestMetadata key {key!r} must be 1-256 characters of {_ALLOWED_CHARACTERS}"
)
if _VALUE_PATTERN.fullmatch(value) is None:
return BedrockRequestMetadataViolation(
"constraint", f"requestMetadata value for key {key!r} must be 0-256 characters of {_ALLOWED_CHARACTERS}"
)
return None
def bedrock_request_metadata_violation(caller_metadata: object) -> BedrockRequestMetadataViolation | None:
"""Why a caller's ``requestMetadata`` breaks Bedrock's rules, in the order Bedrock checks them: a
wrong JSON type before the pair count and character rules. ``None`` means absent or valid."""
text_pairs: Final = _text_pairs(caller_metadata)
entry_count: Final = len(caller_metadata) if isinstance(caller_metadata, Sized) else 0
if caller_metadata is None:
return None
if not isinstance(caller_metadata, Mapping):
return BedrockRequestMetadataViolation("type", "requestMetadata must be a dictionary")
if len(text_pairs) != entry_count:
return BedrockRequestMetadataViolation("type", "requestMetadata keys and values must be strings")
if len(text_pairs) > BEDROCK_REQUEST_METADATA_MAX_PAIRS:
return BedrockRequestMetadataViolation(
"constraint", f"requestMetadata can contain a maximum of {BEDROCK_REQUEST_METADATA_MAX_PAIRS} items"
)
return next((violation for key, value in text_pairs if (violation := _pair_violation(key, value))), None)
def bedrock_request_metadata_is_owned() -> bool:
"""
Whether the proxy OWNS the request-metadata field and header name for this request.
@ -149,25 +200,51 @@ def bedrock_request_metadata_is_owned() -> bool:
return bool(_allowed_fields())
def caller_request_metadata_violations_are_dropped(drop_params: bool) -> bool:
return drop_params or litellm.drop_params is True
def bedrock_request_metadata_is_rewritten(drop_params: bool) -> bool:
return bedrock_request_metadata_is_owned() or caller_request_metadata_violations_are_dropped(drop_params)
def _header_value(pairs: Mapping[str, str]) -> str:
return json.dumps(pairs, separators=(",", ":"))
def _fits_header(pairs: Mapping[str, str]) -> bool:
return len(_header_value(pairs)) <= BEDROCK_REQUEST_METADATA_HEADER_MAX_CHARS
def _longest_fitting_prefix(resolved: Mapping[str, str]) -> dict[str, str]:
pairs: Final = tuple(resolved.items())
return next(
(dict(pairs[:count]) for count in range(len(pairs), 0, -1) if _fits_header(dict(pairs[:count]))),
{},
)
def bedrock_request_metadata_headers(
litellm_params: Mapping[str, object] | None,
) -> tuple[frozenset[str], tuple[tuple[str, str], ...]]:
"""
The signed ``X-Amzn-Bedrock-Request-Metadata`` header for the Invoke paths, which have no
body field for request metadata.
The signed ``X-Amzn-Bedrock-Request-Metadata`` header for an Invoke request whose body carries
no ``requestMetadata`` of its own.
Returns the header names the proxy OWNS and, separately, the pairs to send. Ownership is
reported whenever forwarding is enabled, including when nothing resolves, because a caller
can suppress the resolver (omit the allow-listed fields, or send values that all fail
Bedrock's rules) and an owned-but-empty result must still evict the caller's header rather
than fall back to it.
than fall back to it. Only the leading pairs that fit Bedrock's header limit are sent, so
identity always survives and long ``spend_logs_metadata`` pairs drop off the end.
"""
if not bedrock_request_metadata_is_owned():
return frozenset(), ()
resolved: Final = resolve_bedrock_request_metadata(litellm_params)
if resolved is None:
fitting: Final = {} if resolved is None else _longest_fitting_prefix(resolved)
if not fitting:
return _OWNED_HEADER_NAMES, ()
return _OWNED_HEADER_NAMES, ((BEDROCK_REQUEST_METADATA_HEADER, json.dumps(resolved, separators=(",", ":"))),)
return _OWNED_HEADER_NAMES, ((BEDROCK_REQUEST_METADATA_HEADER, _header_value(fitting)),)
def merge_bedrock_invoke_headers(

73
litellm/llms/bedrock_mantle/chat/transformation.py
View file

@ -10,19 +10,21 @@ Auth: Bearer token (litellm_params.api_key, BEDROCK_MANTLE_API_KEY, or the
BedrockMantleAuthMixin in common_utils.
"""
from collections.abc import AsyncIterator, Iterator
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock_mantle.common_utils import (
BEDROCK_MANTLE_DEFAULT_REGION,
BedrockMantleAuthMixin,
)
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
from litellm.llms.openai.chat.gpt_5_transformation import get_effort_level, is_gpt_reasoning_series_name
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
@ -39,6 +41,51 @@ if TYPE_CHECKING:
)
def _with_chat_reasoning_effort(non_default_params: Mapping[str, object]) -> Mapping[str, object]:
"""Chat Completions takes ``reasoning_effort`` as a level, so a Responses-shaped ``{"effort", "summary"}``
passes only its effort, the way the OpenAI GPT-5 chat config reads it."""
reasoning_effort: Final = non_default_params.get("reasoning_effort")
if not isinstance(reasoning_effort, dict):
return non_default_params
effort: Final = get_effort_level(reasoning_effort)
rest: Final = {key: value for key, value in non_default_params.items() if key != "reasoning_effort"}
return rest if effort is None else {**rest, "reasoning_effort": effort}
def _unsupported_chat_values(non_default_params: Mapping[str, object]) -> tuple[str, ...]:
n: Final = non_default_params.get("n")
reasoning_effort: Final = non_default_params.get("reasoning_effort")
summary: Final = reasoning_effort.get("summary") if isinstance(reasoning_effort, Mapping) else None
return tuple(
value
for value, unsupported in (
(f"n={n}", isinstance(n, int) and n > 1),
(f"reasoning summary {summary}", summary is not None),
)
if unsupported
)
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
return litellm.drop_params is True or litellm_params.get("drop_params") is True
def _chat_request_params(optional_params: Mapping[str, object], drop_params: bool) -> Mapping[str, object]:
unsupported: Final = _unsupported_chat_values(optional_params)
if unsupported and not drop_params:
raise UnsupportedParamsError(
message=f"Bedrock Mantle Chat Completions does not support {', '.join(unsupported)}. To drop unsupported "
"openai params from the call, set `litellm.drop_params = True`",
llm_provider="bedrock_mantle",
status_code=400,
)
servable: Final = _with_chat_reasoning_effort(optional_params)
n: Final = servable.get("n")
return MappingProxyType(
{key: value for key, value in servable.items() if not (key == "n" and isinstance(n, int) and n > 1)}
)
class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
"""
Transformation config for Amazon Bedrock Mantle OpenAI-compatible API.
@ -158,6 +205,28 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
if param != "max_tokens"
}
def transform_request(
self,
model: str,
messages: list[AllMessageValues], # mutable-ok: OpenAIGPTConfig.transform_request signature
optional_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature
) -> dict[str, object]: # mutable-ok: OpenAIGPTConfig.transform_request signature
request_params: Final = dict(_chat_request_params(optional_params, _drops_params(litellm_params)))
return super().transform_request(model, messages, request_params, litellm_params, headers)
def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: Mapping[str, object],
) -> Mapping[str, object]:
return _chat_request_params(
super().transform_extra_body(extra_body, request, model, litellm_params), _drops_params(litellm_params)
)
def _supports_reasoning(self, model: str) -> bool:
try:
return litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider)

4
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -24,6 +24,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
from litellm.llms.base_llm.responses.transformation import ReasoningSummaryLevel
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock_mantle.common_utils import (
@ -73,6 +74,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.BEDROCK_MANTLE
def auto_reasoning_summary(self) -> ReasoningSummaryLevel:
return "auto" if self.use_openai_path else super().auto_reasoning_summary()
def get_error_class(
self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers
) -> BaseLLMException:

204
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -1,10 +1,11 @@
import asyncio
import copy
import inspect
import json
import ssl
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Coroutine, Iterator, Mapping, Sequence
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
from contextlib import asynccontextmanager
from functools import lru_cache, partial
from functools import lru_cache, partial, reduce
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -35,7 +36,7 @@ import litellm.litellm_core_utils
import litellm.types
import litellm.types.utils
from litellm._logging import redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, update_headers_with_filtered_beta
from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@ -49,6 +50,7 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
synthesize_subtitle_document,
)
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
from litellm.litellm_core_utils.realtime_errors import (
@ -215,6 +217,7 @@ if TYPE_CHECKING:
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -467,6 +470,8 @@ async def _aiter_bytes_then_close(response: httpx.Response, *, chunk_size: int)
_DECODED_BODY_STALE_HEADERS: Final[frozenset[str]] = frozenset({"content-encoding", "content-length"})
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
_JSON_OBJECTS: Final = TypeAdapter(list[dict[str, object]])
def _decoded_body_headers(response: httpx.Response) -> httpx.Headers:
@ -2180,11 +2185,13 @@ class BaseLLMHTTPHandler:
api_key: str | None,
model: str,
timeout: float | httpx.Timeout | None = None,
rebuild_without_native_output: Callable[[], Mapping[str, object]] | None = None,
) -> httpx.Response:
max_attempts: Final = max(provider_config.max_retry_on_anthropic_messages_http_error, 1)
litellm_params_dict: Final = dict(litellm_params)
optional_params_dict: Final = dict(litellm_params)
for attempt_idx in range(max_attempts):
recovered_errors: tuple[httpx.HTTPStatusError, ...] = () # rebind-ok: a rebuilt body replays every recovery
while True:
try:
response = await async_httpx_client.post(
url=request_url,
@ -2197,64 +2204,79 @@ class BaseLLMHTTPHandler:
response.raise_for_status()
return response
except httpx.HTTPStatusError as e:
hit_max_attempt = attempt_idx + 1 == max_attempts
rebuild_body = (
rebuild_without_native_output
if rebuild_without_native_output is not None
and provider_config.should_retry_anthropic_messages_without_native_output(
e=e, request_body=request_body, model=model
)
else None
)
hit_max_attempt = len(recovered_errors) + 1 == max_attempts
should_retry = provider_config.should_retry_anthropic_messages_on_http_error(
e=e, litellm_params=litellm_params_dict, request_body=request_body
)
if should_retry and not hit_max_attempt:
if logging_obj.baseline_cache_context is not None:
await logging_obj.invalidate_baseline_cache_estimate("retried_request")
if rebuild_body is None and (hit_max_attempt or not should_retry):
raise self._handle_error(e=e, provider_config=provider_config)
if logging_obj.baseline_cache_context is not None:
await logging_obj.invalidate_baseline_cache_estimate("retried_request")
if rebuild_body is not None:
request_body.clear()
request_body.update(rebuild_body())
for recovered_error in recovered_errors:
provider_config.transform_anthropic_messages_request_on_http_error(
e=recovered_error, request_data=request_body
)
else:
verbose_logger.debug(
"Anthropic /v1/messages: invalid thinking signature; "
"stripping thinking blocks and retrying (attempt %s/%s).",
attempt_idx + 2,
len(recovered_errors) + 2,
max_attempts,
)
provider_config.transform_anthropic_messages_request_on_http_error(e=e, request_data=request_body)
headers, signed_json_body = await sign_request_off_loop_if_aws(
provider_config,
provider_config.sign_request,
headers=headers,
optional_params=optional_params_dict,
request_data=request_body,
api_base=request_url,
api_key=api_key,
stream=stream,
fake_stream=False,
model=model,
)
logging_obj.model_call_details.update(request_body)
continue
raise self._handle_error(e=e, provider_config=provider_config)
recovered_errors = (*recovered_errors, e)
headers, signed_json_body = await sign_request_off_loop_if_aws(
provider_config,
provider_config.sign_request,
headers=headers,
optional_params=optional_params_dict,
request_data=request_body,
api_base=request_url,
api_key=api_key,
stream=stream,
fake_stream=False,
model=model,
)
logging_obj.model_call_details.update(request_body)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
raise RuntimeError("unreachable: anthropic messages HTTP retry loop exited without return")
@staticmethod
def _resolve_anthropic_messages_timeout(
litellm_params: GenericLiteLLMParams,
stream: bool,
custom_llm_provider: str,
) -> float | httpx.Timeout | None:
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
)
from litellm.utils import supports_httpx_timeout
stream_timeout: Final = litellm_params.get("stream_timeout") if stream else None
model_timeout: Final = stream_timeout if stream_timeout is not None else litellm_params.get("timeout")
model_timeout: Final = litellm_params.get("timeout")
request_timeout: Final = litellm_params.get("request_timeout")
global_timeout: Final = get_configured_request_timeout()
if model_timeout is None and request_timeout is None and global_timeout is None:
if stream_timeout is None and model_timeout is None and request_timeout is None and global_timeout is None:
return None
return CompletionTimeout.resolve(
model_timeout,
{"request_timeout": request_timeout},
{"stream_timeout": stream_timeout, "request_timeout": request_timeout},
custom_llm_provider,
global_timeout=global_timeout,
supports_httpx_timeout=supports_httpx_timeout,
stream_kind=StreamKind.LIVE if stream else StreamKind.NOT_STREAMED,
)
async def async_anthropic_messages_handler(
@ -2304,6 +2326,16 @@ class BaseLLMHTTPHandler:
merged_headers.update(extra_headers_from_kwargs)
if provider_specific_headers:
merged_headers.update(provider_specific_headers)
request_params: Final = reduce(
delete_nested_value,
litellm_params.get("additional_drop_params") or (),
anthropic_messages_optional_request_params,
)
if (
request_params.get("safeguards") is not None
and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
):
await aload_beta_headers_config()
(
headers,
api_base,
@ -2311,7 +2343,7 @@ class BaseLLMHTTPHandler:
headers=merged_headers or {},
model=model,
messages=messages,
optional_params=anthropic_messages_optional_request_params,
optional_params=request_params,
litellm_params=dict(litellm_params),
api_key=api_key,
api_base=api_base,
@ -2342,20 +2374,29 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
)
additional_drop_params: Final[list[str]] = litellm_params.get("additional_drop_params") or []
if additional_drop_params:
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
for path in additional_drop_params:
anthropic_messages_optional_request_params = delete_nested_value(
anthropic_messages_optional_request_params, path
)
native_output_fallback_params: Final = anthropic_messages_provider_config.native_output_fallback_litellm_params(
model=model,
anthropic_messages_optional_request_params=_OPTIONAL_PARAMS.validate_python(request_params),
litellm_params=litellm_params,
)
rebuild_without_native_output: Final = (
None
if native_output_fallback_params is None
else partial[Mapping[str, object]](
anthropic_messages_provider_config.transform_anthropic_messages_request,
model=model,
messages=copy.deepcopy(_JSON_OBJECTS.validate_python(messages)),
anthropic_messages_optional_request_params=copy.deepcopy(_JSON_OBJECT.validate_python(request_params)),
litellm_params=native_output_fallback_params,
headers=_JSON_OBJECT.validate_python(headers),
)
)
# Prepare request body
request_body: Final = anthropic_messages_provider_config.transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
anthropic_messages_optional_request_params=request_params,
litellm_params=litellm_params,
headers=headers,
)
@ -2407,6 +2448,13 @@ class BaseLLMHTTPHandler:
},
)
applied_timeout: Final = self._resolve_anthropic_messages_timeout(
litellm_params=litellm_params,
stream=stream or False,
custom_llm_provider=custom_llm_provider,
)
if applied_timeout is not None:
logging_obj.record_applied_timeout(applied_timeout)
response: Final = await self._async_post_anthropic_messages_with_http_error_retry(
async_httpx_client=async_httpx_client,
request_url=request_url,
@ -2419,11 +2467,8 @@ class BaseLLMHTTPHandler:
litellm_params=litellm_params,
api_key=api_key,
model=model,
timeout=self._resolve_anthropic_messages_timeout(
litellm_params=litellm_params,
stream=stream or False,
custom_llm_provider=custom_llm_provider,
),
timeout=applied_timeout,
rebuild_without_native_output=rebuild_without_native_output,
)
# used for logging + cost tracking
@ -2462,7 +2507,7 @@ class BaseLLMHTTPHandler:
held_back_tool_names: Final = self._server_fulfilled_tools_in_request(
logging_obj=logging_obj,
tools=anthropic_messages_optional_request_params.get("tools"),
tools=request_params.get("tools"),
)
initial_response = AgenticAnthropicStreamingIterator(
completion_stream=completion_stream,
@ -2470,7 +2515,7 @@ class BaseLLMHTTPHandler:
model=model,
messages=messages,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
anthropic_messages_optional_request_params=request_params,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs_for_agentic,
@ -2493,7 +2538,7 @@ class BaseLLMHTTPHandler:
model=model,
messages=messages,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
anthropic_messages_optional_request_params=request_params,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs_for_agentic,
@ -6865,6 +6910,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider: str | None = None,
first_message: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs: object,
) -> Exception | None:
"""
@ -6880,6 +6926,35 @@ class BaseLLMHTTPHandler:
"""
_ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
_ws_guardrail_callbacks: list = []
_ws_output_guardrail_callbacks: list = []
try:
import litellm as _litellm
# Use duck-typing so any guardrail that exposes the PII
# masking interface works, not just _OPTIONAL_PresidioPIIMasking.
# This avoids a layering violation (SDK importing from proxy).
_ws_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and callable(getattr(cb, "_unmask_pii_text", None))
and getattr(cb, "output_parse_pii", False)
]
_ws_output_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and getattr(cb, "apply_to_output", False)
]
except Exception as _guardrail_exc:
verbose_logger.warning(
"Responses WebSocket: failed to collect guardrail callbacks — PII masking will be skipped. Error: %s",
_guardrail_exc,
)
if responses_api_provider_config is None or not responses_api_provider_config.supports_native_websocket():
from litellm.responses.streaming_iterator import (
ManagedResponsesWebSocketHandler,
@ -6897,6 +6972,8 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
first_message=first_message,
quota_callbacks=_ws_quota_callbacks,
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
frame_guard=frame_guard,
**kwargs,
)
await handler.run()
@ -6978,36 +7055,6 @@ class BaseLLMHTTPHandler:
if litellm_metadata:
_request_data["litellm_metadata"] = litellm_metadata
_ws_guardrail_callbacks: list = []
_ws_output_guardrail_callbacks: list = []
try:
import litellm as _litellm
# Use duck-typing so any guardrail that exposes the PII
# masking interface works, not just _OPTIONAL_PresidioPIIMasking.
# This avoids a layering violation (SDK importing from proxy).
_ws_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and callable(getattr(cb, "_unmask_pii_text", None))
and getattr(cb, "output_parse_pii", False)
]
_ws_output_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and getattr(cb, "apply_to_output", False)
]
except Exception as _guardrail_exc:
verbose_logger.warning(
"Responses WebSocket: failed to collect guardrail "
"callbacks — PII masking will be skipped. Error: %s",
_guardrail_exc,
)
streaming: Final = ResponsesWebSocketStreaming(
websocket=websocket,
backend_ws=cast(ClientConnection, backend_ws),
@ -7021,6 +7068,7 @@ class BaseLLMHTTPHandler:
authorized_model=model,
custom_llm_provider=custom_llm_provider,
request_defaults=request_defaults,
frame_guard=frame_guard,
)
return await streaming.bidirectional_forward()

2
litellm/llms/fal_ai/cost_calculator.py
View file

@ -87,7 +87,7 @@ def _keyed_rows(model: str, quality: str) -> tuple[tuple[int, int, float], ...]:
suffix: Final = f"/{model}"
return tuple(
(width, height, float(raw_cost))
for key in litellm.model_cost
for key in (*litellm.model_cost,)
if isinstance(key, str) and key.startswith(prefix) and key.endswith(suffix)
for size in (key[len(prefix) : -len(suffix)],)
for dimensions in (_parse_keyed_dimensions(size),)

2
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -544,7 +544,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return cached[2]
index: Final[list[tuple[str, dict]]] = []
for key, model_info in model_cost.items():
for key, model_info in (*model_cost.items(),):
if not key.startswith("fireworks_ai/"):
continue
if not isinstance(model_info, dict):

5
litellm/llms/microsoft_365_copilot/chat/transformation.py
View file

@ -109,6 +109,11 @@ class Microsoft365CopilotChatConfig(BaseConfig):
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseConfig requires a list return
return ["stream", "max_tokens", "max_completion_tokens"]
def should_fake_stream(
self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None
) -> bool:
return True
def transform_request(
self,
model: str,

2
litellm/llms/ollama/common_utils.py
View file

@ -197,7 +197,7 @@ class OllamaModelInfo(BaseLLMModelInfo):
"ollama/" + stripped_model,
"ollama_chat/" + stripped_model,
}
model_cost_keys: Final = {key.lower() for key in model_cost}
model_cost_keys: Final = {key.lower() for key in (*model_cost,)}
return any(name.lower() in model_cost_keys for name in potential_model_names)
@staticmethod

4
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -28,7 +28,9 @@ def _catalogue_declares_default_effort() -> bool:
the one litellm gave before it existed. Scanning costs ~80us on the largest published map and
only on the fallback path, which is noise beside the request it precedes.
"""
return any(isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in litellm.model_cost.values())
return any(
isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in (*litellm.model_cost.values(),)
)
def _normalize_reasoning_effort_for_chat_completion(

1
litellm/llms/openai/responses/count_tokens/token_counter.py
View file

@ -124,6 +124,7 @@ class OpenAITokenCounter(BaseTokenCounter):
error=True,
error_message=request_error,
status_code=400,
rejected_before_provider_call=True,
)
deployment = deployment or {}

6
litellm/llms/openai_like/README.md
View file

@ -63,6 +63,12 @@ That's it! The provider will be automatically loaded and available.
// Optional: Special handling flags
"special_handling": {
"convert_content_list_to_string": true
},
// Optional: Capabilities every model of this provider has, used when a
// model has no cost map row or its row leaves the field unset
"provider_info": {
"supports_function_calling": true
}
}
}

8
litellm/llms/openai_like/json_loader.py
View file

@ -6,7 +6,12 @@ import json
from pathlib import Path
from typing import Final
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.types.utils import ProviderSpecificModelInfo
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
class SimpleProviderConfig:
@ -22,6 +27,9 @@ class SimpleProviderConfig:
self.constraints = data.get("constraints", {})
self.special_handling = data.get("special_handling", {})
self.supported_endpoints = data.get("supported_endpoints", [])
self.provider_info: ProviderSpecificModelInfo = (
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
)
class JSONProviderRegistry:

2
litellm/llms/openai_like/messages/transformation.py
View file

@ -8,6 +8,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.openai_like.json_loader import SimpleProviderConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
@ -57,6 +58,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
headers=normalized,
optional_params=optional_params,
messages=messages,
beta_map_provider=LlmProviders.OPENAI_LIKE.value,
)
return merged, api_base

10
litellm/llms/openai_like/providers.json
View file

@ -230,6 +230,16 @@
"api_key_env": "REKA_API_KEY",
"api_base_env": "REKA_API_BASE"
},
"openinfer": {
"base_url": "https://api.openinfer.ai/v1",
"api_key_env": "OPENINFER_API_KEY",
"api_base_env": "OPENINFER_API_BASE",
"base_class": "openai_gpt",
"provider_info": {
"supports_function_calling": true
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"inferenceapis": {
"base_url": "https://api.inferenceapis.com/v1",
"api_key_env": "INFERENCEAPIS_API_KEY",

18
litellm/llms/vertex_ai/common_utils.py
View file

@ -1278,6 +1278,7 @@ class VertexAITokenCounter(BaseTokenCounter):
) -> TokenCountResponse | None:
import copy
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_publisher
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
VertexAIError as PartnerVertexAIError,
)
@ -1293,6 +1294,23 @@ class VertexAITokenCounter(BaseTokenCounter):
):
return None
without_count_route: Final = (
VertexAIPartnerModels.is_vertex_partner_model(model_to_use) and count_tokens_publisher(model_to_use) is None
)
model_info: Final[Mapping[str, object]] = deployment.get("model_info") or {}
if without_count_route and messages and model_info.get("custom_tokenizer") is not None:
return None
if without_count_route:
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="vertex_ai_partner_models",
error=True,
error_message=f"LiteLLM does not support counting tokens for Vertex AI model {model_to_use}.",
status_code=400,
)
# Check if this is a partner model (Claude, Mistral, etc.)
if VertexAIPartnerModels.is_vertex_partner_model(model_to_use):
# Use partner models token counter

25
litellm/llms/vertex_ai/count_tokens/context.py
View file

@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Final, Literal
from pydantic import JsonValue, TypeAdapter, ValidationError
from typing_extensions import assert_never
from litellm.litellm_core_utils.prompt_templates.factory import chat_completion_tools_for
from litellm.types.llms.anthropic import AnthropicMessagesTextParam
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.vertex_ai import ContentType
@ -75,27 +76,6 @@ def _is_function_tool(tool: Mapping[str, JsonValue]) -> bool:
return isinstance(tool.get("name"), str) or "input_schema" in tool or "function" in tool
def _is_responses_function_tool(tool: Mapping[str, JsonValue]) -> bool:
return tool.get("type") == "function" and "function" not in tool and "input_schema" not in tool
def _chat_tool(tool: Mapping[str, JsonValue]) -> tuple[Mapping[str, object], ...]:
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
if _is_responses_function_tool(tool):
chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
[dict(tool)] # pyright: ignore[reportArgumentType] # validated JSON dict in the Responses function tool shape
)
return tuple(chat_tools)
anthropic_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(
[dict(tool)] # pyright: ignore[reportArgumentType] # validated JSON dict in an Anthropic or chat tool shape
)
return tuple(anthropic_tools)
def _map_function_tools(
tools: object | None,
) -> tuple[Mapping[str, object], ...] | None | VertexGeminiCountContextError:
@ -112,7 +92,7 @@ def _map_function_tools(
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
chat_tools: Final = tuple(
chat_tool for tool in function_tools for chat_tool in _chat_tool(tool)
chat_tool for tool in function_tools for chat_tool in chat_completion_tools_for(tool)
) # comprehension-ok: flattens each tool's translation
try:
mapped_tools: Final = VertexGeminiConfig().map_function(
@ -163,6 +143,7 @@ def count_context_error_response(
error=True,
error_message=error.error_message,
status_code=error.status_code,
rejected_before_provider_call=True,
)

36
litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py
View file

@ -1,5 +1,5 @@
"""
Token counter for Vertex AI Partner Models (Anthropic Claude, Mistral, etc.)
Token counter for Vertex AI Partner Models (Anthropic Claude, Meta Llama)
This handler provides token counting for partner models hosted on Vertex AI.
Unlike Gemini models which use Google's token counting API, partner models use
@ -12,36 +12,30 @@ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
_COUNT_TOKENS_PUBLISHERS: Final = (
("claude", "anthropic"),
("llama", "meta"),
("meta/", "meta"),
)
def count_tokens_publisher(model: str) -> str | None:
return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
class VertexAIPartnerModelsTokenCounter(VertexBase):
"""
Token counter for Vertex AI Partner Models.
Handles token counting for models like Claude (Anthropic), Mistral, etc.
Handles token counting for models like Claude (Anthropic) and Llama (Meta)
that are available through Vertex AI's partner model program.
"""
def _get_publisher_for_model(self, model: str) -> str:
"""
Determine the publisher name for the given model.
Args:
model: The model name (e.g., "claude-3-5-sonnet-20241022")
Returns:
Publisher name to use in the Vertex AI endpoint URL
Raises:
ValueError: If the model is not a recognized partner model
"""
if "claude" in model:
return "anthropic"
elif "mistral" in model or "codestral" in model:
return "mistralai"
elif "llama" in model or "meta/" in model:
return "meta"
else:
publisher: Final = count_tokens_publisher(model)
if publisher is None:
raise ValueError(f"Unknown partner model: {model}")
return publisher
def _build_count_tokens_endpoint(
self,

373
litellm/main.py
View file

@ -77,7 +77,7 @@ from litellm.litellm_core_utils.audio_utils.utils import (
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
maybe_run_chat_completion_agentic_loop,
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_litellm_params import (
AWS_CREDENTIAL_KWARGS_KEYS,
@ -119,7 +119,10 @@ from litellm.llms.base_llm.base_model_iterator import (
from litellm.llms.base_llm.chat.transformation import with_attribution_headers
from litellm.llms.bedrock.common_utils import (
BedrockModelInfo,
NativeOutputFallback,
bedrock_route_for_request,
deployment_model_info,
without_bedrock_native_output_opt_in,
without_bedrock_route_prefix,
)
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
@ -218,6 +221,7 @@ from .litellm_core_utils.streaming_chunk_builder_utils import (
)
from .llms.anthropic.chat import AnthropicChatCompletion
from .llms.anthropic.chat.prompt_cache_breakpoints import (
ClaudePromptCache,
ClaudeRoute,
carries_prompt_cache_breakpoint,
claude_prompt_cache,
@ -232,6 +236,7 @@ from .llms.azure_ai.embed import AzureAIEmbedding
from .llms.bedrock.chat import BedrockConverseLLM
from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig
from .llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback
from .llms.bedrock.embed.embedding import BedrockEmbedding
from .llms.bedrock.image_edit.handler import BedrockImageEdit
from .llms.bedrock.image_generation.image_handler import BedrockImageGeneration
@ -539,7 +544,14 @@ async def acompletion(
mock_timeout = kwargs.get("mock_timeout", None)
if mock_timeout is True:
await _handle_mock_timeout_async(mock_timeout, timeout, model)
stream_timeout: Final = kwargs.get("stream_timeout")
await _handle_mock_timeout_async(
mock_timeout,
stream_timeout
if stream is True and isinstance(stream_timeout, int | float | str | httpx.Timeout)
else timeout,
model,
)
loop: Final = asyncio.get_event_loop()
custom_llm_provider = kwargs.get("custom_llm_provider", None)
@ -1087,6 +1099,11 @@ def mock_completion(
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
_resolve_openai_api_base as _resolve_openai_api_base, # pyright: ignore[reportPrivateUsage] # noqa: PLC0414 # re-export of moved helper
)
from litellm.responses.bridge_check import (
bridge_reasoning_effort,
keep_registered_mode,
str_keyed_mapping,
)
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
responses_api_bridge_check as responses_api_bridge_check, # pyright: ignore[reportUnknownVariableType] # noqa: PLC0414 # re-export of moved function
)
@ -1136,13 +1153,66 @@ def _claude_route(provider_config: BaseConfig | None, reaches_converse: bool) ->
return "anthropic"
def _converse_tools(converse_params: Mapping[str, object] | None, messages: Sequence[object]) -> tuple[bool, bool]:
"""Whether Converse sends a tool in its tool config, and whether it adds one of its own there."""
if converse_params is None:
return False, False
def _converse_attempt_params(
request_params: Mapping[str, object],
special_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
plans_for_the_retry: bool,
) -> tuple[Mapping[str, object], ...]:
"""The params of each Converse attempt to plan for: the first, and the json tool retry a rejected native output
gets when ``plans_for_the_retry``."""
first: Final = _map_request_params(**request_params, model_info=model_info, **special_params)
if not plans_for_the_retry or not AmazonConverseConfig.sends_native_text_format(first):
return (first,)
retry_model_info: Final = deployment_model_info(without_bedrock_native_output_opt_in({"model_info": model_info}))
return (first, _map_request_params(**request_params, model_info=retry_model_info, **special_params))
def _converse_tools(attempts: Sequence[Mapping[str, object]], messages: Sequence[object]) -> tuple[bool, bool]:
"""Whether Converse sends a tool in its tool config on any attempt, and whether it adds one of its own there."""
return (
AmazonConverseConfig.sends_tools(converse_params, messages),
AmazonConverseConfig.adds_a_tool(converse_params, messages),
any(AmazonConverseConfig.sends_tools(params, messages) for params in attempts),
any(AmazonConverseConfig.adds_a_tool(params, messages) for params in attempts),
)
def _claude_prompt_cache_on_route(
messages: Sequence[object],
tools: Sequence[object] | None,
special_params: Mapping[str, object],
request_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
provider_config: BaseConfig | None,
*,
reaches_converse: bool,
drop_params: bool,
model: str,
llm_provider: str,
) -> ClaudePromptCache:
converse_attempts: Final = (
_converse_attempt_params(request_params, special_params, model_info, plans_for_the_retry=drop_params)
if reaches_converse and carries_prompt_cache_breakpoint(messages, tools)
else ()
)
sends_tools, adds_a_tool = _converse_tools(converse_attempts, messages)
return resolved_claude_prompt_cache(
claude_prompt_cache(
messages,
tools,
special_params,
drop_params=drop_params,
tool_history_as_text=reaches_converse and AmazonConverseConfig.sends_tool_history_as_text(sends_tools),
tool_config_cache_control=(
AmazonConverseConfig.tool_config_cache_control(special_params.get("cache_control_injection_points"))
if sends_tools
else None
),
route=_claude_route(provider_config, reaches_converse),
adds_a_tool=adds_a_tool,
pairs_unanswered_tool_calls=litellm.modify_params is True,
),
model=model,
llm_provider=llm_provider,
)
@ -1187,6 +1257,52 @@ def _drop_anthropic_only_tool_keys(tools: list[dict[str, object]] | None) -> lis
return [_without_anthropic_only_tool_keys(tool) if isinstance(tool, dict) else tool for tool in tools]
def _azure_completion_fakes_stream(model: str, base_model: str | None) -> bool:
if not litellm.AzureOpenAIO1Config().is_o_series_model(model=base_model or model):
return False
o_series_handler_config: Final = ProviderConfigManager.get_provider_chat_config(
model=model, provider=LlmProviders.AZURE
)
return o_series_handler_config is not None and o_series_handler_config.should_fake_stream(
model=model, stream=True, custom_llm_provider=LlmProviders.AZURE.value
)
def _completion_fakes_stream(
model: str,
custom_llm_provider: str | None,
provider_config: BaseConfig | None,
optional_params: Mapping[str, object],
*,
base_model: str | None,
bridges_to_responses_api: bool,
) -> bool:
if optional_params.get("fake_stream") is True:
return True
if custom_llm_provider is None:
return False
if bridges_to_responses_api:
responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(
provider=custom_llm_provider, model=model
)
return responses_config is not None and responses_config.should_fake_stream(
model=model, stream=True, custom_llm_provider=custom_llm_provider
)
if (
custom_llm_provider == LlmProviders.VERTEX_AI.value
and get_vertex_ai_model_route(model=model, litellm_params={"base_model": base_model})
== VertexAIModelRoute.GEMMA
):
return True
if isinstance(provider_config, litellm.AmazonAI21Config):
return True
if custom_llm_provider == LlmProviders.AZURE.value:
return _azure_completion_fakes_stream(model, base_model)
return provider_config is not None and provider_config.should_fake_stream(
model=model, stream=True, custom_llm_provider=custom_llm_provider
)
class _ProxyAuthHeadersProvider(Protocol):
def get_auth_headers(self) -> Mapping[str, str]: ...
@ -1258,6 +1374,8 @@ def _register_custom_pricing_for_request(
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
pricing used by sibling deployments of the same backend model. Direct SDK
calls keep the legacy behavior of registering the shared key with pricing.
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
deployment registration, since a call routes by its own ``model_info``.
"""
entry: Final = _build_custom_pricing_entry(
custom_llm_provider=custom_llm_provider,
@ -1267,8 +1385,11 @@ def _register_custom_pricing_for_request(
shared_key: Final = f"{custom_llm_provider}/{model}"
deployment_id: Final = _get_router_deployment_id(kwargs)
if deployment_id is None:
litellm.register_model({shared_key: entry}, persist_across_reloads=False)
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
return
shared_entry: Final = keep_registered_mode(
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
)
litellm.register_model(
{deployment_id: entry},
persist_across_reloads=False,
@ -1277,7 +1398,7 @@ def _register_custom_pricing_for_request(
add_to_provider_model_sets=False,
)
litellm.register_model(
{shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)},
{shared_key: shared_entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
@ -4197,6 +4318,53 @@ _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
_BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
_AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = (
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
"aws_region_name",
)
def _native_output_fallback(
ctx: CompletionDispatchContext,
first_attempt_params: Mapping[str, object],
first_attempt_litellm_params: Mapping[str, object],
) -> NativeOutputFallback:
litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params)
remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params))
return NativeOutputFallback(
optional_params={
**{key: value for key, value in remapped_params.items() if key != "aws_bedrock_client"},
**{
key: first_attempt_params[key]
for key in _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS
if key in first_attempt_params
},
},
litellm_params=litellm_params,
)
def _converse_native_output_fallback(
ctx: CompletionDispatchContext,
first_attempt_params: Mapping[str, object],
first_attempt_litellm_params: Mapping[str, object],
) -> NativeOutputFallback | None:
fallback: Final = _native_output_fallback(ctx, first_attempt_params, first_attempt_litellm_params)
return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback
def _native_output_attempt_timeout(
ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None
) -> float | str | httpx.Timeout | None:
if fallback is None:
return ctx.timeout
retry_timeout: Final = ctx.timeout_for_optional_params(fallback.optional_params)
ctx.logging.record_applied_timeout(retry_timeout)
return retry_timeout
def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResult:
acompletion: Final = ctx.acompletion
api_base: Final = ctx.api_base
@ -4268,22 +4436,36 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
)
elif bedrock_route == "converse":
model = without_bedrock_route_prefix(model)
response = bedrock_converse_chat_completion.completion(
model=model,
messages=messages,
custom_prompt_dict=custom_prompt_dict,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=_get_encoding(),
logging_obj=logging,
extra_headers=headers, # Use merged headers instead of original extra_headers
timeout=timeout,
acompletion=acompletion,
client=client,
api_base=api_base,
api_key=api_key,
converse_model: Final = model
def converse_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult:
return bedrock_converse_chat_completion.completion(
model=converse_model,
messages=messages,
custom_prompt_dict=custom_prompt_dict,
model_response=model_response,
optional_params=optional_params if fallback is None else dict(fallback.optional_params),
litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params),
logger_fn=logger_fn,
encoding=_get_encoding(),
logging_obj=logging,
extra_headers=headers, # Use merged headers instead of original extra_headers
timeout=_native_output_attempt_timeout(ctx, fallback),
acompletion=acompletion,
client=client,
api_base=api_base,
api_key=api_key,
)
converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params)
response = call_with_native_output_fallback(
converse_attempt,
(
partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params))
if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params)
else None
),
converse_model,
)
elif bedrock_route == "converse_like":
model = model.replace("converse_like/", "")
@ -4305,22 +4487,36 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
client=client,
)
else:
response = base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
custom_llm_provider="bedrock",
timeout=timeout,
headers=headers,
encoding=_get_encoding(),
api_key=api_key,
logging_obj=logging,
client=client,
def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult:
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params if fallback is None else dict(fallback.optional_params),
litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params),
custom_llm_provider="bedrock",
timeout=_native_output_attempt_timeout(ctx, fallback),
headers=headers,
encoding=_get_encoding(),
api_key=api_key,
logging_obj=logging,
client=client,
)
invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params)
response = call_with_native_output_fallback(
invoke_attempt,
(
partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params))
if isinstance(provider_config, AmazonAnthropicClaudeConfig)
and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params)
else None
),
model,
)
return response
@ -5599,6 +5795,9 @@ def completion(
custom_llm_provider=custom_llm_provider,
web_search_options=web_search_options,
api_base=api_base,
deployment_model_info=str_keyed_mapping(
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
),
)
if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model):
@ -5621,15 +5820,6 @@ def completion(
"aws_region_name", None
) # support region-based pricing for bedrock
### TIMEOUT LOGIC ###
timeout = CompletionTimeout.resolve(
timeout,
kwargs,
custom_llm_provider,
global_timeout=get_configured_request_timeout(),
supports_httpx_timeout=supports_httpx_timeout,
)
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if (
(input_cost_per_token is not None and output_cost_per_token is not None)
@ -5773,36 +5963,16 @@ def completion(
accepts_summary=provider_config is not None
and provider_config.accepts_reasoning_summary_in_reasoning_effort,
)
converse_params: Final = (
_map_request_params(
**{**request_param_args, "reasoning_effort": provider_reasoning_effort},
model_info=validated_model_info,
**non_default_params,
)
if reaches_converse and carries_prompt_cache_breakpoint(routed_messages, request_tools)
else None
)
converse_sends_tools, converse_adds_a_tool = _converse_tools(converse_params, routed_messages)
claude_cache: Final = (
resolved_claude_prompt_cache(
claude_prompt_cache(
routed_messages,
request_tools,
non_default_params,
drop_params=claude_drop_params,
tool_history_as_text=reaches_converse
and AmazonConverseConfig.sends_tool_history_as_text(converse_sends_tools),
tool_config_cache_control=(
AmazonConverseConfig.tool_config_cache_control(
non_default_params.get("cache_control_injection_points")
)
if converse_sends_tools
else None
),
route=_claude_route(provider_config, reaches_converse),
adds_a_tool=converse_adds_a_tool,
pairs_unanswered_tool_calls=litellm.modify_params is True,
),
_claude_prompt_cache_on_route(
routed_messages,
request_tools,
non_default_params,
{**request_param_args, "reasoning_effort": provider_reasoning_effort},
validated_model_info,
provider_config,
reaches_converse=reaches_converse,
drop_params=claude_drop_params,
model=model,
llm_provider=custom_llm_provider,
)
@ -5831,11 +6001,39 @@ def completion(
"tools": tools if claude_cache is None or claude_cache.tools is None else list(claude_cache.tools),
"messages": messages,
}
optional_params = get_optional_params(
**{**optional_param_args, "reasoning_effort": provider_reasoning_effort},
model_info=validated_model_info,
**non_default_params,
mapping_args: Final = MappingProxyType({**optional_param_args, "reasoning_effort": provider_reasoning_effort})
mapping_special_params: Final = MappingProxyType(dict(non_default_params))
def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]:
return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params)
optional_params = optional_params_for_model_info(validated_model_info)
fakes_stream: Final = partial(
_completion_fakes_stream,
model,
custom_llm_provider,
provider_config,
base_model=base_model if isinstance(base_model, str) else None,
bridges_to_responses_api=bridges_to_responses_api is True,
)
resolve_timeout: Final = partial(
CompletionTimeout.resolve,
timeout,
kwargs,
custom_llm_provider,
global_timeout=get_configured_request_timeout(),
supports_httpx_timeout=supports_httpx_timeout,
)
streams: Final = stream is True
def timeout_for_optional_params(attempt_params: Mapping[str, object]) -> float | httpx.Timeout:
if not streams:
return resolve_timeout(stream_kind=StreamKind.NOT_STREAMED)
return resolve_timeout(stream_kind=StreamKind.FAKED if fakes_stream(attempt_params) else StreamKind.LIVE)
### TIMEOUT LOGIC ###
timeout = timeout_for_optional_params(optional_params)
logging.record_applied_timeout(timeout)
processed_non_default_params: Final = pre_process_non_default_params(
model=model,
passed_params=optional_param_args,
@ -5994,9 +6192,12 @@ def completion(
custom_llm_provider=custom_llm_provider,
web_search_options=web_search_options,
tools=tools,
reasoning_effort=reasoning_effort,
reasoning_effort=bridge_reasoning_effort(reasoning_effort, optional_params.get("extra_body")),
reasoning_summary=_reasoning_summary_for_bridge,
api_base=api_base,
deployment_model_info=str_keyed_mapping(
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
),
)
# Use base_model (the true underlying model) for Azure model-type
@ -6069,6 +6270,7 @@ def completion(
model=model,
model_response=model_response,
optional_params=optional_params,
optional_params_for_model_info=optional_params_for_model_info,
organization=organization,
provider_config=provider_config,
request_params=MappingProxyType(
@ -6079,6 +6281,7 @@ def completion(
temperature=temperature,
text_completion=text_completion,
timeout=timeout,
timeout_for_optional_params=timeout_for_optional_params,
top_p=top_p,
)
if custom_llm_provider == "azure":

9
litellm/messages/dispatch.py
View file

@ -50,6 +50,13 @@ _PYTHON_AMESSAGES: Final = _python_amessages()
_AMESSAGES: Final = signature(_PYTHON_AMESSAGES)
def _native_kwargs(fields: Mapping[str, object], kwargs: Mapping[str, object]) -> Mapping[str, object]:
stream_timeout: Final = kwargs.get("stream_timeout")
if fields.get("stream") is not True or stream_timeout is None:
return kwargs
return {**kwargs, "timeout": stream_timeout}
def _public_request(
legacy: inspect.Signature, args: tuple[object, ...], kwargs: Mapping[str, object]
) -> NativeCall | None:
@ -61,7 +68,7 @@ def _public_request(
max_tokens: Final = fields.get("max_tokens")
if not isinstance(model, str) or messages is None or not isinstance(max_tokens, int):
return None
return native_call(legacy, args, kwargs)
return native_call(legacy, args, _native_kwargs(fields, kwargs))
def _resolved_provider(request: NativeCall) -> str | None:

217
litellm/model_prices_and_context_window_backup.json
View file

@ -2323,6 +2323,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2362,6 +2363,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2401,6 +2403,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2440,6 +2443,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2479,6 +2483,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2518,6 +2523,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2594,6 +2600,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2633,6 +2640,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2672,6 +2680,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2711,6 +2720,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2750,6 +2760,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -2789,6 +2800,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -3016,7 +3028,7 @@
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -3301,7 +3313,7 @@
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -10904,7 +10916,6 @@
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
@ -10973,7 +10984,6 @@
"output_cost_per_token_batches": 4e-05,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
@ -11005,7 +11015,6 @@
"output_cost_per_token_batches": 4e-05,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
@ -12877,6 +12886,7 @@
"supports_prompt_caching": true
},
"azure_ai/ministral-3b": {
"deprecation_date": "2026-12-09",
"input_cost_per_token": 4e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 128000,
@ -12940,6 +12950,7 @@
"supports_vision": true
},
"azure_ai/mistral-medium-2505": {
"deprecation_date": "2026-12-09",
"input_cost_per_token": 4e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12977,6 +12988,7 @@
"supports_tool_choice": true
},
"azure_ai/mistral-small-2503": {
"deprecation_date": "2026-12-09",
"input_cost_per_token": 1e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 128000,
@ -21132,6 +21144,7 @@
"input_cost_per_token": 1.4e-07,
"input_dbu_cost_per_token": 2e-06,
"litellm_provider": "databricks",
"token_count_framing": 97,
"max_input_tokens": 1000000,
"max_output_tokens": 393216,
"max_tokens": 393216,
@ -21154,6 +21167,11 @@
"supports_tool_choice": true,
"supports_vision": false
},
"databricks/system.ai.deepseek-v4-flash-0731": {
"litellm_provider": "databricks",
"mode": "chat",
"token_count_framing": 97
},
"databricks/databricks-deepseek-v4-pro-0813": {
"cache_creation_input_token_cost": 1.31999e-06,
"cache_read_input_token_cost": 1.3202e-07,
@ -22122,6 +22140,7 @@
"input_cost_per_token": 2.99999e-06,
"input_dbu_cost_per_token": 4.2857e-05,
"litellm_provider": "databricks",
"token_count_framing": 100,
"max_input_tokens": 1000000,
"max_output_tokens": 1048576,
"max_tokens": 1048576,
@ -22145,6 +22164,11 @@
"supports_tool_choice": true,
"supports_vision": true
},
"databricks/system.ai.kimi-k3": {
"litellm_provider": "databricks",
"mode": "chat",
"token_count_framing": 100
},
"databricks/databricks-llama-4-maverick": {
"cache_creation_input_token_cost": 5.0001e-07,
"cache_read_input_token_cost": 5.0001e-07,
@ -24683,7 +24707,7 @@
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -24746,6 +24770,7 @@
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"eu.meta.llama3-2-1b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 1.3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -24753,10 +24778,12 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.3e-07,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md",
"supports_function_calling": true,
"supports_tool_choice": false
},
"eu.meta.llama3-2-3b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 1.9e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -24764,6 +24791,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.9e-07,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md",
"supports_function_calling": true,
"supports_tool_choice": false
},
@ -29699,6 +29727,7 @@
},
"vertex_ai/gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"deprecation_date": "2027-09-02",
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -29806,6 +29835,7 @@
"supports_native_streaming": true
},
"vertex_ai/gemini-3.8-live": {
"deprecation_date": "2027-09-25",
"input_cost_per_audio_token": 3e-06,
"input_cost_per_image_token": 1e-06,
"input_cost_per_token": 7.5e-07,
@ -31960,6 +31990,7 @@
},
"gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
"deprecation_date": "2027-09-02",
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -33882,7 +33913,7 @@
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -40514,6 +40545,7 @@
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"meta.llama3-1-405b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 2.4e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -40556,6 +40588,7 @@
"supports_tool_choice": false
},
"meta.llama3-2-11b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 1.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -40569,6 +40602,7 @@
"supports_vision": true
},
"meta.llama3-2-1b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -40576,10 +40610,12 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1e-07,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md",
"supports_function_calling": true,
"supports_tool_choice": false
},
"meta.llama3-2-3b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -40587,10 +40623,12 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md",
"supports_function_calling": true,
"supports_tool_choice": false
},
"meta.llama3-2-90b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -45425,6 +45463,38 @@
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/getting-started/pricing"
},
"perplexity/pplx-decider-v1.2": {
"input_cost_per_token": 2e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
"perplexity/pplx-decider-latest": {
"input_cost_per_token": 2e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
"perplexity/pplx-decider-flash-v1.2": {
"input_cost_per_token": 1e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
"perplexity/pplx-decider-flash-latest": {
"input_cost_per_token": 1e-08,
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
"perplexity/sonar": {
"input_cost_per_token": 1e-06,
"litellm_provider": "perplexity",
@ -48433,6 +48503,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -48467,6 +48538,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -48868,7 +48940,7 @@
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -48947,6 +49019,7 @@
"supports_vision": false
},
"us.meta.llama3-1-405b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 2.4e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -48989,6 +49062,7 @@
"supports_tool_choice": false
},
"us.meta.llama3-2-11b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 1.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -49002,6 +49076,7 @@
"supports_vision": true
},
"us.meta.llama3-2-1b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -49009,10 +49084,12 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1e-07,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md",
"supports_function_calling": true,
"supports_tool_choice": false
},
"us.meta.llama3-2-3b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -49020,10 +49097,12 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md",
"supports_function_calling": true,
"supports_tool_choice": false
},
"us.meta.llama3-2-90b-instruct-v1:0": {
"deprecation_date": "2026-07-07",
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -59370,10 +59449,11 @@
"cache_read_input_token_cost": 1.375e-06,
"output_cost_per_token": 8.25e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-cyber.html",
"use_openai_responses_path": true,
"supported_endpoints": [
"/v1/responses"
@ -59405,7 +59485,7 @@
"output_cost_per_token": 2.2e-05,
"output_cost_per_token_above_272k_tokens": 3.3e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 1000000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
@ -65561,6 +65641,40 @@
"supports_vision": false,
"source": "https://deepinfra.com/pricing"
},
"deepinfra/deepseek-ai/DeepSeek-V4.1-Flash": {
"max_tokens": 131072,
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 4.2e-07,
"cache_read_input_token_cost": 4.2e-09,
"supports_prompt_caching": true,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash"
},
"deepinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"max_tokens": 131072,
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 4.2e-07,
"cache_read_input_token_cost": 4.2e-09,
"supports_prompt_caching": true,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_reasoning": true,
"supports_vision": true,
"source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash-1017"
},
"deepinfra/openai/gpt-oss-120b-Ultra": {
"max_tokens": 131072,
"max_input_tokens": 131072,
@ -67074,6 +67188,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -67107,6 +67222,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -67344,6 +67460,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -67377,6 +67494,7 @@
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
@ -71435,6 +71553,8 @@
"input_cost_per_token_flex": 1.65e-06,
"input_cost_per_token_priority": 5.775e-06,
"litellm_provider": "bedrock_converse",
"token_count_framing": 64,
"token_count_tokenizer": "kimi_k3",
"max_input_tokens": 1000000,
"max_output_tokens": 131072,
"max_tokens": 131072,
@ -71465,6 +71585,8 @@
"input_cost_per_token_flex": 1.5e-06,
"input_cost_per_token_priority": 5.25e-06,
"litellm_provider": "bedrock_converse",
"token_count_framing": 64,
"token_count_tokenizer": "kimi_k3",
"max_input_tokens": 1000000,
"max_output_tokens": 131072,
"max_tokens": 131072,
@ -71495,6 +71617,8 @@
"input_cost_per_token_flex": 1.65e-06,
"input_cost_per_token_priority": 5.775e-06,
"litellm_provider": "bedrock_converse",
"token_count_framing": 64,
"token_count_tokenizer": "kimi_k3",
"max_input_tokens": 1000000,
"max_output_tokens": 131072,
"max_tokens": 131072,
@ -72729,6 +72853,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -72861,6 +72986,7 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_bedrock_native_structured_output": true,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -72993,6 +73119,10 @@
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "bedrock_converse",
"token_count_model": {
"model": "anthropic.claude-opus-5-5",
"region": "us-east-1"
},
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@ -73066,6 +73196,10 @@
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"au.anthropic.claude-sonnet-5-5": {
"token_count_model": {
"model": "au.anthropic.claude-opus-5-5",
"region": "ap-southeast-4"
},
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
@ -73266,6 +73400,10 @@
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "bedrock_converse",
"token_count_model": {
"model": "global.anthropic.claude-opus-5-5",
"region": "us-east-1"
},
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@ -73475,6 +73613,10 @@
"cache_read_input_token_cost": 1.1e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "bedrock_converse",
"token_count_model": {
"model": "us.anthropic.claude-opus-5-5",
"region": "us-east-1"
},
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@ -74275,6 +74417,8 @@
"input_cost_per_token_flex": 1.65e-06,
"input_cost_per_token_priority": 5.775e-06,
"litellm_provider": "bedrock_converse",
"token_count_framing": 64,
"token_count_tokenizer": "kimi_k3",
"max_input_tokens": 1000000,
"max_output_tokens": 131072,
"max_tokens": 131072,
@ -74448,6 +74592,10 @@
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "bedrock_converse",
"token_count_model": {
"model": "anthropic.claude-opus-5-5",
"region": "us-east-1"
},
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@ -74526,6 +74674,10 @@
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"au.anthropic.claude-haiku-5-5": {
"token_count_model": {
"model": "au.anthropic.claude-opus-5-5",
"region": "ap-southeast-4"
},
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
@ -74756,6 +74908,10 @@
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "bedrock_converse",
"token_count_model": {
"model": "global.anthropic.claude-opus-5-5",
"region": "us-east-1"
},
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@ -74903,6 +75059,10 @@
"cache_read_input_token_cost": 1.1e-08,
"input_cost_per_token": 1.1e-07,
"litellm_provider": "bedrock_converse",
"token_count_model": {
"model": "us.anthropic.claude-opus-5-5",
"region": "us-east-1"
},
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@ -75323,6 +75483,7 @@
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-4-argon": {
"deprecation_date": "2027-10-13",
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,
@ -75456,6 +75617,42 @@
"supports_reasoning": true,
"supports_vision": true
},
"openinfer/@oi/GLM-5.3-Flash": {
"max_tokens": 393216,
"max_input_tokens": 393216,
"max_output_tokens": 393216,
"input_cost_per_token": 1.95e-07,
"output_cost_per_token": 1.56e-06,
"litellm_provider": "openinfer",
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://api.openinfer.ai/v1/models"
},
"openinfer/@oi/Qwen3.5-27B": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
"input_cost_per_token": 1.95e-07,
"output_cost_per_token": 1.56e-06,
"litellm_provider": "openinfer",
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://api.openinfer.ai/v1/models"
},
"openinfer/@oi/Qwen3.5-9B": {
"max_tokens": 100000,
"max_input_tokens": 100000,
"max_output_tokens": 100000,
"input_cost_per_token": 5e-06,
"output_cost_per_token": 1.5e-07,
"litellm_provider": "openinfer",
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"source": "https://api.openinfer.ai/v1/models"
},
"xai/grok-voice-3.0": {
"input_cost_per_second": 0.001333333333333,
"input_cost_per_text_message": 0.004,

17
litellm/provider_endpoints_support_backup.json
View file

@ -2731,6 +2731,23 @@
"messages": true,
"responses": true
}
},
"openinfer": {
"display_name": "OpenInfer (`openinfer`)",
"url": "https://docs.litellm.ai/docs/providers/openinfer",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": false
}
}
},
"endpoints": {

75
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -862,12 +862,17 @@ def _raise_unless_oauth2_discovery_server(
def _dcr_bridge_relays_client_registration(mcp_server: MCPServer) -> bool:
"""True when a DCR-bridge server relays client registration to the upstream authorization
server instead of short-circuiting to an admin-configured OAuth client. In the relay arm the
server instead of short-circuiting to the server's own OAuth client. In the relay arm the
upstream holds each client's own registration, so the authorize and token relays pass the
client's ``client_id`` and ``redirect_uri`` through verbatim and the authorization code
returns directly to the client's redirect URI without transiting the gateway. Gateway-side
redirect trust and the ``/callback`` state relay therefore only apply to the short-circuit
arm, where the upstream only knows the gateway's own callback."""
arm, where the upstream only knows the gateway's own callback.
Authorize and token decide per request from the server's current client: once the gateway
holds its shared client (registered at ``/register`` under ``PROXY_BASE_URL``, or entered by an
admin) every caller runs the gateway-callback arm, while a server that still has none keeps
relaying, so clients registered before the shared client existed keep their own binding."""
return bool(
mcp_server.is_dcr_bridge
and mcp_server.effective_registration_url
@ -880,6 +885,27 @@ def _dcr_bridge_relays_client_registration(mcp_server: MCPServer) -> bool:
)
def _register_relays_bridge_client(mcp_server: MCPServer) -> bool:
"""``/register`` relays the caller's own registration only when the gateway cannot keep a shared
client: with ``PROXY_BASE_URL`` set it registers one client bound to its own ``/callback`` and
keeps it as the server's client, so an upstream that restricts redirect domains (Atlassian)
never sees a laptop loopback redirect. Without a database the shared client would live only in
this process, so a database-free gateway keeps relaying as well."""
return _dcr_bridge_relays_client_registration(mcp_server) and (
configured_proxy_base_url() is None or not _shared_client_store_available()
)
def _shared_client_store_available() -> bool:
from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 # avoids circular import
try:
get_prisma_client_or_throw("Database not connected. Cannot keep the shared MCP OAuth client.")
except HTTPException:
return False
return True
def _require_s256_pkce(
code_challenge: str | None,
code_challenge_method: str | None,
@ -1028,6 +1054,17 @@ async def _managed_authorization_scope(
return selected_scope, scope_user_id
async def _hydrate_bridge_shared_client(mcp_server: MCPServer, request: Request) -> None:
"""Whichever worker served ``/register`` persisted the bridge's shared gateway client; a worker
whose catalog snapshot predates that write still holds no ``client_id``, so the gateway-callback
arms load the persisted client before presenting one upstream. Callers that carry their own
client (a gateway-minted ephemeral client sealed into the state or the code) skip this, since the
upstream bound their code to that client."""
if not mcp_server.is_dcr_bridge or mcp_server.client_id:
return
await _reuse_persisted_dcr_client_if_available(mcp_server, f"{get_request_base_url(request)}/callback")
async def authorize_with_server(
request: Request,
mcp_server: MCPServer,
@ -1069,6 +1106,8 @@ async def authorize_with_server(
# calling this for its enforcement side effect, then falls through to the gateway
# /callback flow below, which reads the original code_challenge names.
bridge_challenge, bridge_method = _require_s256_pkce(code_challenge, code_challenge_method)
if ephemeral_dcr_client is None:
await _hydrate_bridge_shared_client(resolved_server, request)
# A gateway-minted ephemeral client is registered against {base}/callback, so its
# flow must run the short-circuit arm; the relay arm is only for clients that
# registered themselves through the front door and hold their own redirect binding.
@ -1233,6 +1272,7 @@ async def exchange_token_with_server(
refresh_token: str | None = None,
scope: str | None = None,
client_token_endpoint_auth_method: MCPTokenEndpointAuthMethod | None = None,
sealed_client: bool = False,
):
raise_if_not_oauth2(mcp_server)
if grant_type not in ("authorization_code", "refresh_token"):
@ -1282,11 +1322,14 @@ async def exchange_token_with_server(
# recovered from a sealed code) must authenticate the way its own registration was granted,
# not the way the server row is configured; callers that carry no method keep the row's method
# as before.
resolved_client_id: Final = resolved_server.client_id if resolved_server.client_id else client_id
resolved_client_secret: Final = resolved_server.client_secret if resolved_server.client_id else client_secret
if not sealed_client:
await _hydrate_bridge_shared_client(resolved_server, request)
server_client_wins: Final = bool(resolved_server.client_id) and not sealed_client
resolved_client_id: Final = resolved_server.client_id if server_client_wins else client_id
resolved_client_secret: Final = resolved_server.client_secret if server_client_wins else client_secret
resolved_auth_method: Final = (
resolved_server.token_endpoint_auth_method
if resolved_server.client_id
if server_client_wins
else (client_token_endpoint_auth_method or resolved_server.token_endpoint_auth_method)
)
try:
@ -1830,10 +1873,13 @@ async def _persist_dcr_client_registration(
empty and otherwise adopts whatever a concurrent registration persisted first.
The client-forwarded token modes (``true_passthrough`` / ``oauth_delegate``) are skipped
unconditionally: the caller holds the upstream token and the gateway must hold no OAuth
client identity for these servers. Persisting here would stamp ``oauth2_flow`` and a
``client_id`` onto a server whose mode promises the gateway stores nothing, making a
fresh pass-through server read as gateway-authorized.
unless the server is a DCR bridge: the caller holds the upstream token and the gateway holds
no OAuth client identity for these servers, so persisting would stamp ``oauth2_flow`` and a
``client_id`` onto a server whose mode promises the gateway stores nothing. A DCR bridge is
the exception by design: its gateway-hosted sign-in registers one shared client bound to the
gateway ``/callback`` (the same app an admin may enter by hand), and that client must survive
for every later caller's authorize and token exchange. ``oauth2_flow`` is still never stamped
on a client-forwarded row.
``redirect_uris`` records what the client is bound to so a later origin change can be
detected as a positive mismatch and trigger re-registration instead of stranding the
@ -1842,7 +1888,7 @@ async def _persist_dcr_client_registration(
``update_mcp_server`` merges credential blobs: a re-registered public client must not
inherit the previous client's secret or auth method.
"""
if mcp_server.is_client_forwarded_token:
if mcp_server.is_client_forwarded_token and not mcp_server.is_dcr_bridge:
return "skipped"
try:
@ -1910,20 +1956,23 @@ async def _persist_dcr_client_registration(
):
await _refresh_persisted_dcr_server(stored)
return "reused"
stamped_flow: Final[Literal["authorization_code"] | None] = (
None if mcp_server.is_client_forwarded_token else "authorization_code"
)
updated_row: Final = await update_mcp_server(
prisma_client=prisma_client,
data=(
UpdateMCPServerRequest(
server_id=mcp_server.server_id,
credentials=credentials,
oauth2_flow="authorization_code",
oauth2_flow=stamped_flow,
token_url=mcp_server.token_url,
)
if mcp_server.token_url
else UpdateMCPServerRequest(
server_id=mcp_server.server_id,
credentials=credentials,
oauth2_flow="authorization_code",
oauth2_flow=stamped_flow,
)
),
touched_by="mcp_oauth_dcr",
@ -2211,7 +2260,7 @@ async def register_client_with_server(
)
return dummy_return
bridge_relay: Final = _dcr_bridge_relays_client_registration(resolved_server)
bridge_relay: Final = _register_relays_bridge_client(resolved_server)
if bridge_relay and not client_redirect_uris:
raise HTTPException(
status_code=400,

3
litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py
View file

@ -248,6 +248,7 @@ class SubjectTokenRefusal(LiteLLMBaseModel):
"invalid_grant",
"invalid_client",
"invalid_scope",
"invalid_target",
"temporarily_unavailable",
]
description: str = Field(min_length=1)
@ -1527,7 +1528,7 @@ def _subject_token_refusal_response(refusal: SubjectTokenRefusal) -> Response:
return _oauth_error(503, refusal.error, refusal.description)
case "invalid_client":
return _oauth_error(401, refusal.error, refusal.description)
case "unsupported_grant_type" | "invalid_request" | "invalid_grant" | "invalid_scope":
case "unsupported_grant_type" | "invalid_request" | "invalid_grant" | "invalid_scope" | "invalid_target":
return _oauth_error(400, refusal.error, refusal.description)
case _:
assert_never(refusal.error)

44
litellm/proxy/_experimental/mcp_server/id_jag_grant.py
View file

@ -23,6 +23,7 @@ from urllib.parse import unquote_plus
import jwt
from fastapi import Request
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.models.user import LiteLLM_UserTable
@ -48,6 +49,7 @@ _INACTIVE_SUBJECT: Final = frozenset(("no_active_key", "unresolvable"))
_REPLAY_KEY_PREFIX: Final = "mcp_gateway_dcr:id_jag:"
SCOPE_MCP: Final = "litellm:mcp"
SCOPE_LLM: Final = "litellm:llm"
_RESOURCE_CLAIM_ADAPTER: Final = TypeAdapter(list[str])
SCOPE_ROUTE_GROUPS: Final = MappingProxyType(
{SCOPE_MCP: ("mcp_inference_routes",), SCOPE_LLM: ("llm_api_routes_without_mcp",)}
)
@ -65,9 +67,9 @@ class GrantedScope:
def granted_scope(claim: object, *, mcp_resource: bool) -> GrantedScope | SubjectTokenRefusal:
"""The routes the assertion buys. With ``aud`` the bare origin the ``scope`` claim decides and
no claim is the full credential. With ``aud`` the MCP resource (``{base}/mcp``) the identity
provider named that resource, so the credential is ceilinged to the MCP routes whatever the
claim says: ``litellm:llm`` cannot be granted and is refused."""
no claim is the full credential. With the MCP resource (``{base}/mcp``) as ``aud`` or as every
``resource``, the identity provider named that resource, so the credential is ceilinged to the
MCP routes whatever the claim says: ``litellm:llm`` cannot be granted and is refused."""
if claim is not None and not isinstance(claim, str):
return SubjectTokenRefusal(error="invalid_scope", description=_SCOPE_DESCRIPTION)
requested: Final = frozenset(claim.split()) if claim else frozenset()
@ -90,6 +92,32 @@ def names_mcp_resource(aud: object, base_url: str | None) -> bool:
return base_url is not None and f"{base_url}/mcp" in audiences and base_url not in audiences
def _resource_claim_values(claim: object) -> tuple[str, ...] | None:
if isinstance(claim, str):
return (claim,)
if not isinstance(claim, list):
return None
try:
values: Final = _RESOURCE_CLAIM_ADAPTER.validate_python(claim, strict=True)
except ValidationError:
return None
return tuple(values) if values else None
def resource_names_mcp(claims: Mapping[str, object], base_url: str | None) -> bool | SubjectTokenRefusal:
"""The signed RFC 8707 ``resource`` claim (a URI or an array of URIs) restricts the credential like an
``aud`` of ``{base}/mcp`` does."""
if "resource" not in claims:
return False
resources: Final = _resource_claim_values(claims["resource"])
if resources is None:
return SubjectTokenRefusal(error="invalid_target", description=_RESOURCE_DESCRIPTION)
served_resources: Final = frozenset(gateway_audiences(base_url))
if any(resource not in served_resources for resource in resources):
return SubjectTokenRefusal(error="invalid_target", description=_RESOURCE_DESCRIPTION)
return frozenset(resources) == frozenset((f"{base_url}/mcp",))
def _route_groups(scopes: Sequence[str]) -> Iterator[str]:
for scope in scopes:
yield from SCOPE_ROUTE_GROUPS[scope]
@ -98,8 +126,10 @@ def _route_groups(scopes: Sequence[str]) -> Iterator[str]:
_SCOPE_DESCRIPTION: Final = (
f"scope must be {SCOPE_MCP} (MCP gateway routes), {SCOPE_LLM} (LLM API routes), or both, space separated"
)
_RESOURCE_DESCRIPTION: Final = "the assertion's resource claim must name this gateway's origin or its /mcp resource"
_MCP_RESOURCE_DESCRIPTION: Final = (
f"the /mcp audience only grants MCP routes ({SCOPE_MCP}); use the bare origin as audience for LLM routes"
f"the /mcp audience or resource only grants MCP routes ({SCOPE_MCP}); use the bare origin as audience and "
"resource for LLM routes"
)
@ -450,8 +480,12 @@ async def identity_from_id_jag(
if isinstance(marker, SubjectTokenRefusal):
return marker
replay_key, replay_ttl_seconds = marker
mcp_resource: Final = resource_names_mcp(verified, prerequisites.base_url)
if isinstance(mcp_resource, SubjectTokenRefusal):
return mcp_resource
scope: Final = granted_scope(
verified.get("scope"), mcp_resource=names_mcp_resource(verified.get("aud"), prerequisites.base_url)
verified.get("scope"),
mcp_resource=mcp_resource or names_mcp_resource(verified.get("aud"), prerequisites.base_url),
)
if isinstance(scope, SubjectTokenRefusal):
return scope

15
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -4531,7 +4531,7 @@ class MCPServerManager:
)
else:
page: Final = (
await client.list_tools_page(params)
await self._fetch_tools_page_with_timeout(client, server, params)
if params is not None
else ListToolsResult(tools=await self._fetch_tools_with_timeout(client, server.name))
)
@ -5621,6 +5621,19 @@ class MCPServerManager:
)
raise_classified_list_failure(e, server_name)
async def _fetch_tools_page_with_timeout(
self, client: MCPClient, server: MCPServer, params: PaginatedRequestParams
) -> ListToolsResult:
deadline: Final = (
MCP_TOOL_LISTING_TIMEOUT if server.timeout is None else max(server.timeout, MCP_TOOL_LISTING_TIMEOUT)
)
try:
with anyio.fail_after(deadline):
return await client.list_tools_page(params)
except TimeoutError as e:
verbose_logger.warning("Timeout while listing tools from %s", server.name)
raise MCPServerListError(ServerListFault(tag="timeout"), server.name) from e
_SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024
def assign_unique_short_prefix(

28
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -14412,6 +14412,20 @@
"description": "Optional field if guardrail requires a 'model' parameter",
"title": "Model"
},
"num_retries": {
"anyOf": [
{
"maximum": 5.0,
"minimum": 0.0,
"type": "integer"
},
{
"type": "null"
}
],
"description": "How many more times to send a guardrail check whose connection timed out waiting for the answer or was dropped after the request went out. Each retry opens a fresh connection and gets the full timeout. Connect failures, HTTP error statuses and BLOCKED verdicts are not retried. Unset or 0 sends each check once, and the most allowed is 5. Implemented by guardrail='generic_guardrail_api', other guardrails ignore it.",
"title": "Num Retries"
},
"on_sensitive_data": {
"anyOf": [
{
@ -17001,6 +17015,20 @@
"description": "If True, logs violations without blocking. Defaults to False if not provided",
"title": "Monitor Mode"
},
"num_retries": {
"anyOf": [
{
"maximum": 5.0,
"minimum": 0.0,
"type": "integer"
},
{
"type": "null"
}
],
"description": "How many more times to send a guardrail check whose connection timed out waiting for the answer or was dropped after the request went out. Each retry opens a fresh connection and gets the full timeout. Connect failures, HTTP error statuses and BLOCKED verdicts are not retried. Unset or 0 sends each check once, and the most allowed is 5. Implemented by guardrail='generic_guardrail_api', other guardrails ignore it.",
"title": "Num Retries"
},
"on_disallowed_action": {
"default": "block",
"description": "Choose whether disallowed tools block the request or get rewritten out of the payload",

6
litellm/proxy/_types.py
View file

@ -563,6 +563,7 @@ class LiteLLMRoutes(enum.Enum):
"/team/metadata_schema",
"/user/info",
"/v2/user/info",
"/management/v1/users/me",
"/model/info",
"/v1/model/info",
"/v2/model/info",
@ -3667,6 +3668,7 @@ class SpendLogsMetadata(TypedDict):
user_api_key_project_alias: str | None
user_api_key_org_id: str | None
user_api_key_user_id: str | None
user_api_key_user_email: ReadOnly[str | None]
user_api_key_team_alias: str | None
spend_logs_metadata: dict | None # special param to log k,v pairs to spendlogs for a call
requester_ip_address: str | None
@ -4827,6 +4829,10 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
"mcp_routes",
"/v1/messages",
"/v1/messages/count_tokens",
"/rag/ingest",
"/v1/rag/ingest",
"/rag/query",
"/v1/rag/query",
]
team_id_default: str | None = Field(
default=None,

3
litellm/proxy/anthropic_endpoints/endpoints.py
View file

@ -48,6 +48,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
openai_error_type,
with_litellm_call_id,
)
from litellm.proxy.common_utils.token_count_settings import count_source_fields
from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
from litellm.router import Router
@ -420,7 +421,7 @@ async def count_tokens(
log_llm_api_exception(e, litellm_call_id)
return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
else:
return {"input_tokens": token_response.total_tokens}
return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)}
@router.post(

4
litellm/proxy/auth/auth_checks.py
View file

@ -4784,7 +4784,7 @@ def check_model_access_helper(
if _model_in_team_aliases(model=model, team_model_aliases=team_model_aliases):
return True
if _model_matches_any_wildcard_pattern_in_list(model=model, allowed_model_list=filtered_models):
if model_matches_any_wildcard_pattern_in_list(model=model, allowed_model_list=filtered_models):
return True
all_model_access: bool = False
@ -7434,7 +7434,7 @@ def is_model_allowed_by_pattern(model: str, allowed_model_pattern: str) -> bool:
return False
def _model_matches_any_wildcard_pattern_in_list(model: str, allowed_model_list: list) -> bool:
def model_matches_any_wildcard_pattern_in_list(model: str, allowed_model_list: Sequence[str]) -> bool:
"""
Returns True if a model matches any wildcard pattern in a list.

17
litellm/proxy/auth/handle_jwt.py
View file

@ -1767,6 +1767,7 @@ class JWTAuthManager:
from litellm.proxy.proxy_server import llm_router
denied_pass_through_team: LiteLLM_TeamTable | None = None
route_denied_team_id: str | None = None # rebind-ok: retain route-denied team while checking all claims
if not team_ids:
if (
@ -1780,7 +1781,7 @@ class JWTAuthManager:
return None, None
any_claim_team_resolved = False
for team_id in team_ids:
for team_id in sorted(team_ids, key=str):
try:
team_object = await get_team_object(
team_id=team_id,
@ -1809,7 +1810,9 @@ class JWTAuthManager:
user_route=route,
litellm_proxy_roles=jwt_handler.litellm_jwtauth,
)
if is_allowed and not JWTAuthManager._team_has_passthrough_route_access(
if not is_allowed:
route_denied_team_id = team_id
elif not JWTAuthManager._team_has_passthrough_route_access(
team_object=team_object,
route=route,
request_method=request_method,
@ -1832,7 +1835,13 @@ class JWTAuthManager:
# Claim resolved but no model access, or fallback disabled — deny.
raise HTTPException(
status_code=403,
detail=f"No team has access to the requested model: {requested_model}. Checked teams={team_ids}. Check `/models` to see all available models.",
detail=(
f"Team {route_denied_team_id} can access model {requested_model} but route {route} "
"is not in litellm_jwtauth.team_allowed_routes"
if route_denied_team_id is not None
else f"No team has access to the requested model: {requested_model}. "
f"Checked teams={team_ids}. Check `/models` to see all available models."
),
)
# No claim team resolved and fallback enabled — defer to fallback.
@ -2003,7 +2012,7 @@ class JWTAuthManager:
status_code=403,
detail=(
f"x-litellm-team-id '{header_value}' does not resolve to a team id or a unique team alias in your "
f"JWT's allowed teams. Allowed team ids: {sorted(allowed_team_ids)}"
f"JWT's allowed teams. Allowed team ids: {sorted(allowed_team_ids, key=str)}"
),
)

2
litellm/proxy/auth/login_utils.py
View file

@ -57,7 +57,7 @@ if TYPE_CHECKING:
from prisma import types as prisma_types
BREACH_RECHECK_INTERVAL: Final = timedelta(hours=24)
PASSWORD_RESET_ALLOWED_ROUTES: Final = ("/user/password/change", "/session/logout")
PASSWORD_RESET_ALLOWED_ROUTES: Final = ("/user/password/change", "/session/logout", "/management/v1/users/me")
PASSWORD_SESSION_METADATA: Final = MappingProxyType({"login_method": "username_password"})

25
litellm/proxy/auth/route_checks.py
View file

@ -72,14 +72,22 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spe
_LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp"))
_AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS))
# Method-scoped because the generic credential CRUD handlers share these paths:
# the route check cannot admit PATCH/DELETE on a name like "user_connections"
# or "*/user_connection*" without also opening the admin CRUD handlers.
_INTERNAL_USER_CREDENTIAL_CONNECTION_ROUTES: Final = (
# Writes that only touch the caller's own credential row (BYOK connections and
# per-user MCP credentials). Method-scoped because the generic credential CRUD
# handlers share these paths: the route check cannot admit PATCH/DELETE on a
# name like "user_connections" without also opening the admin CRUD handlers.
_SELF_SCOPED_CREDENTIAL_ROUTES: Final = (
("GET", "/credentials/user_connections"),
("POST", "/credentials/{credential_name:path}/user_connection/start"),
("POST", "/credentials/{credential_name:path}/user_connection/poll"),
("DELETE", "/credentials/{credential_name:path}/user_connection"),
("POST", "/v1/mcp/server/oauth/{server_id}/register"),
("POST", "/v1/mcp/server/{server_id}/oauth-user-credential"),
("DELETE", "/v1/mcp/server/{server_id}/oauth-user-credential"),
("POST", "/v1/mcp/server/{server_id}/user-credential"),
("DELETE", "/v1/mcp/server/{server_id}/user-credential"),
("POST", "/v1/mcp/server/{server_id}/user-env-vars"),
("DELETE", "/v1/mcp/server/{server_id}/user-env-vars"),
)
@ -349,7 +357,7 @@ class RouteChecks:
_user_role == LitellmUserRoles.INTERNAL_USER.value
and (
RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.internal_user_routes.value)
or RouteChecks.is_internal_user_credential_connection_route(route=route, request=request)
or RouteChecks.is_self_scoped_credential_route(route=route, request=request)
)
or user_is_org_admin(request_data=request_data, user_object=user_obj)
and RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.org_admin_allowed_routes.value)
@ -702,13 +710,13 @@ class RouteChecks:
_get_request_method = get_request_method
@staticmethod
def is_internal_user_credential_connection_route(route: str, request: Request | None) -> bool:
def is_self_scoped_credential_route(route: str, request: Request | None) -> bool:
method: Final = RouteChecks.get_request_method(request)
if method is None:
return False
return any(
method == expected_method and RouteChecks.check_route_access(route=route, allowed_routes=(pattern,))
for expected_method, pattern in _INTERNAL_USER_CREDENTIAL_CONNECTION_ROUTES
for expected_method, pattern in _SELF_SCOPED_CREDENTIAL_ROUTES
)
@staticmethod
@ -1033,6 +1041,9 @@ class RouteChecks:
if route == "/session/logout":
return
if RouteChecks.is_self_scoped_credential_route(route=route, request=request):
return
# Hard-block known write routes regardless of HTTP method (defensive
# — these are POSTs in practice, but pinning them here protects
# against future GET-shaped writes).

9
litellm/proxy/batches_endpoints/endpoints.py
View file

@ -149,6 +149,14 @@ async def _raise_when_input_file_must_be_managed(model: str, credentials: Mappin
)
def _raise_when_routing_headers_disagree(request: Request, body_model: object) -> None:
if (isinstance(body_model, str) and body_model) or request.query_params.get("model"):
return
if len(frozenset(request.headers.getlist("x-litellm-model"))) < 2:
return
raise batch_error(400, "Send one x-litellm-model header. This request has several naming different models")
def _litellm_metadata_of(data: MutableMapping[str, object]) -> MutableMapping[str, object]:
"""The request's litellm_metadata mapping, created on the request when it carries none.
@ -274,6 +282,7 @@ async def create_batch(
data: dict = {}
try:
data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line
_raise_when_routing_headers_disagree(request, data.get("model"))
verbose_proxy_logger.debug(
"Request received by LiteLLM:\n%s",
json.dumps(data, indent=4),

3
litellm/proxy/client/__init__.py
View file

@ -1,6 +1,6 @@
from .chat import ChatClient
from .client import Client
from .exceptions import UnauthorizedError
from .exceptions import StreamError, UnauthorizedError
from .health import HealthManagementClient
from .model_groups import ModelGroupsManagementClient
from .models import ModelsManagementClient
@ -12,6 +12,7 @@ __all__ = [
"HealthManagementClient",
"ModelGroupsManagementClient",
"ModelsManagementClient",
"StreamError",
"UnauthorizedError",
"UsersManagementClient",
]

14
litellm/proxy/client/chat.py
View file

@ -5,13 +5,21 @@ from typing import Any, Final
import requests
from pydantic import ConfigDict, TypeAdapter
from .exceptions import UnauthorizedError
from .exceptions import StreamError, UnauthorizedError
_SSE_LINE: Final[TypeAdapter[bytes | bytearray]] = TypeAdapter(
bytes | bytearray, config=ConfigDict(arbitrary_types_allowed=True, strict=True, hide_input_in_errors=True)
)
def _raise_if_error_frame(chunk: object) -> None:
match chunk:
case {"error": object() as error} if error:
raise StreamError(error)
case _:
return
class ChatClient:
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 600):
"""
@ -143,6 +151,7 @@ class ChatClient:
Raises:
UnauthorizedError: If the request fails with a 401 status code
StreamError: If the server ends the stream with an error event
requests.exceptions.RequestException: If the request fails with any other error
"""
url: Final = f"{self._base_url}/chat/completions"
@ -184,9 +193,10 @@ class ChatClient:
break
try:
chunk = json.loads(data_str)
yield chunk
except json.JSONDecodeError:
continue
_raise_if_error_frame(chunk)
yield chunk
except requests.exceptions.HTTPError as e:
if e.response.status_code == 401:

12
litellm/proxy/client/cli/commands/chat.py
View file

@ -409,10 +409,16 @@ def _stream_response(
console.print(f"\n[red]Error: HTTP {e.response.status_code}[/red]")
try:
error_body: Final[_ErrorBodyView] = {"body": e.response.json()}
console.print(f"[red]{error_body['body'].get('error', {}).get('message', 'Unknown error')}[/red]")
console.print(
error_body["body"].get("error", {}).get("message", "Unknown error"),
style="red",
markup=False,
highlight=False,
emoji=False,
)
except json.JSONDecodeError:
console.print(f"[red]{e.response.text}[/red]")
console.print(e.response.text, style="red", markup=False, highlight=False, emoji=False)
return None
except Exception as e:
console.print(f"\n[red]Error: {e}[/red]")
console.print(f"\nError: {e}", style="red", markup=False, highlight=False, emoji=False)
return None

16
litellm/proxy/client/exceptions.py
View file

@ -25,3 +25,19 @@ class NotFoundError(Exception):
def __init__(self, orig_exception: requests.exceptions.HTTPError | str):
self.orig_exception = _redact_orig_exception(orig_exception)
super().__init__(str(self.orig_exception))
def _stream_error_message(error: object) -> str:
match error:
case {"message": str(message)}:
return message
case _:
return str(error)
class StreamError(Exception):
"""Exception raised when the server ends a stream with an error event."""
def __init__(self, error: object) -> None:
self.error = error
super().__init__(redact_string(_stream_error_message(error)))

52
litellm/proxy/common_request_processing.py
View file

@ -636,7 +636,7 @@ def serialize_http_exception_detail(
msg: Final = detail.get("message")
if isinstance(msg, str):
return msg, detail
return json.dumps(detail), detail
return json.dumps(detail, default=str), detail
return str(detail), None
@ -1117,7 +1117,7 @@ def sse_error_payload(exc: BaseException) -> tuple[int, Mapping[str, object]]:
def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]:
"""The two frames an SSE stream ends with once it can no longer raise."""
return f"data: {json.dumps({'error': error_obj})}\n\n", "data: [DONE]\n\n"
return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n"
def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
@ -1663,19 +1663,26 @@ def _totals_to_zero(response_cost: float | str | None) -> bool:
return False
def _is_cache_hit(litellm_logging_obj: LiteLLMLoggingObj) -> bool:
caching_details: Final = litellm_logging_obj.caching_details
return caching_details is not None and caching_details.get("cache_hit") is True
def _get_cost_breakdown_from_logging_obj(
litellm_logging_obj: LiteLLMLoggingObj | None,
response_cost: float | str | None = None,
) -> CostBreakdownHeaderValues:
"""Extract discount, margin, and per-component cost information from logging object's cost breakdown.
A non-inference call that priced at zero never records a breakdown, so its components are
reported as zero here. Any such call that did price normally (retrieving a background response,
and the cost poller's read of one) reports the breakdown it stored, or nothing at all when the
breakdown has not landed yet.
A cache hit reports every component as zero. A non-inference call that priced at zero never
records a breakdown, so its components are reported as zero too. Any such call that did price
normally (retrieving a background response, and the cost poller's read of one) reports the
breakdown it stored, or nothing at all when the breakdown has not landed yet.
"""
if not litellm_logging_obj or not hasattr(litellm_logging_obj, "cost_breakdown"):
return CostBreakdownHeaderValues()
if _is_cache_hit(litellm_logging_obj):
return _ZERO_COST_BREAKDOWN
cost_breakdown: Final = litellm_logging_obj.cost_breakdown
if not cost_breakdown:
@ -1705,6 +1712,13 @@ def _header_value(value: float | None) -> str | None:
return str(value) if value is not None else None
def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | str:
if hidden_params.get("cache_hit") is True:
return 0.0
response_cost: Final = hidden_params.get("response_cost")
return response_cost if isinstance(response_cost, (int, float, str)) and response_cost else ""
def cost_headers(
response_cost: float | str | None, litellm_logging_obj: LiteLLMLoggingObj | None
) -> Mapping[str, str | None]:
@ -2084,18 +2098,12 @@ class ProxyBaseLLMRequestProcessing:
Build LiteLLM proxy response headers for routes that call the LLM directly
(e.g. Google native :generateContent) instead of base_process_llm_request.
"""
if isinstance(response, dict):
hidden_params = get_hidden_params_dict(response)
else:
hidden_params = getattr(response, "_hidden_params", None) or {}
if not isinstance(hidden_params, dict):
hidden_params = {}
hidden_params: Final = get_hidden_params_dict(response)
model_id: Final = ProxyBaseLLMRequestProcessing._get_model_id_from_response(hidden_params, request_data)
cache_key: Final = hidden_params.get("cache_key", None) or ""
api_base: Final = hidden_params.get("api_base", None) or ""
response_cost: Final = hidden_params.get("response_cost", None) or ""
response_cost: Final = _response_cost_header_value(hidden_params)
fastest_response_batch_completion: Final = hidden_params.get("fastest_response_batch_completion", None)
additional_headers: Final = hidden_params.get("additional_headers", {}) or {}
@ -2640,7 +2648,7 @@ class ProxyBaseLLMRequestProcessing:
cache_key=hidden_params.get("cache_key") or "",
api_base=hidden_params.get("api_base") or "",
version=version,
response_cost=hidden_params.get("response_cost") or "",
response_cost=_response_cost_header_value(hidden_params),
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
fastest_response_batch_completion=hidden_params.get("fastest_response_batch_completion"),
request_data=self.data,
@ -2685,13 +2693,17 @@ class ProxyBaseLLMRequestProcessing:
hold the attribute at all, and Google :generateContent carries
``_hidden_params`` but no synchronously-populated ``response_cost``. In both
cases the cost is read back from the logging object instead, recomputing from
the same calculator only when it has not been stored yet.
the same calculator only when it has not been stored yet. A response served
from the cache costs 0, which the calculator cannot tell from a fresh call, and any
other zero or unknown cost comes back empty so no cost header is sent.
"""
if _is_cache_hit(logging_obj):
return 0.0
stored_cost: Final = logging_obj.model_call_details.get("response_cost")
if isinstance(stored_cost, (int, float)):
return float(stored_cost)
return float(stored_cost) or ""
recomputed_cost: Final = logging_obj.response_cost_calculator(result=response)
return recomputed_cost if isinstance(recomputed_cost, (int, float)) else ""
return recomputed_cost or "" if isinstance(recomputed_cost, (int, float)) else ""
def _debug_log_request_payload(self) -> None:
"""Log request payload at DEBUG level, truncating if too large."""
@ -2900,7 +2912,7 @@ class ProxyBaseLLMRequestProcessing:
cache_key, api_base, response_cost = (
hidden_params.get("cache_key", None) or "",
hidden_params.get("api_base", None) or "",
hidden_params.get("response_cost", None) or "",
_response_cost_header_value(hidden_params),
)
fastest_response_batch_completion, additional_headers = (
hidden_params.get("fastest_response_batch_completion", None),
@ -3170,7 +3182,7 @@ class ProxyBaseLLMRequestProcessing:
recover_response_cost: Final = not response_cost and hidden_params.get("response_cost") is None
computed_cost_for_headers: Final = (
self._response_cost_from_logging_obj(response=response, logging_obj=logging_obj) or ""
self._response_cost_from_logging_obj(response=response, logging_obj=logging_obj)
if recover_response_cost
else response_cost
)

Some files were not shown because too many files have changed in this diff Show more