scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 128

Merge branch 'main' into litellm_langfuse_otel_metadata_keys

Browse source
This commit is contained in:
yucheng 2026-10-19 00:54:00 -07:00
commit 0df6eb1739
767 changed files with 49837 additions and 4658 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
.github/e2e-stack/assert_tests_ran.py vendored
View file

@ -1,69 +0,0 @@
import os
import re
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Final
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
from coverage_registry.management_cases import MANAGEMENT_CASES
def main() -> int:
selected: Final = tuple(sys.argv[2:])
try:
report: Final = ET.parse(Path(sys.argv[1])).getroot()
except (ET.ParseError, OSError):
_ = sys.stdout.write("::error::could not read the test execution report\n")
return 1
cases: Final = tuple(report.iter("testcase"))
expected_count: Final = os.environ.get("E2E_REQUIRED_TEST_COUNT")
passed: Final = frozenset(
case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
)
missing: Final = tuple(path for path in selected if path not in passed)
required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
passed_nodes: Final = frozenset(
prop.get("value")
for case in cases
if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
for prop in case.findall("./properties/property")
if prop.get("name") == "management_node"
)
missing_nodes: Final = required_nodes - passed_nodes
for node in sorted(missing_nodes):
_ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
for path in selected:
collected: Final = sum(case.get("file") == path for case in cases)
skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
_ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
for case in cases:
if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error", "skipped")):
continue
outcome = "skipped" if case.find("skipped") is not None else "failed"
_ = sys.stdout.write(f" {outcome}: {case.get('classname', '')}::{case.get('name', '')}\n")
for prop in case.findall("./properties/property"):
name = prop.get("name", "")
value = prop.get("value", "")
if name in ("oauth_failure_phase", "oauth_exception_type", "oauth_frame") and re.fullmatch(
r"[A-Za-z0-9_.:<>-]{1,240}", value
):
_ = sys.stdout.write(f" {name}: {value}\n")
if expected_count is not None and (
len(cases) != int(expected_count) or any(case.find("skipped") is not None for case in cases)
):
_ = sys.stdout.write("::error::required test count was not met or a required case was skipped\n")
return 1
if (
selected
and not missing
and not missing_nodes
and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
):
return 0
_ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
return 1
if __name__ == "__main__":
sys.exit(main())

4
.github/workflows/test-linting.yml vendored
View file

@ -539,8 +539,8 @@ jobs:
- name: test_unit_passed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
- name: test_e2e_idp_stack
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_idp_stack.py
- name: test_e2e_metadata
env:

180
.github/workflows/test-mcp-oauth-e2e.yml vendored
View file

@ -1,180 +0,0 @@
name: MCP OAuth happy path
on:
pull_request:
paths:
- '.github/workflows/test-mcp-oauth-e2e.yml'
- '.github/e2e-stack/**'
- 'tests/e2e/*.py'
- 'tests/e2e/pytest.ini'
- 'tests/e2e/idp_realm.json'
- 'tests/e2e/mcp/**'
- 'litellm/experimental_mcp_client/**'
- 'litellm/proxy/_experimental/mcp_server/**'
- 'litellm/proxy/auth/**'
- 'litellm/proxy/management_endpoints/mcp_management_endpoints.py'
- 'litellm/proxy/_types.py'
- 'litellm/types/mcp_server/mcp_server_manager.py'
- 'litellm/proxy/management_endpoints/*sso*.py'
- 'litellm/proxy/management_endpoints/sso/**'
- 'litellm/proxy/common_utils/encrypt_decrypt_utils.py'
- 'litellm/proxy/proxy_server.py'
- 'litellm/proxy/schema.prisma'
- 'ui/litellm-dashboard/src/app/connect/**'
- 'ui/litellm-dashboard/src/app/mcp/oauth/**'
- 'pyproject.toml'
- 'uv.lock'
workflow_dispatch:
permissions: {}
concurrency:
group: mcp-oauth-${{ github.ref }}
cancel-in-progress: true
jobs:
oauth:
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
environment: e2e-changed
timeout-minutes: 45
permissions:
contents: read
id-token: write
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: litellm
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U litellm"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_HOST: 127.0.0.1
DATABASE_PORT: '5432'
DATABASE_USER: litellm
DATABASE_PASSWORD: dbpassword9090
DATABASE_NAME: litellm
DATABASE_URL: postgresql://litellm:dbpassword9090@127.0.0.1:5432/litellm
E2E_KEYCLOAK_URL: http://127.0.0.1:8081
E2E_KEYCLOAK_ADMIN_USER: admin
E2E_KEYCLOAK_ADMIN_PASSWORD: e2e-ephemeral-idp-not-a-secret
E2E_FIXTURE_MODE: live
E2E_PROVIDER_CACHE: '0'
E2E_MCP_OAUTH_LIVE: '1'
E2E_REQUIRED_TEST_COUNT: '4'
steps:
- name: Checkout the tested source
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
persist-credentials: false
- name: Require and materialize the upstream login
env:
STORAGE_STATE: ${{ secrets.E2E_LINEAR_STORAGE_STATE_B64 }}
run: |
umask 077
python3 - <<'PY'
import base64
import json
import os
import secrets
from pathlib import Path
encoded = os.environ.get("STORAGE_STATE", "")
if not encoded:
raise SystemExit("E2E_LINEAR_STORAGE_STATE_B64 is required; capture and provision a test-account login")
state = json.loads(base64.b64decode(encoded, validate=True))
if not isinstance(state, dict) or not state.get("cookies"):
raise SystemExit("The captured login must contain browser cookies")
directory = Path(os.environ["RUNNER_TEMP"]) / "mcp-oauth-private"
directory.mkdir(mode=0o700)
path = directory / "linear-state.json"
path.write_text(json.dumps(state))
with open(os.environ["GITHUB_ENV"], "a") as output:
output.write(f"E2E_LINEAR_STORAGE_STATE={path}\n")
for name in ("LITELLM_MASTER_KEY", "LITELLM_SALT_KEY"):
value = "sk-e2e-" + secrets.token_hex(24)
print(f"::add-mask::{value}")
output.write(f"{name}={value}\n")
PY
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.13'
- uses: ./.github/actions/setup-uv-with-retries
with:
version: '0.10.9'
- uses: ./.github/actions/cache-cargo-build
- name: Install the frozen E2E environment
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra proxy-runtime --extra extra_proxy --group ci --group proxy-dev --group e2e-dev
uv run --no-sync python scripts/prisma_generate_if_needed.py
uv run --no-sync playwright install --with-deps chromium
- name: Configure license access
id: aws
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
with:
role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
aws-region: us-east-1
role-session-name: mcp-oauth-${{ github.run_id }}
role-duration-seconds: 900
output-env-credentials: false
output-credentials: true
- name: Load the E2E license
env:
AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
AWS_DEFAULT_REGION: us-east-1
run: |
license="$(aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license --query SecretString --output text)"
test -n "${license}"
echo "::add-mask::${license}"
echo "LITELLM_LICENSE=${license}" >> "${GITHUB_ENV}"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
- name: Build the gateway consent UI at the tested commit
run: |
cd ui/litellm-dashboard
../../scripts/with_dashboard_node.sh npm ci
../../scripts/with_dashboard_node.sh npm run build
mkdir -p ../../litellm/proxy/_experimental/out
cp -r out/. ../../litellm/proxy/_experimental/out/
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name index.html | while read -r page; do
mkdir -p "${page%.html}"
mv "${page}" "${page%.html}/index.html"
done
- name: Prepare the isolated database and IdP
run: |
umask 077
bash .github/e2e-stack/start-idp.sh
uv run --no-sync python migrations/run.py > "${RUNNER_TEMP}/mcp-oauth-private/migrations.log" 2>&1
- name: Run every required OAuth variant without retries
run: |
umask 077
uv run --no-sync pytest -c tests/e2e/pytest.ini tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
--rootdir=. --reruns 0 --tb=short -o junit_family=xunit1 \
--junitxml="${RUNNER_TEMP}/mcp-oauth-private/results.xml" \
> "${RUNNER_TEMP}/mcp-oauth-private/pytest.log" 2>&1
- name: Report JUnit results and reject skipped or missing cases
if: always()
run: |
uv run --no-sync python .github/e2e-stack/assert_tests_ran.py \
"${RUNNER_TEMP}/mcp-oauth-private/results.xml" tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py
- name: Remove private login and logs
if: always()
run: |
docker rm -f e2e-keycloak >/dev/null 2>&1 || true
rm -rf "${RUNNER_TEMP}/mcp-oauth-private"

1
.github/workflows/test-unit.yml vendored
View file

@ -658,6 +658,7 @@ jobs:
- shard: proxy-extras
test-path: >-
tests/unit/litellm_proxy_extras
tests/unit/db_scripts
python-version: "3.12"
workers: 2
reruns: 0

1
backend/routes/allowlist.py
View file

@ -165,6 +165,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
{
"/admin",
"/metrics",
"/swagger", # API documentation static assets belong to the backend
"/mcp", # lazily-mounted MCP sub-app serves on the backend component
}

201
db_scripts/strip_vector_store_internal_metadata.py Normal file
View file

@ -0,0 +1,201 @@
import argparse
import os
import sys
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from itertools import accumulate, repeat, takewhile
from typing import Final, Literal, TextIO
import httpx
from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
from litellm.vector_stores.utils import is_litellm_written_metadata_key
DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
LIST_PAGE_SIZE: Final = "100"
class VectorStore(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
id: str
metadata: dict[str, JsonValue] | None = None
class VectorStorePage(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
data: tuple[VectorStore, ...]
has_more: bool = False
class OpenAIErrorBody(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
message: str = ""
class OpenAIErrorResponse(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
error: OpenAIErrorBody
@dataclass(frozen=True, slots=True)
class Options:
api_key: str
api_base: str
vector_store_ids: tuple[str, ...]
extra_keys: frozenset[str]
apply: bool
@dataclass(frozen=True, slots=True)
class StoreResult:
store_id: str
matched_keys: tuple[str, ...]
status: Literal["dry_run", "updated", "failed"]
error: str = ""
def is_proxy_written_key(key: str, extra_keys: frozenset[str]) -> bool:
return is_litellm_written_metadata_key(key) or key in extra_keys
def matching_keys(store: VectorStore, extra_keys: frozenset[str]) -> tuple[str, ...]:
return tuple(sorted(key for key in (store.metadata or {}) if is_proxy_written_key(key, extra_keys)))
def fetch_page(client: httpx.Client, after: str | None) -> VectorStorePage:
params: Final = {"limit": LIST_PAGE_SIZE} if after is None else {"limit": LIST_PAGE_SIZE, "after": after}
return VectorStorePage.model_validate_json(client.get("/vector_stores", params=params).raise_for_status().content)
def fetch_next_page(client: httpx.Client, page: VectorStorePage | None) -> VectorStorePage | None:
if page is None or not page.has_more or not page.data:
return None
return fetch_page(client, after=page.data[-1].id)
def list_all_stores(client: httpx.Client) -> Iterator[VectorStore]:
pages: Final = accumulate(
repeat(None), lambda page, _: fetch_next_page(client, page), initial=fetch_page(client, after=None)
)
for page in takewhile(lambda page: page is not None, pages):
yield from page.data
def retrieve_stores(client: httpx.Client, store_ids: tuple[str, ...]) -> Iterator[VectorStore]:
for store_id in store_ids:
yield VectorStore.model_validate_json(client.get(f"/vector_stores/{store_id}").raise_for_status().content)
def describe_http_error(response: httpx.Response) -> str:
try:
message: Final = OpenAIErrorResponse.model_validate_json(response.content).error.message
except ValidationError:
return f"HTTP {response.status_code}"
return f"HTTP {response.status_code}: {message}"
def delete_keys(client: httpx.Client, store_id: str, keys: tuple[str, ...]) -> StoreResult:
try:
response: Final = client.post(f"/vector_stores/{store_id}", json={"metadata": dict.fromkeys(keys)})
except httpx.TransportError as error:
return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=type(error).__name__)
if response.is_success:
return StoreResult(store_id=store_id, matched_keys=keys, status="updated")
return StoreResult(store_id=store_id, matched_keys=keys, status="failed", error=describe_http_error(response))
def clean_store(client: httpx.Client, store: VectorStore, options: Options) -> StoreResult | None:
keys: Final = matching_keys(store, options.extra_keys)
if not keys:
return None
if not options.apply:
return StoreResult(store_id=store.id, matched_keys=keys, status="dry_run")
return delete_keys(client, store.id, keys)
def format_result(result: StoreResult) -> str:
key_list: Final = ", ".join(result.matched_keys)
count: Final = len(result.matched_keys)
match result.status:
case "dry_run":
return f"{result.store_id}: would delete {count} keys: {key_list}"
case "updated":
return f"{result.store_id}: deleted {count} keys: {key_list}"
case "failed":
return f"{result.store_id}: failed to delete {count} keys ({result.error}): {key_list}"
def clean_and_report(
client: httpx.Client, stores: Iterator[VectorStore], options: Options, out: TextIO
) -> Iterator[StoreResult | None]:
for store in stores:
result: Final = clean_store(client, store, options)
if result is not None:
print(format_result(result), file=out)
yield result
def run(options: Options, client: httpx.Client, out: TextIO) -> int:
stores: Final = (
retrieve_stores(client, options.vector_store_ids) if options.vector_store_ids else list_all_stores(client)
)
scanned: Final = tuple(clean_and_report(client, stores, options, out))
results: Final = tuple(result for result in scanned if result is not None)
updated: Final = sum(1 for result in results if result.status == "updated")
failed: Final = sum(1 for result in results if result.status == "failed")
mode: Final = "applied" if options.apply else "dry run, pass --apply to delete"
print(
f"scanned={len(scanned)} matched={len(results)} updated={updated} failed={failed} ({mode})",
file=out,
)
return 1 if failed else 0
def parse_options(argv: Sequence[str] | None) -> Options:
parser: Final = argparse.ArgumentParser(
description="Remove LiteLLM proxy-written keys from OpenAI vector store metadata. Dry run by default."
)
parser.add_argument("--api-key", default=os.environ.get("OPENAI_API_KEY"), help="defaults to $OPENAI_API_KEY")
parser.add_argument("--api-base", default=DEFAULT_API_BASE)
parser.add_argument("--vector-store-id", action="append", default=[], help="repeatable, defaults to every store")
parser.add_argument("--extra-key", action="append", default=[], help="repeatable, another key name to delete")
parser.add_argument("--apply", action="store_true", help="delete the matching keys instead of listing them")
args: Final = parser.parse_args(argv)
if not args.api_key:
parser.error("--api-key or OPENAI_API_KEY is required")
return Options(
api_key=args.api_key,
api_base=args.api_base.rstrip("/"),
vector_store_ids=tuple(args.vector_store_id),
extra_keys=frozenset(args.extra_key),
apply=args.apply,
)
def main(argv: Sequence[str] | None = None, transport: httpx.BaseTransport | None = None) -> int:
options: Final = parse_options(argv)
with httpx.Client(
base_url=options.api_base,
headers={"Authorization": f"Bearer {options.api_key}"},
timeout=60.0,
transport=transport,
) as client:
try:
return run(options, client, sys.stdout)
except httpx.HTTPStatusError as error:
sys.stderr.write(f"could not read vector stores: {describe_http_error(error.response)}\n")
return 2
except httpx.TransportError as error:
sys.stderr.write(f"could not reach {options.api_base}: {type(error).__name__}\n")
return 2
except ValidationError:
sys.stderr.write(f"unexpected vector store response shape from {options.api_base}\n")
return 2
if __name__ == "__main__":
sys.exit(main())

71
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -3,6 +3,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
"""
import asyncio
import json
from collections.abc import Mapping, Sequence
from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta, timezone
@ -227,7 +228,7 @@ class CheckBatchCost:
"so it will no longer be polled"
)
async def _claim_job_for_costing(self, job: "ManagedObjectRow") -> bool:
async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool:
"""
Atomically flip batch_processed from false to true, returning whether this pod won
the row. Every pod and uvicorn worker schedules its own poller against the shared
@ -241,10 +242,16 @@ class CheckBatchCost:
"""
if not self._has_batch_processed_column:
return True
from litellm_enterprise.proxy.hooks.managed_files import postgres_storable_json
try:
claimed: Final = await _managed_object_table(self.prisma_client).update_many(
where={"id": job.id, "batch_processed": False},
data={"batch_processed": True},
data={
"batch_processed": True,
"status": response.status if response.status != "completed" else "complete",
"file_object": postgres_storable_json(response),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
@ -253,7 +260,9 @@ class CheckBatchCost:
return False
return claimed > 0
async def _release_job_claim(self, job: "ManagedObjectRow") -> None:
async def _release_job_claim(
self, job: "ManagedObjectRow", original_status: str | None, original_file_object: object
) -> None:
"""Give a claimed row back once billing it failed, so a later poll cycle retries it.
Safe to match on batch_processed=True: while this poller is active the retrieve
@ -265,7 +274,11 @@ class CheckBatchCost:
try:
await _managed_object_table(self.prisma_client).update_many(
where={"id": job.id, "batch_processed": True},
data={"batch_processed": False},
data={
"batch_processed": False,
"status": original_status,
"file_object": json.dumps(original_file_object),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
@ -564,8 +577,6 @@ class CheckBatchCost:
@staticmethod
def _get_input_file_id(job: "ManagedObjectRow") -> Optional[str]:
import json
from litellm.types.utils import LiteLLMBatch
file_object: Final = job.file_object
@ -718,8 +729,7 @@ class CheckBatchCost:
)
except Exception as _e:
verbose_proxy_logger.warning(
f"CheckBatchCost: failed to create managed file ID for "
f"{_file_attr}={_raw_file_id!r}: {_e}"
f"CheckBatchCost: failed to create managed file ID for {_file_attr}={_raw_file_id!r}: {_e}"
)
# Pass the deployment's router-registered pricing (litellm_params custom
@ -755,8 +765,9 @@ class CheckBatchCost:
failed_requests=output_file_result.failed_requests + error_file_failed_requests,
)
)
logging_model: Final = batch_result.models[0] if batch_result.models else litellm_model_name or model_id
logging_obj = LiteLLMLogging(
model=batch_result.models[0],
model=logging_model,
messages=[{"role": "user", "content": "<retrieve_batch>"}],
stream=False,
call_type="aretrieve_batch",
@ -787,7 +798,9 @@ class CheckBatchCost:
custom_llm_provider=str(llm_provider) if llm_provider else None,
)
if not await self._claim_job_for_costing(job):
original_status: Final = job.status
original_file_object: Final = job.file_object
if not await self._claim_job_for_costing(job, response.model_copy(update={"usage": batch_result.usage})):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
"in this window, so its cost is already being tracked there"
@ -806,7 +819,7 @@ class CheckBatchCost:
batch_completion_cost=batch_result.completion_cost,
)
except Exception:
await self._release_job_claim(job)
await self._release_job_claim(job, original_status, original_file_object)
raise
# Record batch duration (completed_at - created_at)
@ -1099,10 +1112,7 @@ class CheckBatchCost:
continue
## RETRIEVE THE BATCH JOB OUTPUT FILE
if (
response.status in PROVIDER_TERMINAL_BATCH_STATUSES
and response.output_file_id is not None
):
if response.status in PROVIDER_TERMINAL_BATCH_STATUSES and response.output_file_id is not None:
try:
tracked = await self._track_completed_batch_cost(
job=job,
@ -1133,24 +1143,19 @@ class CheckBatchCost:
# Track this job for the final metrics summary
processed_models.append(tracked)
# mark the job as complete
try:
update_data: dict = {
"status": response.status if response.status != "completed" else "complete",
"file_object": response.model_dump_json(),
}
if self._has_batch_processed_column:
update_data["batch_processed"] = True
await _managed_object_table(self.prisma_client).update(
where={"id": job.id},
data=update_data,
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
if not self._has_batch_processed_column:
# nothing claimed the row, so it stays in the poll page and its lines wait for that retry
if not self._has_batch_processed_column:
try:
await _managed_object_table(self.prisma_client).update(
where={"id": job.id},
data={
"status": response.status if response.status != "completed" else "complete",
"file_object": response.model_dump_json(),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
continue
self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)

8
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -416,7 +416,7 @@ def _storable_json_value(value: JsonValue) -> JsonValue:
return value
def _postgres_storable_json(file_object: BaseModel) -> str:
def postgres_storable_json(file_object: BaseModel) -> str:
dumped: Final = file_object.model_dump_json()
if "\\u0000" not in dumped:
return dumped
@ -650,7 +650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
# FIX: Update status and file_object on every operation to keep state in sync
update_columns: Final = {
"file_object": _postgres_storable_json(stored_object),
"file_object": postgres_storable_json(stored_object),
"status": file_object.status,
"updated_by": user_api_key_dict.user_id,
}
@ -666,7 +666,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
data={
"create": {
"unified_object_id": unified_object_id,
"file_object": _postgres_storable_json(stored_object),
"file_object": postgres_storable_json(stored_object),
"model_object_id": model_object_id,
"file_purpose": file_purpose,
"created_by": resolve_resource_owner_id(user_api_key_dict),
@ -702,7 +702,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"unified_object_id": response_id,
"model_object_id": decrypted_response_id(response_id),
"file_purpose": "response",
"file_object": _postgres_storable_json(_without_model_output(response)),
"file_object": postgres_storable_json(_without_model_output(response)),
"status": PENDING_RESPONSE_STATUSES[0],
"created_by": resolve_resource_owner_id(user_api_key_dict),
"team_id": user_api_key_dict.team_id,

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.77"
version = "0.1.78"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.77"
version = "0.1.78"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.109"
version = "0.4.110"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.109"
version = "0.4.110"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

1
litellm-rust/Cargo.lock generated
View file

@ -4083,6 +4083,7 @@ dependencies = [
name = "litellm-inference-responses"
version = "0.1.0"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-util",
"litellm-auth",

1
litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs
View file

@ -94,6 +94,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
"docker_model_runner",
"ragflow",
"pinstripes",
"openinfer",
"darkbloom",
"meta",
"cognition",

71
litellm-rust/crates/inference-messages/tests/messages/request.rs
View file

@ -343,6 +343,77 @@ async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
assert_eq!(sent, expected);
}
#[rstest]
#[case::anthropic("anthropic", &[AnthropicBeta::DangerousToolUse20260903])]
#[case::deepseek("deepseek", &[])]
#[case::minimax("minimax", &[])]
#[tokio::test]
async fn safeguards_add_the_dangerous_tool_use_beta_only_where_the_beta_map_has_a_column(
call: MessagesCall,
#[case] provider: &str,
#[case] derived: &[AnthropicBeta],
) {
let upstream = upstream([message_response()]).await;
let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
..call
},
json!({"safeguards": safeguards}),
))
.await;
let request = only_request(&upstream).await;
let expected: BetaSet = derived
.iter()
.cloned()
.chain([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
.collect();
assert_eq!(sent_betas(&request), expected);
assert_eq!(
request.body_json::<Value>().unwrap()["safeguards"],
safeguards
);
}
#[rstest]
#[tokio::test]
async fn null_safeguards_are_dropped_instead_of_sent_without_their_beta(
call: MessagesCall,
#[values("anthropic", "azure_ai", "bedrock_mantle", "deepseek", "minimax")] provider: &str,
) {
let upstream = upstream([message_response()]).await;
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
..call
},
json!({"safeguards": null}),
))
.await;
let request = only_request(&upstream).await;
assert_eq!(
sent_betas(&request),
BetaSet::from_iter([AnthropicBeta::Other("caller-beta-2025-01-01".to_string())])
);
assert!(
!request
.body_json::<Map<String, Value>>()
.unwrap()
.contains_key("safeguards")
);
}
#[rstest]
#[tokio::test]
async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: MessagesCall) {

1
litellm-rust/crates/inference-responses/AGENTS.md
View file

@ -2,3 +2,4 @@
- This crate owns Responses API call orchestration. Payloads belong in `litellm-llms-types`, transformations in `llms/src/base_llm/responses` and `llms/src/<provider>/responses`
- The HTTP route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks, cached through `litellm_inference::caching::execute_streaming`
- WebSocket sessions (`src/websocket.rs`) stay separate from the HTTP call driver because a connection can accept multiple requests while receiving events
- `src/ids.rs` owns the LiteLLM-managed ids handed back to callers (`resp_`, `encitem_`, `litellm_enc:`, `cntr_`) and must stay fully compatible with `litellm/responses/utils.py`

1
litellm-rust/crates/inference-responses/Cargo.toml
View file

@ -6,6 +6,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
bytes.workspace = true
futures-util.workspace = true
litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }

499
litellm-rust/crates/inference-responses/src/ids.rs Normal file
View file

@ -0,0 +1,499 @@
//! LiteLLM-managed identifiers on Responses payloads, fully compatible with `litellm/responses/utils.py`
//!
//! LiteLLM rewrites some ids in a Responses API result before handing it back, so that a later
//! request carrying those ids can be routed to the provider and deployment that produced them,
//! and then restored to what the provider originally issued. The native and Python paths serve
//! the same traffic, so an id minted by either one must decode on the other.
//!
//! Each managed id is a fixed prefix followed by standard, padded base64 of a `;`-separated
//! plaintext:
//!
//! | Id | Applied to | Decoded plaintext |
//! |---|---|---|
//! | `resp_<b64>` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` |
//! | `encitem_<b64>` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` |
//! | `litellm_enc:<b64>;{content}` | that item's `encrypted_content` | `<b64>` is `model_id:{model_id}`, and the original content follows the `;` unencoded |
//! | `cntr_<b64>` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` |
//!
//! For example, OpenAI's `resp_0a1b2c3d4e5f` served without a deployment id becomes
//! `resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm`,
//! which decodes to `litellm:custom_llm_provider:openai;model_id:None;response_id:resp_0a1b2c3d4e5f`.
//!
//! How they fit together:
//! - `resp_` lets a follow-up's `previous_response_id` find the provider and deployment again.
//! [`encode_response`] stamps it on every response, unless the id is already managed.
//! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only
//! decrypts under the API key that produced it, so the router pins replays to that deployment.
//! They are stamped only when the router sets `encrypted_content_affinity_enabled` and a
//! deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's,
//! carry the deployment in the wrapped content alone.
//! - `cntr_` lets container calls find the provider and deployment that own a container. A
//! container id that already names a provider is left as it is.
//! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the
//! request input back into the provider's own bytes before the request is sent. The routing
//! decision itself stays in Python's `EncryptedContentAffinityCheck`.
//!
//! Python's quirks are kept on purpose:
//! - A missing provider or `model_id` is written as `None` in `resp_` ids (an f-string of
//! `None`), but as an empty string in `cntr_` ids. [`decode_container_id`] reads both as absent.
//! - Decoding follows Python's non-strict `base64.b64decode`: characters outside the alphabet
//! are skipped, trailing bits are allowed, and padding stripped in transit is added back.
//! - A `resp_` id counts as managed when it decodes to at least three `;`-separated parts.
use base64::{
Engine, alphabet,
engine::{GeneralPurpose, general_purpose::PAD},
};
use litellm_llms_types::formats::responses::ResponsesApiResponse;
use serde_json::{Map, Value};
const RESPONSE_PREFIX: &str = "resp_";
const ITEM_PREFIX: &str = "encitem_";
const CONTAINER_PREFIX: &str = "cntr_";
const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:";
/// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and
/// non-zero trailing bits are accepted
const LENIENT: GeneralPurpose = GeneralPurpose::new(
&alphabet::STANDARD,
PAD.with_decode_allow_trailing_bits(true),
);
/// Where a response came from, as encoded into the ids LiteLLM hands back
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct IdScope {
pub provider: Option<String>,
pub model_id: Option<String>,
pub encrypted_content_affinity: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DecodedContainerId {
pub provider: Option<String>,
pub model_id: Option<String>,
pub container_id: String,
}
/// Rewrites the response id, encrypted-content items and container ids the way the Python
/// Responses path does before returning to the caller. An already managed id is left untouched
pub fn encode_response(response: &mut ResponsesApiResponse, scope: &IdScope) {
if is_managed_response_id(&response.id) {
return;
}
response.id = response_id(scope, &response.id);
if let Some(model_id) = scope
.model_id
.as_deref()
.filter(|_| scope.encrypted_content_affinity)
.filter(|model_id| !model_id.is_empty())
{
response
.output
.iter_mut()
.filter_map(Value::as_object_mut)
.for_each(|item| encode_encrypted_item(item, model_id));
}
response
.output
.iter_mut()
.for_each(|item| encode_containers(item, scope));
}
/// Restores encrypted-content item ids and contents that LiteLLM encoded on an earlier response,
/// so the provider receives what it originally issued
pub fn restore_input(input: &mut Value) {
let Some(items) = input.as_array_mut() else {
return;
};
for item in items.iter_mut().filter_map(Value::as_object_mut) {
if let Some(Value::String(id)) = item.get_mut("id")
&& let Some((_, original)) = decode_encrypted_item_id(id)
{
*id = original;
}
if let Some(Value::String(content)) = item.get_mut("encrypted_content")
&& let Some(original) = unwrap_encrypted_content(content)
{
*content = original;
}
}
}
pub fn response_id(scope: &IdScope, id: &str) -> String {
let assembled = format!(
"litellm:custom_llm_provider:{};model_id:{};response_id:{id}",
python_str(scope.provider.as_deref()),
python_str(scope.model_id.as_deref()),
);
format!("{RESPONSE_PREFIX}{}", LENIENT.encode(assembled))
}
pub fn is_managed_response_id(id: &str) -> bool {
decode_text(&id.replace(RESPONSE_PREFIX, ""))
.is_some_and(|decoded| decoded.split(';').count() >= 3)
}
pub fn encrypted_item_id(model_id: &str, id: &str) -> String {
format!(
"{ITEM_PREFIX}{}",
LENIENT.encode(format!("litellm:model_id:{model_id};item_id:{id}"))
)
}
pub fn decode_encrypted_item_id(id: &str) -> Option<(String, String)> {
let decoded = decode_text(&padded(id.strip_prefix(ITEM_PREFIX)?))?;
let (model_id, item_id) = decoded.split_once(';')?;
Some((
model_id.replace("litellm:model_id:", ""),
item_id.replace("item_id:", ""),
))
}
pub fn wrap_encrypted_content(content: &str, model_id: &str) -> String {
format!(
"{WRAPPED_CONTENT_PREFIX}{};{content}",
LENIENT.encode(format!("model_id:{model_id}"))
)
}
/// The original content of a wrapped `encrypted_content`, or `None` when it is not wrapped
pub fn unwrap_encrypted_content(content: &str) -> Option<String> {
if !content.starts_with(WRAPPED_CONTENT_PREFIX) {
return None;
}
let (metadata, original) = content.split_once(';')?;
decode_text(&padded(&metadata.replace(WRAPPED_CONTENT_PREFIX, "")))?;
Some(original.to_owned())
}
pub fn container_id(scope: &IdScope, id: &str) -> String {
let assembled = format!(
"litellm:custom_llm_provider:{};model_id:{};container_id:{id}",
scope.provider.as_deref().unwrap_or_default(),
scope.model_id.as_deref().unwrap_or_default(),
);
format!("{CONTAINER_PREFIX}{}", LENIENT.encode(assembled))
}
pub fn decode_container_id(id: &str) -> DecodedContainerId {
let unmanaged = || DecodedContainerId {
provider: None,
model_id: None,
container_id: id.to_owned(),
};
if !id.starts_with(CONTAINER_PREFIX) {
return unmanaged();
}
let Some(decoded) = decode_text(&id.replace(CONTAINER_PREFIX, "")) else {
return unmanaged();
};
let parsed = decoded
.strip_prefix("litellm:custom_llm_provider:")
.and_then(|rest| rest.split_once(';'))
.and_then(|(provider, rest)| {
let (model_id, rest) = rest.strip_prefix("model_id:")?.split_once(';')?;
let container_id = rest.strip_prefix("container_id:")?;
(!container_id.is_empty() && !container_id.contains('\n')).then_some((
provider,
model_id,
container_id,
))
});
let Some((provider, model_id, container_id)) = parsed else {
return unmanaged();
};
let present = |value: &str| (!matches!(value, "" | "None")).then(|| value.to_owned());
DecodedContainerId {
provider: present(provider),
model_id: present(model_id),
container_id: container_id.to_owned(),
}
}
fn encode_encrypted_item(item: &mut Map<String, Value>, model_id: &str) {
let Some(Value::String(content)) = item.get_mut("encrypted_content") else {
return;
};
if content.is_empty() {
return;
}
*content = wrap_encrypted_content(content, model_id);
if let Some(Value::String(id)) = item.get_mut("id")
&& !id.is_empty()
{
*id = encrypted_item_id(model_id, id);
}
}
fn encode_containers(item: &mut Value, scope: &IdScope) {
let Some(item) = item.as_object_mut() else {
return;
};
encode_container_field(item, scope);
if let Some(nested) = item
.get_mut("code_interpreter_call")
.and_then(Value::as_object_mut)
{
encode_container_field(nested, scope);
}
if item.get("type").and_then(Value::as_str) != Some("message") {
return;
}
let Some(content) = item.get_mut("content").and_then(Value::as_array_mut) else {
return;
};
content
.iter_mut()
.filter_map(|part| part.get_mut("annotations").and_then(Value::as_array_mut))
.flatten()
.for_each(|annotation| encode_containers(annotation, scope));
}
fn encode_container_field(item: &mut Map<String, Value>, scope: &IdScope) {
if let Some(Value::String(id)) = item.get_mut("container_id")
&& decode_container_id(id).provider.is_none()
{
*id = container_id(scope, id);
}
}
fn python_str(value: Option<&str>) -> &str {
value.unwrap_or("None")
}
fn padded(value: &str) -> String {
let missing = value.len() % 4;
let padding = if missing == 0 { 0 } else { 4 - missing };
format!("{value}{}", "=".repeat(padding))
}
fn decode_text(value: &str) -> Option<String> {
let alphabet: String = value
.chars()
.filter(|character| {
character.is_ascii_alphanumeric() || matches!(character, '+' | '/' | '=')
})
.collect();
String::from_utf8(LENIENT.decode(alphabet).ok()?).ok()
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
fn scope(provider: Option<&str>, model_id: Option<&str>, affinity: bool) -> IdScope {
IdScope {
provider: provider.map(str::to_owned),
model_id: model_id.map(str::to_owned),
encrypted_content_affinity: affinity,
}
}
fn response(id: &str, output: Value) -> ResponsesApiResponse {
serde_json::from_value(json!({"id": id, "model": "gpt-6.1-sol", "output": output})).unwrap()
}
#[rstest]
#[case::openai_without_model_id(
Some("openai"),
None,
"resp_0a1b2c3d4e5f",
"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm"
)]
#[case::openrouter_with_model_id(
Some("openrouter"),
Some("deploy-1"),
"gen-123",
"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5yb3V0ZXI7bW9kZWxfaWQ6ZGVwbG95LTE7cmVzcG9uc2VfaWQ6Z2VuLTEyMw=="
)]
fn response_ids_match_python(
#[case] provider: Option<&str>,
#[case] model_id: Option<&str>,
#[case] id: &str,
#[case] expected: &str,
) {
assert_eq!(response_id(&scope(provider, model_id, false), id), expected);
}
#[rstest]
#[case::raw_short_id("resp_0a1b2c3d4e5f", false)]
#[case::raw_long_id("resp_68a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3", false)]
#[case::two_segments("resp_YTti", false)]
#[case::three_segments("resp_YTtiO2M=", true)]
#[case::managed(
"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOng=",
true
)]
fn managed_response_ids_match_python(#[case] id: &str, #[case] expected: bool) {
assert_eq!(is_managed_response_id(id), expected);
}
#[test]
fn encrypted_items_match_python() {
assert_eq!(
encrypted_item_id("deploy-1", "rs_0a1b"),
"encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI="
);
assert_eq!(
wrap_encrypted_content("gAAAA", "deploy-1"),
"litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA"
);
}
#[rstest]
#[case::padded("encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", Some(("deploy-1", "rs_0a1b")))]
#[case::padding_stripped_in_transit("encitem_bGl0ZWxsbTptb2RlbF9pZDptO2l0ZW1faWQ6cnNfMQ", Some(("m", "rs_1")))]
#[case::not_base64("encitem_notbase64!", None)]
#[case::unmanaged("rs_0a1b", None)]
fn encrypted_item_ids_decode_like_python(
#[case] id: &str,
#[case] expected: Option<(&str, &str)>,
) {
assert_eq!(
decode_encrypted_item_id(id),
expected.map(|(model_id, item_id)| (model_id.to_owned(), item_id.to_owned()))
);
}
#[rstest]
#[case::unpadded_metadata("litellm_enc:bW9kZWxfaWQ6bQ;gAAAA", Some("gAAAA"))]
#[case::no_separator("litellm_enc:nosemicolon", None)]
#[case::unwrapped("gAAAA", None)]
fn encrypted_content_unwraps_like_python(
#[case] content: &str,
#[case] expected: Option<&str>,
) {
assert_eq!(unwrap_encrypted_content(content).as_deref(), expected);
}
#[rstest]
#[case::with_model_id(
Some("deploy-1"),
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpkZXBsb3ktMTtjb250YWluZXJfaWQ6Y250cl9hYmM="
)]
#[case::without_model_id(
None,
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfYWJj"
)]
fn container_ids_match_python(#[case] model_id: Option<&str>, #[case] expected: &str) {
assert_eq!(
container_id(&scope(Some("openai"), model_id, false), "cntr_abc"),
expected
);
}
#[rstest]
#[case::raw("cntr_abc", None, None, "cntr_abc")]
#[case::managed(
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDptO2NvbnRhaW5lcl9pZDpjbnRyXzE=",
Some("openai"),
Some("m"),
"cntr_1"
)]
#[case::empty_segments(
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOjttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfMQ==",
None,
None,
"cntr_1"
)]
#[case::python_none_provider(
"cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOk5vbmU7bW9kZWxfaWQ6O2NvbnRhaW5lcl9pZDp4",
None,
None,
"x"
)]
fn container_ids_decode_like_python(
#[case] id: &str,
#[case] provider: Option<&str>,
#[case] model_id: Option<&str>,
#[case] container_id: &str,
) {
assert_eq!(
decode_container_id(id),
DecodedContainerId {
provider: provider.map(str::to_owned),
model_id: model_id.map(str::to_owned),
container_id: container_id.to_owned(),
}
);
}
#[test]
fn encodes_response_items_under_affinity() {
let mut encoded = response(
"resp_1",
json!([
{"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
{"type": "reasoning", "encrypted_content": "gBBBB", "summary": []},
{"type": "code_interpreter_call", "id": "ci_1", "container_id": "cntr_abc"},
{"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
{"type": "container_file_citation", "container_id": "cntr_abc"}
]}]}
]),
);
let scope = scope(Some("openai"), Some("deploy-1"), true);
encode_response(&mut encoded, &scope);
let container = container_id(&scope, "cntr_abc");
assert_eq!(encoded.id, response_id(&scope, "resp_1"));
assert_eq!(
Value::Array(encoded.output),
json!([
{
"type": "reasoning",
"id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
"encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
"summary": []
},
{"type": "reasoning", "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gBBBB", "summary": []},
{"type": "code_interpreter_call", "id": "ci_1", "container_id": container},
{"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
{"type": "container_file_citation", "container_id": container}
]}]}
])
);
}
#[rstest]
#[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))]
#[case::no_model_id(scope(Some("openai"), None, true))]
fn leaves_encrypted_items_without_affinity(#[case] scope: IdScope) {
let output = json!([{"type": "reasoning", "id": "rs_1", "encrypted_content": "gAAAA", "summary": []}]);
let mut encoded = response("resp_1", output.clone());
encode_response(&mut encoded, &scope);
assert_eq!(Value::Array(encoded.output), output);
}
#[test]
fn leaves_managed_responses_untouched() {
let scope = scope(Some("openai"), Some("deploy-1"), true);
let id = response_id(&scope, "resp_1");
let output = json!([{"type": "code_interpreter_call", "container_id": "cntr_abc"}]);
let mut encoded = response(&id, output.clone());
encode_response(&mut encoded, &scope);
assert_eq!((encoded.id, Value::Array(encoded.output)), (id, output));
}
#[test]
fn restores_encrypted_items_in_input() {
let mut input = json!([
{
"type": "reasoning",
"id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
"encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
"summary": []
},
{"role": "user", "content": "hi"}
]);
restore_input(&mut input);
assert_eq!(
input,
json!([
{"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
{"role": "user", "content": "hi"}
])
);
}
}

1
litellm-rust/crates/inference-responses/src/lib.rs
View file

@ -1,6 +1,7 @@
use litellm_host::observation::ObservationSender;
pub use litellm_inference::RouteError as Error;
pub mod ids;
pub mod route;
pub mod types;
pub mod websocket;

11
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -195,9 +195,10 @@ impl AnthropicBeta {
fn rejected_by(&self) -> &'static [BetaProvider] {
match self {
Self::AdvancedToolUse20251120 | Self::ContextManagement20250627 => {
&[BetaProvider::BedrockConverse]
Self::AdvancedToolUse20251120 => {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
Self::AdvisorTool20260301 | Self::Compact20260904 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
@ -231,12 +232,11 @@ impl AnthropicBeta {
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::DangerousToolUse20260903 => {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
BetaProvider::AzureAi,
@ -244,6 +244,7 @@ impl AnthropicBeta {
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::FineGrainedToolStreaming20250514 => {
&[BetaProvider::AzureAi, BetaProvider::VertexAi]

2
litellm-rust/crates/llms/src/anthropic/beta_headers.rs
View file

@ -76,7 +76,7 @@ mod tests {
#[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))]
#[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))]
#[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))]
#[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))]
#[case::databricks(BetaProvider::Databricks, None)]
#[case::bedrock_converse(BetaProvider::BedrockConverse, None)]
fn filter_renames_per_host(
#[case] provider: BetaProvider,

91
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -109,7 +109,10 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
merge_beta_headers(
headers,
provider_feature_betas(request, BetaProvider::Anthropic),
)
}
}
@ -136,12 +139,20 @@ pub(crate) fn transform_messages_request(
messages: strip_encrypted_reasoning_blocks(messages),
params: MessagesOptionalParams {
context_management,
extra: without_null_safeguards(request.params.extra),
..request.params
},
..request
})
}
fn without_null_safeguards(extra: Map<String, Value>) -> Map<String, Value> {
extra
.into_iter()
.filter(|(key, value)| key != "safeguards" || !value.is_null())
.collect()
}
pub(crate) fn update_headers_with_anthropic_beta(
headers: Headers,
request: &MessagesRequest,
@ -149,12 +160,34 @@ pub(crate) fn update_headers_with_anthropic_beta(
merge_beta_headers(headers, feature_betas(request))
}
/// The betas a request needs on a host other than the first-party API: the Anthropic set
/// filtered and renamed through the host's `BetaProvider` policy.
pub(crate) fn update_headers_with_safeguards_beta(
headers: Headers,
request: &MessagesRequest,
provider: BetaProvider,
) -> Headers {
merge_beta_headers(
headers,
feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)),
)
}
/// The betas a request needs on a host with a column in the beta map, filtered and renamed
/// through that column.
pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet {
feature_betas(request)
.iter()
.filter_map(|beta| beta.on(provider))
on_provider(
feature_betas(request).union(mapped_host_betas(&request.params)),
provider,
)
}
fn on_provider(betas: BetaSet, provider: BetaProvider) -> BetaSet {
betas.iter().filter_map(|beta| beta.on(provider)).collect()
}
fn mapped_host_betas(params: &MessagesOptionalParams) -> BetaSet {
carries_safeguards(params)
.then_some(AnthropicBeta::DangerousToolUse20260903)
.into_iter()
.collect()
}
@ -206,6 +239,13 @@ fn uses_structured_output(params: &MessagesOptionalParams) -> bool {
.is_some_and(|config| config.format.is_some())
}
fn carries_safeguards(params: &MessagesOptionalParams) -> bool {
params
.extra
.get("safeguards")
.is_some_and(|value| !value.is_null())
}
fn messages_carry_output_config(messages: &[Message]) -> bool {
messages
.iter()
@ -1025,10 +1065,44 @@ mod tests {
json!({"messages": [{"role": "user", "content": "hi", "output_config": null}]}),
&["per-turn-control-2026-07-01"]
)]
#[case::safeguards(
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
&[]
)]
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
assert_eq!(feature_betas(&request(fields)), betas(expected));
}
#[rstest]
#[case::anthropic(BetaProvider::Anthropic, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
#[case::vertex_ai(BetaProvider::VertexAi, json!([{"type": "dangerous_tool_use"}]), &["dangerous-tool-use-2026-09-03"])]
#[case::empty_list(BetaProvider::Anthropic, json!([]), &["dangerous-tool-use-2026-09-03"])]
#[case::not_a_list(BetaProvider::Anthropic, json!(1), &["dangerous-tool-use-2026-09-03"])]
#[case::null(BetaProvider::Anthropic, Value::Null, &[])]
#[case::bedrock_converse_has_a_null_cell(BetaProvider::BedrockConverse, json!([{"type": "dangerous_tool_use"}]), &[])]
fn safeguards_add_the_dangerous_tool_use_beta_where_the_beta_map_has_a_cell(
#[case] provider: BetaProvider,
#[case] safeguards: Value,
#[case] expected: &[&str],
) {
assert_eq!(
provider_feature_betas(&request(json!({"safeguards": safeguards})), provider),
betas(expected)
);
}
#[rstest]
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
assert_eq!(
update_headers_with_anthropic_beta(
client.clone(),
&request(json!({"safeguards": [{"type": "dangerous_tool_use"}]}))
),
client
);
}
#[rstest]
#[case::no_betas(&[("x-api-key", "k"), ("anthropic-version", "2023-06-01")], json!({}), &[("x-api-key", "k"), ("anthropic-version", "2023-06-01")])]
#[case::blank_beta_header(&[("Anthropic-Beta", " , "), ("x-api-key", "k")], json!({}), &[("Anthropic-Beta", " , "), ("x-api-key", "k")])]
@ -1067,6 +1141,11 @@ mod tests {
json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}),
&[("anthropic-beta", "claude-code-20250219,context-management-2025-06-27,effort-2025-11-24,interleaved-thinking-2025-05-14,per-turn-control-2026-07-01")],
)]
#[case::safeguards_add_the_dangerous_tool_use_beta_to_the_client_betas(
&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")],
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
&[("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")],
)]
fn request_headers_merge_the_feature_betas(
#[case] input: &[(&str, &str)],
#[case] fields: Value,

12
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -2,7 +2,7 @@ use litellm_auth::SecretValue;
use litellm_http::request::{has_bearer_auth, has_header};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
@ -12,7 +12,7 @@ use crate::{
common_utils::API_KEY_PLACEMENT,
messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
transformation::{transform_messages_request, update_headers_with_safeguards_beta},
},
},
azure_ai::common_utils::{
@ -104,7 +104,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
update_headers_with_safeguards_beta(headers, request, BetaProvider::AzureAi)
}
}
@ -544,6 +544,12 @@ mod tests {
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
)]
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
#[case::safeguards_need_the_dangerous_tool_use_beta(
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
&[("anthropic-beta", "claude-code-20250219")],
&[("x-api-key", "k"), ("anthropic-beta", "claude-code-20250219,dangerous-tool-use-2026-09-03")]
)]
#[case::null_safeguards_need_no_beta(json!({"safeguards": null}), &[], &[("x-api-key", "k")])]
fn request_headers_carry_the_anthropic_feature_betas(
#[case] fields: serde_json::Value,
#[case] forwarded: &[(&str, &str)],

21
litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs
View file

@ -6,7 +6,7 @@ use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{header_value, with_header, without_headers};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
use serde_json::Value;
@ -15,7 +15,7 @@ use crate::{
Error, ErrorDetail,
anthropic::messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
transformation::{transform_messages_request, update_headers_with_safeguards_beta},
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
@ -174,7 +174,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
None => headers,
};
update_headers_with_anthropic_beta(headers, request)
update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic)
}
fn wire_body(&self, body: Value) -> Value {
@ -356,6 +356,21 @@ mod tests {
);
}
#[rstest]
#[case::safeguards(json!([{"type": "dangerous_tool_use"}]), Some("dangerous-tool-use-2026-09-03"))]
#[case::null_safeguards(Value::Null, None)]
fn safeguards_need_the_dangerous_tool_use_beta(
#[case] safeguards: Value,
#[case] expected: Option<&str>,
) {
let request = serde_json::from_value(json!({"model": "claude-test", "max_tokens": 16, "messages": [], "safeguards": safeguards})).unwrap();
let headers = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.request_headers(vec![], &request);
assert_eq!(
header_value(&headers, "anthropic-beta").map(|value| value.to_string()),
expected.map(str::to_string)
);
}
#[rstest]
#[case::default(json!({}), false)]
#[case::override_empty(json!({"claude_platform_unsupported_params": []}), true)]

23
litellm-rust/crates/llms/src/deepseek/messages/transformation.rs
View file

@ -451,4 +451,27 @@ mod tests {
json!({"type": "enabled", "budget_tokens": 1024})
);
}
#[rstest]
#[case::without_client_betas(&[])]
#[case::with_client_betas(&[("anthropic-beta", "claude-code-20250219,interleaved-thinking-2025-05-14")])]
fn safeguards_and_client_betas_reach_deepseek_as_sent(#[case] client: &[(&str, &str)]) {
let safeguards = json!([{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}]);
let body = json!({
"model": "deepseek-v4-pro",
"max_tokens": 100,
"messages": [{"role": "user", "content": "hi"}],
"safeguards": safeguards
});
let headers: Headers = client
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect();
assert_eq!(
DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG
.request_headers(headers.clone(), &request_from(body.clone())),
headers
);
assert_eq!(transformed(body)["safeguards"], safeguards);
}
}

19
litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs
View file

@ -213,23 +213,14 @@ fn has_web_search_tool(tools: Option<&[Recognized<MessagesTool>]>) -> bool {
}
/// Python's Vertex `validate_anthropic_messages_environment` beta set: the shared feature
/// betas under Vertex's policy, plus the web search and safeguards betas Vertex needs spelled
/// out.
/// betas under Vertex's policy, plus the web search beta Vertex needs spelled out.
fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
let params = &request.params;
provider_feature_betas(request, BetaProvider::VertexAi).union(
[
has_web_search_tool(params.tools.as_deref())
.then_some(AnthropicBeta::WebSearch20250305),
params
.extra
.get("safeguards")
.is_some_and(|value| !value.is_null())
.then_some(AnthropicBeta::DangerousToolUse20260903),
]
.into_iter()
.flatten()
.collect(),
has_web_search_tool(params.tools.as_deref())
.then_some(AnthropicBeta::WebSearch20250305)
.into_iter()
.collect(),
)
}

73
litellm-rust/crates/python-bridge/src/routes/responses/host.rs
View file

@ -1,22 +1,41 @@
use std::convert::Infallible;
use crate::routes::inference::InferenceHost;
use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned};
use litellm_inference_responses::{Error, route::Responses, types::ResponsesCall};
use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py};
use litellm_inference_responses::{
Error,
ids::{self, IdScope},
route::Responses,
types::ResponsesCall,
};
use pyo3::{
gc::{PyTraverseError, PyVisit},
prelude::*,
types::PyDict,
};
pub(super) struct ResponsesPythonHost(pub InferenceHost);
pub(super) struct ResponsesPythonHost {
host: InferenceHost,
ids: IdScope,
}
impl ResponsesPythonHost {
pub fn new(host: InferenceHost) -> Self {
Self {
host,
ids: IdScope::default(),
}
}
}
pub(super) fn project(
host: &mut InferenceHost,
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
) -> PyResult<ResponsesCall> {
let call = host.project(py, arguments, "input")?;
let mut call = host.project(py, arguments, "input")?;
ids::restore_input(&mut call.input);
let optional_params = call
.params
.into_iter()
@ -50,6 +69,39 @@ pub(super) fn project(
})
}
/// The provider and deployment Python encodes into the ids it returns: the resolved provider and
/// `litellm_metadata.model_info.id`
fn id_scope(
host: &InferenceHost,
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
call: &ResponsesCall,
) -> PyResult<IdScope> {
let metadata = host.argument(py, arguments, "litellm_metadata")?;
let encrypted_content_affinity = match &metadata {
Some(metadata) => metadata
.call_method1("get", ("encrypted_content_affinity_enabled",))?
.is_truthy()?,
None => false,
};
let metadata = metadata
.map(|value| from_py::<serde_json::Value>(&value))
.transpose()?
.unwrap_or_default();
let model_id = match metadata.pointer("/model_info/id") {
None | Some(serde_json::Value::Null) => None,
Some(serde_json::Value::String(id)) => Some(id.clone()),
Some(other) => Some(other.to_string()),
};
Ok(IdScope {
provider: get_custom_llm_provider(&call.model, call.custom_llm_provider.as_deref())
.map(|resolved| resolved.custom_llm_provider.to_owned())
.or_else(|| call.custom_llm_provider.clone()),
model_id,
encrypted_content_affinity,
})
}
impl PythonBinding for ResponsesPythonHost {
type Protocol = Responses;
type Failure = PyErr;
@ -59,7 +111,8 @@ impl PythonBinding for ResponsesPythonHost {
py: Python<'_>,
arguments: &Bound<'_, PyDict>,
) -> Result<ResponsesCall, InvokeError<Error>> {
let call = project(&mut self.0, py, arguments).map_err(InvokeError::Python)?;
let call = project(&mut self.host, py, arguments).map_err(InvokeError::Python)?;
self.ids = id_scope(&self.host, py, arguments, &call).map_err(InvokeError::Python)?;
if call
.optional_params
.get("stream")
@ -77,7 +130,9 @@ impl PythonBinding for ResponsesPythonHost {
py: Python<'_>,
response: <Responses as litellm_host::protocol::Protocol>::Response,
) -> PyResult<Py<PyAny>> {
self.0.response(py, &response)
let mut response = response;
ids::encode_response(&mut response, &self.ids);
self.host.response(py, &response)
}
fn encode_stream_head(
@ -103,7 +158,7 @@ impl PythonBinding for ResponsesPythonHost {
}
fn map_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
self.0.error(py, error)
self.host.error(py, error)
}
fn host_error(error: &PyErr) -> Error {
Error::InvalidRequest(error.to_string().into())
@ -122,9 +177,9 @@ impl PythonHostCalls<Responses> for ResponsesPythonHost {
impl PythonOwned for ResponsesPythonHost {
fn close(&mut self, _: Python<'_>) {
self.0.close();
self.host.close();
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
self.0.traverse(visit)
self.host.traverse(visit)
}
}

2
litellm-rust/crates/python-bridge/src/routes/responses/mod.rs
View file

@ -62,7 +62,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py
));
}
let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost(host))
run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
}
#[pyfunction]

7
litellm/__init__.py
View file

@ -676,6 +676,7 @@ qwen_ai_platform_models: Set = set()
moonshot_models: Set = set()
publicai_models: Set = set()
darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
@ -742,7 +743,7 @@ def is_openai_finetune_model(key: str) -> bool:
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
for key, value in model_cost_map.items():
for key, value in tuple(model_cost_map.items()):
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
open_ai_chat_completion_models.add(key)
elif value.get("litellm_provider") == "text-completion-openai":
@ -945,6 +946,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
publicai_models.add(key)
elif value.get("litellm_provider") == "darkbloom":
darkbloom_models.add(key)
elif value.get("litellm_provider") == "openinfer":
openinfer_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
@ -1113,6 +1116,7 @@ model_list = list(
| moonshot_models
| publicai_models
| darkbloom_models
| openinfer_models
| v0_models
| morph_models
| lambda_ai_models
@ -1229,6 +1233,7 @@ def _build_models_by_provider() -> dict:
"moonshot": moonshot_models,
"publicai": publicai_models,
"darkbloom": darkbloom_models,
"openinfer": openinfer_models,
"v0": v0_models,
"morph": morph_models,
"lambda_ai": lambda_ai_models,

2
litellm/_logging.py
View file

@ -1000,6 +1000,8 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
lg.handlers.clear() # remove any existing handlers
lg.addHandler(handler) # add JSON formatter handler
lg.propagate = False # prevent bubbling to parent/root
if (configured_level := os.getenv("LITELLM_LOG")) is not None:
logging.getLogger().setLevel(resolve_log_level(configured_level))
def _get_uvicorn_json_log_config():

22
litellm/anthropic_beta_headers_config.json
View file

@ -220,36 +220,36 @@
},
"databricks": {
"advisor-tool-2026-03-01": null,
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"advanced-tool-use-2025-11-20": null,
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"code-execution-2025-08-25": null,
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"dangerous-tool-use-2026-09-03": null,
"dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": "fast-mode-2026-02-01",
"files-api-2025-04-14": "files-api-2025-04-14",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"structured-output-2024-03-01": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"oauth-2025-04-20": "oauth-2025-04-20",
"oauth-2025-04-20": null,
"output-128k-2025-02-19": "output-128k-2025-02-19",
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": "web-search-2025-03-05"
}
}

45
litellm/anthropic_beta_headers_manager.py
View file

@ -22,17 +22,28 @@ Environment Variables:
- LITELLM_ANTHROPIC_BETA_HEADERS_URL: Custom URL for remote config (optional)
"""
import asyncio
import json
import os
import threading
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from functools import cache
from importlib.resources import files
from typing import Final
from typing import Final, TypeAlias
import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
_BETA_HEADERS_CONFIG_LOCK: Final = threading.Lock()
_BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="litellm-beta-headers-config")
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
class GetAnthropicBetaHeadersConfig:
@ -174,13 +185,27 @@ def _load_beta_headers_config() -> dict:
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
# Get the URL from environment or use default
from litellm import anthropic_beta_headers_url
with _BETA_HEADERS_CONFIG_LOCK:
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
_BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
verbose_logger.debug("Loaded and cached beta headers config")
# Get the URL from environment or use default
from litellm import anthropic_beta_headers_url
return _BETA_HEADERS_CONFIG
_BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
verbose_logger.debug("Loaded and cached beta headers config")
return _BETA_HEADERS_CONFIG
async def aload_beta_headers_config() -> None:
if _BETA_HEADERS_CONFIG is None:
await asyncio.get_running_loop().run_in_executor(_BETA_HEADERS_CONFIG_LOADER, _load_beta_headers_config)
@cache
def _bundled_beta_headers_config() -> BetaHeadersConfig:
return _BETA_HEADERS_CONFIG_ADAPTER.validate_python(GetAnthropicBetaHeadersConfig.load_local_beta_headers_config())
def reload_beta_headers_config() -> dict:
@ -288,6 +313,14 @@ def is_beta_header_supported(
return beta_header in provider_mapping and provider_mapping[beta_header] is not None
def is_beta_header_supported_by_bundled_config(beta_header: str, provider: str) -> bool:
config: Final = _bundled_beta_headers_config()
aliases: Final = config.get("provider_aliases", {})
canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
provider_mapping: Final = config.get(canonical_provider, {})
return isinstance(provider_mapping, Mapping) and provider_mapping.get(beta_header) is not None
def get_provider_beta_header(
anthropic_beta_header: str,
provider: str,

16
litellm/batches/batch_utils.py
View file

@ -794,11 +794,17 @@ def _batch_response_was_successful(
message batch results lines report ``result.type == "succeeded"``; Bedrock
batch output lines report ``modelOutput`` (and no ``error``).
"""
if custom_llm_provider == "anthropic":
return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
if custom_llm_provider == "bedrock":
return batch_job_output_file.get("modelOutput") is not None and batch_job_output_file.get("error") is None
_response: Final[dict[str, object]] = cast( # cast-ok: batch response bodies come from provider JSON
dict[str, object], batch_job_output_file.get("response", None) or {}
envelope_key: Final = "result" if custom_llm_provider == "anthropic" else "response"
envelope: Final = batch_job_output_file.get(envelope_key)
if not isinstance(envelope, Mapping):
if envelope is not None:
verbose_logger.warning("batch output %s must be an object, treating the line as unsuccessful", envelope_key)
return False
response: Final = cast(Mapping[object, object], envelope) # cast-ok: mapping shape validated above
return (
response.get("type") == "succeeded"
if custom_llm_provider == "anthropic"
else response.get("status_code") == 200
)
return _response.get("status_code", None) == 200

8
litellm/caching/caching_handler.py
View file

@ -346,6 +346,7 @@ class LLMCachingHandler:
if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
verbose_logger.debug("Checking Async Cache")
logging_obj.reset_caching_details()
cached_result = await self._retrieve_from_cache(
call_type=call_type,
kwargs=kwargs,
@ -357,7 +358,7 @@ class LLMCachingHandler:
verbose_logger.debug("Cache Hit!")
cache_hit: Final = True
end_time: Final = datetime.datetime.now()
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model,
custom_llm_provider=kwargs.get("custom_llm_provider", None),
api_base=kwargs.get("api_base", None),
@ -366,7 +367,7 @@ class LLMCachingHandler:
cache_duration_ms: Final = (cache_check_end_time - cache_check_start_time) * 1000
self._update_litellm_logging_obj_environment(
logging_obj=logging_obj,
model=model,
model=provider_model,
kwargs=kwargs,
cached_result=cached_result,
is_async=True,
@ -381,7 +382,7 @@ class LLMCachingHandler:
call_type=call_type,
kwargs=kwargs,
logging_obj=logging_obj,
model=model,
model=provider_model,
custom_llm_provider=kwargs.get("custom_llm_provider", None),
args=args,
)
@ -467,6 +468,7 @@ class LLMCachingHandler:
new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
print_verbose("Checking Sync Cache")
logging_obj.reset_caching_details()
with response_cache_phase("get"):
cached_result = litellm.cache.get_cache(**new_kwargs)
if is_response_without_output(cached_result):

39
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -41,7 +41,11 @@ from litellm.responses.sse_output_recovery import (
record_output_item_chunk,
record_output_text_chunk,
)
from litellm.responses.utils import ResponsesAPIRequestUtils, normalize_responses_api_stream_options
from litellm.responses.utils import (
ResponsesAPIRequestUtils,
auto_reasoning_summary,
normalize_responses_api_stream_options,
)
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionReasoningItem,
@ -176,6 +180,16 @@ def _without_audio_input_parts(input_items: list[object]) -> list[object]:
return [_without_audio_input_parts_in_item(item) for item in input_items]
def _effort_with_requested_summary(nested_effort: object, top_level_effort: object) -> object:
match top_level_effort, nested_effort:
case {"summary": str() as summary}, str() as effort:
return {"effort": effort, "summary": summary}
case {"summary": str() as summary}, Mapping() as reasoning if "summary" not in reasoning:
return {**reasoning, "summary": summary}
case _:
return nested_effort
def _served_provider(litellm_params: Mapping[str, object]) -> str | None:
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
return custom_llm_provider if isinstance(custom_llm_provider, str) else None
@ -953,6 +967,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
optional_params: dict,
responses_api_request: "ResponsesAPIOptionalRequestParams",
drop_params: bool = False,
model: str | None = None,
custom_llm_provider: str | None = None,
) -> None:
"""Map optional_params into responses_api_request (mutates in place)."""
for key, value in optional_params.items():
@ -984,7 +1000,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif key == "previous_response_id":
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
responses_api_request["reasoning"] = self.map_reasoning_effort(value)
responses_api_request["reasoning"] = self.map_reasoning_effort(value, model, custom_llm_provider)
elif key == "web_search_options":
self._add_web_search_tool(responses_api_request, value)
@ -1100,6 +1116,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
optional_params,
responses_api_request,
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
model=model,
custom_llm_provider=_served_provider(litellm_params),
)
stream: Final = optional_params.get("stream") or litellm_params.get("stream", False)
@ -1705,14 +1723,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Extract supported params from extra_body and merge into optional_params
extra_body_copy: Final = extra_body.copy()
top_level_effort: Final = cast(object, optional_params.get("reasoning_effort")) # cast-ok: upcast
for key, value in extra_body_copy.items():
if key in supported_responses_api_params:
# Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort)
optional_params[key] = extra_body.pop(key)
extra_body.pop(key)
optional_params[key] = (
_effort_with_requested_summary(cast(object, value), top_level_effort) # cast-ok: upcast
if key == "reasoning_effort"
else value
)
return optional_params
def map_reasoning_effort(self, reasoning_effort: object) -> Reasoning:
def map_reasoning_effort(
self, reasoning_effort: object, model: str | None = None, custom_llm_provider: str | None = None
) -> Reasoning:
# If dict is passed, convert it directly to Reasoning object
if isinstance(reasoning_effort, dict):
return Reasoning(
@ -1726,7 +1752,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
return (
Reasoning(effort=reasoning_effort, summary="detailed")
Reasoning(
effort=reasoning_effort,
summary="detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider),
)
if auto_summary_enabled
else Reasoning(effort=reasoning_effort)
)

3
litellm/constants.py
View file

@ -1030,6 +1030,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"https://api.prisminference.com/v1",
"https://api.reka.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
"https://api.openinfer.ai/v1",
]
@ -1098,6 +1099,7 @@ openai_compatible_providers: Final[list] = [
"docker_model_runner",
"ragflow",
"pinstripes", # Pinstripes - JSON-configured provider
"openinfer",
"darkbloom",
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
"cognition",
@ -1631,6 +1633,7 @@ OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_to
CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"

11
litellm/integrations/custom_guardrail.py
View file

@ -1139,6 +1139,17 @@ class CustomGuardrail(CustomLogger):
"""
return False
def supports_structured_messages(self) -> bool:
"""Whether this guardrail scans every row of ``structured_messages`` and returns texts that agree with them.
Guardrails that scan only part of the rows (the latest user turn, the
last row, the turns after the last assistant turn, or no tool results),
derive their returned texts from the rows, or cannot rewrite rows whose
content is a list of parts, override this to return False. Translation handlers that can rewrite the request from ``texts``
alone, like Bedrock Converse, then hand them only ``texts``.
"""
return True
def _suppressed_by_auto_router_compression(self) -> bool:
"""True when an auto router's own compression policy suppresses this guardrail.

43
litellm/integrations/custom_logger.py
View file

@ -912,9 +912,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
This is useful for logging payloads that contain sensitive information.
"""
import litellm
from litellm import Choices, Message, ModelResponse
from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS
from litellm.litellm_core_utils.redact_messages import redacted_litellm_params
from litellm.litellm_core_utils.redact_messages import (
redacted_litellm_params,
redacted_standard_logging_payload,
)
turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False)
excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
@ -940,41 +942,14 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS)
}
# Handle turn_off_message_logging - redact messages and responses (if not already excluded)
if turn_off_message_logging and not self.redacts_messages_itself():
redacted_str: Final = "redacted-by-litellm"
if "messages" not in (excluded_fields or ()) and standard_logging_object_copy.get("messages") is not None:
standard_logging_object_copy["messages"] = [Message(content=redacted_str).model_dump()]
if "response" not in (excluded_fields or ()) and standard_logging_object_copy.get("response") is not None:
response: Final = standard_logging_object_copy["response"]
# Check if this is a ResponsesAPIResponse (has "output" field)
if isinstance(response, dict) and "output" in response:
# Make a copy to avoid modifying the original
from copy import deepcopy
response_copy: Final = deepcopy(response)
# Redact content in output array
if isinstance(response_copy.get("output"), list):
for output_item in response_copy["output"]:
if isinstance(output_item, dict) and "content" in output_item:
if isinstance(output_item["content"], list):
# Redact text in content items
for content_item in output_item["content"]:
if isinstance(content_item, dict) and "text" in content_item:
content_item["text"] = redacted_str
standard_logging_object_copy["response"] = response_copy
else:
# Standard ModelResponse format
model_response: Final = ModelResponse(choices=[Choices(message=Message(content=redacted_str))])
model_response_dict: Final = model_response.model_dump()
standard_logging_object_copy["response"] = model_response_dict
return {
**model_call_details,
**redacted_params,
"standard_logging_object": standard_logging_object_copy,
"standard_logging_object": (
redacted_standard_logging_payload(standard_logging_object_copy)
if turn_off_message_logging and not self.redacts_messages_itself()
else standard_logging_object_copy
),
}
async def get_proxy_server_request_from_cold_storage_with_object_key(

18
litellm/integrations/opentelemetry.py
View file

@ -6,9 +6,10 @@ from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
from typing import TYPE_CHECKING, Any, Final, cast
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -106,6 +107,14 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
# Cap on credential-scoped providers held at once; each one owns an exporter thread.
_MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
class _CacheHitView(TypedDict, total=False):
cache_hit: ReadOnly[object]
saved_cache_cost: ReadOnly[object]
_CACHE_HIT_VIEW: Final = TypeAdapter(_CacheHitView)
# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
@ -2434,6 +2443,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
litellm_params=litellm_params,
)
# Cost breakdown tracking
cache_view: Final = _CACHE_HIT_VIEW.validate_python(standard_logging_payload)
cache_hit: Final = cache_view.get("cache_hit") is True
cost_breakdown: Final[CostBreakdown | None] = standard_logging_payload.get("cost_breakdown")
if cost_breakdown:
for key, value in cost_breakdown.items():
@ -2443,6 +2454,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
key=f"gen_ai.cost.{key}",
value=value,
)
self.safe_set_attribute(span=span, key=LiteLLM.CACHE_HIT, value=cache_hit)
if cache_hit:
saved_cache_cost: Final = cache_view.get("saved_cache_cost")
if isinstance(saved_cache_cost, (int, float)):
self.safe_set_attribute(span=span, key=LiteLLM.SAVED_CACHE_COST, value=saved_cache_cost)
#############################################
########## LLM Request Attributes ###########
#############################################

34
litellm/integrations/otel/emitter.py
View file

@ -101,23 +101,19 @@ def _carried_keys(
return frozenset(key for key, _value in attributes)
def attribute_budget(span: Span, reserved: int, overwrites: frozenset[str] = frozenset()) -> int | None:
"""How many mapped attributes fit on ``span`` next to what it already carries and ``reserved`` more.
def carried_keys(span: Span) -> frozenset[str]:
"""The attribute keys ``span`` already holds, empty for a span that exposes none."""
if not isinstance(span, ReadableSpan):
return frozenset()
return _carried_keys(span.attributes or ())
``overwrites`` are the mapped keys already present on ``span``: setting one
replaces the value in place and consumes no slot against the limit, so only
the genuinely new pre-existing keys reduce the budget. Counting the
overwritten ones too reserves slots the fit can never spend and sheds
indexed message attributes for nothing.
"""
def attribute_budget(span: Span, reserved: int) -> int | None:
"""How many new attribute keys fit on ``span`` next to what it already carries and ``reserved`` more."""
limit: Final = span_attribute_limit(span)
if limit is None:
return None
if not isinstance(span, ReadableSpan):
return limit - reserved
existing_keys: Final = _carried_keys(span.attributes or ())
fresh: Final = len(existing_keys - overwrites) if overwrites else len(existing_keys)
return limit - fresh - reserved
return limit - len(carried_keys(span)) - reserved
def stamp_error(
@ -304,10 +300,16 @@ class SpanEmitter:
)
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
reserved: Final = len(stamped_later.keys() - mapped.keys())
overwrites: Final = carried_keys(span) & mapped.keys()
# The Langfuse request metadata sheds first, so a span at its attribute limit loses nothing else to it.
budget: Final = attribute_budget(span, reserved, frozenset(mapped))
for key, value in fit_indexed_messages(mapped, budget, LANGFUSE_REQUEST_METADATA_ATTRIBUTES).items():
span.set_attribute(key, value)
fitted: Final = fit_indexed_messages(
MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
attribute_budget(span, reserved),
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
)
for key, value in mapped.items():
if key in overwrites or key in fitted:
span.set_attribute(key, value)
if error:
stamped: Final = stamp_error(span, error)
if stamped is not None and self._event_recorder is not None and role is SpanRole.LLM_CALL:

8
litellm/integrations/otel/logger.py
View file

@ -538,8 +538,12 @@ class OpenTelemetryV2(CustomLogger):
# logger is a success/failure callback only, so ``pre_call`` never reaches it
# and no carrier exists. The payload plus the request-level provider-handoff
# stamp (``upstream_started``) is the affirmative signal of a real call; a
# gate rejection carries ``is_no_upstream_call`` and gets no span.
if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
# upstream stamp, since the payload proves the call completed.
if carrier is None and (
call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
):
return None
try:
return self._finish_carrier(carrier, call, start_time, end_time)

2
litellm/integrations/otel/mappers/genai.py
View file

@ -91,6 +91,8 @@ class GenAIMapper:
f"{LiteLLM.COST_PREFIX}margin_fixed_amount": lambda d: d.cost.margin_fixed_amount,
f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent,
f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount,
LiteLLM.CACHE_HIT: lambda d: d.cache_hit,
LiteLLM.SAVED_CACHE_COST: lambda d: d.saved_cache_cost,
LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
LiteLLM.REQUEST_ROUTE: lambda d: d.request_route,
LiteLLM.REQUEST_PURPOSE: lambda d: d.request_purpose,

5
litellm/integrations/otel/model/metadata.py
View file

@ -260,6 +260,10 @@ class LLMCallEvent:
# ``InternalCallOrigin`` such as ``autorouter_classifier``), ``None`` for the
# caller's own provider attempt.
purpose: str | None
# True for a response served from the litellm response cache. A cache hit
# never hands off to a provider, so ``upstream_started`` stays False even
# though the call completed and is worth a span.
cache_hit: bool
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
@ -284,6 +288,7 @@ class LLMCallEvent:
upstream_started=kwargs.get("api_call_start_time") is not None,
upstream_start_seconds=_epoch_seconds(kwargs.get("api_call_start_time")),
purpose=internal_call_origin(payload, kwargs),
cache_hit=bool(payload.get("cache_hit")) if payload else False, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
provisional_span_name=f"{operation.value} {model}".strip(),
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
trace=trace,

7
litellm/integrations/otel/model/payloads.py
View file

@ -448,6 +448,8 @@ class LLMCallSpanData:
embedding_output: EmbeddingOutput | None = None
promoted_metadata: Mapping[str, str] = field(default_factory=lambda: _EMPTY_METADATA)
routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({}))
cache_hit: bool | None = None
saved_cache_cost: float | None = None
@classmethod
def from_standard_logging_payload(
@ -463,6 +465,7 @@ class LLMCallSpanData:
metadata_keys: tuple[str, ...] = (),
) -> LLMCallSpanData:
params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {})
cache_hit: Final = as_bool(payload.get("cache_hit")) is True # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
# The single parse of the request's metadata — the request-vs-provider
# model split, the response model, api base, and identity all come from
# here rather than being re-derived from the raw payload dicts.
@ -498,7 +501,7 @@ class LLMCallSpanData:
error=_parse_error(payload),
response_cost=as_float(payload.get("response_cost")),
cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))),
server=ServerInfo.from_api_base(context.api_base),
server=None if cache_hit else ServerInfo.from_api_base(context.api_base),
identity=context.identity,
request_metadata=exported_request_metadata(
{
@ -522,6 +525,8 @@ class LLMCallSpanData:
session_id=session_id or None,
embedding_output=embedding_output if capture_content else None,
routing_attributes=routing_decision_attributes(decision),
cache_hit=cache_hit,
saved_cache_cost=as_float(payload.get("saved_cache_cost")) if cache_hit else None, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown
)

2
litellm/integrations/otel/model/semconv.py
View file

@ -340,6 +340,8 @@ class LiteLLM:
# semconv key for an MCP server's *name* (the convention uses ``server.address``
# for its network location), so it lives under the vendor namespace.
MCP_SERVER_NAME: Final = "litellm.mcp.server.name"
CACHE_HIT: Final = "litellm.cache_hit"
SAVED_CACHE_COST: Final = "litellm.saved_cache_cost"
class Metric:

124
litellm/litellm_core_utils/completion_timeout.py
View file

@ -2,15 +2,25 @@
from __future__ import annotations
from collections.abc import Callable
from collections.abc import Callable, Mapping
from enum import Enum
from functools import partial
from types import MappingProxyType
from typing import Final
import httpx
from openai import Timeout as SDKTimeout
from typing_extensions import assert_never
from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS
class StreamKind(Enum):
NOT_STREAMED = "not_streamed"
LIVE = "live"
FAKED = "faked"
class CompletionTimeout:
"""Resolves HTTP timeout for ``completion()`` from model vs global settings."""
@ -36,17 +46,91 @@ class CompletionTimeout:
return COMPLETION_HTTP_FALLBACK_SECONDS
return float(global_timeout)
@staticmethod
def _longer_limit(first: float | None, second: float | None) -> float | None:
return None if first is None or second is None else max(first, second)
@staticmethod
def _longer(first: float | httpx.Timeout, second: float | httpx.Timeout) -> float | httpx.Timeout:
if not isinstance(first, httpx.Timeout) and not isinstance(second, httpx.Timeout):
return max(first, second)
first_limits: Final = httpx.Timeout(first)
second_limits: Final = httpx.Timeout(second)
longer: Final = CompletionTimeout._longer_limit
return httpx.Timeout(
connect=longer(first_limits.connect, second_limits.connect),
read=longer(first_limits.read, second_limits.read),
write=longer(first_limits.write, second_limits.write),
pool=longer(first_limits.pool, second_limits.pool),
)
@staticmethod
def for_stream(
stream_kind: StreamKind, request_timeout: float | httpx.Timeout, stream_timeout: float | httpx.Timeout | None
) -> float | httpx.Timeout:
"""
``stream_timeout`` bounds each read of a live stream, so it replaces the request timeout there. A fake
stream is one non-streaming request whose first byte arrives when generation ends, so it takes the longer
of each limit and never fails sooner than the same call would with either one alone.
"""
if stream_timeout is None:
return request_timeout
match stream_kind:
case StreamKind.NOT_STREAMED:
return request_timeout
case StreamKind.LIVE:
return stream_timeout
case StreamKind.FAKED:
return CompletionTimeout._longer(request_timeout, stream_timeout)
case _:
assert_never(stream_kind)
@staticmethod
def as_limit(timeout: object) -> float | httpx.Timeout:
if isinstance(timeout, (httpx.Timeout, SDKTimeout)):
return CompletionTimeout.normalize(timeout)
if isinstance(timeout, (float, int, str)):
return float(timeout)
raise TypeError(f"timeout must be a number or an httpx.Timeout, got {type(timeout).__name__}")
@staticmethod
def stream_request_kwargs(non_stream_timeout: object, stream_timeout: object) -> Mapping[str, object]:
if stream_timeout is None:
return MappingProxyType({"timeout": non_stream_timeout})
return MappingProxyType(
{
"timeout": None
if non_stream_timeout is None
else CompletionTimeout.for_stream(
StreamKind.FAKED,
CompletionTimeout.as_limit(non_stream_timeout),
CompletionTimeout.as_limit(stream_timeout),
),
"stream_timeout": stream_timeout,
}
)
@staticmethod
def _coerce(
timeout: object, custom_llm_provider: str, supports_httpx_timeout: Callable[[str], bool]
) -> float | httpx.Timeout:
limit: Final = CompletionTimeout.as_limit(timeout)
if not isinstance(limit, httpx.Timeout) or supports_httpx_timeout(custom_llm_provider):
return limit
return float(limit.read) if limit.read is not None else COMPLETION_HTTP_FALLBACK_SECONDS
@staticmethod
def resolve(
model_timeout: float | str | httpx.Timeout | SDKTimeout | None,
kwargs: dict,
kwargs: Mapping[str, object],
custom_llm_provider: str,
*,
global_timeout: float | str | None,
supports_httpx_timeout: Callable[[str], bool],
stream_kind: StreamKind,
) -> float | httpx.Timeout:
"""
Resolution order (first non-None wins):
Request timeout resolution order (first non-None wins):
1. ``model_timeout`` (call argument / merged ``litellm_params``)
2. ``kwargs["timeout"]``
@ -54,24 +138,20 @@ class CompletionTimeout:
4. ``global_timeout`` (the explicitly-configured ``litellm.request_timeout``),
or 600 when nothing was configured.
A stream then applies ``kwargs["stream_timeout"]`` per :meth:`for_stream`.
Coerce :class:`httpx.Timeout` when the provider does not support it.
"""
resolved: float | str | httpx.Timeout | SDKTimeout
if model_timeout is not None:
resolved = model_timeout
elif kwargs.get("timeout") is not None:
resolved = kwargs["timeout"]
elif kwargs.get("request_timeout") is not None:
resolved = kwargs["request_timeout"]
else:
resolved = CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
if isinstance(resolved, (httpx.Timeout, SDKTimeout)):
normalized: Final = CompletionTimeout.normalize(resolved)
if supports_httpx_timeout(custom_llm_provider):
return normalized
read_timeout: Final = normalized.read
return (
float(read_timeout) if read_timeout is not None else COMPLETION_HTTP_FALLBACK_SECONDS
) # default 10 min timeout
return float(resolved)
coerce: Final = partial(
CompletionTimeout._coerce,
custom_llm_provider=custom_llm_provider,
supports_httpx_timeout=supports_httpx_timeout,
)
non_stream_candidates: Final = (model_timeout, kwargs.get("timeout"), kwargs.get("request_timeout"))
explicit: Final = next((candidate for candidate in non_stream_candidates if candidate is not None), None)
request_timeout: Final = coerce(
explicit if explicit is not None else CompletionTimeout._fallback_when_no_explicit_timeout(global_timeout)
)
stream_timeout: Final = None if stream_kind is StreamKind.NOT_STREAMED else kwargs.get("stream_timeout")
return CompletionTimeout.for_stream(
stream_kind, request_timeout, None if stream_timeout is None else coerce(stream_timeout)
)

52
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -1196,6 +1196,9 @@ def _map_sagemaker_exception(
)
_VERTEX_403_STATUS_CODE_PATTERN: Final = re.compile(r"(?<![0-9A-Za-z])403(?![0-9A-Za-z])")
def _map_vertex_exception(
*,
model: str,
@ -1206,6 +1209,8 @@ def _map_vertex_exception(
exception_provider: str,
extra_information: str,
) -> None:
status_code: Final[object] = getattr(original_exception, "status_code", None)
body_code: Final[int | None] = _get_body_error_code(error_str)
if "Vertex AI API has not been used in project" in error_str or "Unable to find your project" in error_str:
raise BadRequestError(
message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
@ -1252,7 +1257,11 @@ def _map_vertex_exception(
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
elif "403" in error_str:
elif (
(not isinstance(status_code, int) or status_code in (403, 500))
and body_code in (None, 403)
and _VERTEX_403_STATUS_CODE_PATTERN.search(error_str)
):
raise BadRequestError(
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
@ -1306,7 +1315,7 @@ def _map_vertex_exception(
elif (
isinstance(getattr(original_exception, "status_code", None), int)
and 500 <= original_exception.status_code < 600
and _get_body_error_code(error_str) == 429
and body_code == 429
):
# upstream gateway wraps a 429 inside a 5xx envelope
# e.g. HTTP 500/503 with {"error":{"code":429,...}}.
@ -2413,6 +2422,35 @@ def _map_exception_by_status(
)
_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_URL_SCHEME: Final = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*")
_HOST_WITH_OPTIONAL_PORT: Final = re.compile(
r"(?:(?:[A-Za-z0-9-]+\.)+[A-Za-z0-9-]+|localhost|\[[0-9A-Fa-f:.]+\])(?::\d+)?"
)
def with_deployment_api_base(litellm_params: object) -> Mapping[str, object]:
params: Final = as_str_mapping(litellm_params) or _NO_LITELLM_PARAMS
metadata: Final = as_str_mapping(params.get("metadata"))
deployment_api_base: Final = None if metadata is None else metadata.get("api_base")
if not isinstance(deployment_api_base, str) or not deployment_api_base:
return params
return MappingProxyType({**params, "api_base": deployment_api_base})
def _api_base_without_credentials(api_base: str) -> str:
scheme, separator, _ = api_base.partition("://")
scheme_prefix: Final = scheme + separator if separator and _URL_SCHEME.fullmatch(scheme) else ""
authority_and_rest: Final = api_base[len(scheme_prefix) :]
authority: Final = re.split(r"[/?#]", authority_and_rest, maxsplit=1)[0]
after_authority: Final = authority_and_rest[len(authority) :]
host: Final = authority.rpartition("@")[2]
if "@" in after_authority and _HOST_WITH_OPTIONAL_PORT.fullmatch(host) is None:
return ""
path: Final = re.split(r"[?#]", after_authority, maxsplit=1)[0]
return f"{scheme_prefix}{host}{path}"
def _is_guardrail_block(original_exception: Exception) -> bool:
from litellm.integrations.custom_guardrail import is_guardrail_intervention
@ -2483,8 +2521,9 @@ def exception_type(
if isinstance(custom_llm_provider, str) and len(custom_llm_provider) > 0:
exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception"
if _api_base:
extra_information += f"\nAPI Base: `{_api_base}`"
shown_api_base: Final = _api_base_without_credentials(_api_base) if _api_base else ""
if shown_api_base:
extra_information += f"\nAPI Base: `{shown_api_base}`"
if messages and len(messages) > 0 and litellm.redact_messages_in_exceptions is False:
extra_information += f"\nMessages: `{messages}`"
@ -2512,7 +2551,8 @@ def exception_type(
################################################################################
if (
"Request Timeout Error" in error_str
isinstance(original_exception, httpx.TimeoutException)
or "Request Timeout Error" in error_str
or "Request timed out" in error_str
or "Timed out generating response" in error_str
or "The read operation timed out" in error_str
@ -2520,7 +2560,7 @@ def exception_type(
exception_mapping_worked = True
raise Timeout(
message=f"APITimeoutError - Request timed out. Error_str: {error_str}",
message=f"APITimeoutError - Request timed out. Error_str: {error_str or type(mappable_exception).__name__}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,

58
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -324,10 +324,16 @@ def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _Fetc
return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable:
try:
return urllib.parse.urlsplit(url)
except ValueError as error:
return ModelCostMapReloadUnavailable(reason=f"invalid URL {url}: {error}")
def _attempt_fetch_s3(parts: urllib.parse.SplitResult, url: str, timeout: int) -> _FetchAttemptOutcome:
from botocore.exceptions import BotoCoreError, ClientError
parts: Final = urllib.parse.urlsplit(url)
bucket: Final = parts.netloc
key: Final = parts.path.lstrip("/")
if not bucket or not key:
@ -343,9 +349,11 @@ def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return await asyncio.to_thread(_attempt_fetch_s3, url, timeout)
return await asyncio.to_thread(_attempt_fetch_s3, parts, url, timeout)
try:
response: Final = await client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -354,9 +362,11 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return _attempt_fetch_s3(url, timeout)
return _attempt_fetch_s3(parts, url, timeout)
try:
response: Final = client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -547,22 +557,39 @@ def get_model_cost_map_loaded_at() -> "datetime | None":
return _cost_map_source_info.loaded_at
def _expand_model_aliases(model_cost: dict) -> dict:
class AliasedCostEntry(dict[str, object]):
__slots__ = ("names",)
def __init__(self, entry: Mapping[str, object], names: tuple[str, ...]) -> None:
super().__init__(entry)
self.names: tuple[str, ...] = names
def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:
return MappingProxyType(
{
owner: (owner, *(alias for alias, alias_owner in alias_owners.items() if alias_owner == owner))
for owner in dict.fromkeys(alias_owners.values())
}
)
def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]:
"""
Expand ``aliases`` lists in model cost entries into top-level entries.
Each alias gets a reference to the **same** dict object as the canonical
entry (zero memory overhead). The ``aliases`` key is removed from the
entry so downstream code never sees it.
Each alias gets a reference to the **same** ``AliasedCostEntry`` as the
canonical entry, which carries every name it is served under. The
``aliases`` key is removed from the entry so downstream code never sees it.
If an alias collides with an existing canonical entry the alias is
skipped and a warning is logged.
"""
aliases_to_add: Final[dict[str, dict]] = {}
alias_owners: Final[dict[str, str]] = {}
keys_with_aliases: Final[list[str]] = []
for model_name, model_info in model_cost.items():
aliases: list | None = model_info.get("aliases")
aliases: object = model_info.get("aliases")
if aliases is None:
continue
keys_with_aliases.append(model_name)
@ -584,7 +611,7 @@ def _expand_model_aliases(model_cost: dict) -> dict:
model_name,
)
continue
if alias in aliases_to_add:
if alias in alias_owners:
verbose_logger.warning(
"LiteLLM model alias conflict: alias '%s' (from '%s') "
"was already claimed by another entry — skipping.",
@ -592,13 +619,14 @@ def _expand_model_aliases(model_cost: dict) -> dict:
model_name,
)
continue
aliases_to_add[alias] = model_info # same dict reference
alias_owners[alias] = model_name
# Remove the ``aliases`` key from entries so it doesn't pollute model info
for key in keys_with_aliases:
model_cost[key].pop("aliases", None)
model_cost.update(aliases_to_add)
for owner, names in _alias_groups(alias_owners).items():
model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names)))
return model_cost

10
litellm/litellm_core_utils/get_provider_specific_headers.py
View file

@ -9,6 +9,7 @@ class ProviderSpecificHeaderUtils:
def get_provider_specific_headers(
provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
custom_llm_provider: str | None,
native_anthropic_messages: bool = False,
) -> Mapping[str, str]:
"""
Get the provider specific headers for the given custom llm provider.
@ -17,7 +18,8 @@ class ProviderSpecificHeaderUtils:
carries its own comma-separated provider list, so headers that are safe for several
providers and headers that are safe for exactly one can travel on the same request
without sharing a scope. Entries whose provider list does not contain
`custom_llm_provider` contribute nothing.
`custom_llm_provider` contribute nothing, and entries marked
`native_anthropic_messages_only` contribute only to a provider's native Messages route.
Returns:
Dict: The provider specific headers for the given custom llm provider
@ -31,6 +33,12 @@ class ProviderSpecificHeaderUtils:
matched_headers: Final = {}
for scoped_header in scoped_headers:
if (
"native_anthropic_messages_only" in scoped_header
and scoped_header["native_anthropic_messages_only"]
and not native_anthropic_messages
):
continue
stored_providers = scoped_header.get("custom_llm_provider", "")
provider_list = [p.strip() for p in stored_providers.split(",")]
if custom_llm_provider in provider_list:

201
litellm/litellm_core_utils/litellm_logging.py
View file

@ -13,12 +13,13 @@ import sys
import time
import traceback
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime as dt_object
from functools import lru_cache
from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
from httpx import Response
from httpx import Response, Timeout
from pydantic import BaseModel, JsonValue, TypeAdapter
import litellm
@ -558,6 +559,12 @@ def mask_api_base_credentials(api_base: str) -> str:
return api_base[:key_end] + "*" * 5 + api_base[-4:]
@dataclass(frozen=True, slots=True)
class UncachedPrice:
cost: float | None
cost_breakdown: CostBreakdown | None
def _timestamp_seconds(moment: object) -> float | None:
if isinstance(moment, datetime.datetime):
return moment.timestamp()
@ -627,15 +634,15 @@ class Logging(LiteLLMLoggingBaseClass):
self,
model: str,
messages,
stream,
stream: bool | None,
call_type,
start_time: datetime.datetime,
litellm_call_id: str,
function_id: str,
litellm_trace_id: str | None = None,
dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
applied_guardrails: list[str] | None = None,
@ -664,6 +671,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.message_copy_duration_ms: float = (time.time() - _copy_start) * 1000
self.callback_duration_ms: float = 0.0
self.stream = stream
self.applied_timeout: float | Timeout | None = None
self.start_time = start_time # log the call start time
self.call_type = call_type
self.relays_llm_passthrough: Final = call_type in (
@ -710,8 +718,10 @@ class Logging(LiteLLMLoggingBaseClass):
# Initialize dynamic callbacks
self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks
self.dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = dynamic_success_callbacks
self.dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = (
self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_success_callbacks
)
self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_async_success_callbacks
)
self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks
@ -778,6 +788,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.defer_async_logging: bool = False
self.enqueue_deferred_logging: Callable[[], None] | None = None
self._async_success_scheduled: bool = False
self._sync_success_scheduled: bool = False
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
@ -811,6 +822,10 @@ class Logging(LiteLLMLoggingBaseClass):
new_call.sync_streaming_chunks = []
return new_call
def reset_caching_details(self) -> None:
"""Forget an earlier cache lookup's result, so a call that looks up again reports only its latest."""
self.caching_details = None
def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
self.response_timing_metrics = dict(timing_metrics)
@ -1045,6 +1060,11 @@ class Logging(LiteLLMLoggingBaseClass):
if "custom_llm_provider" in self.model_call_details:
self.custom_llm_provider = self.model_call_details["custom_llm_provider"]
def record_applied_timeout(self, timeout: float | Timeout) -> None:
self.applied_timeout = timeout
self.litellm_params["timeout"] = timeout
self.model_call_details["litellm_params"] = self.litellm_params
def update_from_kwargs(
self,
kwargs: dict,
@ -1890,6 +1910,7 @@ class Logging(LiteLLMLoggingBaseClass):
cache_hit: bool | None = None,
litellm_model_name: str | None = None,
router_model_id: str | None = None,
ignore_stamped_cost: bool = False,
) -> float | None:
"""
Calculate response cost using result + logging object variables.
@ -1925,7 +1946,9 @@ class Logging(LiteLLMLoggingBaseClass):
):
hidden_params: Final = result_hidden_params
if (
"response_cost" in hidden_params and hidden_params["response_cost"] is not None
not ignore_stamped_cost
and "response_cost" in hidden_params
and hidden_params["response_cost"] is not None
): # use cost if already calculated
self._record_zero_cost_diagnostic(
priced_result,
@ -2051,6 +2074,20 @@ class Logging(LiteLLMLoggingBaseClass):
_response_cost_calculator = response_cost_calculator
def uncached_price(self, result: object) -> UncachedPrice:
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
debug_key: Final = "response_cost_failure_debug_information"
had_failure_debug_info: Final = debug_key in self.model_call_details
try:
cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown)
finally:
self.cost_breakdown = previous_cost_breakdown
self.billed_token_rates = previous_billed_token_rates
if not had_failure_debug_info:
self.model_call_details.pop(debug_key, None)
def _record_zero_cost_diagnostic(
self,
result: object,
@ -2284,7 +2321,7 @@ class Logging(LiteLLMLoggingBaseClass):
**kwargs,
)
if not self._should_run_sync_callbacks_for_async_calls():
if not self.stream or not self._should_run_sync_callbacks_for_async_calls():
return
executor.submit(
@ -2412,7 +2449,7 @@ class Logging(LiteLLMLoggingBaseClass):
self,
event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"],
) -> None:
if self.stream is not None and self.stream is True and event_type in ["async_success", "sync_success"]:
if self.stream is True and event_type in ["async_success", "sync_success"]:
"""
Ignore check on stream, as there can be multiple chunks
"""
@ -2909,6 +2946,8 @@ class Logging(LiteLLMLoggingBaseClass):
discarded as a duplicate of the in-progress create.
"""
self.model_call_details.pop("has_logged_async_success", None)
self.model_call_details.pop("has_logged_sync_success", None)
self._sync_success_scheduled = False
litellm_params = self.model_call_details.get("litellm_params")
if not isinstance(litellm_params, dict):
return
@ -3031,7 +3070,7 @@ class Logging(LiteLLMLoggingBaseClass):
complete_streaming_response: (
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
) = None
if "complete_streaming_response" in self.model_call_details:
if "complete_streaming_response" in self.model_call_details and not for_async_request:
return # break out of this.
complete_streaming_response = self._get_assembled_streaming_response(
result=result,
@ -3097,6 +3136,11 @@ class Logging(LiteLLMLoggingBaseClass):
dynamic_success_callbacks=self.dynamic_success_callbacks,
global_callbacks=litellm.success_callback,
)
delivery_callbacks: Final = (
self._get_completed_success_callbacks(asynchronous=False)
if for_async_request and not self.stream
else callbacks
)
## REDACT MESSAGES ##
result = redact_message_input_output_from_logging(
@ -3104,7 +3148,8 @@ class Logging(LiteLLMLoggingBaseClass):
result=result,
)
## LOGGING HOOK ##
for callback in callbacks:
hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
for callback in hook_callbacks:
if isinstance(callback, CustomGuardrail):
from litellm.types.guardrails import GuardrailEventHooks
@ -3134,7 +3179,7 @@ class Logging(LiteLLMLoggingBaseClass):
call_details_view: Final = MappingProxyType[str, object](call_details)
self.mark_logging_complete(event_type="sync_success")
for callback in callbacks:
for callback in delivery_callbacks:
try:
should_run = self.should_run_callback(
callback=callback,
@ -3406,7 +3451,12 @@ class Logging(LiteLLMLoggingBaseClass):
)
if (
isinstance(callback, CustomLogger)
and is_sync_request
and (is_sync_request or (for_async_request and not self.stream))
and not (
for_async_request
and self.stream
and callback in (self.dynamic_async_success_callbacks or ())
)
and self.call_type
!= CallTypes.pass_through.value # pass-through endpoints call async_log_success_event
): # custom logger class
@ -3423,7 +3473,7 @@ class Logging(LiteLLMLoggingBaseClass):
result = call_details_view["complete_response"]
callback.log_success_event(
kwargs=call_details,
kwargs=callback.redact_standard_logging_payload_from_model_call_details(call_details),
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3665,6 +3715,9 @@ class Logging(LiteLLMLoggingBaseClass):
dynamic_success_callbacks=self.dynamic_async_success_callbacks,
global_callbacks=litellm._async_success_callback,
)
delivery_callbacks: Final = (
self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks
)
result = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
@ -3673,7 +3726,8 @@ class Logging(LiteLLMLoggingBaseClass):
## LOGGING HOOK ##
for callback in callbacks:
hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
for callback in hook_callbacks:
try:
if isinstance(callback, CustomGuardrail):
from litellm.types.guardrails import GuardrailEventHooks
@ -3711,7 +3765,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.mark_logging_complete(event_type="async_success")
for callback in callbacks:
for callback in delivery_callbacks:
# check if callback can run for this request
litellm_params = self.model_call_details.get("litellm_params", {})
should_run = self.should_run_callback(
@ -3846,6 +3900,13 @@ class Logging(LiteLLMLoggingBaseClass):
traceback.format_exc(),
)
self._handle_callback_failure(callback=callback)
if not self.stream:
try:
self.handle_sync_success_callbacks_for_async_calls(
result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit
)
except RuntimeError:
verbose_logger.exception("Could not schedule synchronous success callbacks")
await flush_post_call_redis_batches()
def _handle_callback_failure(self, callback: object):
@ -4269,6 +4330,10 @@ class Logging(LiteLLMLoggingBaseClass):
if self._should_run_sync_callbacks_for_async_calls() is False:
return
if not self.stream and self._sync_success_scheduled:
return
if not self.stream:
self._sync_success_scheduled = True
executor.submit(
self.success_handler,
result,
@ -4287,11 +4352,57 @@ class Logging(LiteLLMLoggingBaseClass):
) -> None:
executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
def _get_completed_success_callbacks(
self, *, asynchronous: bool
) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
sync_callbacks: Final = self.get_combined_callback_list(
self.dynamic_success_callbacks, litellm.success_callback
)
async_callbacks: Final = self.get_combined_callback_list(
self.dynamic_async_success_callbacks,
litellm._async_success_callback, # pyright: ignore[reportPrivateUsage] # shared logging registry
)
legacy_sync_callbacks: Final = self._remove_internal_litellm_callbacks(
self._remove_internal_custom_logger_callbacks(sync_callbacks)
)
selected: Final = async_callbacks if asynchronous else sync_callbacks
other: Final = sync_callbacks if asynchronous else async_callbacks
return tuple(
callback
for callback in self.get_combined_callback_list(selected, other)
if (
(
(
getattr(callback.async_log_success_event, "__func__", callback.async_log_success_event)
is not CustomLogger.async_log_success_event
)
== asynchronous
and (
asynchronous
or getattr(callback.log_success_event, "__func__", callback.log_success_event)
is not CustomLogger.log_success_event
)
)
if isinstance(callback, CustomLogger)
else callback in selected
and (
asynchronous
or (
callback not in async_callbacks
and callback in legacy_sync_callbacks
and not (isinstance(callback, str) and callback in INTERNAL_CALLBACK_NAMES)
)
)
)
)
def _should_run_sync_callbacks_for_async_calls(self) -> bool:
"""
Returns:
- bool: True if sync callbacks should be run for async calls. eg. `langfuse`, `s3`
"""
if not self.stream:
return bool(self._get_completed_success_callbacks(asynchronous=False))
_combined_sync_callbacks: Final = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_success_callbacks,
global_callbacks=litellm.success_callback,
@ -4321,7 +4432,11 @@ class Logging(LiteLLMLoggingBaseClass):
_filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks)
return len(_filtered_failure_callbacks) > 0
def get_combined_callback_list(self, dynamic_success_callbacks: list | None, global_callbacks: list) -> list:
def get_combined_callback_list(
self,
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None,
global_callbacks: list[str | Callable[..., object] | CustomLogger],
) -> list[str | Callable[..., object] | CustomLogger]:
if dynamic_success_callbacks is None:
return list(global_callbacks)
return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
@ -6909,6 +7024,36 @@ def _autorouter_savings_for_payload(
return None
_CACHE_HIT_ZEROED_AMOUNTS: Final = MappingProxyType(
{
"cache_read_cost": CostBreakdown(cache_read_cost=0.0),
"cache_creation_cost": CostBreakdown(cache_creation_cost=0.0),
"reasoning_cost": CostBreakdown(reasoning_cost=0.0),
"original_cost": CostBreakdown(original_cost=0.0),
"discount_amount": CostBreakdown(discount_amount=0.0),
"margin_fixed_amount": CostBreakdown(margin_fixed_amount=0.0),
"margin_total_amount": CostBreakdown(margin_total_amount=0.0),
}
)
def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
"""Every amount at 0, keeping the pricing basis and rates a fresh call would have had, which savings reports read."""
zero_costs: Final = CostBreakdown(input_cost=0.0, output_cost=0.0, total_cost=0.0, tool_usage_cost=0.0)
if served is None:
return zero_costs
zeroed_additional_costs: Final = (
CostBreakdown(additional_costs=dict.fromkeys(served["additional_costs"], 0.0))
if "additional_costs" in served
else CostBreakdown()
)
return functools.reduce(
lambda breakdown, zeroed: breakdown | zeroed,
(_CACHE_HIT_ZEROED_AMOUNTS[key] for key in served if key in _CACHE_HIT_ZEROED_AMOUNTS),
served | zero_costs | zeroed_additional_costs,
)
def get_standard_logging_object_payload(
kwargs: dict | None,
init_response_obj: object,
@ -6992,17 +7137,10 @@ def get_standard_logging_object_payload(
"user", None
) # maintain backwards compatibility with old request body check
saved_cache_cost: float = 0.0
if cache_hit is True or served_from_cache(init_response_obj):
id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id
if cache_hit is True:
saved_cache_cost = (
logging_obj.response_cost_calculator(
result=init_response_obj,
cache_hit=False,
)
or 0.0
)
uncached_price: Final = logging_obj.uncached_price(result=init_response_obj) if cache_hit is True else None
saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0
## Get model cost information ##
base_model = get_base_model_from_metadata(model_call_details=kwargs)
@ -7019,7 +7157,7 @@ def get_standard_logging_object_payload(
# clean up litellm hidden params
clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
if clean_hidden_params["response_cost"] is None and raw_response_cost is not None:
if cache_hit is True or (clean_hidden_params["response_cost"] is None and raw_response_cost is not None):
clean_hidden_params["response_cost"] = llm_response_cost
if clean_hidden_params["litellm_overhead_time_ms"] is None and status == "success":
# /v1/messages dict results and the bridge stream wrappers keep it on the logging object;
@ -7081,7 +7219,14 @@ def get_standard_logging_object_payload(
):
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
request_cost_breakdown: Final = cost_breakdown_with_guardrail(
(
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
if uncached_price is not None
else logging_obj.cost_breakdown
),
guardrail_cost,
)
captured_baseline: Final = logging_obj.baseline_observation
autorouter_savings: Final = (
None

27
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -5533,6 +5533,11 @@ def _bedrock_tools_pt(
return tool_block_list
def bedrock_tool_blocks(tools: Sequence[Mapping[str, object]], model: str | None) -> tuple[BedrockToolBlock, ...]:
"""Bedrock Converse ``toolConfig.tools`` entries for OpenAI or Anthropic tool definitions."""
return tuple(_bedrock_tools_pt([dict(tool) for tool in tools], model=model))
def _is_system_message(message: Mapping[str, object]) -> bool:
role: Final = message.get("role")
return isinstance(role, str) and "system" in role
@ -6166,3 +6171,25 @@ def resolve_structured_messages(
if structured:
return [msg if isinstance(msg, dict) else msg.model_dump() for msg in structured]
return None
def is_flat_responses_tool(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type")
return tool_type in ("function", "custom") and tool_type not in tool and "input_schema" not in tool
def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
if is_flat_responses_tool(tool):
chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
[dict(tool)] # pyright: ignore[reportArgumentType] # a JSON dict in a flat Responses tool shape
)
return tuple(chat_tools)
anthropic_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai(
[dict(tool)] # pyright: ignore[reportArgumentType] # a JSON dict in an Anthropic or chat tool shape
)
return tuple(anthropic_tools)

4
litellm/litellm_core_utils/served_output_texts.py
View file

@ -126,13 +126,13 @@ def _chat_stream_choice_text(chunks: Sequence[ModelResponseStream], index: int)
content
for chunk in chunks
for choice in chunk.choices
if choice.index == index and isinstance(content := choice.delta.content, str)
if (choice.index or 0) == index and isinstance(content := choice.delta.content, str)
)
return "".join(contents) if contents else None
def _chat_stream_texts(chunks: Sequence[ModelResponseStream]) -> ServedTexts | None:
choice_count: Final = max((choice.index + 1 for chunk in chunks for choice in chunk.choices), default=0)
choice_count: Final = max(((choice.index or 0) + 1 for chunk in chunks for choice in chunk.choices), default=0)
texts: Final = tuple(_chat_stream_choice_text(chunks, index) for index in range(choice_count))
return texts if any(text is not None for text in texts) else None

44
litellm/litellm_core_utils/streaming_handler.py
View file

@ -45,9 +45,9 @@ from litellm.types.utils import (
)
from litellm.types.utils import GenericStreamingChunk as GChunk
from ..exceptions import OpenAIError
from ..exceptions import OpenAIError, Timeout
from .core_helpers import map_finish_reason, process_response_headers
from .exception_mapping_utils import exception_type
from .exception_mapping_utils import exception_type, with_deployment_api_base
from .llm_response_utils.get_api_base import get_api_base
from .rules import Rules
@ -88,6 +88,16 @@ def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, o
setattr(response, key, value)
def _is_transport_timeout(error: BaseException) -> bool:
return isinstance(error, httpx.TimeoutException) or (
isinstance(error, Timeout) and isinstance(error.__cause__, httpx.TimeoutException)
)
def _skips_stream_fallback(status_code: int | None) -> bool:
return status_code is not None and 400 <= status_code < 500 and status_code != 429
def is_async_iterable(obj: object) -> bool:
"""
Check if an object is an async iterable (can be used with 'async for').
@ -1579,11 +1589,7 @@ class CustomStreamWrapper:
except Exception as e:
traceback.format_exc()
setattr(e, "message", str(e))
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=e,
)
raise self._map_stream_exception(e)
def set_logging_event_loop(self, loop):
"""
@ -2370,6 +2376,14 @@ class CustomStreamWrapper:
if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta):
self._emitted_disqualifying_content = True
def _map_stream_exception(self, original_exception: Exception) -> Exception:
return exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=original_exception,
extra_kwargs=with_deployment_api_base(self.logging_obj.model_call_details.get("litellm_params")),
)
def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
"""
Common error handling for both __next__ and __anext__.
@ -2380,6 +2394,8 @@ class CustomStreamWrapper:
429 (rate-limit) is explicitly exempted from the 4xx filter because
it is transient and the Router should switch to another model group.
So is a transport read timeout, which maps to a 408 Timeout, including
one an inner Responses stream already mapped for the chat bridge.
An error an inner stream already wrapped (the chat-to-Responses bridge
consumes a Responses stream) is rebuilt around the provider exception
@ -2414,13 +2430,7 @@ class CustomStreamWrapper:
mapped_exception: Exception = e
else:
try:
mapped_exception = exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider,
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
mapped_exception = self._map_stream_exception(e)
except Exception as mapping_error:
mapped_exception = mapping_error
self._restore_consumer_correlation_context()
@ -2450,9 +2460,9 @@ class CustomStreamWrapper:
# Raise non-retriable client errors directly (skip fallback).
# Exception: 429 (rate-limit) IS retriable/transient — allow it
# through so the Router can switch to a different model group.
if mapped_status_code is not None and 400 <= mapped_status_code < 500 and mapped_status_code != 429:
raise mapped_exception
if original_status_code is not None and 400 <= original_status_code < 500 and original_status_code != 429:
if not _is_transport_timeout(e) and (
_skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
):
raise mapped_exception
raise MidStreamFallbackError(

62
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -30,6 +30,7 @@ from litellm.llms.anthropic.pass_through.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
is_provider_native_tool_dict,
)
from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.pass_through.stream_assembly import (
build_complete_streaming_response,
build_usage_only_response_from_chunks,
@ -1456,6 +1457,9 @@ class AnthropicMessagesHandler(BaseTranslation):
return _SSEFieldRewrite("delta", "text", next(replacements))
cls._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta)
cls._redact_ended_stream_message_delta(
responses_so_far, kept_tool_use_ids=frozenset(cls._streamed_tool_use_ids(responses_so_far).values())
)
@classmethod
def _write_ended_stream_tool_call_rewrites(
@ -1523,6 +1527,64 @@ class AnthropicMessagesHandler(BaseTranslation):
return None
cls._rewrite_ended_stream_events(responses_so_far, rewrite_tool_use)
streamed_tool_use_ids: Final = cls._streamed_tool_use_ids(responses_so_far)
cls._redact_ended_stream_message_delta(
responses_so_far,
kept_tool_use_ids=frozenset(
tool_use_id for index, tool_use_id in streamed_tool_use_ids.items() if index not in rewrites_by_block
),
)
@classmethod
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
"""The id of each buffered ``tool_use`` block, by its content block index."""
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
tool_use_blocks: Final = (
(event.get("index"), block)
for event in events
if event.get("type") == "content_block_start"
and (block := as_str_mapping(event.get("content_block"))) is not None
and block.get("type") == "tool_use"
)
return MappingProxyType(
{
index: tool_use_id
for index, block in tool_use_blocks
if isinstance(index, int) and isinstance(tool_use_id := block.get("id"), str)
}
)
@classmethod
def _redact_ended_stream_message_delta(
cls,
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
*,
kept_tool_use_ids: frozenset[str],
) -> None:
"""Hold the buffered ``message_delta`` to the rules a re-emitted rewrite follows: a verdict survives
only for a tool call delivered unchanged, and no free-text ``explanation`` describes the text the
guardrail rewrote."""
def rewrite_stop_details(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
delta: Final = as_str_mapping(event.get("delta")) if event.get("type") == "message_delta" else None
if delta is None or "stop_details" not in delta:
return None
return _SSEFieldRewrite(
"delta", "stop_details", redacted_stop_details(delta["stop_details"], keep_explanations=False)
)
def rewrite_safeguard_results(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
delta: Final = as_str_mapping(event.get("delta")) if event.get("type") == "message_delta" else None
if delta is None or "safeguard_results" not in delta:
return None
return _SSEFieldRewrite(
"delta",
"safeguard_results",
filtered_safeguard_results(delta["safeguard_results"], kept_tool_use_ids, keep_explanations=False),
)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_stop_details)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_safeguard_results)
@staticmethod
def _rewrite_ended_stream_events(

10
litellm/llms/anthropic/pass_through/adapters/handler.py
View file

@ -501,7 +501,11 @@ class LiteLLMMessagesToCompletionTransformationHandler:
return None
if resolved_provider == "litellm_proxy":
return None
from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary, responses_api_bridge_check
from litellm.responses.bridge_check import (
chat_target_accepts_reasoning_summary,
responses_api_bridge_check,
str_keyed_mapping,
)
if requested_summary is not None and chat_target_accepts_reasoning_summary(local_model, resolved_provider):
return _ChatTargetReasoning(effort=effort, summary=requested_summary)
@ -510,6 +514,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
web_search_options: Final = completion_kwargs.get("web_search_options")
tools: Final = completion_kwargs.get("tools")
client_summary: Final = thinking.get("summary")
model_info, _ = responses_api_bridge_check(
model=local_model,
custom_llm_provider=resolved_provider,
@ -517,8 +522,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
cast(OpenAIWebSearchOptions, web_search_options) if isinstance(web_search_options, dict) else None
),
tools=cast("list[dict[str, object]]", tools) if isinstance(tools, list) else None,
reasoning_effort=reasoning_effort,
reasoning_effort=effort if client_summary is None else {"effort": effort, "summary": client_summary},
api_base=resolved_api_base,
deployment_model_info=str_keyed_mapping(completion_kwargs.get("model_info")),
)
return None if model_info.get("mode") == "responses" else _ChatTargetReasoning(effort=effort)

11
litellm/llms/anthropic/pass_through/adapters/transformation.py
View file

@ -14,6 +14,7 @@ from litellm.llms.anthropic.pass_through.utils import (
json_copy_or_original,
prompt_cache_key_from_user_id,
)
from litellm.responses.utils import auto_reasoning_summary
# OpenAI has a 64-character limit for function/tool names
# Anthropic does not have this limit, so we need to truncate long names
@ -736,7 +737,7 @@ class LiteLLMAnthropicMessagesAdapter:
if reasoning_effort:
return {
"reasoning_effort": LiteLLMAnthropicMessagesAdapter._apply_reasoning_summary_wrapping(
reasoning_effort, thinking
reasoning_effort, thinking, model, None
)
}
return {}
@ -745,6 +746,8 @@ class LiteLLMAnthropicMessagesAdapter:
def _apply_reasoning_summary_wrapping(
reasoning_effort: str,
thinking: Mapping[str, object],
model: str,
custom_llm_provider: str | None,
) -> Any:
"""
Apply the reasoning_effort/summary wrapping rules shared by every
@ -762,7 +765,7 @@ class LiteLLMAnthropicMessagesAdapter:
if summary:
return {"effort": reasoning_effort, "summary": summary}
if is_reasoning_auto_summary_enabled():
return {"effort": reasoning_effort, "summary": "detailed"}
return {"effort": reasoning_effort, "summary": auto_reasoning_summary(model, custom_llm_provider)}
return reasoning_effort
def translate_anthropic_tool_choice_to_openai(
@ -1174,7 +1177,9 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs["reasoning_effort"] = (
reasoning_effort
if is_claude_target
else self._apply_reasoning_summary_wrapping(reasoning_effort, cast(dict[str, object], thinking))
else self._apply_reasoning_summary_wrapping(
reasoning_effort, cast(dict[str, object], thinking), model, custom_llm_provider
)
)
def _translate_output_format_to_openai(

15
litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py
View file

@ -12,6 +12,7 @@ import json
from collections.abc import Mapping
from typing import Any, Final, cast
from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -34,8 +35,13 @@ class FakeAnthropicMessagesStreamIterator:
- message_stop
"""
def __init__(self, response: AnthropicMessagesResponse):
def __init__(
self,
response: AnthropicMessagesResponse,
message_delta_fields: Mapping[str, object] | None = None,
):
self.response = response
self.message_delta_fields = message_delta_fields
self.chunks = self._create_streaming_chunks()
self.current_index = 0
@ -175,14 +181,9 @@ class FakeAnthropicMessagesStreamIterator:
cache_read_input_tokens: Final = usage.get("cache_read_input_tokens")
if cache_read_input_tokens is not None:
delta_usage["cache_read_input_tokens"] = cache_read_input_tokens
safeguard_results: Final = self.response.get("safeguard_results")
message_delta: Final = {
"type": "message_delta",
"delta": {
"stop_reason": self.response.get("stop_reason"),
"stop_sequence": self.response.get("stop_sequence"),
**({"safeguard_results": safeguard_results} if safeguard_results is not None else {}),
},
"delta": anthropic_message_delta(response_dict, self.message_delta_fields),
"usage": delta_usage,
}
chunks.append(f"event: message_delta\ndata: {json.dumps(message_delta)}\n\n".encode())

91
litellm/llms/anthropic/pass_through/messages/handler.py
View file

@ -7,13 +7,14 @@
import asyncio
import contextvars
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
from functools import partial
from typing import Final, cast
from pydantic import TypeAdapter
import litellm
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
@ -39,8 +40,7 @@ from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler
from ..safeguards import (
SafeguardsEvaluator,
native_route_classifies,
read_classifier_model_setting,
requested_dangerous_tool_use,
proxy_serves_safeguards,
with_safeguard_results,
)
from .interceptors import get_messages_interceptors
@ -55,15 +55,19 @@ _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object])
_SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]])
def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool:
from litellm.responses.bridge_check import responses_api_bridge_check
def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool:
from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping
model_info, _ = responses_api_bridge_check(model=model, custom_llm_provider=custom_llm_provider)
model_info, _ = responses_api_bridge_check(
model=model,
custom_llm_provider=custom_llm_provider,
deployment_model_info=str_keyed_mapping(deployment_model_info),
)
return model_info.get("mode") == "responses"
def _responses_mode_is_lost_by_prefix_strip(
requested_model: str, resolved_model: str, custom_llm_provider: str
requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object
) -> bool:
"""Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
@ -73,15 +77,16 @@ def _responses_mode_is_lost_by_prefix_strip(
"""
if requested_model == resolved_model:
return False
return _bridges_to_responses_api(requested_model, custom_llm_provider) and not _bridges_to_responses_api(
resolved_model, custom_llm_provider
)
return _bridges_to_responses_api(
requested_model, custom_llm_provider, deployment_model_info
) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info)
def _should_route_to_responses_api(
custom_llm_provider: str | None,
requested_model: str | None = None,
resolved_model: str | None = None,
deployment_model_info: object = None,
) -> bool:
"""Return True when the request should use the Responses API path.
@ -94,16 +99,24 @@ def _should_route_to_responses_api(
return True
if custom_llm_provider is None or requested_model is None or resolved_model is None:
return False
return _responses_mode_is_lost_by_prefix_strip(requested_model, resolved_model, custom_llm_provider)
return _responses_mode_is_lost_by_prefix_strip(
requested_model, resolved_model, custom_llm_provider, deployment_model_info
)
async def _with_native_safeguard_results(
pending: Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[bytes]],
evaluator: SafeguardsEvaluator,
proxy_classifies: Callable[[], Awaitable[bool]],
build_evaluator: Callable[[], SafeguardsEvaluator | None],
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
from .streaming_iterator import AnthropicMessagesStreamingResponse, SafeguardedAnthropicSSEStream
response: Final = await pending
if not await proxy_classifies():
return response
evaluator: Final = await asyncio.to_thread(build_evaluator)
if evaluator is None:
return response
if isinstance(response, AnthropicMessagesStreamingResponse):
return AnthropicMessagesStreamingResponse(
completion_stream=SafeguardedAnthropicSSEStream(response, evaluator),
@ -114,6 +127,10 @@ async def _with_native_safeguard_results(
return SafeguardedAnthropicSSEStream(response, evaluator)
def _native_safeguards_evaluator(kwargs: Mapping[str, object], messages: object) -> SafeguardsEvaluator | None:
return request_safeguards_evaluator(kwargs, _SAFEGUARD_MESSAGES.validate_python(messages))
def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
"""Whether the deployment opted into forwarding /v1/messages untranslated.
@ -132,7 +149,7 @@ def _deployment_supports_cache_control_ttl(model_info: object) -> bool:
def _native_messages_config(
model: str, custom_llm_provider: str | None, model_info: object
model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None
) -> BaseAnthropicMessagesConfig | None:
provider_config: Final = (
ProviderConfigManager.get_provider_anthropic_messages_config(
@ -142,8 +159,11 @@ def _native_messages_config(
if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]
else None
)
if provider_config is not None or not _deployment_passes_through_anthropic_messages(model_info):
return provider_config
native_config: Final = (
provider_config if provider_config is not None and provider_config.serves_api_base(model, api_base) else None
)
if native_config is not None or not _deployment_passes_through_anthropic_messages(model_info):
return native_config
from litellm.llms.openai_like.messages.transformation import (
OpenAILikeAnthropicMessagesConfig,
)
@ -151,12 +171,26 @@ def _native_messages_config(
return OpenAILikeAnthropicMessagesConfig(cache_control_ttl=_deployment_supports_cache_control_ttl(model_info))
def proxy_classifies_tool_uses(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> bool:
async def _proxy_classifies(model: str, safeguards: object, native_config: BaseAnthropicMessagesConfig | None) -> bool:
if not proxy_serves_safeguards(safeguards):
return False
if native_config is None:
return True
if native_config.should_filter_anthropic_beta_headers():
await aload_beta_headers_config()
return not native_config.supports_safeguards() or native_route_classifies(model, safeguards)
async def proxy_classifies_tool_uses(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> bool:
"""Whether this proxy, rather than the upstream, answers ``safeguards`` for a request routed this way."""
safeguards: Final = kwargs.get("safeguards")
if _native_messages_config(model, custom_llm_provider, kwargs.get("model_info")) is not None:
return native_route_classifies(model, safeguards)
return requested_dangerous_tool_use(safeguards) is not None and read_classifier_model_setting() is not None
api_base: Final = kwargs.get("api_base")
return await _proxy_classifies(
model,
kwargs.get("safeguards"),
_native_messages_config(
model, custom_llm_provider, kwargs.get("model_info"), api_base if isinstance(api_base, str) else None
),
)
####### ENVIRONMENT VARIABLES ###################
@ -601,12 +635,12 @@ def anthropic_messages_handler(
)
anthropic_messages_provider_config: Final = _native_messages_config(
model, custom_llm_provider, kwargs.get("model_info")
model, custom_llm_provider, kwargs.get("model_info"), api_base
)
if anthropic_messages_provider_config is None:
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
if kwargs.get("compaction") is None and _should_route_to_responses_api(
custom_llm_provider, original_model, model
custom_llm_provider, original_model, model, kwargs.get("model_info")
):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=max_tokens,
@ -684,11 +718,6 @@ def anthropic_messages_handler(
safeguard_kwargs: Final = (
_SAFEGUARD_KWARGS.validate_python(kwargs) if is_async is True and "safeguards" in kwargs else None
)
safeguards_evaluator: Final = (
request_safeguards_evaluator(safeguard_kwargs, _SAFEGUARD_MESSAGES.validate_python(messages))
if safeguard_kwargs is not None and native_route_classifies(model, safeguard_kwargs.get("safeguards"))
else None
)
native_response: Final = base_llm_http_handler.anthropic_messages_handler(
model=model,
messages=cast( # cast-ok: anthropic messages are dicts at runtime
@ -709,6 +738,10 @@ def anthropic_messages_handler(
stream=stream,
kwargs=kwargs,
)
if safeguards_evaluator is None or not isinstance(native_response, Coroutine):
if safeguard_kwargs is None or not isinstance(native_response, Coroutine):
return native_response
return _with_native_safeguard_results(native_response, safeguards_evaluator)
return _with_native_safeguard_results(
native_response,
partial(_proxy_classifies, model, safeguard_kwargs.get("safeguards"), anthropic_messages_provider_config),
partial(_native_safeguards_evaluator, safeguard_kwargs, messages),
)

2
litellm/llms/anthropic/pass_through/messages/mcp_handler.py
View file

@ -136,7 +136,7 @@ async def anthropic_messages_with_mcp(
)
custom_llm_provider: Final = kwargs.get("custom_llm_provider")
classified_here: Final = proxy_classifies_tool_uses(
classified_here: Final = await proxy_classifies_tool_uses(
model, custom_llm_provider if isinstance(custom_llm_provider, str) else None, kwargs
)
loop_call_args: Final[Mapping[str, object]] = (

26
litellm/llms/anthropic/pass_through/messages/streaming_iterator.py
View file

@ -18,7 +18,11 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
from litellm.llms.anthropic.pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE
from litellm.llms.anthropic.pass_through.messages.utils import (
INCOMPLETE_STREAM_ERROR_MESSAGE,
MESSAGE_DELTA_OPTIONAL_KEYS,
anthropic_message_delta,
)
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
from litellm.proxy.pass_through_endpoints.success_handler import (
@ -280,30 +284,32 @@ def anthropic_messages_response_as_sse_events(response: AnthropicMessagesRespons
content_events: Final = (
event for index, block in enumerate(content_blocks) for event in _anthropic_content_block_events(index, block)
)
# A real message_start always carries a null stop_reason/stop_sequence and
# a zero output_tokens - those are only known once generation finishes, so
# copying the completed response's final values here would let a client
# treat the message as already finished, or double-count output tokens.
# A real message_start always carries a null stop_reason/stop_sequence/stop_details,
# a zero output_tokens and no safeguard_results - those are only known once
# generation finishes, so copying the completed response's final values here
# would let a client treat the message as already finished, or double-count
# output tokens.
message_start_usage: Final = {
**(response.get("usage") or {}),
"output_tokens": 0,
}
message_start_fields: Final = {
key: value for key, value in response.items() if key not in MESSAGE_DELTA_OPTIONAL_KEYS
}
message_start_payload: Final = {
"type": "message_start",
"message": {
**response,
**message_start_fields,
"content": [],
"stop_reason": None,
"stop_sequence": None,
"stop_details": None,
"usage": message_start_usage,
},
}
message_delta_payload: Final = {
"type": "message_delta",
"delta": {
"stop_reason": response.get("stop_reason"),
"stop_sequence": response.get("stop_sequence"),
},
"delta": anthropic_message_delta(response),
"usage": response.get("usage") or {},
}
return (

70
litellm/llms/anthropic/pass_through/messages/transformation.py
View file

@ -3,9 +3,12 @@ from typing import ClassVar, Final
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -109,6 +112,27 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
def should_filter_anthropic_beta_headers(self) -> bool:
return self._resolved_provider != "anthropic"
def supports_safeguards(self) -> bool:
"""
Whether the upstream platform answers the ``safeguards`` body field itself. A platform
whose beta header map column marks the dangerous-tool-use beta unsupported does not, and
a platform the map does not list is trusted to unless its config says otherwise.
"""
return not self.should_filter_anthropic_beta_headers() or (
ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
not in get_unsupported_headers(self._resolved_provider)
)
def drops_safeguards(self, safeguards: object, drop_params: bool) -> bool:
"""
``safeguards`` and the dangerous-tool-use beta leave together for a platform that does not
answer them, when the proxy classifies in its place or drop_params is set. Otherwise both
are forwarded as sent and the platform answers its own 400, like any unsupported param.
"""
return not self.supports_safeguards() and (
drop_params or litellm.drop_params is True or proxy_serves_safeguards(safeguards)
)
def _remove_scope_from_cache_control(self, anthropic_messages_request: dict) -> None:
"""
Remove `scope` field from cache_control blocks.
@ -207,7 +231,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
ids were found without tool_result blocks immediately after") while
consecutive user turns merge upstream.
Billing-header system blocks are stripped from the top-level ``system``
field regardless of whether anything was hoisted.
field when ``should_strip_billing_metadata`` says so, regardless of
whether anything was hoisted.
Subclasses whose upstream rejects the role opt in by calling this from
their ``transform_anthropic_messages_request``; the first-party Anthropic
@ -241,7 +266,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
)
for block in as_system_content_blocks(source)
]
filtered_system: Final = self._filter_billing_headers_from_system(system_content)
filtered_system: Final = (
self._filter_billing_headers_from_system(system_content)
if self.should_strip_billing_metadata()
else system_content
)
if filtered_system:
anthropic_messages_request["system"] = filtered_system
else:
@ -591,6 +620,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
status_code=400,
)
if "safeguards" in anthropic_messages_optional_request_params and self.drops_safeguards(
anthropic_messages_optional_request_params["safeguards"], # pyright: ignore[reportUnknownArgumentType] # the request params arrive as a bare dict
litellm_params.drop_params is True,
):
anthropic_messages_optional_request_params.pop("safeguards") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
self._translate_reasoning_effort_to_anthropic(
model=model,
optional_params=anthropic_messages_optional_request_params,
@ -683,9 +718,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
self,
model: str,
httpx_response: httpx.Response,
request_body: dict,
request_body: dict[str, object], # mutable-ok: mirrors the base signature
litellm_logging_obj: LiteLLMLoggingObj,
) -> AsyncIterator:
) -> AsyncIterator[bytes]:
"""Helper function to handle Anthropic streaming responses using the existing logging handlers"""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
BaseAnthropicMessagesStreamingIterator,
@ -702,12 +737,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
litellm_logging_obj=litellm_logging_obj,
)
@staticmethod
def _update_headers_with_anthropic_beta(
self,
headers: dict,
optional_params: dict,
custom_llm_provider: str = "anthropic",
messages: Sequence[object] = (),
beta_map_provider: str | None = None,
drop_params: bool = False,
) -> dict:
"""
Auto-inject anthropic-beta headers based on features used.
@ -724,8 +761,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params: Optional parameters including tools, context_management, output_format, speed
custom_llm_provider: Provider name for looking up correct tool search header
messages: Request messages, scanned for per-message output_config
beta_map_provider: Provider whose beta map column gates safeguards, defaults to custom_llm_provider
drop_params: Whether the deployment or request set drop_params
"""
beta_values: Final[set] = set()
beta_values: Final[set[str]] = set()
existing_beta: Final = tuple(
piece.strip()
@ -775,6 +814,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if optional_params.get("speed") == "fast":
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value)
if optional_params.get("safeguards") is not None and is_beta_header_supported_by_bundled_config(
ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value,
beta_map_provider or custom_llm_provider,
):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
@ -796,9 +841,20 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
else ()
)
all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas)
dangerous_tool_use: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
forwarded_beta_values: Final = ( # pyright: ignore[reportUnknownVariableType] # derived from the bare beta set
all_beta_values - {dangerous_tool_use}
if dangerous_tool_use in all_beta_values
and self.drops_safeguards(
optional_params.get("safeguards"), # pyright: ignore[reportUnknownArgumentType] # the optional params arrive as a bare dict
drop_params,
)
else all_beta_values
)
if not all_beta_values:
return headers
merged: Final = {key: value for key, value in headers.items() if key.lower() != "anthropic-beta"}
merged["anthropic-beta"] = ",".join(sorted(all_beta_values))
if forwarded_beta_values:
merged["anthropic-beta"] = ",".join(sorted(forwarded_beta_values))
return merged

91
litellm/llms/anthropic/pass_through/messages/utils.py
View file

@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Final, cast, get_type_hints
from pydantic import JsonValue
from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.core_helpers import as_str_mapping
@ -49,6 +49,95 @@ def refusal_stop_details(explanation: str | None) -> AnthropicStopDetails:
return AnthropicStopDetails(type="refusal", category=None, explanation=explanation)
MESSAGE_DELTA_OPTIONAL_KEYS: Final = ("stop_details", "safeguard_results")
def anthropic_message_delta(
response: Mapping[str, object], optional_fields: Mapping[str, object] | None = None
) -> Mapping[str, object]:
"""
The ``delta`` of the ``message_delta`` event synthesized from a complete
Anthropic Messages response. A real stream carries ``stop_details`` and
``safeguard_results`` there, so a client that asked to stream reads them
from this event and nowhere else. ``optional_fields`` replaces the
response's own when it was rebuilt from a stream that carried them.
"""
optional_source: Final = response if optional_fields is None else optional_fields
present_optional: Final = {
key: value for key in MESSAGE_DELTA_OPTIONAL_KEYS if (value := optional_source.get(key)) is not None
}
return {
"stop_reason": response.get("stop_reason"),
"stop_sequence": response.get("stop_sequence"),
**present_optional,
}
_SAFEGUARD_RESULT_ITEMS: Final = TypeAdapter(tuple[object, ...])
def redacted_stop_details(raw: object, keep_explanations: bool) -> object:
"""``stop_details`` as a rewritten stream may deliver it: its free-text ``explanation`` can quote
the text a guardrail rewrote, so it is nulled unless the rewrite changed no text."""
stop_details: Final = as_str_mapping(raw)
if keep_explanations or stop_details is None or "explanation" not in stop_details:
return raw
return {**stop_details, "explanation": None}
def _without_explanation(raw: object) -> object:
mapping: Final = as_str_mapping(raw)
if mapping is None:
return raw
return {key: value for key, value in mapping.items() if key != "explanation"}
def _status_without_explanations(raw: object) -> object:
status: Final = as_str_mapping(_without_explanation(raw))
if status is None:
return raw
tool_uses: Final = as_str_mapping(status.get("tool_uses"))
if tool_uses is None:
return status
return {
**status,
"tool_uses": {tool_use_id: _without_explanation(verdict) for tool_use_id, verdict in tool_uses.items()},
}
def _result_without_explanations(raw: object) -> object:
result: Final = as_str_mapping(_without_explanation(raw))
if result is None:
return raw
if "status" not in result:
return result
return {**result, "status": _status_without_explanations(result["status"])}
def _kept_tool_use_verdicts(raw: object, kept_tool_use_ids: frozenset[str]) -> object:
result: Final = as_str_mapping(raw)
status: Final = as_str_mapping(result.get("status")) if result is not None else None
tool_uses: Final = as_str_mapping(status.get("tool_uses")) if status is not None else None
if result is None or status is None or tool_uses is None:
return raw
kept_tool_uses: Final = {
tool_use_id: verdict for tool_use_id, verdict in tool_uses.items() if tool_use_id in kept_tool_use_ids
}
return {**result, "status": {**status, "tool_uses": kept_tool_uses}}
def filtered_safeguard_results(raw: object, kept_tool_use_ids: frozenset[str], keep_explanations: bool) -> object:
"""``safeguard_results`` as a rewritten stream may deliver them: a ``tool_uses`` verdict survives only
for a tool call the client gets with the input the classifier judged, and the free-text ``explanation`` of a
result, its status and each verdict is dropped unless the rewrite changed no text."""
try:
results: Final = _SAFEGUARD_RESULT_ITEMS.validate_python(raw)
except ValidationError:
return raw if keep_explanations else None
kept: Final = tuple(_kept_tool_use_verdicts(result, kept_tool_use_ids) for result in results)
return kept if keep_explanations else tuple(_result_without_explanations(result) for result in kept)
def _mapping_field(container: object, key: str) -> object | None:
"""One key of a raw provider payload, or None when the payload is not a mapping."""
if not isinstance(container, Mapping):

2
litellm/llms/anthropic/pass_through/responses_adapters/handler.py
View file

@ -16,6 +16,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
)
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
from litellm.responses.utils import auto_reasoning_summary
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
AllAnthropicMessageValues,
@ -188,6 +189,7 @@ def _build_responses_kwargs(
include_encrypted_reasoning=_provider_returns_encrypted_reasoning(
model, forwarded_kwargs.get("custom_llm_provider")
),
auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")),
)
# Normalize reasoning effort based on model capabilities

8
litellm/llms/anthropic/pass_through/responses_adapters/transformation.py
View file

@ -544,6 +544,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
def translate_thinking_to_reasoning(
thinking: dict[str, object],
output_config: dict[str, object] | None = None,
auto_summary_level: str = "detailed",
) -> dict[str, object] | None:
"""
Convert Anthropic thinking param to Responses API reasoning param.
@ -576,7 +577,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if summary:
result["summary"] = summary
elif auto_summary:
result["summary"] = "detailed"
result["summary"] = auto_summary_level
return result
@classmethod
@ -591,6 +592,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
self,
anthropic_request: AnthropicMessagesRequest,
include_encrypted_reasoning: bool = True,
auto_summary_level: str = "detailed",
) -> dict[str, Any]:
"""
Translate a full Anthropic /v1/messages request dict to
@ -599,7 +601,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
``include_encrypted_reasoning`` asks the provider for ``reasoning.encrypted_content``
on every call, so a reasoning model's items can be replayed intact next turn even
when the client sent no ``thinking`` block; pass False for a provider whose
Responses API rejects ``include``.
Responses API rejects ``include``. ``auto_summary_level`` is the ``reasoning.summary`` the
opt-in auto summary injects when the client sent none.
"""
model: Final[str] = anthropic_request["model"]
messages_list: Final = cast(
@ -670,6 +673,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
reasoning: Final = self.translate_thinking_to_reasoning(
thinking,
output_config=cast(dict[str, Any] | None, output_config),
auto_summary_level=auto_summary_level,
)
if reasoning:
responses_kwargs["reasoning"] = reasoning

10
litellm/llms/anthropic/pass_through/safeguards.py
View file

@ -273,12 +273,12 @@ def read_classifier_model_setting() -> str | None:
return configured if isinstance(configured, str) and configured else None
def proxy_serves_safeguards(safeguards: object) -> bool:
return requested_dangerous_tool_use(safeguards) is not None and read_classifier_model_setting() is not None
def native_route_classifies(model: str, safeguards: object) -> bool:
return (
"claude" not in model.lower()
and requested_dangerous_tool_use(safeguards) is not None
and read_classifier_model_setting() is not None
)
return "claude" not in model.lower() and proxy_serves_safeguards(safeguards)
def build_safeguards_evaluator(

25
litellm/llms/anthropic/pass_through/stream_assembly.py
View file

@ -1,16 +1,18 @@
import codecs
import json
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import as_str_mapping, map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.chat.handler import (
ModelResponseIterator as AnthropicModelResponseIterator,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.pass_through.messages.utils import MESSAGE_DELTA_OPTIONAL_KEYS
from litellm.types.utils import (
Choices,
Message,
@ -447,6 +449,27 @@ def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None:
)
def anthropic_message_delta_fields(all_chunks: Sequence[object]) -> Mapping[str, object]:
"""The ``stop_details`` and ``safeguard_results`` of the stream's ``message_delta``.
The assembled OpenAI shape has no slot for them, and a client that asked to stream reads them
from that event and nowhere else (a Claude Code auto-mode session takes its classifier verdict
from ``safeguard_results``), so a caller re-emitting a rewritten response merges them back in.
"""
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return MappingProxyType({})
message_delta: Final = next(
(event for event in reversed(_parsed_sse_events(sse_stream)) if event.get("type") == "message_delta"), None
)
delta: Final = as_str_mapping(message_delta.get("delta")) if message_delta is not None else None
if delta is None:
return MappingProxyType({})
return MappingProxyType(
{key: value for key in MESSAGE_DELTA_OPTIONAL_KEYS if (value := delta.get(key)) is not None}
)
def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
"""Whether raw SSE frames are Anthropic Messages events.

30
litellm/llms/base_llm/anthropic_messages/transformation.py
View file

@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -138,6 +138,15 @@ class BaseAnthropicMessagesConfig(ABC):
"""
return True
def supports_safeguards(self) -> bool:
return True
def serves_api_base(self, model: str, api_base: str | None) -> bool:
return True
def is_native_messages_route(self) -> bool:
return True
def handles_web_search_natively(self) -> bool:
"""
Whether the upstream this config routes to executes ``web_search`` tools
@ -180,7 +189,7 @@ class BaseAnthropicMessagesConfig(ABC):
return 2
def should_retry_anthropic_messages_on_http_error(
self, e: httpx.HTTPStatusError, litellm_params: dict, request_body: object = None
self, e: httpx.HTTPStatusError, litellm_params: Mapping[str, object], request_body: object = None
) -> bool:
"""
When True, async_anthropic_messages_handler will transform the request body
@ -202,7 +211,9 @@ class BaseAnthropicMessagesConfig(ABC):
) and is_anthropic_thinking_binding_mismatch_error(error_text)
return not caller_asked_for_this_error and is_anthropic_invalid_thinking_block_error(error_text)
def transform_anthropic_messages_request_on_http_error(self, e: httpx.HTTPStatusError, request_data: dict) -> dict:
def transform_anthropic_messages_request_on_http_error(
self, e: httpx.HTTPStatusError, request_data: dict[str, object]
) -> dict[str, object]:
"""
Mutates request_data in place when retrying after a recoverable HTTP error.
"""
@ -214,3 +225,16 @@ class BaseAnthropicMessagesConfig(ABC):
if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
return request_data
def native_output_fallback_litellm_params(
self,
model: str,
anthropic_messages_optional_request_params: Mapping[str, object],
litellm_params: GenericLiteLLMParams,
) -> GenericLiteLLMParams | None:
return None
def should_retry_anthropic_messages_without_native_output(
self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str
) -> bool:
return False

8
litellm/llms/base_llm/responses/transformation.py
View file

@ -1,7 +1,7 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
import httpx
@ -26,6 +26,8 @@ else:
LiteLLMLoggingObj = Any
BaseLLMException = Any
ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
class BaseResponsesAPIConfig(ABC):
def __init__(self):
@ -91,6 +93,10 @@ class BaseResponsesAPIConfig(ABC):
def get_supported_openai_params(self, model: str) -> list:
pass
def auto_reasoning_summary(self) -> ReasoningSummaryLevel:
"""The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
return "detailed"
@abstractmethod
def map_openai_params(
self,

1
litellm/llms/baseten/chat.py
View file

@ -54,6 +54,7 @@ class BasetenConfig(OpenAIGPTConfig):
return [
"max_tokens",
"max_completion_tokens",
"reasoning_effort",
"response_format",
"seed",
"stop",

24
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -42,14 +42,22 @@ def make_sync_call(
if client is None:
client = get_httpx_client() # Create a new client if none provided
response: Final = client.post(
api_base,
headers=headers,
data=data,
stream=not fake_stream,
logging_obj=logging_obj,
timeout=timeout,
)
try:
response: Final = client.post(
api_base,
headers=headers,
data=data,
stream=not fake_stream,
logging_obj=logging_obj,
timeout=timeout,
)
except httpx.HTTPStatusError as err:
raise BedrockError(
status_code=err.response.status_code,
message=error_response_text(err.response),
headers=err.response.headers,
response=err.response,
) from err
if response.status_code != 200:
raise BedrockError(

353
litellm/llms/bedrock/chat/converse_to_openai.py Normal file
View file

@ -0,0 +1,353 @@
import json
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import accumulate, chain, groupby, repeat, takewhile
from typing import Final, TypeAlias, TypeVar
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAssistantToolCall,
ChatCompletionSystemMessage,
ChatCompletionTextObject,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolMessage,
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
ChatCompletionUserMessage,
)
JsonPath: TypeAlias = tuple[str | int, ...]
_Node: Final = TypeVar("_Node")
_OBJECT: Final = TypeAdapter(Mapping[str, object])
_ITEMS: Final = TypeAdapter(Sequence[object])
class _ConverseModel(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
class _GuardText(_ConverseModel):
text: str = ""
class _GuardContent(_ConverseModel):
text: _GuardText | None = None
class _ToolUse(_ConverseModel):
tool_use_id: str = Field(default="", alias="toolUseId")
name: str = ""
tool_input: object = Field(default=None, alias="input")
class _ToolResultContent(_ConverseModel):
text: str = ""
json_value: object = Field(default=None, alias="json")
class _ToolResult(_ConverseModel):
tool_use_id: str = Field(default="", alias="toolUseId")
content: tuple[_ToolResultContent, ...] = ()
class _ContentBlock(_ConverseModel):
text: str = ""
guard_content: _GuardContent | None = Field(default=None, alias="guardContent")
tool_use: _ToolUse | None = Field(default=None, alias="toolUse")
tool_result: _ToolResult | None = Field(default=None, alias="toolResult")
class _Message(_ConverseModel):
role: str = ""
content: tuple[_ContentBlock, ...] = ()
class _ToolInputSchema(_ConverseModel):
json_schema: Mapping[str, object] | None = Field(default=None, alias="json")
class _ToolSpec(_ConverseModel):
name: str = ""
description: str = ""
input_schema: _ToolInputSchema | None = Field(default=None, alias="inputSchema")
class _Tool(_ConverseModel):
tool_spec: _ToolSpec | None = Field(default=None, alias="toolSpec")
class _ToolConfig(_ConverseModel):
tools: tuple[_Tool, ...] = ()
class ConverseRequest(_ConverseModel):
system: tuple[_ContentBlock, ...] = ()
messages: tuple[_Message, ...] = ()
tool_config: _ToolConfig | None = Field(default=None, alias="toolConfig")
additional_model_request_fields: object = Field(default=None, alias="additionalModelRequestFields")
@dataclass(frozen=True, slots=True)
class TextSlot:
path: JsonPath
text: str
@dataclass(frozen=True, slots=True)
class JsonSlot:
path: JsonPath
value: object
Slot: TypeAlias = TextSlot | JsonSlot
@dataclass(frozen=True, slots=True)
class ConverseRow:
message: AllMessageValues
content: tuple[Slot, ...]
calls: tuple[JsonSlot, ...] = ()
def parse_converse_request(body: Mapping[str, object]) -> ConverseRequest | None:
try:
return ConverseRequest.model_validate(body)
except ValidationError:
verbose_proxy_logger.debug("Converse body not convertible to OpenAI messages")
return None
def as_json_object(value: object) -> Mapping[str, object] | None:
try:
return _OBJECT.validate_python(value)
except ValidationError:
return None
def as_json_array(value: object) -> Sequence[object] | None:
try:
return _ITEMS.validate_python(value)
except ValidationError:
return None
def breadth_first(root: _Node, children: Callable[[_Node], tuple[_Node, ...]]) -> Iterator[_Node]:
def next_level(level: tuple[_Node, ...], _: None) -> tuple[_Node, ...]:
return tuple(chain.from_iterable(map(children, level)))
return chain.from_iterable(takewhile(bool, accumulate(repeat(None), next_level, initial=(root,))))
def _json_children(node: tuple[JsonPath, object]) -> tuple[tuple[JsonPath, object], ...]:
path, value = node
fields: Final = as_json_object(value)
if fields is not None:
return tuple(((*path, key), child) for key, child in fields.items())
items: Final = as_json_array(value)
return () if items is None else tuple(((*path, index), child) for index, child in enumerate(items))
def _text_leaf(node: tuple[JsonPath, object]) -> TextSlot | None:
path, value = node
return TextSlot(path, value) if isinstance(value, str) and value else None
def _flat_text_order(root: tuple[JsonPath, object]) -> Iterator[tuple[JsonPath, object]]:
"""
The nodes under ``root`` in the order the flat texts list their strings: each container's own strings in field
order, then its nested containers, the last one first.
"""
pending: Final = [root]
while pending:
children = _json_children(pending.pop())
yield from (child for child in children if isinstance(child[1], str))
pending.extend(child for child in children if not isinstance(child[1], str))
def _string_leaves(path: JsonPath, value: object) -> tuple[TextSlot, ...]:
leaves: Final = map(_text_leaf, _flat_text_order((path, value)))
return tuple(leaf for leaf in leaves if leaf is not None)
def _block_slot(path: JsonPath, block: _ContentBlock) -> TextSlot | None:
if block.text:
return TextSlot((*path, "text"), block.text)
guard_text: Final = block.guard_content.text if block.guard_content else None
return (
TextSlot((*path, "guardContent", "text", "text"), guard_text.text) if guard_text and guard_text.text else None
)
def _text_slots(path: JsonPath, blocks: Iterable[tuple[int, _ContentBlock]]) -> tuple[TextSlot, ...]:
slots: Final = (_block_slot((*path, index), block) for index, block in blocks)
return tuple(slot for slot in slots if slot is not None)
def _text_parts(slots: Sequence[TextSlot]) -> tuple[ChatCompletionTextObject, ...]:
return tuple(ChatCompletionTextObject(type="text", text=slot.text) for slot in slots)
def _slot_text(slot: Slot) -> str:
match slot:
case TextSlot():
return slot.text
case JsonSlot():
return json.dumps(slot.value)
def _tool_part_slot(path: JsonPath, part: _ToolResultContent) -> Slot | None:
if part.text:
return TextSlot((*path, "text"), part.text)
return None if part.json_value is None else JsonSlot((*path, "json"), part.json_value)
def _tool_result_row(path: JsonPath, result: _ToolResult) -> ConverseRow:
slots: Final = tuple(
slot
for index, part in enumerate(result.content)
if (slot := _tool_part_slot((*path, "content", index), part)) is not None
)
message: Final = ChatCompletionToolMessage(
role="tool", tool_call_id=result.tool_use_id, content="\n".join(map(_slot_text, slots))
)
return ConverseRow(message, slots)
def _user_group(
path: JsonPath, is_tool_result: bool, group: Sequence[tuple[int, _ContentBlock]]
) -> tuple[ConverseRow, ...]:
if is_tool_result:
return tuple(
_tool_result_row((*path, index, "toolResult"), result)
for index, block in group
if (result := block.tool_result) is not None
)
slots: Final = _text_slots(path, group)
return (
(ConverseRow(ChatCompletionUserMessage(role="user", content=list(_text_parts(slots))), slots),) if slots else ()
)
def _user_rows(path: JsonPath, blocks: Sequence[_ContentBlock]) -> tuple[ConverseRow, ...]:
groups: Final = groupby(enumerate(blocks), key=lambda item: item[1].tool_result is not None)
return tuple(
chain.from_iterable(_user_group(path, is_tool_result, tuple(group)) for is_tool_result, group in groups)
)
def _tool_call(tool_use: _ToolUse) -> ChatCompletionAssistantToolCall:
return ChatCompletionAssistantToolCall(
id=tool_use.tool_use_id,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=tool_use.name,
arguments="{}" if tool_use.tool_input is None else json.dumps(tool_use.tool_input),
),
)
def _assistant_rows(
path: JsonPath, blocks: Sequence[_ContentBlock], include_tool_calls: bool
) -> tuple[ConverseRow, ...]:
slots: Final = _text_slots(path, enumerate(blocks))
uses: Final = (
tuple((index, tool_use) for index, block in enumerate(blocks) if (tool_use := block.tool_use) is not None)
if include_tool_calls
else ()
)
if uses:
message: Final = ChatCompletionAssistantMessage(
role="assistant",
content=list(_text_parts(slots)) or None,
tool_calls=[_tool_call(tool_use) for _, tool_use in uses],
)
calls: Final = tuple(
JsonSlot((*path, index, "toolUse", "input"), tool_use.tool_input) for index, tool_use in uses
)
return (ConverseRow(message, slots, calls),)
if not slots:
return ()
return (ConverseRow(ChatCompletionAssistantMessage(role="assistant", content=list(_text_parts(slots))), slots),)
def _message_rows(index: int, message: _Message, include_tool_calls: bool) -> tuple[ConverseRow, ...]:
path: Final = ("messages", index, "content")
if message.role == "assistant":
return _assistant_rows(path, message.content, include_tool_calls)
return _user_rows(path, message.content)
def _system_rows(blocks: Sequence[_ContentBlock]) -> tuple[ConverseRow, ...]:
slots: Final = _text_slots(("system",), enumerate(blocks))
return (
(ConverseRow(ChatCompletionSystemMessage(role="system", content=list(_text_parts(slots))), slots),)
if slots
else ()
)
def _user_row(slots: Sequence[Slot]) -> ConverseRow:
parts: Final = [ChatCompletionTextObject(type="text", text=_slot_text(slot)) for slot in slots]
return ConverseRow(ChatCompletionUserMessage(role="user", content=parts), tuple(slots))
def _with_request_fields(rows: tuple[ConverseRow, ...], value: object, user_turn: int) -> tuple[ConverseRow, ...]:
"""
``rows`` with the ``additionalModelRequestFields`` strings after the text of the latest user row, so a guardrail
that judges only the latest user message reads both. When no user row has text they get a user row of their own
at ``user_turn``, where the rows of the latest user turn that is not only tool results start.
"""
slots: Final = _string_leaves(("additionalModelRequestFields",), value)
if not slots:
return rows
latest: Final = next((index for index in reversed(range(len(rows))) if rows[index].message["role"] == "user"), None)
if latest is None:
return (*rows[:user_turn], _user_row(slots), *rows[user_turn:])
return (*rows[:latest], _user_row((*rows[latest].content, *slots)), *rows[latest + 1 :])
def _is_user_turn(message: _Message) -> bool:
return message.role != "assistant" and any(block.tool_result is None for block in message.content)
def converse_rows(request: ConverseRequest, include_tool_calls: bool) -> tuple[ConverseRow, ...]:
system: Final = _system_rows(request.system)
messages: Final = tuple(
_message_rows(index, message, include_tool_calls) for index, message in enumerate(request.messages)
)
starts: Final = tuple(accumulate(map(len, messages), initial=len(system)))
rows: Final = (*system, *chain.from_iterable(messages))
user_turn: Final = next(
(starts[index] for index in reversed(range(len(messages))) if _is_user_turn(request.messages[index])),
len(rows),
)
return _with_request_fields(rows, request.additional_model_request_fields, user_turn)
@dataclass(frozen=True, slots=True)
class ConverseTool:
index: int
param: ChatCompletionToolParam
def converse_tools(request: ConverseRequest) -> tuple[ConverseTool, ...]:
tools: Final = request.tool_config.tools if request.tool_config else ()
return tuple(
ConverseTool(
index,
ChatCompletionToolParam(
type="function",
function=ChatCompletionToolParamFunctionChunk(
name=spec.name,
description=spec.description,
parameters=dict(spec.input_schema.json_schema),
),
),
)
for index, tool in enumerate(tools)
if (spec := tool.tool_spec) and spec.name and spec.input_schema and spec.input_schema.json_schema is not None
)

119
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -12,7 +12,7 @@ from itertools import chain
from typing import TYPE_CHECKING, Final, Literal, cast, overload
import httpx
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
@ -61,12 +61,16 @@ from litellm.llms.bedrock.common_utils import (
bedrock_cleaned_output_schema,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_native_output_is_opt_in,
bedrock_supports_tool_search,
deployment_model_info,
requests_output_config_format,
)
from litellm.llms.bedrock.request_metadata import (
bedrock_request_metadata_headers,
bedrock_request_metadata_is_owned,
bedrock_request_metadata_is_rewritten,
bedrock_request_metadata_violation,
caller_request_metadata_violations_are_dropped,
merge_bedrock_invoke_headers,
resolve_bedrock_request_metadata,
)
@ -168,6 +172,18 @@ def _tools_the_model_accepts(
return [tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) for tool in tools]
_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
def _has_native_text_format(output_config: object) -> bool:
if not isinstance(output_config, Mapping):
return False
try:
return _OUTPUT_CONFIG_BLOCK.validate_python(output_config).get("textFormat") is not None
except ValidationError:
return False
class AmazonConverseConfig(BaseConfig):
"""
Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
@ -354,77 +370,15 @@ class AmazonConverseConfig(BaseConfig):
and v is not None
}
def _validate_request_metadata(self, metadata: dict) -> None:
"""
Validate requestMetadata according to AWS Bedrock Converse API constraints.
Constraints:
- Maximum of 16 items
- Keys: 1-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{1,256}
- Values: 0-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{0,256}
"""
import re
if not isinstance(metadata, dict):
def _validate_request_metadata(self, metadata: object) -> None:
violation: Final = bedrock_request_metadata_violation(metadata)
if violation is not None:
raise litellm.exceptions.BadRequestError(
message="requestMetadata must be a dictionary",
message=violation.message,
model="bedrock",
llm_provider="bedrock",
)
if len(metadata) > 16:
raise litellm.exceptions.BadRequestError(
message="requestMetadata can contain a maximum of 16 items",
model="bedrock",
llm_provider="bedrock",
)
key_pattern: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
value_pattern: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
for key, value in metadata.items():
if not isinstance(key, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata keys must be strings",
model="bedrock",
llm_provider="bedrock",
)
if not isinstance(value, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata values must be strings",
model="bedrock",
llm_provider="bedrock",
)
if len(key) == 0 or len(key) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata key length must be 1-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if len(value) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata value length must be 0-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if not key_pattern.match(key):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
if not value_pattern.match(value):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
@staticmethod
def _is_openai_gpt_reasoning_model(model: str) -> bool:
return re.search(r"openai\.gpt-\d", model) is not None
@ -1244,7 +1198,7 @@ class AmazonConverseConfig(BaseConfig):
elif param == "context_management" and isinstance(value, (dict, list)):
self._map_context_management_param(value, optional_params)
if param == "requestMetadata":
self._map_request_metadata_param(value, optional_params)
self._map_request_metadata_param(value, optional_params, drop_params)
if param == "service_tier" and isinstance(value, str):
self._map_service_tier_param(value, optional_params)
@ -1277,9 +1231,10 @@ class AmazonConverseConfig(BaseConfig):
return optional_params
def _map_request_metadata_param(self, value: object, optional_params: dict) -> None:
def _map_request_metadata_param(self, value: object, optional_params: dict, drop_params: bool) -> None:
if value is not None and isinstance(value, dict):
self._validate_request_metadata(value)
if not caller_request_metadata_violations_are_dropped(drop_params):
self._validate_request_metadata(value)
optional_params["requestMetadata"] = value
def _map_context_management_param(self, value: dict | list, optional_params: dict) -> None:
@ -1392,6 +1347,22 @@ class AmazonConverseConfig(BaseConfig):
optional_params["json_mode"] = True
return optional_params
@staticmethod
def sends_native_text_format(optional_params: Mapping[str, object]) -> bool:
return _has_native_text_format(optional_params.get("outputConfig"))
def uses_opt_in_native_output(
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> bool:
native_output_requested: Final = (
optional_params.get("json_mode") is True and self.sends_native_text_format(optional_params)
) or requests_output_config_format(optional_params)
return native_output_requested and bedrock_native_output_is_opt_in(
bedrock_capability_model(model, litellm_params.get("base_model")),
self.custom_llm_provider or "bedrock_converse",
deployment_model_info(litellm_params),
)
def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict):
"""
Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget.
@ -1737,7 +1708,7 @@ class AmazonConverseConfig(BaseConfig):
# Extract requestMetadata before processing other parameters
request_metadata: Final = inference_params.pop("requestMetadata", None)
if request_metadata is not None:
if request_metadata is not None and not caller_request_metadata_violations_are_dropped(drop_params):
self._validate_request_metadata(request_metadata)
output_config: OutputConfigBlock | None = inference_params.pop("outputConfig", None)
@ -2164,7 +2135,7 @@ class AmazonConverseConfig(BaseConfig):
request_metadata: Final = resolve_bedrock_request_metadata(
litellm_params=litellm_params, caller_metadata=_data.get("requestMetadata")
)
if bedrock_request_metadata_is_owned():
if bedrock_request_metadata_is_rewritten(litellm_params.get("drop_params") is True):
_data.pop("requestMetadata", None)
if request_metadata is not None:
_data["requestMetadata"] = request_metadata
@ -2244,7 +2215,7 @@ class AmazonConverseConfig(BaseConfig):
request_metadata: Final = resolve_bedrock_request_metadata(
litellm_params=litellm_params, caller_metadata=_data.get("requestMetadata")
)
if bedrock_request_metadata_is_owned():
if bedrock_request_metadata_is_rewritten(litellm_params.get("drop_params") is True):
_data.pop("requestMetadata", None)
if request_metadata is not None:
_data["requestMetadata"] = request_metadata

11
litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py
View file

@ -95,7 +95,12 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
model_info: Mapping[str, object] | None = None,
) -> dict:
return AmazonConverseConfig.map_openai_params(
self, non_default_params, optional_params, model, drop_params, model_info
self,
{key: value for key, value in non_default_params.items() if key != "requestMetadata"},
optional_params,
model,
drop_params,
model_info,
)
def transform_request(
@ -110,7 +115,9 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
self,
model=model,
messages=messages,
optional_params=_without_tool_config_injection_points(optional_params),
optional_params=_without_tool_config_injection_points(
{key: value for key, value in optional_params.items() if key != "requestMetadata"}
),
litellm_params=litellm_params,
headers=headers,
)

6
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -31,6 +31,7 @@ from litellm.llms.bedrock.common_utils import (
normalize_bedrock_opus_output_config_effort,
normalize_custom_field_on_tools,
normalize_tool_input_schema_types_for_bedrock_invoke,
requests_opt_in_native_output,
strip_unsupported_bedrock_invoke_output_config_keys,
tools_without_eager_input_streaming,
)
@ -102,6 +103,11 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
def uses_opt_in_native_output(
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> bool:
return requests_opt_in_native_output(model, optional_params, deployment_model_info(litellm_params))
def map_openai_params(
self,
non_default_params: dict,

62
litellm/llms/bedrock/chat/native_output_fallback.py Normal file
View file

@ -0,0 +1,62 @@
from collections.abc import Callable, Coroutine
from typing import Final, TypeAlias
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.common_utils import (
NativeOutputFallback,
is_bedrock_native_output_rejection,
warn_bedrock_native_output_rejected,
)
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
def _fallback_for(
error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
) -> NativeOutputFallback | None:
if not is_bedrock_native_output_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:
warn_bedrock_native_output_rejected(model, error.message)
return retry
async def _await_with_native_output_fallback(
first_attempt: Coroutine[object, object, NativeOutputResponse],
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory,
model: str,
) -> NativeOutputResponse:
try:
return await first_attempt
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
if retry is None:
raise
second_attempt: Final = attempt(retry)
return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt
def call_with_native_output_fallback(
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory | None,
model: str,
) -> NativeOutputAttemptResult:
if fallback is None:
return attempt(None)
try:
first_attempt: Final = attempt(None)
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
if retry is None:
raise
return attempt(retry)
if isinstance(first_attempt, Coroutine):
return _await_with_native_output_fallback(first_attempt, attempt, fallback, model)
return first_attempt

81
litellm/llms/bedrock/common_utils.py
View file

@ -11,6 +11,7 @@ import os
import re
import warnings
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
@ -184,7 +185,7 @@ def _get_local_model_cost_map() -> dict:
return GetModelCostMap.load_local_model_cost_map()
def pop_bedrock_invoke_output_config_format(request_body: dict) -> dict | None:
def pop_bedrock_invoke_output_config_format(request_body: dict[str, object]) -> dict[str, object] | None:
"""
Remove and return Anthropic's nested ``output_config.format`` field.
@ -205,8 +206,8 @@ def pop_bedrock_invoke_output_config_format(request_body: dict) -> dict | None:
def convert_bedrock_invoke_output_format_to_inline_schema(
output_format: dict,
request_body: dict,
output_format: Mapping[str, object],
request_body: dict[str, object],
) -> None:
"""
Embed an Anthropic structured-output schema into the last user message.
@ -292,6 +293,77 @@ def bedrock_chat_uses_native_structured_output(
)
def bedrock_native_output_is_opt_in(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return not _bedrock_model_supports(
model, "supports_native_structured_output", custom_llm_provider
) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return {
**litellm_params,
"model_info": {**deployment_model_info(litellm_params), "supports_bedrock_native_structured_output": False},
}
_BEDROCK_NATIVE_OUTPUT_UNSUPPORTED: Final = "output_config.format: Extra inputs are not permitted"
_BEDROCK_NATIVE_OUTPUT_REJECTIONS: Final = (
_BEDROCK_NATIVE_OUTPUT_UNSUPPORTED,
"compilation exceeded limits: Schema is too complex",
"output_config.format.schema:",
)
def is_bedrock_native_output_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and any(rejection in error_text for rejection in _BEDROCK_NATIVE_OUTPUT_REJECTIONS)
def warn_bedrock_native_output_rejected(model: str, error_text: str) -> None:
opt_out_advice: Final = (
" Set `supports_bedrock_native_structured_output: false` in this deployment's model_info to skip the "
"rejected call."
if _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED in error_text
else ""
)
verbose_logger.warning(
"Bedrock rejected native structured output for model=%s, so litellm sent the request again without it.%s",
model,
opt_out_advice,
)
@dataclass(frozen=True, slots=True)
class NativeOutputFallback:
optional_params: Mapping[str, object]
litellm_params: Mapping[str, object]
def requests_output_config_format(optional_params: Mapping[str, object]) -> bool:
match optional_params:
case {"output_config": {"format": object()}}:
return True
case _:
return False
def requests_opt_in_native_output(
model: str, anthropic_optional_params: Mapping[str, object], model_info: Mapping[str, object] | None
) -> bool:
native_output_requested: Final = isinstance(
anthropic_optional_params.get("output_format"), Mapping
) or requests_output_config_format(anthropic_optional_params)
return native_output_requested and bedrock_native_output_is_opt_in(model, model_info=model_info)
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
def json_object_or_none(value: object) -> Mapping[str, object] | None:
return _JSON_OBJECT.validate_python(value) if isinstance(value, Mapping) else None
BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64
@ -358,7 +430,8 @@ def apply_bedrock_invoke_structured_output(
"""
legacy_output_format: Final = request_body.pop("output_format", None)
output_config_format: Final = pop_bedrock_invoke_output_config_format(request_body)
schema_format: Final = legacy_output_format if isinstance(legacy_output_format, dict) else output_config_format
legacy_format: Final = json_object_or_none(legacy_output_format)
schema_format: Final = legacy_format if legacy_format is not None else output_config_format
if schema_format is None:
return

2
litellm/llms/bedrock/count_tokens/handler.py
View file

@ -48,7 +48,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model)
# Get AWS region using existing LiteLLM function
aws_region_name: Final = self._get_aws_region_name(
aws_region_name: Final = self.get_aws_region_name(
optional_params=litellm_params,
model=resolved_model,
model_id=None,

502
litellm/llms/bedrock/count_tokens/kimi_k3.py Normal file
View file

@ -0,0 +1,502 @@
"""
Exact Bedrock token counts for Kimi K3, which has no CountTokens support.
Renders the Converse body litellm sends through Moonshot's Kimi K3 chat template and counts it with the
official tokenizer, downloaded once from Hugging Face at a pinned revision and checked against its SHA-256.
"""
import asyncio
import base64
import copy
import hashlib
import json
import threading
import time
from collections.abc import Callable, Iterable, Mapping, Sequence
from concurrent.futures import Future
from dataclasses import dataclass
from enum import Enum
from itertools import chain, groupby
from pathlib import Path
from typing import Annotated, Final, Literal, TypeAlias
import regex
import tiktoken
from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, model_validator
from typing_extensions import Self
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.factory import (
chat_completion_tools_for,
is_flat_responses_tool,
)
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
KIMI_K3_REPO: Final = "moonshotai/Kimi-K3"
KIMI_K3_REVISION: Final = "f831ab66814297da540d832a5235f8e904f29d06"
KIMI_K3_RANKS_FILE: Final = "tiktoken.model"
KIMI_K3_RANKS_SHA256: Final = "b6c497a7469b33ced9c38afb1ad6e47f03f5e5dc05f15930799210ec050c5103"
_PATTERN: Final = "|".join(
(
r"[\p{Han}]+",
r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?",
r"\p{N}{1,3}",
r" ?[^\s\p{L}\p{N}]+[\r\n]*",
r"\s*[\r\n]+",
r"\s+(?!\S)",
r"\s+",
)
)
# Ends chunks only where the pattern above always splits, so their counts sum to the whole text's count while BPE
# stays linear. Only a run of over 1024 characters with no such split, like 2000 spaces, is cut mid-piece.
_CHUNK: Final = regex.compile(
r"(?s).{1,1024}(?:(?<=\S)(?= \S)|(?<=\p{N})(?=\P{N})|(?<=[^\s\p{N}])(?=\p{N})|(?<=\p{L})(?=[^\p{L}\p{M}'])"
r"|(?<=[\r\n])(?=\S)|\Z)|.{1024}\p{N}*"
)
@dataclass(frozen=True, slots=True)
class KimiK3TokenizerSource:
ranks_path: Callable[[], str]
sha256: str
def _hugging_face_ranks_path() -> str:
import huggingface_hub
return huggingface_hub.hf_hub_download(
repo_id=KIMI_K3_REPO,
filename=KIMI_K3_RANKS_FILE,
revision=KIMI_K3_REVISION,
local_files_only=litellm.disable_hf_tokenizer_download is True,
)
HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256)
def _loaded(source: KimiK3TokenizerSource) -> tiktoken.Encoding | str:
try:
data: Final = Path(source.ranks_path()).read_bytes()
except Exception as e: # noqa: BLE001 # a download or read failure of any kind falls back to the estimate
return type(e).__name__
if hashlib.sha256(data).hexdigest() != source.sha256:
return "SHA-256 mismatch"
rows: Final = (line.split() for line in data.splitlines() if line)
return tiktoken.Encoding(
name="kimi_k3",
pat_str=_PATTERN,
mergeable_ranks={base64.b64decode(token): int(rank) for token, rank in rows},
special_tokens={},
)
def load_kimi_k3_encoding(source: KimiK3TokenizerSource) -> tiktoken.Encoding | None:
loaded: Final = _loaded(source)
if isinstance(loaded, tiktoken.Encoding):
return loaded
verbose_logger.warning(
"Kimi K3 tokenizer unavailable (%s), so Kimi K3 token counts are estimated until a later count loads it. "
"Allow downloads of %s from Hugging Face repo %s at revision %s, or put it in the Hugging Face cache",
loaded,
KIMI_K3_RANKS_FILE,
KIMI_K3_REPO,
KIMI_K3_REVISION,
)
return None
def _started_load(source: KimiK3TokenizerSource) -> "Future[tiktoken.Encoding | None]":
encoding: Final = Future[tiktoken.Encoding | None]()
threading.Thread(
target=lambda: encoding.set_result(load_kimi_k3_encoding(source)), name="litellm-kimi-k3-tokenizer", daemon=True
).start()
return encoding
@dataclass(frozen=True, slots=True)
class _EncodingLoad:
encoding: "Future[tiktoken.Encoding | None]"
started_at: float
is_retry: bool
def retry_due(self, now: float, retry_after_seconds: float) -> bool:
return self.encoding.done() and self.encoding.result() is None and now - self.started_at >= retry_after_seconds
@dataclass(slots=True)
class KimiK3Encodings:
source: KimiK3TokenizerSource
retry_after_seconds: float
clock: Callable[[], float] = time.monotonic
current: _EncodingLoad | None = None
def _load(self) -> _EncodingLoad:
now: Final = self.clock()
current: Final = self.current
if current is not None and not current.retry_due(now, self.retry_after_seconds):
return current
load: Final = _EncodingLoad(encoding=_started_load(self.source), started_at=now, is_retry=current is not None)
self.current = load
return load
async def encoding(self, wait_seconds: float) -> tiktoken.Encoding | None:
load: Final = self._load()
remaining: Final = 0 if load.is_retry else load.started_at + wait_seconds - self.clock()
if load.encoding.done() or remaining <= 0:
return load.encoding.result() if load.encoding.done() else None
await asyncio.wait((asyncio.wrap_future(load.encoding),), timeout=remaining)
if load.encoding.done():
return load.encoding.result()
verbose_logger.warning(
"Kimi K3 tokenizer still loading after %s s, so Kimi K3 token counts are estimated until it loads. "
"Check that %s can be downloaded from Hugging Face repo %s at revision %s, or put it in the Hugging Face cache",
wait_seconds,
KIMI_K3_RANKS_FILE,
KIMI_K3_REPO,
KIMI_K3_REVISION,
)
return None
class _Converse(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
class _Text(_Converse):
text: str
class _CachePoint(_Converse):
cache_point: JsonValue = Field(alias="cachePoint")
class _ToolUse(_Converse):
tool_use_id: str = Field(alias="toolUseId")
name: str
arguments: Mapping[str, JsonValue] = Field(alias="input")
class _ToolUseBlock(_Converse):
tool_use: _ToolUse = Field(alias="toolUse")
class _ToolResult(_Converse):
tool_use_id: str = Field(alias="toolUseId")
content: tuple[_Text, ...]
status: str | None = None
class _ToolResultBlock(_Converse):
tool_result: _ToolResult = Field(alias="toolResult")
class _ReasoningText(_Converse):
text: str
signature: str | None = None
class _Reasoning(_Converse):
reasoning_text: _ReasoningText = Field(alias="reasoningText")
class _ReasoningBlock(_Converse):
reasoning_content: _Reasoning = Field(alias="reasoningContent")
class _UserMessage(_Converse):
role: Literal["user"]
content: tuple[_Text | _ToolResultBlock | _CachePoint, ...]
class _AssistantMessage(_Converse):
role: Literal["assistant"]
content: tuple[_Text | _ToolUseBlock | _ReasoningBlock | _CachePoint, ...]
def calls(self) -> tuple[_ToolUse, ...]:
return tuple(block.tool_use for block in self.content if isinstance(block, _ToolUseBlock))
class _InputSchema(_Converse):
definition: JsonValue = Field(alias="json")
class _ToolSpec(_Converse):
name: str
description: str | None = None
input_schema: _InputSchema = Field(alias="inputSchema")
def declared(self) -> Mapping[str, JsonValue]:
described: Final[Mapping[str, JsonValue]] = (
{} if self.description is None else {"description": self.description}
)
return {"name": self.name, **described, "parameters": self.input_schema.definition}
class _ToolSpecBlock(_Converse):
tool_spec: _ToolSpec = Field(alias="toolSpec")
class _ToolConfig(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
tools: tuple[_ToolSpecBlock | _CachePoint, ...]
_Message: TypeAlias = Annotated[_UserMessage | _AssistantMessage, Field(discriminator="role")]
def _result_ids(message: _Message) -> frozenset[str]:
if isinstance(message, _AssistantMessage):
return frozenset()
return frozenset(block.tool_result.tool_use_id for block in message.content if isinstance(block, _ToolResultBlock))
def _call_ids(message: _Message | None) -> frozenset[str]:
if not isinstance(message, _AssistantMessage):
return frozenset()
return frozenset(call.tool_use_id for call in message.calls())
class _ConverseRequest(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
messages: tuple[_Message, ...]
system: tuple[_Text | _CachePoint, ...] = ()
tool_config: _ToolConfig | None = Field(default=None, alias="toolConfig")
@model_validator(mode="after")
def _results_answer_the_previous_calls(self) -> Self:
pairs: Final = zip((None, *self.messages), self.messages, strict=False)
if not all(_result_ids(message) <= _call_ids(previous) for previous, message in pairs):
raise ValueError("a toolResult does not answer a toolUse of the previous assistant message")
return self
class _Control(Enum):
OPEN = "<|open|>"
CLOSE = "<|close|>"
SEP = "<|sep|>"
END_OF_MSG = "<|end_of_msg|>"
_Segment: TypeAlias = str | _Control
_Segments: TypeAlias = tuple[_Segment, ...]
def _open(tag: str, attrs: Iterable[tuple[str, str]] = ()) -> _Segments:
escaped: Final = (
(f" {key}", '="', value.replace("&", "&amp;").replace('"', "&quot;"), '"') for key, value in attrs
)
return (_Control.OPEN, tag, *chain.from_iterable(escaped), _Control.SEP)
def _close(tag: str) -> _Segments:
return (_Control.CLOSE, tag, _Control.SEP)
def _message(attrs: Iterable[tuple[str, str]], body: Iterable[_Segment]) -> _Segments:
return (*_open("message", attrs), *body, *_close("message"), _Control.END_OF_MSG)
def _tool_declare(specs: Sequence[_ToolSpec]) -> _Segments:
tools: Final = json.dumps(
[{"type": "function", "function": spec.declared()} for spec in specs],
ensure_ascii=False,
separators=(",", ":"),
sort_keys=True,
)
return _message(
(("role", "system"), ("type", "tool-declare")),
(f"# Tools\nHere are the available tools, described in JSONSchema.\n\n```json\n{tools}\n```",),
)
def _text(blocks: Iterable[_Converse]) -> str:
return "".join(block.text for block in blocks if isinstance(block, _Text))
def _json_type(value: JsonValue) -> str:
match value:
case bool():
return "boolean"
case None:
return "null"
case int() | float():
return "number"
case str():
return "string"
case dict():
return "object"
case _:
return "array"
def _argument(key: str, value: JsonValue) -> _Segments:
text: Final = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
return (*_open("argument", (("key", key), ("type", _json_type(value)))), text, *_close("argument"))
def _call(index: int, call: _ToolUse) -> _Segments:
arguments: Final = chain.from_iterable(_argument(key, value) for key, value in call.arguments.items())
return (*_open("call", (("tool", call.name), ("index", str(index)))), *arguments, *_close("call"))
def _tool_calls(calls: Sequence[_ToolUse]) -> _Segments:
if not calls:
return ()
rendered: Final = chain.from_iterable(_call(index, call) for index, call in enumerate(calls, start=1))
return (*_open("tools"), *rendered, *_close("tools"))
def _assistant_message(message: _AssistantMessage) -> _Segments:
reasoning: Final = "".join(
block.reasoning_content.reasoning_text.text for block in message.content if isinstance(block, _ReasoningBlock)
)
return _message(
(("role", "assistant"),),
(
*_open("think"),
reasoning,
*_close("think"),
*_open("response"),
_text(message.content),
*_close("response"),
*_tool_calls(message.calls()),
),
)
def _tool_messages(results: Sequence[_ToolResult], previous: _AssistantMessage) -> _Segments:
names: Final = {call.tool_use_id: call.name for call in previous.calls()}
return tuple(
chain.from_iterable(
_message(
(("role", "tool"), ("tool", names[result.tool_use_id]), ("index", str(index))),
(_text(result.content),),
)
for index, result in enumerate(results, start=1)
)
)
def _user_run(blocks: Sequence[_Text | _ToolResultBlock], previous: _Message | None) -> _Segments:
results: Final = tuple(block.tool_result for block in blocks if isinstance(block, _ToolResultBlock))
if results and isinstance(previous, _AssistantMessage):
return _tool_messages(results, previous)
return _message((("role", "user"),), (_text(blocks),))
def _user_messages(message: _UserMessage, previous: _Message | None) -> _Segments:
blocks: Final = tuple(block for block in message.content if not isinstance(block, _CachePoint))
runs: Final = groupby(blocks, key=lambda block: isinstance(block, _ToolResultBlock))
return tuple(chain.from_iterable(_user_run(tuple(run), previous) for _, run in runs))
def _turn(previous: _Message | None, message: _Message) -> _Segments:
if isinstance(message, _AssistantMessage):
return _assistant_message(message)
return _user_messages(message, previous)
def _rendered(request: _ConverseRequest) -> _Segments:
specs: Final = (
()
if request.tool_config is None
else tuple(block.tool_spec for block in request.tool_config.tools if isinstance(block, _ToolSpecBlock))
)
system: Final = _text(request.system)
turns: Final = chain.from_iterable(
_turn(previous, message) for previous, message in zip((None, *request.messages), request.messages, strict=False)
)
return (
*(_tool_declare(specs) if specs else ()),
*(_message((("role", "system"),), (system,)) if system else ()),
*turns,
*_open("message", (("role", "assistant"),)),
*_open("think"),
)
def _text_tokens(encoding: tiktoken.Encoding, text: str) -> int:
spans: Final = (match.span() for match in _CHUNK.finditer(text))
return sum(len(encoding.encode_ordinary(text[start:end])) for start, end in spans)
class _TextPart(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
type: Literal["text"]
class _TextOnlyMessage(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
content: str | tuple[_TextPart, ...] | None = None
def _text_only(message: AllMessageValues) -> bool:
try:
_TextOnlyMessage.model_validate(message)
except ValidationError:
return False
return True
def _as_converse_input(tool: ChatCompletionToolParam) -> tuple[Mapping[str, object], ...]:
return chat_completion_tools_for(tool) if is_flat_responses_tool(tool) else (tool,)
def _converse_request(
model: str, messages: Sequence[AllMessageValues], tools: Sequence[ChatCompletionToolParam] | None
) -> _ConverseRequest | None:
if not all(_text_only(message) for message in messages):
return None
try:
chat_tools: Final = tuple(
chat_tool for tool in tools or () for chat_tool in _as_converse_input(tool)
) # comprehension-ok: flattens each tool's translation
return _ConverseRequest.model_validate(
litellm.AmazonConverseConfig().transform_request(
model=model,
messages=copy.deepcopy(list(messages)),
optional_params={"tools": copy.deepcopy(list(chat_tools))} if chat_tools else {},
litellm_params={},
headers={},
)
)
except Exception: # noqa: BLE001 # a body litellm cannot send to Bedrock, or cannot render, falls back to the estimate
return None
def kimi_k3_tokens(
encoding: tiktoken.Encoding,
model: str,
messages: Sequence[AllMessageValues],
tools: Sequence[ChatCompletionToolParam] | None,
) -> int | None:
request: Final = _converse_request(model, messages, tools)
if request is None:
return None
return sum(
1 if isinstance(segment, _Control) else _text_tokens(encoding, segment) for segment in _rendered(request)
)
def kimi_k3_anthropic_tokens(encoding: tiktoken.Encoding, model: str, fields: Mapping[str, object]) -> int | None:
body: Final = {**fields, "model": model, "max_tokens": 1}
try:
translated, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
copy.deepcopy(body), # pyright: ignore[reportArgumentType] # raw /v1/messages fields
custom_llm_provider="bedrock",
)
except Exception: # noqa: BLE001 # a request the adapter cannot translate falls back to the estimate
return None
match translated:
case {"tools": tools}:
return kimi_k3_tokens(encoding, model, translated["messages"], tools)
case _:
return kimi_k3_tokens(encoding, model, translated["messages"], None)
def kimi_k3_template_tokens(message_count: int, has_tools: bool) -> int:
return 14 + 13 * max(message_count - 1, 0) + (40 if has_tools else 0)

2
litellm/llms/bedrock/count_tokens/mantle_handler.py
View file

@ -52,7 +52,7 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
) -> dict[str, JsonValue]:
try:
request: Final = _COUNT_REQUEST.validate_python(request_data)
aws_region_name: Final = self._get_aws_region_name(
aws_region_name: Final = self.get_aws_region_name(
optional_params=litellm_params, model=resolved_model, model_id=None
)
endpoint_url: Final = self.get_mantle_count_tokens_endpoint(aws_region_name)

8
litellm/llms/bedrock/files/transformation.py
View file

@ -66,8 +66,10 @@ from litellm.utils import get_llm_provider, get_optional_params
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import (
BedrockError,
deployment_model_info,
merge_bedrock_aws_request_params,
resolve_s3_encryption_key_id,
without_bedrock_native_output_opt_in,
)
S3_SIGNED_REQUEST_HEADERS_PARAM: Final = "_s3_signed_request_headers"
@ -989,10 +991,12 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
)
config: Final = AmazonAnthropicClaudeConfig()
litellm_params: Final = without_bedrock_native_output_opt_in({})
mapped_params = get_optional_params(
model=_invoke_route_model(model),
custom_llm_provider="bedrock",
messages=messages,
model_info=deployment_model_info(litellm_params),
**MappingProxyType(
{
k: v
@ -1007,7 +1011,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
optional_params={
k: v for k, v in mapped_params.items() if k not in _MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS
},
litellm_params={},
litellm_params=dict(litellm_params),
headers={},
)
@ -1021,7 +1025,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
converse_config: Final = AmazonConverseConfig()
mapped_params = converse_config.map_openai_params(
non_default_params=optional_params,
non_default_params={k: v for k, v in optional_params.items() if k != "requestMetadata"},
optional_params={},
model=model,
drop_params=False,

30
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -40,12 +40,16 @@ from litellm.llms.bedrock.common_utils import (
bedrock_supports_tool_search,
ensure_bedrock_anthropic_messages_tool_names,
get_anthropic_beta_from_headers,
is_bedrock_native_output_rejection,
normalize_bedrock_opus_output_config_effort,
normalize_custom_field_on_tools,
normalize_tool_input_schema_types_for_bedrock_invoke,
requests_opt_in_native_output,
requests_output_config_format,
strip_unsupported_bedrock_invoke_output_config_keys,
supports_bedrock_prompt_cache_ttl,
tools_without_eager_input_streaming,
warn_bedrock_native_output_rejected,
)
from litellm.llms.bedrock.request_metadata import (
bedrock_request_metadata_headers,
@ -94,6 +98,9 @@ class AmazonAnthropicClaudeMessagesConfig(
def beta_headers_provider(self) -> str:
return self.custom_llm_provider or "bedrock"
def should_strip_billing_metadata(self) -> bool:
return True
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
return True
@ -111,6 +118,29 @@ class AmazonAnthropicClaudeMessagesConfig(
BaseAnthropicMessagesConfig.__init__(self, **kwargs)
AmazonInvokeConfig.__init__(self, **kwargs)
def native_output_fallback_litellm_params(
self,
model: str,
anthropic_messages_optional_request_params: Mapping[str, object],
litellm_params: GenericLiteLLMParams,
) -> GenericLiteLLMParams | None:
model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info)
if not requests_opt_in_native_output(model, anthropic_messages_optional_request_params, model_info):
return None
return litellm_params.model_copy(
update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}}
)
def should_retry_anthropic_messages_without_native_output(
self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str
) -> bool:
if not requests_output_config_format(request_body) or not is_bedrock_native_output_rejection(
e.response.status_code, e.response.text
):
return False
warn_bedrock_native_output_rejected(model, e.response.text)
return True
def validate_anthropic_messages_environment(
self,
headers: dict,

4
litellm/llms/bedrock/messages/mantle_transformation.py
View file

@ -176,9 +176,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
self,
model: str,
httpx_response: httpx.Response,
request_body: dict,
request_body: dict[str, object], # mutable-ok: mirrors the base signature
litellm_logging_obj: LiteLLMLoggingObj,
) -> AsyncIterator:
) -> AsyncIterator[bytes]:
return AnthropicMessagesConfig.get_async_streaming_response_iterator(
self,
model=model,

431
litellm/llms/bedrock/passthrough/guardrail_translation/handler.py
View file

@ -5,10 +5,12 @@ import struct
from binascii import crc32
from collections.abc import AsyncGenerator, AsyncIterable, Callable, Mapping, Sequence
from dataclasses import dataclass
from functools import partial, reduce
from itertools import accumulate, chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias, TypeVar
from pydantic import TypeAdapter, ValidationError
from pydantic import InstanceOf, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -19,10 +21,31 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
StreamTransformSink,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
scoped_structured_message_indices,
unappliable_request_rewrite,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.chat.converse_to_openai import (
ConverseRow,
JsonPath,
as_json_array,
as_json_object,
converse_rows,
converse_tools,
parse_converse_request,
)
from litellm.llms.bedrock.passthrough.guardrail_translation.structured_edits import (
Edit,
ReplacedTools,
ReturnedTool,
ToolResultText,
structured_edits,
tool_edits,
)
from litellm.proxy.guardrails.auto_router_compression import is_compression_guardrail
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
@ -62,7 +85,32 @@ def _generic_passthrough_handler() -> "PassThroughEndpointHandler":
return PassThroughEndpointHandler()
def _reads_flat_texts_only(guardrail: "CustomGuardrail") -> bool:
"""Whether `guardrail` gets only the flat texts on Converse, without rows or tools."""
return is_compression_guardrail(guardrail) or not guardrail.supports_structured_messages()
_StringHolder = tuple[Any, str | int]
_LeafIdentity: TypeAlias = tuple[int, str | int]
@dataclass(frozen=True, slots=True)
class _Assign:
holder: _StringHolder
value: object
@dataclass(frozen=True, slots=True)
class _Remove:
holder: _StringHolder
_Write: TypeAlias = _Assign
_ContentHolder: TypeAlias = tuple[object, str]
_BlankDrop: TypeAlias = tuple[_ContentHolder, frozenset[int]]
_BODY_BY_REFERENCE: Final = TypeAdapter(InstanceOf[dict[str, object]])
_OBJECT_BY_REFERENCE: Final = TypeAdapter(InstanceOf[Mapping[str, object]])
_ARRAY_BY_REFERENCE: Final = TypeAdapter(InstanceOf[list[object]])
def _collect_strings(node: object, holders: list[_StringHolder]) -> None:
@ -98,10 +146,17 @@ def _collect_block_text(block: dict, holders: list[_StringHolder]) -> None:
holders.append((block, "text"))
def _request_field_holders(body: Mapping[str, object]) -> tuple[_StringHolder, ...]:
holders: Final = list[_StringHolder]()
_collect_strings(body.get("additionalModelRequestFields"), holders)
return tuple(holders)
def _extract_converse_texts(
body: dict,
skip_system: bool,
skip_tool: bool,
scan_only_tool_results: bool = False,
) -> tuple[list[str], list[_StringHolder]]:
"""
Walk a Bedrock Converse request body and collect text content.
@ -121,22 +176,25 @@ def _extract_converse_texts(
inflating guardrail usage for a request whose only prompt is one user
message. No other guardrail translation handler puts tool definitions in
``texts``; the chat and messages handlers carry them in the structured
``tools`` input instead, which this handler does not populate because a
Bedrock ``toolSpec`` is not the OpenAI tool shape those consumers expect.
``tools`` input instead, which this handler also populates by converting
each Bedrock ``toolSpec`` to the OpenAI tool shape.
``additionalModelRequestFields`` is treated differently on purpose. Bedrock
gives ``toolConfig.tools`` a fixed schema whose contents are tool metadata by
contract, while ``additionalModelRequestFields`` is free-form and defined by
the target model, so what it carries cannot be classified without knowing
that model. Scanning it stays the fail-closed default.
that model. Scanning it stays the fail-closed default. Its strings come
last, in the same order as they follow the latest user message in the rows.
"""
holders: Final[list[_StringHolder]] = []
if not skip_system:
system_holders: Final = list[_StringHolder]()
if not skip_system and not scan_only_tool_results:
for block in body.get("system") or []:
if isinstance(block, dict):
_collect_block_text(block, holders)
_collect_block_text(block, system_holders)
request_field_holders: Final = () if scan_only_tool_results else _request_field_holders(body)
message_holders: Final = list[_StringHolder]()
for message in body.get("messages") or []:
if not isinstance(message, dict):
continue
@ -145,23 +203,273 @@ def _extract_converse_texts(
continue
if skip_tool and ("toolUse" in block or "toolResult" in block):
continue
_collect_block_text(block, holders)
if scan_only_tool_results and "toolResult" not in block:
continue
_collect_block_text(block, message_holders)
tool_use = block.get("toolUse")
if isinstance(tool_use, dict):
_collect_strings(tool_use.get("input"), holders)
_collect_strings(tool_use.get("input"), message_holders)
tool_result = block.get("toolResult")
if isinstance(tool_result, dict):
for inner in tool_result.get("content") or []:
if isinstance(inner, dict):
_collect_block_text(inner, holders)
_collect_strings(inner.get("json"), holders)
_collect_strings(body.get("additionalModelRequestFields"), holders)
_collect_block_text(inner, message_holders)
_collect_strings(inner.get("json"), message_holders)
holders: Final = [*system_holders, *message_holders, *request_field_holders]
texts: Final = [container[key] for container, key in holders]
return texts, holders
@dataclass(frozen=True, slots=True)
class _Scope:
skip_system: bool
skip_tool: bool
scan_only_tool_results: bool
def _texts_applied_rows(body: Mapping[str, object], texts: Sequence[str], scope: _Scope) -> tuple[ConverseRow, ...]:
rewritten: Final = _BODY_BY_REFERENCE.validate_python(json.loads(json.dumps(body)))
_, holders = _extract_converse_texts(rewritten, scope.skip_system, scope.skip_tool, scope.scan_only_tool_results)
_write_back_texts(texts, holders)
request: Final = parse_converse_request(rewritten)
return converse_rows(request, include_tool_calls=not scope.skip_tool) if request else ()
def _texts_follow_rows(
body: Mapping[str, object],
rows: Sequence[ConverseRow],
texts: Sequence[str],
guardrailed_texts: Sequence[str],
changed_rows: frozenset[int],
scope: _Scope,
) -> bool:
"""Whether the returned texts sit in the flat layout and only touch rows the vendor also changed."""
if len(guardrailed_texts) != len(texts):
return False
view: Final = _texts_applied_rows(body, guardrailed_texts, scope)
if len(view) != len(rows):
return False
rows_changed_by_texts: Final = frozenset(
index for index, (row, viewed) in enumerate(zip(rows, view)) if row.message != viewed.message
)
return rows_changed_by_texts <= changed_rows
def _child(node: object, key: str | int) -> object:
try:
match key:
case str():
return _OBJECT_BY_REFERENCE.validate_python(node).get(key)
case int():
items: Final = _ARRAY_BY_REFERENCE.validate_python(node)
return items[key] if 0 <= key < len(items) else None
except ValidationError:
return None
def _holder(body: object, path: JsonPath) -> _StringHolder | None:
container: Final = reduce(_child, path[:-1], body)
return (container, path[-1]) if isinstance(_child(container, path[-1]), str) else None
def _leaf_identity(holder: _StringHolder) -> _LeafIdentity:
container, key = holder
return id(container), key
def _changed_leaves(
holders: Sequence[_StringHolder], texts: Sequence[str], guardrailed_texts: Sequence[str]
) -> frozenset[_LeafIdentity]:
return frozenset(
_leaf_identity(holder) for holder, before, after in zip(holders, texts, guardrailed_texts) if after != before
)
def _tool_result_write(body: Mapping[str, object], edit: ToolResultText) -> _Write | None:
container: Final = reduce(_child, edit.content[:-1], body)
blocks: Final = as_json_array(_child(container, edit.content[-1]))
if blocks is None:
return None
dropped: Final = frozenset(edit.indices[1:])
return _Assign(
(container, edit.content[-1]),
[
{"text": edit.text} if index == edit.indices[0] else block
for index, block in enumerate(blocks)
if index not in dropped
],
)
def _is_cache_point(entry: object) -> bool:
fields: Final = as_json_object(entry)
return fields is not None and "cachePoint" in fields
def _tool_name(entry: object) -> object:
return _child(_child(entry, "toolSpec"), "name")
def _entries_after(unsent: Sequence[tuple[int, object]], anchor: int) -> tuple[object, ...]:
return tuple(entry for at, entry in unsent if at == anchor)
def _sent_anchor(sent: frozenset[int], anchor: int, index: int) -> int:
return index if index in sent else anchor
def _replaced(entry: ReturnedTool) -> int | None:
return entry if isinstance(entry, int) else entry.replaces
def _replaced_tools_write(body: Mapping[str, object], edit: ReplacedTools) -> _Write | None:
"""
Rebuild ``toolConfig.tools`` in the order the guardrail returned. Entries it was never sent (cache points, system
tools) stay right after the tool they followed, and a cache point goes away with its tool. Refuses when
``toolChoice`` names a tool that is no longer there, and creates ``toolConfig`` when the request had none. A
``toolConfig`` left with no tool stays until the request is sent, so later guardrails still see the tool history.
"""
tool_config: Final = _child(body, "toolConfig")
tools: Final = () if tool_config is None else as_json_array(_child(tool_config, "tools"))
if tools is None:
return None
anchors: Final = tuple(accumulate(range(len(tools)), partial(_sent_anchor, edit.sent), initial=-1))[1:]
unsent: Final = tuple(
(anchor, tool) for index, (anchor, tool) in enumerate(zip(anchors, tools)) if index not in edit.sent
)
replaced: Final = frozenset(_replaced(entry) for entry in edit.tools)
placed: Final = chain.from_iterable(
(tools[entry], *_entries_after(unsent, entry))
if isinstance(entry, int)
else (*entry.blocks, *(() if entry.replaces is None else _entries_after(unsent, entry.replaces)))
for entry in edit.tools
)
orphans: Final = tuple(tool for at, tool in unsent if at not in replaced and at >= 0 and not _is_cache_point(tool))
rebuilt: Final = [*_entries_after(unsent, -1), *placed, *orphans]
chosen: Final = _child(_child(_child(tool_config, "toolChoice"), "tool"), "name")
if isinstance(chosen, str) and all(_tool_name(tool) != chosen for tool in rebuilt):
return None
if tool_config is None:
return None if all(map(_is_cache_point, rebuilt)) else _Assign((body, "toolConfig"), {"tools": rebuilt})
return _Assign((tool_config, "tools"), rebuilt)
def _resolved(body: Mapping[str, object], edit: Edit) -> _Write | None:
match edit:
case ToolResultText():
return _tool_result_write(body, edit)
case ReplacedTools():
return _replaced_tools_write(body, edit)
case (path, text):
holder: Final = _holder(body, path)
return None if holder is None else _Assign(holder, text)
def _kept_write(write: _Write, flat_changes: frozenset[_LeafIdentity]) -> bool:
return _leaf_identity(write.holder) not in flat_changes
def _holder_edits(
body: Mapping[str, object], edits: Sequence[Edit], flat_changes: frozenset[_LeafIdentity]
) -> tuple[_Write, ...] | None:
resolved: Final = tuple(_resolved(body, edit) for edit in edits)
if any(write is None for write in resolved):
return None
return tuple(write for write in resolved if write is not None and _kept_write(write, flat_changes))
def _apply(writes: Sequence[_Assign | _Remove]) -> None:
for write in writes:
match write:
case _Assign(holder=(container, key), value=value):
container[key] = value
case _Remove(holder=(container, key)):
del container[key]
def _block_text_holder(block: object) -> _ContentHolder | None:
if isinstance(_child(block, "text"), str):
return block, "text"
guarded: Final = _child(_child(block, "guardContent"), "text")
return (guarded, "text") if isinstance(_child(guarded, "text"), str) else None
def _final_texts(
holders: Sequence[_StringHolder], applied_texts: Sequence[str], writes: Sequence[_Write]
) -> Mapping[_LeafIdentity, object]:
flat: Final = {_leaf_identity(holder): text for holder, text in zip(holders, applied_texts)}
structured: Final = {_leaf_identity(write.holder): write.value for write in writes}
return MappingProxyType({**flat, **structured})
def _blanked(final: Mapping[_LeafIdentity, object], block: object) -> bool:
holder: Final = _block_text_holder(block)
if holder is None:
return False
container, key = holder
value: Final = final.get((id(container), key))
return isinstance(value, str) and value != _child(container, key) and not value.strip()
def _blank_blocks(final: Mapping[_LeafIdentity, object], holder: _ContentHolder) -> frozenset[int]:
container, key = holder
return frozenset(id(block) for block in as_json_array(_child(container, key)) or () if _blanked(final, block))
def _empties_turn(drop: _BlankDrop) -> bool:
(container, key), dropped = drop
blocks: Final = as_json_array(_child(container, key)) or ()
return bool(dropped) and all(id(block) in dropped or _is_cache_point(block) for block in blocks)
def _turn_drop(final: Mapping[_LeafIdentity, object], message: object) -> _BlankDrop:
return (message, "content"), _blank_blocks(final, (message, "content"))
def _blank_drops(body: Mapping[str, object], final: Mapping[_LeafIdentity, object]) -> tuple[_BlankDrop, ...]:
"""The system and message text blocks the guardrail left blank, except in a turn they would leave empty."""
turns: Final = (_turn_drop(final, message) for message in as_json_array(body.get("messages")) or ())
system: Final = ((body, "system"), _blank_blocks(final, (body, "system")))
return tuple(drop for drop in (system, *(turn for turn in turns if not _empties_turn(turn))) if drop[1])
def _blank_drop_write(drop: _BlankDrop) -> _Assign | _Remove:
holder, dropped = drop
container, key = holder
kept: Final = [block for block in as_json_array(_child(container, key)) or () if id(block) not in dropped]
return _Assign(holder, kept) if not all(map(_is_cache_point, kept)) else _Remove(holder)
def _edits_drop_a_text_change(
body: Mapping[str, object],
edits: Sequence[Edit],
texts: Sequence[str],
guardrailed_texts: Sequence[str],
scope: _Scope,
) -> bool:
"""
Whether the row and tool edits alone leave out a change the guardrail made in its returned texts. Texts of the
length that was sent are read position by position, so a changed string whose field the edits leave at its
original value is dropped. Texts of another length cannot be placed, so what they mask cannot be checked either,
while an empty list leaves every text as it was.
"""
if not guardrailed_texts:
return False
if len(guardrailed_texts) != len(texts):
return True
rewritten: Final = _BODY_BY_REFERENCE.validate_python(json.loads(json.dumps(body)))
_, holders = _extract_converse_texts(rewritten, scope.skip_system, scope.skip_tool, scope.scan_only_tool_results)
writes: Final = _holder_edits(rewritten, edits, frozenset())
if writes is None:
return True
_apply(writes)
edited: Final = [container[key] for container, key in holders]
return any(
returned != before and after == before for before, returned, after in zip(texts, guardrailed_texts, edited)
)
def _extract_converse_output_texts(
content_blocks: Sequence[object],
) -> tuple[list[str], list[_StringHolder]]:
@ -196,8 +504,8 @@ def _extract_converse_output_texts(
def _write_back_texts(
guardrailed_texts: list[str],
holders: list[_StringHolder],
guardrailed_texts: Sequence[str],
holders: Sequence[_StringHolder],
) -> None:
if len(guardrailed_texts) < len(holders):
verbose_proxy_logger.warning(
@ -982,16 +1290,39 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
if not isinstance(body, dict) or not isinstance(body.get("messages"), list):
return data
request_body: Final = _BODY_BY_REFERENCE.validate_python(body)
skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply)
skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
reads_rows: Final = not _reads_flat_texts_only(guardrail_to_apply)
scope: Final = _Scope(
skip_system=effective_skip_system_message_for_guardrail(guardrail_to_apply),
skip_tool=effective_skip_tool_message_for_guardrail(guardrail_to_apply),
scan_only_tool_results=effective_scan_only_tool_results_for_guardrail(guardrail_to_apply),
)
texts, holders = _extract_converse_texts(
request_body,
scope.skip_system,
scope.skip_tool,
scope.scan_only_tool_results,
)
request: Final = parse_converse_request(request_body) if reads_rows else None
rows: Final = converse_rows(request, include_tool_calls=not scope.skip_tool) if request else ()
scoped_indices: Final = scoped_structured_message_indices(
tuple(row.message for row in rows),
scan_only_tool_results=scope.scan_only_tool_results,
skip_system=scope.skip_system,
skip_tool=scope.skip_tool,
)
texts, holders = _extract_converse_texts(body, skip_system, skip_tool)
if not texts:
if not texts and not scoped_indices:
return data
inputs: Final = GenericGuardrailAPIInputs(texts=texts)
sent_messages: Final = [rows[index].message for index in scoped_indices]
inputs: Final = GenericGuardrailAPIInputs(texts=list(texts))
if sent_messages:
inputs["structured_messages"] = json.loads(json.dumps(sent_messages))
tools: Final = converse_tools(request) if request and not scope.scan_only_tool_results else ()
if tools:
inputs["tools"] = copy.deepcopy([tool.param for tool in tools])
model: Final = data.get("model")
if model:
inputs["model"] = model
@ -1004,8 +1335,60 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
)
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
if guardrailed_texts:
_write_back_texts(guardrailed_texts, holders)
guardrailed_messages: Final = guardrailed_inputs.get("structured_messages")
rewritten_rows: Final = (
guardrailed_messages
if sent_messages and isinstance(guardrailed_messages, list) and guardrailed_messages != sent_messages
else None
)
changed_rows: Final[frozenset[int]] = (
frozenset()
if rewritten_rows is None
else frozenset(
index
for index, sent, returned in zip(scoped_indices, sent_messages, rewritten_rows)
if sent != returned
)
)
texts_follow_rows: Final = rewritten_rows is None or _texts_follow_rows(
request_body, rows, texts, guardrailed_texts, changed_rows, scope
)
applied_texts: Final = guardrailed_texts if texts_follow_rows else texts
row_edits: Final = (
()
if rewritten_rows is None
else structured_edits(
rows, scoped_indices, rewritten_rows, partial(_texts_applied_rows, request_body, applied_texts, scope)
)
)
returned_tools: Final = guardrailed_inputs.get("tools")
tool_changes: Final = (
()
if returned_tools is None or not reads_rows
else tool_edits(tools, returned_tools, model if isinstance(model, str) else None)
)
holder_edits: Final = (
None
if row_edits is None
or tool_changes is None
or (
not texts_follow_rows
and _edits_drop_a_text_change(
request_body, (*row_edits, *tool_changes), texts, guardrailed_texts, scope
)
)
else _holder_edits(
request_body, (*row_edits, *tool_changes), _changed_leaves(holders, texts, applied_texts)
)
)
if holder_edits is None:
raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
if holder_edits or (applied_texts and list(applied_texts) != texts):
blank_drops: Final = _blank_drops(request_body, _final_texts(holders, applied_texts, holder_edits))
if applied_texts:
_write_back_texts(applied_texts, holders)
_apply(holder_edits)
_apply(tuple(_blank_drop_write(drop) for drop in blank_drops))
return data

337
litellm/llms/bedrock/passthrough/guardrail_translation/structured_edits.py Normal file
View file

@ -0,0 +1,337 @@
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from typing import Final, TypeAlias
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.prompt_templates.factory import bedrock_tool_blocks
from litellm.llms.bedrock.chat.converse_to_openai import (
ConverseRow,
ConverseTool,
JsonPath,
JsonSlot,
Slot,
TextSlot,
as_json_array,
as_json_object,
breadth_first,
)
from litellm.types.llms.bedrock import BedrockToolBlock
from litellm.types.llms.openai import ChatCompletionTextObject
@dataclass(frozen=True, slots=True)
class ToolResultText:
content: JsonPath
indices: tuple[int, ...]
text: str
@dataclass(frozen=True, slots=True)
class ConvertedTool:
replaces: int | None
blocks: tuple[BedrockToolBlock, ...]
ReturnedTool: TypeAlias = int | ConvertedTool
@dataclass(frozen=True, slots=True)
class ReplacedTools:
sent: frozenset[int]
tools: tuple[ReturnedTool, ...]
TextEdit: TypeAlias = tuple[JsonPath, str]
Edit: TypeAlias = TextEdit | ToolResultText | ReplacedTools
_JsonPair: TypeAlias = tuple[JsonPath, object, object]
_TEXT_PARTS: Final = TypeAdapter(tuple[ChatCompletionTextObject, ...])
_PARSED_JSON: Final = TypeAdapter(object)
def _without(mapping: Mapping[str, object], key: str) -> tuple[tuple[str, object], ...]:
return tuple(sorted(((name, value) for name, value in mapping.items() if name != key), key=lambda item: item[0]))
def _without_content(message: Mapping[str, object]) -> tuple[tuple[str, object], ...]:
return _without(message, "content")
def _text_parts(message: Mapping[str, object]) -> tuple[str, ...] | None:
content: Final = message.get("content")
if isinstance(content, str):
return (content,)
try:
parts: Final = _TEXT_PARTS.validate_python(content)
except ValidationError:
return None
return tuple(part["text"] for part in parts)
def _merged(children: Sequence[tuple[Edit, ...] | None]) -> tuple[Edit, ...] | None:
if any(child is None for child in children):
return None
return tuple(chain.from_iterable(child for child in children if child is not None))
def _paired_children(path: JsonPath, original: object, returned: object) -> tuple[_JsonPair, ...] | None:
original_object: Final = as_json_object(original)
returned_object: Final = as_json_object(returned)
if original_object is not None and returned_object is not None:
if original_object.keys() != returned_object.keys():
return None
return tuple(((*path, key), value, returned_object[key]) for key, value in original_object.items())
original_items: Final = as_json_array(original)
returned_items: Final = as_json_array(returned)
if original_items is None or returned_items is None or len(original_items) != len(returned_items):
return None
return tuple(((*path, index), value, returned_items[index]) for index, value in enumerate(original_items))
def _pair_children(pair: _JsonPair | None) -> tuple[_JsonPair | None, ...]:
if pair is None:
return ()
path, original, returned = pair
if original == returned or (isinstance(original, str) and isinstance(returned, str)):
return ()
children: Final = _paired_children(path, original, returned)
return (None,) if children is None else children
def _leaf_edit(pair: _JsonPair | None) -> TextEdit | None:
match pair:
case (path, str() as original, str() as returned) if original != returned:
return path, returned
case _:
return None
def _json_edits(path: JsonPath, original: object, returned: object) -> tuple[TextEdit, ...] | None:
pairs: Final = tuple(breadth_first((path, original, returned), _pair_children))
if any(pair is None for pair in pairs):
return None
return tuple(edit for pair in pairs if (edit := _leaf_edit(pair)) is not None)
def _slot_edits(slot: Slot, text: str) -> tuple[Edit, ...] | None:
match slot:
case TextSlot():
return () if text == slot.text else ((slot.path, text),)
case JsonSlot():
try:
parsed: Final = _PARSED_JSON.validate_json(text)
except ValidationError:
return None
return _json_edits(slot.path, slot.value, parsed)
def _call_function(call: object) -> Mapping[str, object] | None:
fields: Final = as_json_object(call)
return as_json_object(fields.get("function")) if fields is not None else None
def _call_edits(slot: JsonSlot, original: object, returned: object) -> tuple[Edit, ...] | None:
if original == returned:
return ()
original_call: Final = as_json_object(original)
returned_call: Final = as_json_object(returned)
original_function: Final = _call_function(original)
returned_function: Final = _call_function(returned)
if original_call is None or returned_call is None or original_function is None or returned_function is None:
return None
arguments: Final = returned_function.get("arguments")
if (
not isinstance(arguments, str)
or _without(returned_call, "function") != _without(original_call, "function")
or _without(returned_function, "arguments") != _without(original_function, "arguments")
):
return None
return _slot_edits(slot, arguments)
def _calls_edits(slots: Sequence[JsonSlot], original: object, returned: object) -> tuple[Edit, ...] | None:
original_calls: Final = as_json_array(original)
returned_calls: Final = as_json_array(returned)
if original_calls is None or returned_calls is None or len(original_calls) != len(returned_calls):
return None
return _merged(tuple(_call_edits(*call) for call in zip(slots, original_calls, returned_calls)))
def _carries_no_text(content: object) -> bool:
match content:
case None | "" | []:
return True
case _:
return False
def _content_texts(row: ConverseRow, returned: Mapping[str, object]) -> tuple[str, ...] | None:
content: Final = returned.get("content")
sent: Final = _text_parts(row.message)
if (
isinstance(content, str)
and len(row.content) > 1
and sent is not None
and all("\n" not in text for text in sent)
):
return tuple(content.split("\n"))
return _text_parts(returned)
def _content_edits(row: ConverseRow, original: object, returned: Mapping[str, object]) -> tuple[Edit, ...] | None:
if returned.get("content") == original:
return ()
if not row.content and _carries_no_text(returned.get("content")):
return ()
texts: Final = _content_texts(row, returned)
if texts is None or len(texts) != len(row.content):
return None
return _merged(tuple(_slot_edits(slot, text) for slot, text in zip(row.content, texts)))
def _field(message: Mapping[str, object], key: str) -> object:
return message.get(key)
def _row_edits(row: ConverseRow, returned: Mapping[str, object]) -> tuple[Edit, ...] | None:
if returned == row.message:
return ()
calls: Final = _field(row.message, "tool_calls")
call_edits: Final = _calls_edits(row.calls, calls, returned.get("tool_calls")) if row.calls else ()
comparable: Final = {**returned, "tool_calls": calls} if row.calls else returned
if call_edits is None or _without_content(comparable) != _without_content(row.message):
return None
return _merged((call_edits, _content_edits(row, _field(row.message, "content"), comparable)))
def _same_content(expected: Mapping[str, object], returned: Mapping[str, object]) -> bool:
if expected.get("content") == returned.get("content"):
return True
expected_parts: Final = _text_parts(expected)
returned_parts: Final = _text_parts(returned)
return (
expected_parts is not None
and returned_parts is not None
and "\n".join(expected_parts) == "\n".join(returned_parts)
)
def _reproduces(expected: ConverseRow, returned: Mapping[str, object] | None) -> bool:
return (
returned is not None
and _without_content(returned) == _without_content(expected.message)
and _same_content(expected.message, returned)
)
def _tool_result_text(row: ConverseRow, returned: Mapping[str, object]) -> ToolResultText | None:
texts: Final = _text_parts(returned)
indices: Final = tuple(index for slot in row.content if isinstance(index := slot.path[-2], int))
if (
row.message["role"] != "tool"
or not indices
or texts is None
or _without_content(returned) != _without_content(row.message)
):
return None
return ToolResultText(row.content[0].path[:-2], indices, "\n".join(texts))
def _unmapped_edits(
row: ConverseRow, view: ConverseRow, returned: Mapping[str, object] | None
) -> tuple[Edit, ...] | None:
if _reproduces(view, returned):
return ()
rewrite: Final = _tool_result_text(row, returned) if returned is not None and view == row else None
return None if rewrite is None else (rewrite,)
def structured_edits(
rows: Sequence[ConverseRow],
scoped_indices: Sequence[int],
returned: Sequence[object],
texts_applied_rows: Callable[[], tuple[ConverseRow, ...]],
) -> tuple[Edit, ...] | None:
if len(returned) != len(scoped_indices):
return None
returned_rows: Final = tuple(as_json_object(message) for message in returned)
proposed: Final = tuple(
_row_edits(rows[index], row) if row is not None else None for index, row in zip(scoped_indices, returned_rows)
)
unmapped: Final = tuple(
(index, row) for index, row, edits in zip(scoped_indices, returned_rows, proposed) if edits is None
)
view: Final = texts_applied_rows() if unmapped else ()
if unmapped and len(view) != len(rows):
return None
fallbacks: Final = tuple(_unmapped_edits(rows[index], view[index], row) for index, row in unmapped)
return _merged((*(edits for edits in proposed if edits is not None), *fallbacks))
def _spec_edit(index: int, edit: TextEdit) -> TextEdit | None:
path, text = edit
match path:
case ("function", "description"):
return ("toolConfig", "tools", index, "toolSpec", "description"), text
case ("function", "parameters", *schema_path):
return ("toolConfig", "tools", index, "toolSpec", "inputSchema", "json", *schema_path), text
case _:
return None
def _tool_edits(tool: ConverseTool | None, returned: object) -> tuple[Edit, ...] | None:
edits: Final = None if tool is None else _json_edits((), tool.param, returned)
if tool is None or edits is None:
return None
spec_edits: Final = tuple(_spec_edit(tool.index, edit) for edit in edits)
return None if None in spec_edits else tuple(edit for edit in spec_edits if edit is not None)
def _tool_name(tool: object) -> object:
function: Final = _call_function(tool)
return None if function is None else function.get("name")
def _named(first_by_name: Mapping[str, ConverseTool], returned: object) -> ConverseTool | None:
name: Final = _tool_name(returned)
return first_by_name.get(name) if isinstance(name, str) else None
def _bedrock_blocks(returned: object, model: str | None) -> tuple[BedrockToolBlock, ...] | None:
tool: Final = as_json_object(returned)
function: Final = _call_function(returned)
if (
tool is None
or function is None
or not isinstance(function.get("name"), str)
or ("parameters" in function and as_json_object(function["parameters"]) is None)
):
return None
return bedrock_tool_blocks((tool,), model)
def _returned_tool(
tool: ConverseTool | None, returned: object, model: str | None
) -> tuple[ReturnedTool, tuple[Edit, ...]] | None:
edits: Final = _tool_edits(tool, returned)
if tool is not None and edits is not None:
return tool.index, edits
blocks: Final = _bedrock_blocks(returned, model)
return None if blocks is None else (ConvertedTool(None if tool is None else tool.index, blocks), ())
def tool_edits(tools: Sequence[ConverseTool], returned: object, model: str | None) -> tuple[Edit, ...] | None:
returned_tools: Final = as_json_array(returned)
if returned_tools is None:
return None
first_by_name: Final = {tool.param["function"]["name"]: tool for tool in reversed(tools)}
planned: Final = tuple(_returned_tool(_named(first_by_name, item), item, model) for item in returned_tools)
placed: Final = tuple(plan for plan in planned if plan is not None)
if len(placed) != len(planned):
return None
entries: Final = tuple(entry for entry, _ in placed)
edits: Final = tuple(chain.from_iterable(edits for _, edits in placed))
if entries == tuple(tool.index for tool in tools):
return edits
return (*edits, ReplacedTools(frozenset(tool.index for tool in tools), entries))

174
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -10,13 +10,25 @@ from httpx import Response
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import AWS_ERROR_TYPE_HEADER
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector
from litellm.llms.bedrock.chat.converse_to_openai import as_json_array, as_json_object
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
from litellm.types.utils import ModelResponseStream
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError, BedrockEventStreamDecoderBase, BedrockModelInfo
from ..request_metadata import (
BEDROCK_REQUEST_METADATA_HEADER,
BedrockRequestMetadataViolation,
bedrock_request_metadata_headers,
bedrock_request_metadata_is_owned,
bedrock_request_metadata_violation,
caller_request_metadata_violations_are_dropped,
merge_bedrock_invoke_headers,
resolve_bedrock_request_metadata,
)
if TYPE_CHECKING:
from botocore.eventstream import EventStreamMessage
@ -28,10 +40,152 @@ if TYPE_CHECKING:
BEDROCK_STREAMING_ACTIONS: Final = frozenset({"invoke-with-response-stream", "converse-stream"})
_BEDROCK_CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"})
_BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stream"})
def _bedrock_endpoint_action(endpoint: str) -> str:
return endpoint.partition("?")[0].rstrip("/").rsplit("/", 1)[-1]
def is_bedrock_streaming_endpoint(endpoint: str) -> bool:
return endpoint.partition("?")[0].rstrip("/").rsplit("/", 1)[-1] in BEDROCK_STREAMING_ACTIONS
return _bedrock_endpoint_action(endpoint) in BEDROCK_STREAMING_ACTIONS
def _caller_request_metadata_violation(
action: str, body: Mapping[str, object]
) -> BedrockRequestMetadataViolation | None:
if action in _BEDROCK_INVOKE_ACTIONS and "requestMetadata" in body and body["requestMetadata"] is None:
return BedrockRequestMetadataViolation("type", "requestMetadata must be a dictionary")
return bedrock_request_metadata_violation(body.get("requestMetadata"))
def _aws_error_type(action: str, violation: BedrockRequestMetadataViolation) -> str:
if action in _BEDROCK_CONVERSE_ACTIONS and violation.kind == "type":
return "SerializationException"
return "ValidationException"
def _caller_request_metadata(action: str, body: Mapping[str, object], litellm_params: Mapping[str, object]) -> object:
violation: Final = (
None
if caller_request_metadata_violations_are_dropped(litellm_params.get("drop_params") is True)
else _caller_request_metadata_violation(action, body)
)
if violation is not None:
raise BedrockError(
status_code=400,
message=violation.message,
headers={AWS_ERROR_TYPE_HEADER: _aws_error_type(action, violation)},
)
return body.get("requestMetadata")
def _with_body_request_metadata(
action: str, body: Mapping[str, object], litellm_params: Mapping[str, object]
) -> Mapping[str, object]:
resolved: Final = resolve_bedrock_request_metadata(
litellm_params, caller_metadata=_caller_request_metadata(action, body, litellm_params)
)
without_caller: Final = {key: value for key, value in body.items() if key != "requestMetadata"}
return without_caller if resolved is None else {**without_caller, "requestMetadata": resolved}
def _without_owned_request_metadata_header(headers: Mapping[str, str]) -> Mapping[str, str]:
if not bedrock_request_metadata_is_owned():
return headers
return {name: value for name, value in headers.items() if name.lower() != BEDROCK_REQUEST_METADATA_HEADER.lower()}
def _with_request_metadata(
action: str | None,
headers: Mapping[str, str],
body: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[Mapping[str, str], Mapping[str, object]]:
if action in _BEDROCK_CONVERSE_ACTIONS:
return headers, _with_body_request_metadata(action, body, litellm_params)
if action in _BEDROCK_INVOKE_ACTIONS and "requestMetadata" in body:
return _without_owned_request_metadata_header(headers), _with_body_request_metadata(
action, body, litellm_params
)
if action in _BEDROCK_INVOKE_ACTIONS:
return _with_invoke_request_metadata(headers, body, litellm_params)
return headers, body
def _with_invoke_request_metadata(
headers: Mapping[str, str],
body: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> tuple[Mapping[str, str], Mapping[str, object]]:
owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
return merge_bedrock_invoke_headers(dict(headers), (), metadata_headers, owned_names), body
def _is_cache_point(entry: object) -> bool:
fields: Final = as_json_object(entry)
return fields is not None and "cachePoint" in fields
def _tool_result_part(item: object) -> str | None:
fields: Final = as_json_object(item)
if fields is None:
return None
text: Final = fields.get("text")
if isinstance(text, str):
return text
return json.dumps(fields["json"]) if "json" in fields else None
def _tool_result_text(tool_result: Mapping[str, object]) -> str:
parts: Final = as_json_array(tool_result.get("content")) or ()
rendered: Final = "\n".join(part for item in parts if (part := _tool_result_part(item)) is not None).strip()
return rendered or "<non-text tool result omitted>"
def _block_field(block: object, key: str) -> Mapping[str, object] | None:
fields: Final = as_json_object(block)
return None if fields is None else as_json_object(fields.get(key))
def _as_text_block(block: object) -> object:
tool_use: Final = _block_field(block, "toolUse")
if tool_use is not None:
arguments: Final = json.dumps(tool_use.get("input"))
return {"text": f"[tool call {tool_use.get('toolUseId')}: {tool_use.get('name')}({arguments})]"}
tool_result: Final = _block_field(block, "toolResult")
if tool_result is None:
return block
return {"text": f"[tool result for {tool_result.get('toolUseId')}: {_tool_result_text(tool_result)}]"}
def _has_tool_block(content: Sequence[object]) -> bool:
return any(
_block_field(block, "toolUse") is not None or _block_field(block, "toolResult") is not None for block in content
)
def _as_text_message(message: object) -> object:
fields: Final = as_json_object(message)
content: Final = None if fields is None else as_json_array(fields.get("content"))
if fields is None or content is None or not _has_tool_block(content):
return message
return {**fields, "content": [_as_text_block(block) for block in content]}
def _converse_body_to_send(body: Mapping[str, object]) -> Mapping[str, object]:
"""
``body`` without a Converse ``toolConfig`` that holds no tool, with every ``toolUse`` and ``toolResult`` turned into
text, since Bedrock rejects tool blocks without a tool. Any other ``body`` comes back as it is.
"""
tool_config: Final = as_json_object(body.get("toolConfig"))
tools: Final = None if tool_config is None else as_json_array(tool_config.get("tools"))
if tools is None or not all(map(_is_cache_point, tools)):
return body
messages: Final = as_json_array(body.get("messages"))
rest: Final = {key: value for key, value in body.items() if key != "toolConfig"}
return rest if messages is None else {**rest, "messages": [_as_text_message(message) for message in messages]}
_TEXT_ONLY_DELTA_FIELDS: Final = frozenset({"content", "role"})
@ -284,18 +438,26 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
def sign_request(
self,
headers: dict,
litellm_params: dict,
request_data: dict | None,
headers: dict[str, str],
litellm_params: dict[str, object],
request_data: dict[str, object] | None,
api_base: str,
model: str | None = None,
) -> tuple[dict, bytes | None]:
action: Final = _bedrock_endpoint_action(api_base)
body: Final[Mapping[str, object]] = request_data or {}
optional_params: Final = litellm_params.copy()
signed_headers, signed_body = _with_request_metadata(
action=action if bedrock_request_metadata_is_owned() else None,
headers=headers,
body=_converse_body_to_send(body) if action in _BEDROCK_CONVERSE_ACTIONS else body,
litellm_params=litellm_params,
)
return self._sign_request(
service_name="bedrock",
headers=headers,
headers=dict(signed_headers),
optional_params=optional_params,
request_data=request_data or {},
request_data=dict(signed_body),
api_base=api_base,
model=model,
api_key=self.get_bedrock_bearer_token(optional_params),

129
litellm/llms/bedrock/request_metadata.py
View file

@ -8,17 +8,22 @@ sets ``litellm.bedrock_request_metadata_fields`` (``litellm_settings`` on the pr
Two properties are load-bearing for that billing record and are asserted by the tests:
proxy identity is resolved first so it can never be evicted by caller-supplied pairs, and the
whole ``user_api_key_`` prefix is reserved so a caller cannot write a proxy-authoritative
looking key. Values that break Bedrock's constraints are dropped rather than sanitised or
rejected, because an operator flipping this setting on must not turn a working request into a
400 and a silently rewritten attribution key is worse than an absent one.
looking key. Identity and ``spend_logs_metadata`` values that break Bedrock's constraints are
dropped rather than sanitised or rejected, because an operator flipping this setting on must not
turn a working request into a 400 and a silently rewritten attribution key is worse than an
absent one. For the same reason an Invoke header that would pass Bedrock's 8,500-character limit
keeps only the leading pairs that fit. A caller's own malformed ``requestMetadata`` is rejected with
a 400 unless ``drop_params`` is set, in which case its bad pairs are dropped the same way, with or
without the setting.
"""
from __future__ import annotations
import json
import re
from collections.abc import Mapping
from typing import Final
from collections.abc import Mapping, Sized
from dataclasses import dataclass
from typing import Final, Literal
import litellm
@ -26,15 +31,27 @@ BEDROCK_REQUEST_METADATA_HEADER: Final = "X-Amzn-Bedrock-Request-Metadata"
BEDROCK_REQUEST_METADATA_MAX_PAIRS: Final = 16
BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX: Final = "user_api_key_"
BEDROCK_REQUEST_METADATA_CLIENT_FIELD: Final = "spend_logs_metadata"
BEDROCK_REQUEST_METADATA_HEADER_MAX_CHARS: Final = 8500
_METADATA_PARAM_NAMES: Final[tuple[str, ...]] = ("metadata", "litellm_metadata")
_KEY_PATTERN: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
_VALUE_PATTERN: Final = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
_ALLOWED_CHARACTERS: Final = r"[a-zA-Z0-9\s:_@$#=/+,.-]"
_KEY_PATTERN: Final = re.compile(rf"{_ALLOWED_CHARACTERS}{{1,256}}", re.ASCII)
_VALUE_PATTERN: Final = re.compile(rf"{_ALLOWED_CHARACTERS}{{0,256}}", re.ASCII)
_OWNED_HEADER_NAMES: Final[frozenset[str]] = frozenset((BEDROCK_REQUEST_METADATA_HEADER.lower(),))
@dataclass(frozen=True, slots=True)
class BedrockRequestMetadataViolation:
kind: Literal["type", "constraint"]
message: str
def _is_forwardable(key: str, value: str) -> bool:
return _KEY_PATTERN.match(key) is not None and _VALUE_PATTERN.match(value) is not None
return _KEY_PATTERN.fullmatch(key) is not None and _VALUE_PATTERN.fullmatch(value) is not None
def _is_reserved(key: str) -> bool:
return key.strip().lower().startswith(BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX)
def _text_pairs(source: object) -> tuple[tuple[str, str], ...]:
@ -99,13 +116,16 @@ def _client_pairs(
candidates: Final = tuple(
(key, value)
for key, value in (*_text_pairs(caller_metadata), *spend_logs_pairs)
if not key.startswith(BEDROCK_REQUEST_METADATA_IDENTITY_PREFIX) and _is_forwardable(key, value)
if not _is_reserved(key) and _is_forwardable(key, value)
)
return tuple(
pair
for index, pair in enumerate(candidates)
if pair[0] not in tuple(earlier for earlier, _ in candidates[:index])
)[:budget]
first_index: Final = {key: index for index, (key, _) in reversed(tuple(enumerate(candidates)))}
return tuple(pair for index, pair in enumerate(candidates) if first_index[pair[0]] == index)[:budget]
def _forwardable_caller_pairs(caller_metadata: object) -> tuple[tuple[str, str], ...]:
return tuple(pair for pair in _text_pairs(caller_metadata) if _is_forwardable(*pair))[
:BEDROCK_REQUEST_METADATA_MAX_PAIRS
]
def resolve_bedrock_request_metadata(
@ -113,17 +133,18 @@ def resolve_bedrock_request_metadata(
caller_metadata: object = None,
) -> dict[str, str] | None:
"""
Resolve the ``requestMetadata`` pairs to send to Bedrock, or ``None`` when the feature is
off or nothing survives Bedrock's constraints. The result is a plain dict because it is
written straight onto the Converse body, which Bedrock types as ``dict[str, str]``.
Resolve the ``requestMetadata`` pairs to send to Bedrock, or ``None`` when nothing survives
Bedrock's constraints. The result is a plain dict because it is written straight onto the
Converse body, which Bedrock types as ``dict[str, str]``.
``caller_metadata`` is any ``requestMetadata`` the caller passed explicitly. It has already
been validated (and rejected with a 400) by the Converse transformation, so it is only
filtered here for the reserved identity prefix and the remaining slot budget.
``caller_metadata`` is any ``requestMetadata`` the caller passed explicitly. Callers reject a
malformed one with a 400 unless ``drop_params`` is set, so here its pairs are only filtered for
Bedrock's rules, the reserved identity prefix, and the remaining slot budget. With the feature
off only Bedrock's rules apply, since no identity is written for a caller to forge.
"""
allowed_fields: Final = _allowed_fields()
if not allowed_fields:
return None
return dict(_forwardable_caller_pairs(caller_metadata)) or None
sources: Final = _metadata_sources(litellm_params)
identity: Final = _identity_pairs(sources, allowed_fields)
client: Final = _client_pairs(
@ -136,6 +157,36 @@ def resolve_bedrock_request_metadata(
return resolved or None
def _pair_violation(key: str, value: str) -> BedrockRequestMetadataViolation | None:
if _KEY_PATTERN.fullmatch(key) is None:
return BedrockRequestMetadataViolation(
"constraint", f"requestMetadata key {key!r} must be 1-256 characters of {_ALLOWED_CHARACTERS}"
)
if _VALUE_PATTERN.fullmatch(value) is None:
return BedrockRequestMetadataViolation(
"constraint", f"requestMetadata value for key {key!r} must be 0-256 characters of {_ALLOWED_CHARACTERS}"
)
return None
def bedrock_request_metadata_violation(caller_metadata: object) -> BedrockRequestMetadataViolation | None:
"""Why a caller's ``requestMetadata`` breaks Bedrock's rules, in the order Bedrock checks them: a
wrong JSON type before the pair count and character rules. ``None`` means absent or valid."""
text_pairs: Final = _text_pairs(caller_metadata)
entry_count: Final = len(caller_metadata) if isinstance(caller_metadata, Sized) else 0
if caller_metadata is None:
return None
if not isinstance(caller_metadata, Mapping):
return BedrockRequestMetadataViolation("type", "requestMetadata must be a dictionary")
if len(text_pairs) != entry_count:
return BedrockRequestMetadataViolation("type", "requestMetadata keys and values must be strings")
if len(text_pairs) > BEDROCK_REQUEST_METADATA_MAX_PAIRS:
return BedrockRequestMetadataViolation(
"constraint", f"requestMetadata can contain a maximum of {BEDROCK_REQUEST_METADATA_MAX_PAIRS} items"
)
return next((violation for key, value in text_pairs if (violation := _pair_violation(key, value))), None)
def bedrock_request_metadata_is_owned() -> bool:
"""
Whether the proxy OWNS the request-metadata field and header name for this request.
@ -149,25 +200,51 @@ def bedrock_request_metadata_is_owned() -> bool:
return bool(_allowed_fields())
def caller_request_metadata_violations_are_dropped(drop_params: bool) -> bool:
return drop_params or litellm.drop_params is True
def bedrock_request_metadata_is_rewritten(drop_params: bool) -> bool:
return bedrock_request_metadata_is_owned() or caller_request_metadata_violations_are_dropped(drop_params)
def _header_value(pairs: Mapping[str, str]) -> str:
return json.dumps(pairs, separators=(",", ":"))
def _fits_header(pairs: Mapping[str, str]) -> bool:
return len(_header_value(pairs)) <= BEDROCK_REQUEST_METADATA_HEADER_MAX_CHARS
def _longest_fitting_prefix(resolved: Mapping[str, str]) -> dict[str, str]:
pairs: Final = tuple(resolved.items())
return next(
(dict(pairs[:count]) for count in range(len(pairs), 0, -1) if _fits_header(dict(pairs[:count]))),
{},
)
def bedrock_request_metadata_headers(
litellm_params: Mapping[str, object] | None,
) -> tuple[frozenset[str], tuple[tuple[str, str], ...]]:
"""
The signed ``X-Amzn-Bedrock-Request-Metadata`` header for the Invoke paths, which have no
body field for request metadata.
The signed ``X-Amzn-Bedrock-Request-Metadata`` header for an Invoke request whose body carries
no ``requestMetadata`` of its own.
Returns the header names the proxy OWNS and, separately, the pairs to send. Ownership is
reported whenever forwarding is enabled, including when nothing resolves, because a caller
can suppress the resolver (omit the allow-listed fields, or send values that all fail
Bedrock's rules) and an owned-but-empty result must still evict the caller's header rather
than fall back to it.
than fall back to it. Only the leading pairs that fit Bedrock's header limit are sent, so
identity always survives and long ``spend_logs_metadata`` pairs drop off the end.
"""
if not bedrock_request_metadata_is_owned():
return frozenset(), ()
resolved: Final = resolve_bedrock_request_metadata(litellm_params)
if resolved is None:
fitting: Final = {} if resolved is None else _longest_fitting_prefix(resolved)
if not fitting:
return _OWNED_HEADER_NAMES, ()
return _OWNED_HEADER_NAMES, ((BEDROCK_REQUEST_METADATA_HEADER, json.dumps(resolved, separators=(",", ":"))),)
return _OWNED_HEADER_NAMES, ((BEDROCK_REQUEST_METADATA_HEADER, _header_value(fitting)),)
def merge_bedrock_invoke_headers(

73
litellm/llms/bedrock_mantle/chat/transformation.py
View file

@ -10,19 +10,21 @@ Auth: Bearer token (litellm_params.api_key, BEDROCK_MANTLE_API_KEY, or the
BedrockMantleAuthMixin in common_utils.
"""
from collections.abc import AsyncIterator, Iterator
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock_mantle.common_utils import (
BEDROCK_MANTLE_DEFAULT_REGION,
BedrockMantleAuthMixin,
)
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
from litellm.llms.openai.chat.gpt_5_transformation import get_effort_level, is_gpt_reasoning_series_name
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
@ -39,6 +41,51 @@ if TYPE_CHECKING:
)
def _with_chat_reasoning_effort(non_default_params: Mapping[str, object]) -> Mapping[str, object]:
"""Chat Completions takes ``reasoning_effort`` as a level, so a Responses-shaped ``{"effort", "summary"}``
passes only its effort, the way the OpenAI GPT-5 chat config reads it."""
reasoning_effort: Final = non_default_params.get("reasoning_effort")
if not isinstance(reasoning_effort, dict):
return non_default_params
effort: Final = get_effort_level(reasoning_effort)
rest: Final = {key: value for key, value in non_default_params.items() if key != "reasoning_effort"}
return rest if effort is None else {**rest, "reasoning_effort": effort}
def _unsupported_chat_values(non_default_params: Mapping[str, object]) -> tuple[str, ...]:
n: Final = non_default_params.get("n")
reasoning_effort: Final = non_default_params.get("reasoning_effort")
summary: Final = reasoning_effort.get("summary") if isinstance(reasoning_effort, Mapping) else None
return tuple(
value
for value, unsupported in (
(f"n={n}", isinstance(n, int) and n > 1),
(f"reasoning summary {summary}", summary is not None),
)
if unsupported
)
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
return litellm.drop_params is True or litellm_params.get("drop_params") is True
def _chat_request_params(optional_params: Mapping[str, object], drop_params: bool) -> Mapping[str, object]:
unsupported: Final = _unsupported_chat_values(optional_params)
if unsupported and not drop_params:
raise UnsupportedParamsError(
message=f"Bedrock Mantle Chat Completions does not support {', '.join(unsupported)}. To drop unsupported "
"openai params from the call, set `litellm.drop_params = True`",
llm_provider="bedrock_mantle",
status_code=400,
)
servable: Final = _with_chat_reasoning_effort(optional_params)
n: Final = servable.get("n")
return MappingProxyType(
{key: value for key, value in servable.items() if not (key == "n" and isinstance(n, int) and n > 1)}
)
class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
"""
Transformation config for Amazon Bedrock Mantle OpenAI-compatible API.
@ -158,6 +205,28 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
if param != "max_tokens"
}
def transform_request(
self,
model: str,
messages: list[AllMessageValues], # mutable-ok: OpenAIGPTConfig.transform_request signature
optional_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature
) -> dict[str, object]: # mutable-ok: OpenAIGPTConfig.transform_request signature
request_params: Final = dict(_chat_request_params(optional_params, _drops_params(litellm_params)))
return super().transform_request(model, messages, request_params, litellm_params, headers)
def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: Mapping[str, object],
) -> Mapping[str, object]:
return _chat_request_params(
super().transform_extra_body(extra_body, request, model, litellm_params), _drops_params(litellm_params)
)
def _supports_reasoning(self, model: str) -> bool:
try:
return litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider)

4
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -24,6 +24,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
from litellm.llms.base_llm.responses.transformation import ReasoningSummaryLevel
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock_mantle.common_utils import (
@ -73,6 +74,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.BEDROCK_MANTLE
def auto_reasoning_summary(self) -> ReasoningSummaryLevel:
return "auto" if self.use_openai_path else super().auto_reasoning_summary()
def get_error_class(
self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers
) -> BaseLLMException:

205
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -1,10 +1,11 @@
import asyncio
import copy
import inspect
import json
import ssl
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Coroutine, Iterator, Mapping, Sequence
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
from contextlib import asynccontextmanager
from functools import lru_cache, partial
from functools import lru_cache, partial, reduce
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -35,7 +36,7 @@ import litellm.litellm_core_utils
import litellm.types
import litellm.types.utils
from litellm._logging import redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, update_headers_with_filtered_beta
from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@ -49,6 +50,7 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
synthesize_subtitle_document,
)
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
from litellm.litellm_core_utils.realtime_errors import (
@ -215,6 +217,7 @@ if TYPE_CHECKING:
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -467,6 +470,8 @@ async def _aiter_bytes_then_close(response: httpx.Response, *, chunk_size: int)
_DECODED_BODY_STALE_HEADERS: Final[frozenset[str]] = frozenset({"content-encoding", "content-length"})
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
_JSON_OBJECTS: Final = TypeAdapter(list[dict[str, object]])
def _decoded_body_headers(response: httpx.Response) -> httpx.Headers:
@ -2180,11 +2185,13 @@ class BaseLLMHTTPHandler:
api_key: str | None,
model: str,
timeout: float | httpx.Timeout | None = None,
rebuild_without_native_output: Callable[[], Mapping[str, object]] | None = None,
) -> httpx.Response:
max_attempts: Final = max(provider_config.max_retry_on_anthropic_messages_http_error, 1)
litellm_params_dict: Final = dict(litellm_params)
optional_params_dict: Final = dict(litellm_params)
for attempt_idx in range(max_attempts):
recovered_errors: tuple[httpx.HTTPStatusError, ...] = () # rebind-ok: a rebuilt body replays every recovery
while True:
try:
response = await async_httpx_client.post(
url=request_url,
@ -2197,64 +2204,79 @@ class BaseLLMHTTPHandler:
response.raise_for_status()
return response
except httpx.HTTPStatusError as e:
hit_max_attempt = attempt_idx + 1 == max_attempts
rebuild_body = (
rebuild_without_native_output
if rebuild_without_native_output is not None
and provider_config.should_retry_anthropic_messages_without_native_output(
e=e, request_body=request_body, model=model
)
else None
)
hit_max_attempt = len(recovered_errors) + 1 == max_attempts
should_retry = provider_config.should_retry_anthropic_messages_on_http_error(
e=e, litellm_params=litellm_params_dict, request_body=request_body
)
if should_retry and not hit_max_attempt:
if logging_obj.baseline_cache_context is not None:
await logging_obj.invalidate_baseline_cache_estimate("retried_request")
if rebuild_body is None and (hit_max_attempt or not should_retry):
raise self._handle_error(e=e, provider_config=provider_config)
if logging_obj.baseline_cache_context is not None:
await logging_obj.invalidate_baseline_cache_estimate("retried_request")
if rebuild_body is not None:
request_body.clear()
request_body.update(rebuild_body())
for recovered_error in recovered_errors:
provider_config.transform_anthropic_messages_request_on_http_error(
e=recovered_error, request_data=request_body
)
else:
verbose_logger.debug(
"Anthropic /v1/messages: invalid thinking signature; "
"stripping thinking blocks and retrying (attempt %s/%s).",
attempt_idx + 2,
len(recovered_errors) + 2,
max_attempts,
)
provider_config.transform_anthropic_messages_request_on_http_error(e=e, request_data=request_body)
headers, signed_json_body = await sign_request_off_loop_if_aws(
provider_config,
provider_config.sign_request,
headers=headers,
optional_params=optional_params_dict,
request_data=request_body,
api_base=request_url,
api_key=api_key,
stream=stream,
fake_stream=False,
model=model,
)
logging_obj.model_call_details.update(request_body)
continue
raise self._handle_error(e=e, provider_config=provider_config)
recovered_errors = (*recovered_errors, e)
headers, signed_json_body = await sign_request_off_loop_if_aws(
provider_config,
provider_config.sign_request,
headers=headers,
optional_params=optional_params_dict,
request_data=request_body,
api_base=request_url,
api_key=api_key,
stream=stream,
fake_stream=False,
model=model,
)
logging_obj.model_call_details.update(request_body)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
raise RuntimeError("unreachable: anthropic messages HTTP retry loop exited without return")
@staticmethod
def _resolve_anthropic_messages_timeout(
litellm_params: GenericLiteLLMParams,
stream: bool,
custom_llm_provider: str,
) -> float | httpx.Timeout | None:
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
)
from litellm.utils import supports_httpx_timeout
stream_timeout: Final = litellm_params.get("stream_timeout") if stream else None
model_timeout: Final = stream_timeout if stream_timeout is not None else litellm_params.get("timeout")
model_timeout: Final = litellm_params.get("timeout")
request_timeout: Final = litellm_params.get("request_timeout")
global_timeout: Final = get_configured_request_timeout()
if model_timeout is None and request_timeout is None and global_timeout is None:
if stream_timeout is None and model_timeout is None and request_timeout is None and global_timeout is None:
return None
return CompletionTimeout.resolve(
model_timeout,
{"request_timeout": request_timeout},
{"stream_timeout": stream_timeout, "request_timeout": request_timeout},
custom_llm_provider,
global_timeout=global_timeout,
supports_httpx_timeout=supports_httpx_timeout,
stream_kind=StreamKind.LIVE if stream else StreamKind.NOT_STREAMED,
)
async def async_anthropic_messages_handler(
@ -2292,6 +2314,7 @@ class BaseLLMHTTPHandler:
provider_specific_headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
provider_specific_header=provider_specific_header,
custom_llm_provider=custom_llm_provider,
native_anthropic_messages=anthropic_messages_provider_config.is_native_messages_route(),
)
forwarded_headers: Final = kwargs.get("headers", None)
# Also check for extra_headers in kwargs (from config or direct calls)
@ -2304,6 +2327,16 @@ class BaseLLMHTTPHandler:
merged_headers.update(extra_headers_from_kwargs)
if provider_specific_headers:
merged_headers.update(provider_specific_headers)
request_params: Final = reduce(
delete_nested_value,
litellm_params.get("additional_drop_params") or (),
anthropic_messages_optional_request_params,
)
if (
request_params.get("safeguards") is not None
and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
):
await aload_beta_headers_config()
(
headers,
api_base,
@ -2311,7 +2344,7 @@ class BaseLLMHTTPHandler:
headers=merged_headers or {},
model=model,
messages=messages,
optional_params=anthropic_messages_optional_request_params,
optional_params=request_params,
litellm_params=dict(litellm_params),
api_key=api_key,
api_base=api_base,
@ -2342,20 +2375,29 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
)
additional_drop_params: Final[list[str]] = litellm_params.get("additional_drop_params") or []
if additional_drop_params:
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
for path in additional_drop_params:
anthropic_messages_optional_request_params = delete_nested_value(
anthropic_messages_optional_request_params, path
)
native_output_fallback_params: Final = anthropic_messages_provider_config.native_output_fallback_litellm_params(
model=model,
anthropic_messages_optional_request_params=_OPTIONAL_PARAMS.validate_python(request_params),
litellm_params=litellm_params,
)
rebuild_without_native_output: Final = (
None
if native_output_fallback_params is None
else partial[Mapping[str, object]](
anthropic_messages_provider_config.transform_anthropic_messages_request,
model=model,
messages=copy.deepcopy(_JSON_OBJECTS.validate_python(messages)),
anthropic_messages_optional_request_params=copy.deepcopy(_JSON_OBJECT.validate_python(request_params)),
litellm_params=native_output_fallback_params,
headers=_JSON_OBJECT.validate_python(headers),
)
)
# Prepare request body
request_body: Final = anthropic_messages_provider_config.transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
anthropic_messages_optional_request_params=request_params,
litellm_params=litellm_params,
headers=headers,
)
@ -2407,6 +2449,13 @@ class BaseLLMHTTPHandler:
},
)
applied_timeout: Final = self._resolve_anthropic_messages_timeout(
litellm_params=litellm_params,
stream=stream or False,
custom_llm_provider=custom_llm_provider,
)
if applied_timeout is not None:
logging_obj.record_applied_timeout(applied_timeout)
response: Final = await self._async_post_anthropic_messages_with_http_error_retry(
async_httpx_client=async_httpx_client,
request_url=request_url,
@ -2419,11 +2468,8 @@ class BaseLLMHTTPHandler:
litellm_params=litellm_params,
api_key=api_key,
model=model,
timeout=self._resolve_anthropic_messages_timeout(
litellm_params=litellm_params,
stream=stream or False,
custom_llm_provider=custom_llm_provider,
),
timeout=applied_timeout,
rebuild_without_native_output=rebuild_without_native_output,
)
# used for logging + cost tracking
@ -2462,7 +2508,7 @@ class BaseLLMHTTPHandler:
held_back_tool_names: Final = self._server_fulfilled_tools_in_request(
logging_obj=logging_obj,
tools=anthropic_messages_optional_request_params.get("tools"),
tools=request_params.get("tools"),
)
initial_response = AgenticAnthropicStreamingIterator(
completion_stream=completion_stream,
@ -2470,7 +2516,7 @@ class BaseLLMHTTPHandler:
model=model,
messages=messages,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
anthropic_messages_optional_request_params=request_params,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs_for_agentic,
@ -2493,7 +2539,7 @@ class BaseLLMHTTPHandler:
model=model,
messages=messages,
anthropic_messages_provider_config=anthropic_messages_provider_config,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
anthropic_messages_optional_request_params=request_params,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs_for_agentic,
@ -6865,6 +6911,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider: str | None = None,
first_message: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs: object,
) -> Exception | None:
"""
@ -6880,6 +6927,35 @@ class BaseLLMHTTPHandler:
"""
_ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
_ws_guardrail_callbacks: list = []
_ws_output_guardrail_callbacks: list = []
try:
import litellm as _litellm
# Use duck-typing so any guardrail that exposes the PII
# masking interface works, not just _OPTIONAL_PresidioPIIMasking.
# This avoids a layering violation (SDK importing from proxy).
_ws_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and callable(getattr(cb, "_unmask_pii_text", None))
and getattr(cb, "output_parse_pii", False)
]
_ws_output_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and getattr(cb, "apply_to_output", False)
]
except Exception as _guardrail_exc:
verbose_logger.warning(
"Responses WebSocket: failed to collect guardrail callbacks — PII masking will be skipped. Error: %s",
_guardrail_exc,
)
if responses_api_provider_config is None or not responses_api_provider_config.supports_native_websocket():
from litellm.responses.streaming_iterator import (
ManagedResponsesWebSocketHandler,
@ -6897,6 +6973,8 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
first_message=first_message,
quota_callbacks=_ws_quota_callbacks,
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
frame_guard=frame_guard,
**kwargs,
)
await handler.run()
@ -6978,36 +7056,6 @@ class BaseLLMHTTPHandler:
if litellm_metadata:
_request_data["litellm_metadata"] = litellm_metadata
_ws_guardrail_callbacks: list = []
_ws_output_guardrail_callbacks: list = []
try:
import litellm as _litellm
# Use duck-typing so any guardrail that exposes the PII
# masking interface works, not just _OPTIONAL_PresidioPIIMasking.
# This avoids a layering violation (SDK importing from proxy).
_ws_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and callable(getattr(cb, "_unmask_pii_text", None))
and getattr(cb, "output_parse_pii", False)
]
_ws_output_guardrail_callbacks = [
cb
for cb in _litellm.callbacks
if callable(getattr(cb, "check_pii", None))
and callable(getattr(cb, "get_presidio_settings_from_request_data", None))
and getattr(cb, "apply_to_output", False)
]
except Exception as _guardrail_exc:
verbose_logger.warning(
"Responses WebSocket: failed to collect guardrail "
"callbacks — PII masking will be skipped. Error: %s",
_guardrail_exc,
)
streaming: Final = ResponsesWebSocketStreaming(
websocket=websocket,
backend_ws=cast(ClientConnection, backend_ws),
@ -7021,6 +7069,7 @@ class BaseLLMHTTPHandler:
authorized_model=model,
custom_llm_provider=custom_llm_provider,
request_defaults=request_defaults,
frame_guard=frame_guard,
)
return await streaming.bidirectional_forward()

2
litellm/llms/databricks/chat/transformation.py
View file

@ -278,7 +278,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
litellm_params: dict,
stream: bool | None = None,
) -> str:
use_ai_gateway: Final = model.removeprefix("databricks/").count(".") >= 2
use_ai_gateway: Final = self.uses_ai_gateway(model)
api_base = self._get_api_base(api_base, use_ai_gateway=use_ai_gateway)
url_base: Final = api_base.rstrip("/") if use_ai_gateway else api_base
complete_url: Final = f"{url_base}/chat/completions"

81
litellm/llms/databricks/common_utils.py
View file

@ -12,12 +12,16 @@ Authentication priority:
import os
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final, Literal
from urllib.parse import urlsplit, urlunsplit
from pydantic import BaseModel, ConfigDict
import litellm
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
_DEFAULT_M2M_TOKEN_LIFETIME_SECONDS: Final = 3600
@ -26,6 +30,28 @@ class DatabricksException(BaseLLMException):
pass
DATABRICKS_ANTHROPIC_MESSAGES_PATH: Final = "/anthropic/v1/messages"
DATABRICKS_SERVING_ENDPOINTS_MESSAGES_PATH: Final = f"/serving-endpoints{DATABRICKS_ANTHROPIC_MESSAGES_PATH}"
DATABRICKS_AI_GATEWAY_ANTHROPIC_PATH: Final = "/ai-gateway/anthropic"
DATABRICKS_AI_GATEWAY_MESSAGES_PATH: Final = f"{DATABRICKS_AI_GATEWAY_ANTHROPIC_PATH}/v1/messages"
DATABRICKS_WORKSPACE_ROUTE_PREFIXES: Final = (
"/serving-endpoints",
"/serving-endpoints/anthropic",
DATABRICKS_SERVING_ENDPOINTS_MESSAGES_PATH,
"/ai-gateway/mlflow/v1",
DATABRICKS_AI_GATEWAY_ANTHROPIC_PATH,
DATABRICKS_AI_GATEWAY_MESSAGES_PATH,
)
DATABRICKS_NATIVE_MESSAGES_MODEL_PREFIXES: Final = ("databricks-claude-", "system.ai.claude-")
DATABRICKS_ENDPOINT_PATHS: Final = MappingProxyType(
{
"chat_completions": "/chat/completions",
"embeddings": "/embeddings",
"anthropic_messages": DATABRICKS_ANTHROPIC_MESSAGES_PATH,
}
)
class OAuthM2MToken(BaseModel):
model_config = ConfigDict(frozen=True)
@ -144,6 +170,47 @@ class DatabricksBase:
return redacted
@staticmethod
def resolve_api_key(api_key: str | None) -> str | None:
return api_key or litellm.api_key or litellm.databricks_key or get_secret_str("DATABRICKS_API_KEY")
@staticmethod
def resolve_api_base(api_base: str | None) -> str | None:
return api_base or litellm.api_base or os.getenv("DATABRICKS_API_BASE")
@staticmethod
def serves_native_anthropic_messages(model: str) -> bool:
return model.removeprefix("databricks/").lower().startswith(DATABRICKS_NATIVE_MESSAGES_MODEL_PREFIXES)
@staticmethod
def uses_ai_gateway(model: str) -> bool:
return model.removeprefix("databricks/").count(".") >= 2
@staticmethod
def recognizes_workspace_route(api_base: str, model: str) -> bool:
route: Final = urlsplit(api_base).path.rstrip("/")
return route.endswith(DATABRICKS_WORKSPACE_ROUTE_PREFIXES) or (
route == "" and DatabricksBase.uses_ai_gateway(model)
)
@staticmethod
def workspace_root(api_base: str) -> str:
normalized: Final = api_base.rstrip("/")
route_prefix: Final = next(
(prefix for prefix in DATABRICKS_WORKSPACE_ROUTE_PREFIXES if normalized.endswith(prefix)), ""
)
return normalized.removesuffix(route_prefix)
@staticmethod
def configured_user_agent(litellm_params: Mapping[str, object]) -> str | None:
configured: Final = (
litellm_params.get("user_agent")
or litellm_params.get("databricks_user_agent")
or os.getenv("LITELLM_USER_AGENT")
or os.getenv("DATABRICKS_USER_AGENT")
)
return configured if isinstance(configured, str) else None
@staticmethod
def _build_user_agent(custom_user_agent: str | None = None) -> str:
"""
@ -327,11 +394,11 @@ class DatabricksBase:
self,
api_key: str | None,
api_base: str | None,
endpoint_type: Literal["chat_completions", "embeddings"],
endpoint_type: Literal["chat_completions", "embeddings", "anthropic_messages"],
custom_endpoint: bool | None,
headers: dict | None,
custom_user_agent: str | None = None,
) -> tuple[str, dict]:
) -> tuple[str, dict[str, str]]: # mutable-ok: becomes the outbound headers
"""
Validate and configure the Databricks environment.
@ -343,7 +410,7 @@ class DatabricksBase:
Args:
api_key: Personal access token (PAT)
api_base: Databricks workspace URL with /serving-endpoints
endpoint_type: Type of endpoint (chat_completions or embeddings)
endpoint_type: Type of endpoint (chat_completions, embeddings or anthropic_messages)
custom_endpoint: Whether using a custom endpoint URL
headers: Existing headers dict
custom_user_agent: Optional custom user agent to prefix
@ -410,9 +477,5 @@ class DatabricksBase:
# Debug logging with redaction (never log actual tokens)
verbose_logger.debug("Databricks request headers: %s", self.redact_headers_for_logging(headers))
if endpoint_type == "chat_completions" and custom_endpoint is not True:
api_base = f"{api_base}/chat/completions"
elif endpoint_type == "embeddings" and custom_endpoint is not True:
api_base = f"{api_base}/embeddings"
return api_base, headers
endpoint_path: Final = "" if custom_endpoint is True else DATABRICKS_ENDPOINT_PATHS[endpoint_type]
return f"{api_base}{endpoint_path}", headers

194
litellm/llms/databricks/messages/transformation.py Normal file
View file

@ -0,0 +1,194 @@
import json
from collections.abc import AsyncIterator, Mapping
from typing import Final
import httpx
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.databricks.common_utils import (
DATABRICKS_AI_GATEWAY_MESSAGES_PATH,
DATABRICKS_ANTHROPIC_MESSAGES_PATH,
DATABRICKS_SERVING_ENDPOINTS_MESSAGES_PATH,
DatabricksBase,
)
from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS
from litellm.types.router import GenericLiteLLMParams
DATABRICKS_UNSUPPORTED_SERVER_TOOL_PREFIXES: Final = ("web_search_", "web_fetch_", "code_execution_")
DROP_CONTAINER_WARNING: Final = (
"Dropping `container` for model=%s. The Databricks Anthropic Messages route rejects it, "
"and the chat completions bridge it replaces never forwarded it."
)
BEDROCK_INVOCATION_METRICS_FIELD: Final = "amazon-bedrock-invocationMetrics"
def _is_unsupported_server_tool(tool: object) -> bool:
tool_type: Final = (as_str_mapping(tool) or {}).get("type")
return isinstance(tool_type, str) and tool_type.startswith(DATABRICKS_UNSUPPORTED_SERVER_TOOL_PREFIXES)
def _tool_name(tool: object) -> object:
return (as_str_mapping(tool) or {}).get("name")
def _without_unsupported_server_tools(params: Mapping[str, object]) -> dict[str, object]:
if not isinstance(params.get("tools"), list):
return dict(params)
sent_tools: Final = tuple(iter_items(params.get("tools")))
kept_tools: Final = [tool for tool in sent_tools if not _is_unsupported_server_tool(tool)]
dropped_names: Final = frozenset(_tool_name(tool) for tool in sent_tools if _is_unsupported_server_tool(tool))
forced_tool: Final = _tool_name(params.get("tool_choice"))
keeps_tool_choice: Final = (
"tool_choice" in params and bool(kept_tools) and (forced_tool is None or forced_tool not in dropped_names)
)
remaining: Final = {key: value for key, value in params.items() if key not in ("tools", "tool_choice")}
return (
remaining
| ({"tools": kept_tools} if kept_tools else {})
| ({"tool_choice": params["tool_choice"]} if keeps_tool_choice else {})
)
def _without_invocation_metrics(sse_line: bytes) -> bytes:
if not sse_line.startswith(b"data:") or BEDROCK_INVOCATION_METRICS_FIELD.encode() not in sse_line:
return sse_line
event: Final = as_str_mapping(json.loads(sse_line.removeprefix(b"data:")))
if event is None:
return sse_line
kept_fields: Final = {key: value for key, value in event.items() if key != BEDROCK_INVOCATION_METRICS_FIELD}
return b"data: " + json.dumps(kept_fields, separators=(",", ":")).encode()
async def _strip_invocation_metrics(sse_stream: AsyncIterator[bytes]) -> AsyncIterator[bytes]:
partial_line: Final = bytearray()
async for chunk in sse_stream:
partial_line.extend(chunk)
if complete_end := partial_line.rfind(b"\n") + 1:
yield b"\n".join(
_without_invocation_metrics(line) for line in bytes(partial_line[:complete_end]).split(b"\n")
)
del partial_line[:complete_end]
if partial_line:
yield _without_invocation_metrics(bytes(partial_line))
class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig):
"""
Forwards /v1/messages for Claude models to the workspace's native Anthropic
route (``/serving-endpoints/anthropic/v1/messages`` for a serving endpoint,
``/ai-gateway/anthropic/v1/messages`` for a Unity Catalog model service), so
Anthropic-only request fields and response fields (``safeguards`` and
``safeguard_results``) survive instead of being bridged through chat completions.
"""
@property
def custom_llm_provider(self) -> str | None:
return "databricks"
def handles_web_search_natively(self) -> bool:
return False
def drops_safeguards(self, safeguards: object, drop_params: bool) -> bool:
return not self.supports_safeguards()
def serves_api_base(self, model: str, api_base: str | None) -> bool:
resolved_api_base: Final = self.resolve_api_base(api_base)
return resolved_api_base is None or self.recognizes_workspace_route(resolved_api_base, model)
def get_async_streaming_response_iterator(
self,
model: str,
httpx_response: httpx.Response,
request_body: dict[str, object], # mutable-ok: mirrors the base signature
litellm_logging_obj: LiteLLMLoggingObj,
) -> AsyncIterator[bytes]:
return _strip_invocation_metrics(
super().get_async_streaming_response_iterator(
model=model,
httpx_response=httpx_response,
request_body=request_body,
litellm_logging_obj=litellm_logging_obj,
)
)
def validate_anthropic_messages_environment(
self,
headers: dict[str, str], # mutable-ok: mirrors the base signature
model: str,
messages: list[object], # mutable-ok: mirrors the base signature
optional_params: dict[str, object], # mutable-ok: mirrors the base signature
litellm_params: dict[str, object], # mutable-ok: mirrors the base signature
api_key: str | None = None,
api_base: str | None = None,
) -> tuple[dict[str, str], str | None]: # mutable-ok: mirrors the base signature
client_anthropic_headers: Final = {
key: value for key, value in headers.items() if key.lower() in ANTHROPIC_API_HEADERS
}
messages_api_base, auth_headers = self.databricks_validate_environment( # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict
api_key=self.resolve_api_key(api_key),
api_base=self.resolve_api_base(api_base),
endpoint_type="anthropic_messages",
custom_endpoint=False,
headers={key: value for key, value in headers.items() if key not in client_anthropic_headers},
custom_user_agent=self.configured_user_agent(litellm_params),
)
workspace_api_base: Final = messages_api_base.removesuffix(DATABRICKS_ANTHROPIC_MESSAGES_PATH)
normalized_headers: Final = {
("content-type" if key.lower() == "content-type" else key): value
for key, value in {**auth_headers, **client_anthropic_headers}.items()
}
finalized_headers: Final[dict[str, str]] = self._finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers
headers=normalized_headers,
optional_params=optional_params,
messages=messages,
)
return finalized_headers, workspace_api_base
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: dict[str, object], # mutable-ok: mirrors the base signature
litellm_params: dict[str, object], # mutable-ok: mirrors the base signature
stream: bool | None = None,
) -> str:
messages_path: Final = (
DATABRICKS_AI_GATEWAY_MESSAGES_PATH
if self.uses_ai_gateway(model)
else DATABRICKS_SERVING_ENDPOINTS_MESSAGES_PATH
)
return f"{self.workspace_root(self._get_api_base(api_base))}{messages_path}"
def transform_anthropic_messages_request(
self,
model: str,
messages: list[dict[str, object]], # mutable-ok: mirrors the base signature
anthropic_messages_optional_request_params: dict[str, object], # mutable-ok: mirrors the base signature
litellm_params: GenericLiteLLMParams,
headers: dict[str, str], # mutable-ok: mirrors the base signature
) -> dict[str, object]: # mutable-ok: mirrors the base signature
if "container" in anthropic_messages_optional_request_params:
litellm.verbose_logger.warning(DROP_CONTAINER_WARNING, model)
workspace_params: Final = {
key: value for key, value in anthropic_messages_optional_request_params.items() if key != "container"
}
drops_unsupported_params: Final = litellm.drop_params is True or litellm_params.drop_params is True
request_params: Final = (
_without_unsupported_server_tools(workspace_params) if drops_unsupported_params else workspace_params
)
anthropic_messages_request: Final[dict[str, object]] = super().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared base returns a bare dict # mutable-ok: becomes the outbound request body
model=model,
messages=messages,
anthropic_messages_optional_request_params=request_params,
litellm_params=litellm_params,
headers=headers,
)
self._normalize_system_role_messages(anthropic_messages_request, model=model) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict
self._remove_scope_from_cache_control(anthropic_messages_request) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict
return anthropic_messages_request

2
litellm/llms/fal_ai/cost_calculator.py
View file

@ -87,7 +87,7 @@ def _keyed_rows(model: str, quality: str) -> tuple[tuple[int, int, float], ...]:
suffix: Final = f"/{model}"
return tuple(
(width, height, float(raw_cost))
for key in litellm.model_cost
for key in (*litellm.model_cost,)
if isinstance(key, str) and key.startswith(prefix) and key.endswith(suffix)
for size in (key[len(prefix) : -len(suffix)],)
for dimensions in (_parse_keyed_dimensions(size),)

2
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -544,7 +544,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return cached[2]
index: Final[list[tuple[str, dict]]] = []
for key, model_info in model_cost.items():
for key, model_info in (*model_cost.items(),):
if not key.startswith("fireworks_ai/"):
continue
if not isinstance(model_info, dict):

5
litellm/llms/microsoft_365_copilot/chat/transformation.py
View file

@ -109,6 +109,11 @@ class Microsoft365CopilotChatConfig(BaseConfig):
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseConfig requires a list return
return ["stream", "max_tokens", "max_completion_tokens"]
def should_fake_stream(
self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None
) -> bool:
return True
def transform_request(
self,
model: str,

2
litellm/llms/ollama/common_utils.py
View file

@ -197,7 +197,7 @@ class OllamaModelInfo(BaseLLMModelInfo):
"ollama/" + stripped_model,
"ollama_chat/" + stripped_model,
}
model_cost_keys: Final = {key.lower() for key in model_cost}
model_cost_keys: Final = {key.lower() for key in (*model_cost,)}
return any(name.lower() in model_cost_keys for name in potential_model_names)
@staticmethod

4
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -28,7 +28,9 @@ def _catalogue_declares_default_effort() -> bool:
the one litellm gave before it existed. Scanning costs ~80us on the largest published map and
only on the fallback path, which is noise beside the request it precedes.
"""
return any(isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in litellm.model_cost.values())
return any(
isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in (*litellm.model_cost.values(),)
)
def _normalize_reasoning_effort_for_chat_completion(

1
litellm/llms/openai/responses/count_tokens/token_counter.py
View file

@ -124,6 +124,7 @@ class OpenAITokenCounter(BaseTokenCounter):
error=True,
error_message=request_error,
status_code=400,
rejected_before_provider_call=True,
)
deployment = deployment or {}

6
litellm/llms/openai_like/README.md
View file

@ -63,6 +63,12 @@ That's it! The provider will be automatically loaded and available.
// Optional: Special handling flags
"special_handling": {
"convert_content_list_to_string": true
},
// Optional: Capabilities every model of this provider has, used when a
// model has no cost map row or its row leaves the field unset
"provider_info": {
"supports_function_calling": true
}
}
}

8
litellm/llms/openai_like/json_loader.py
View file

@ -6,7 +6,12 @@ import json
from pathlib import Path
from typing import Final
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.types.utils import ProviderSpecificModelInfo
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
class SimpleProviderConfig:
@ -22,6 +27,9 @@ class SimpleProviderConfig:
self.constraints = data.get("constraints", {})
self.special_handling = data.get("special_handling", {})
self.supported_endpoints = data.get("supported_endpoints", [])
self.provider_info: ProviderSpecificModelInfo = (
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
)
class JSONProviderRegistry:

9
litellm/llms/openai_like/messages/transformation.py
View file

@ -8,6 +8,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.openai_like.json_loader import SimpleProviderConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
@ -32,6 +33,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
super().__init__()
self._cache_control_ttl: Final = cache_control_ttl
def is_native_messages_route(self) -> bool:
return False
def validate_anthropic_messages_environment(
self,
headers: dict[str, str],
@ -57,6 +61,8 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
headers=normalized,
optional_params=optional_params,
messages=messages,
beta_map_provider=LlmProviders.OPENAI_LIKE.value,
drop_params=litellm_params.get("drop_params") is True, # pyright: ignore[reportUnknownMemberType] # the litellm params arrive as a bare dict
)
return merged, api_base
@ -130,6 +136,9 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig):
def should_strip_billing_metadata(self) -> bool:
return True
def supports_safeguards(self) -> bool:
return False
def _resolve_api_key(self, api_key: str | None) -> str | None:
return api_key or get_secret_str(self._provider.api_key_env) or litellm.api_key

10
litellm/llms/openai_like/providers.json
View file

@ -230,6 +230,16 @@
"api_key_env": "REKA_API_KEY",
"api_base_env": "REKA_API_BASE"
},
"openinfer": {
"base_url": "https://api.openinfer.ai/v1",
"api_key_env": "OPENINFER_API_KEY",
"api_base_env": "OPENINFER_API_BASE",
"base_class": "openai_gpt",
"provider_info": {
"supports_function_calling": true
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"inferenceapis": {
"base_url": "https://api.inferenceapis.com/v1",
"api_key_env": "INFERENCEAPIS_API_KEY",

Some files were not shown because too many files have changed in this diff Show more