scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_decrease_anys_opus5_0826

Browse source 
# Conflicts:
#	basedpyright-code-budget.json
#	litellm/llms/soniox/common_utils.py
#	ruff-strict-budget.json
#	type-discipline-budget.json
This commit is contained in:
mateo-berri 2026-08-29 03:29:39 -07:00
commit 47d8ce6d10
641 changed files with 67048 additions and 5943 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
.circleci/config.yml
View file

@ -2421,45 +2421,6 @@ jobs:
- wait_for_service:
url: http://localhost:4000
timeout: "300"
# Add Ruby installation and testing before the existing Node.js and Python tests
- run:
name: Install Ruby and Bundler
command: |
# Clone RVM at pinned tag and verify the commit SHA matches the
# published tag before running its install script.
RVM_VERSION="1.29.12"
RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81"
git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm
RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)"
if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then
echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2
exit 1
fi
# Import RVM signing keys (used by `rvm install` to verify Ruby tarballs)
gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
# Install RVM from the verified checkout. The install script
# sources `scripts/functions/installer` using paths relative to
# its own working directory, so it must be run from /tmp/rvm.
(cd /tmp/rvm && ./install --path "$HOME/.rvm")
source "$HOME/.rvm/scripts/rvm"
# Install Ruby 3.2.2 (RVM verifies the tarball PGP signature)
rvm install 3.2.2
rvm use 3.2.2 --default
# Install latest Bundler
gem install bundler
- run:
name: Run Ruby tests
command: |
source $HOME/.rvm/scripts/rvm
cd tests/pass_through_tests/ruby_passthrough_tests
bundle install
bundle exec rspec
no_output_timeout: 30m
# Install Node.js directly from nodejs.org with SHA256 verification,
# instead of piping NodeSource's setup_24.x apt-repo installer into
# sudo bash (which runs a mutable upstream script unattended).

18
.github/workflows/check-ui-api-types.yml vendored
View file

@ -83,6 +83,24 @@ jobs:
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Regenerate the lazy OpenAPI snapshot
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
- name: Fail if the lazy OpenAPI snapshot is stale
if: steps.changes.outputs.relevant == 'true'
run: |
if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
echo ""
echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
echo "To fix, run from the repo root:"
echo " uv run python -m litellm.proxy._lazy_openapi_snapshot"
echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
exit 1
fi
echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
- name: Set up Node.js
if: steps.changes.outputs.relevant == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0

68
.github/workflows/sync-together-ai-models.yml vendored Normal file
View file

@ -0,0 +1,68 @@
name: Sync Together AI model registry
on:
schedule:
- cron: "30 6 * * *"
workflow_dispatch:
permissions:
contents: write
pull-requests: write
jobs:
sync_together_ai_models:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: litellm_internal_staging
persist-credentials: false
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Look for an already-open sync PR
id: existing
run: |
open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
--jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
if [ -n "$open_pr" ]; then
echo "An open sync PR already exists on branch $open_pr; skipping this run."
fi
env:
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
- name: Run the sync
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
env:
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
- name: Regenerate the JSON schema
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create a pull request when the registry changed
if: steps.existing.outputs.open_pr == ''
run: |
if git diff --quiet; then
echo "Registry already in sync; no PR needed."
exit 0
fi
branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b "$branch"
git add model_prices_and_context_window.json \
litellm/model_prices_and_context_window_backup.json \
model_prices_and_context_window.schema.json
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
gh auth setup-git
git push origin "$branch"
gh pr create --title "feat(models): sync together_ai model registry" \
--body-file "$RUNNER_TEMP/pr_body.md" \
--head "$branch" \
--base litellm_internal_staging
env:
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}

2
.github/workflows/test-terraform-provider.yml vendored
View file

@ -114,4 +114,4 @@ jobs:
- name: Audit provider endpoints against the schema
working-directory: terraform/provider
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json" -coverage-allowlist ./tools/endpointaudit/coverage_allowlist.txt

1
.github/workflows/test-unit.yml vendored
View file

@ -141,6 +141,7 @@ jobs:
test-path: >-
tests/test_litellm/proxy/analytics_endpoints
tests/test_litellm/proxy/management_endpoints
tests/test_litellm/proxy/list_api
tests/test_litellm/proxy/memory
tests/test_litellm/proxy/guardrails
tests/test_litellm/proxy/management_helpers

24
basedpyright-code-budget.json
View file

@ -1,15 +1,15 @@
{
"reportAny": {
"limit": 17259
"limit": 17271
},
"reportArgumentType": {
"limit": 2551
"limit": 2544
},
"reportAssignmentType": {
"limit": 320
"limit": 319
},
"reportAttributeAccessIssue": {
"limit": 483
"limit": 480
},
"reportCallIssue": {
"limit": 112
@ -24,13 +24,13 @@
"limit": 19
},
"reportExplicitAny": {
"limit": 5482
"limit": 5486
},
"reportFunctionMemberAccess": {
"limit": 7
},
"reportGeneralTypeIssues": {
"limit": 150
"limit": 101
},
"reportIncompatibleMethodOverride": {
"limit": 56
@ -57,7 +57,7 @@
"limit": 5658
},
"reportMissingTypeArgument": {
"limit": 15427
"limit": 15425
},
"reportMissingTypeStubs": {
"limit": 40
@ -99,19 +99,19 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44528
"limit": 44526
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38746
"limit": 38721
},
"reportUnknownParameterType": {
"limit": 19780
"limit": 19778
},
"reportUnknownVariableType": {
"limit": 30299
"limit": 30292
},
"reportUnnecessaryCast": {
"limit": 117
@ -123,7 +123,7 @@
"limit": 5
},
"reportUnnecessaryIsInstance": {
"limit": 831
"limit": 829
},
"reportUntypedBaseClass": {
"limit": 0

14
ci_cd/generate_model_prices_schema.py
View file

@ -73,6 +73,11 @@ ARRAY_KEYS: dict[str, JsonSchema] = {
"description": "Output modalities the model can produce.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
},
"reasoning_effort_levels": {
"type": "array",
"description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
"items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
},
"supported_regions": {
"type": "array",
"description": "Cloud regions the model is available in ('global' or region ids).",
@ -215,6 +220,15 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
"enum": ["low", "medium", "high", "max", "xhigh"],
},
"default_reasoning_effort": {
"type": "string",
"description": (
"Reasoning effort the provider applies when the request omits reasoning_effort. "
"Gates whether a non-default temperature or the top_p/logprobs sampling params are "
"accepted, which hold only when the effort resolves to 'none'."
),
"enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
},
"comment": STRING,
"audio_transcription_config": STRING,
}

5
db_scripts/partition_spend_logs.sql
View file

@ -10,6 +10,11 @@
-- partitioned, so existing installs are unaffected until you run this.
--
-- IMPORTANT
-- * After partitioning, `prisma db push` (including the proxy's
-- --use_prisma_db_push startup mode) is NOT supported: it tries to rewrite
-- the primary key back to ("request_id"), which Postgres rejects on a
-- partitioned table. The proxy detects this and exits with guidance.
-- Use the default startup path (`prisma migrate deploy`) instead.
-- * Test on a staging copy first and take a backup.
-- * Postgres cannot convert a populated table to partitioned in place, so this
-- renames the old table aside and creates a fresh partitioned table.

124
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -38,6 +38,8 @@ from litellm.repositories.verification_token_repository import VerificationToken
if TYPE_CHECKING:
from prisma import models as prisma_models
from litellm import Router
router = APIRouter()
_OBJECT_PERMISSION_PAYLOAD: Final = TypeAdapter(dict[str, object])
@ -217,6 +219,114 @@ def _check_team_project_limits(
)
def _project_models_missing_positive_quota(
models: list[str] | None,
rpm_limits: Mapping[str, object] | None,
tpm_limits: Mapping[str, object] | None,
) -> list[str]:
"""Return the models that lack a positive `rpm` AND `tpm` quota.
A valid quota is a positive integer; null, zero, and negative are rejected
because downstream rate limiters treat a non-positive limit as immediately
exhausted (every request blocked).
"""
def _is_positive(value: object) -> bool:
return isinstance(value, int) and not isinstance(value, bool) and value > 0
rpm = rpm_limits or {}
tpm = tpm_limits or {}
return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))]
def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
def _project_models_expanding_at_request_time(
models: Sequence[str] | None, access_group_names: frozenset[str]
) -> tuple[str, ...]:
"""Entries project auth expands to many concrete models (`all-proxy-models`, `*` patterns,
access groups). The rate limiter looks quotas up by the exact requested model name, so a
quota keyed on one of these entries is never applied."""
return tuple(
model
for model in (models or ())
if model == SpecialModelNames.all_proxy_models.value or "*" in model or model in access_group_names
)
def _raise_on_project_models_expanding_at_request_time(
models: Sequence[str] | None, access_group_names: frozenset[str]
) -> None:
expanding: Final = _project_models_expanding_at_request_time(models, access_group_names)
if not expanding:
return
raise HTTPException(
status_code=400,
detail={
"error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead."
},
)
def _raise_on_missing_project_model_quota(
data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset()
) -> None:
"""Require a positive `rpm`/`tpm` quota for every model on project CREATE.
`model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request
model's `set_model_info` validator, so they are read from there.
Only invoked when `general_settings.enforce_project_model_quota` is enabled
(default off), so it is opt-in and does not change behavior for existing users.
"""
_raise_on_project_models_expanding_at_request_time(data.models, access_group_names)
metadata = data.metadata or {}
missing = _project_models_missing_positive_quota(
data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit")
)
if not missing:
return
raise HTTPException(
status_code=400,
detail={
"error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
},
)
def _raise_on_missing_project_model_quota_on_update(
data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset()
) -> None:
"""Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE.
`/project/update` replaces `models` and `metadata` when they are provided, so the
check runs on what the project WILL look like: a partial update that doesn't touch
models/quota keeps the existing values, while one that adds a model or clears a
model's quota must leave every resulting model with a positive limit.
Only invoked when `general_settings.enforce_project_model_quota` is enabled
(default off), so it is opt-in and does not change behavior for existing users.
"""
resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or [])
resulting_metadata = (
data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {})
)
_raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names)
missing = _project_models_missing_positive_quota(
resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit")
)
if not missing:
return
raise HTTPException(
status_code=400,
detail={
"error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
},
)
async def _create_budget_for_project(
data: NewProjectRequest,
user_id: str | None,
@ -362,7 +472,9 @@ async def new_project(
```
"""
from litellm.proxy.proxy_server import (
general_settings,
litellm_proxy_admin_name,
llm_router,
premium_user,
prisma_client,
)
@ -409,6 +521,10 @@ async def new_project(
data=data,
)
# Opt-in (default off): require rpm/tpm for every model added to the project.
if general_settings.get("enforce_project_model_quota", False):
_raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router))
# Check if user has permission to create projects for this team
# only team admins can create projects for their team
has_permission = await _check_user_permission_for_project(
@ -546,7 +662,9 @@ async def update_project(
```
"""
from litellm.proxy.proxy_server import (
general_settings,
litellm_proxy_admin_name,
llm_router,
premium_user,
prisma_client,
user_api_key_cache,
@ -650,6 +768,12 @@ async def update_project(
data=data,
)
# Opt-in (default off): require rpm/tpm for every model the update would leave on the project.
if general_settings.get("enforce_project_model_quota", False):
_raise_on_missing_project_model_quota_on_update(
data, existing_project, _router_access_group_names(llm_router)
)
# Prepare update data
update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.60"
version = "0.1.61"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.60"
version = "0.1.61"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

18
litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

22
litellm-proxy-extras/litellm_proxy_extras/migrations/20260828000000_shadow_eval_cost_comparison/migration.sql Normal file
View file

@ -0,0 +1,22 @@
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cost" DOUBLE PRECISION;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "shadow_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cache_hit" BOOLEAN NOT NULL DEFAULT false;
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalFunnel" (
"job_id" TEXT NOT NULL,
"not_sampled" INTEGER NOT NULL DEFAULT 0,
"unjudgeable" INTEGER NOT NULL DEFAULT 0,
"shed" INTEGER NOT NULL DEFAULT 0,
"withheld" INTEGER NOT NULL DEFAULT 0,
CONSTRAINT "LiteLLM_ShadowEvalFunnel_pkey" PRIMARY KEY ("job_id")
);

21
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -1527,12 +1533,27 @@ model LiteLLM_ShadowEvalAttempt {
confidence Float?
judge_cost Float @default(0)
shadow_cost Float @default(0)
real_cost Float? // NULL = row predates cost measurement; comparisons read only measured rows
real_classifier_cost Float @default(0)
shadow_classifier_cost Float @default(0)
real_cache_hit Boolean @default(false)
error String?
created_at DateTime @default(now())
@@index([job_id])
}
// Per-leg sampling funnel counters the attempt rows cannot derive: requests an
// admitting job saw but did not judge. attempted = the leg's attempt rows; the
// leg's eligible traffic = not_sampled + unjudgeable + shed + withheld + attempted.
model LiteLLM_ShadowEvalFunnel {
job_id String @id
not_sampled Int @default(0)
unjudgeable Int @default(0)
shed Int @default(0)
withheld Int @default(0)
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

141
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -40,6 +40,65 @@ def _get_prisma_env() -> dict:
_MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
)
_SPEND_LOGS_PK_CLAUSE_RE = re.compile(
r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"'
r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$',
re.IGNORECASE,
)
PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
"LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), "
"so its primary key must include the partition key (\"startTime\"). `prisma db push` "
"reconciles the database against schema.prisma, which declares the unpartitioned "
"primary key (\"request_id\"), and Postgres rejects that rewrite with: unique "
"constraint on partitioned table must include all partitioning columns. Start the "
"proxy without --use_prisma_db_push so it uses `prisma migrate deploy`, which only "
"applies shipped migrations and leaves the partitioned primary key alone."
)
def _without_sql_comments(statement: str) -> str:
return "\n".join(
line
for line in statement.splitlines()
if line.strip() and not line.strip().startswith("--")
).strip()
def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
if not prefix_match:
return statement
kept = tuple(
clause.strip()
for clause in statement[prefix_match.end():].split(",\n")
if not _SPEND_LOGS_PK_CLAUSE_RE.match(clause.strip())
)
if not kept:
return None
return statement[: prefix_match.end()] + ",\n".join(kept)
def filter_partitioned_spend_logs_diff(diff_sql: str) -> str:
"""Drop statements from a `prisma migrate diff` script that fight the
SpendLogs partitioning runbook (db_scripts/partition_spend_logs.sql): the
primary-key rewrite on "LiteLLM_SpendLogs", which Postgres rejects on a
partitioned table, and drops of runbook artifacts such as
"LiteLLM_SpendLogs_legacy"."""
kept = tuple(
filtered
for statement in diff_sql.split(";")
for bare in (_without_sql_comments(statement),)
if bare and not _SPEND_LOGS_ARTIFACT_DROP_RE.match(bare)
for filtered in (_without_spend_logs_pk_clauses(bare),)
if filtered is not None
)
return "".join(f"{statement};\n\n" for statement in kept)
def _migration_timestamp(name: str) -> int:
"""Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name.
@ -355,7 +414,24 @@ class ProxyExtrasDBManager:
return
logger.info(f"Migration diff created at {diff_sql_path}")
if ProxyExtrasDBManager.spend_logs_is_partitioned():
filtered_sql = filter_partitioned_spend_logs_diff(
diff_sql_path.read_text()
)
diff_sql_path.write_text(filtered_sql)
logger.info(
"LiteLLM_SpendLogs is partitioned; removed its primary-key "
"rewrite and partitioning artifacts from the drift script"
)
if not filtered_sql.strip():
logger.info("Drift script is empty after filtering; nothing to apply")
if not mark_all_applied:
return
ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
return
# 2. Run prisma db execute to apply the migration
applied_ok = False
try:
logger.info("Running prisma db execute to apply the migration diff...")
result = subprocess.run(
@ -376,6 +452,7 @@ class ProxyExtrasDBManager:
)
logger.info(f"prisma db execute stdout: {result.stdout}")
logger.info("✅ Migration diff applied successfully")
applied_ok = True
except subprocess.CalledProcessError as e:
logger.warning(f"Failed to apply migration diff: {e.stderr}")
except subprocess.TimeoutExpired:
@ -384,6 +461,16 @@ class ProxyExtrasDBManager:
# 3. Mark all migrations as applied
if not mark_all_applied:
return
if not applied_ok:
logger.warning(
"Drift script failed to apply; NOT marking migrations as "
"applied so a later migration run can retry them"
)
return
ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
@staticmethod
def _mark_migrations_applied(migrations_dir: str) -> None:
migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
logger.info(f"Resolving {len(migration_names)} migrations")
for migration_name in migration_names:
@ -410,6 +497,55 @@ class ProxyExtrasDBManager:
f"Failed to resolve migration {migration_name}: {e.stderr}"
)
@staticmethod
def spend_logs_is_partitioned() -> bool:
"""True when the connected database's LiteLLM_SpendLogs is a
partitioned table in Prisma's target schema (the `schema` URL param,
falling back to Prisma's default target, public), i.e. the operator
ran db_scripts/partition_spend_logs.sql. Returns False when psycopg is
unavailable or the database cannot be reached, preserving the
pre-existing behavior in those cases."""
database_url = os.getenv("DATABASE_URL")
if not database_url:
return False
try:
import psycopg
except ImportError:
return False
cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
try:
with psycopg.connect(
cleaned_url, connect_timeout=10, autocommit=True
) as conn:
row = conn.execute(
"SELECT 1 "
"FROM pg_partitioned_table pt "
"JOIN pg_class c ON c.oid = pt.partrelid "
"JOIN pg_namespace n ON n.oid = c.relnamespace "
"WHERE c.relname = 'LiteLLM_SpendLogs' "
" AND n.nspname = %s",
(
ProxyExtrasDBManager._prisma_schema_param(database_url)
or "public",
),
).fetchone()
except (psycopg.OperationalError, psycopg.DatabaseError):
return False
return row is not None
@staticmethod
def _prisma_schema_param(url: str) -> Optional[str]:
"""The `schema` query param Prisma uses to pick its target schema,
or None when the URL does not set one."""
from urllib.parse import urlparse, parse_qsl
return next(
(v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
None,
)
@staticmethod
def _strip_prisma_query_params(url: str) -> str:
"""Remove Prisma-specific query params (connection_limit, pool_timeout,
@ -528,7 +664,8 @@ class ProxyExtrasDBManager:
migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
if not use_migrate:
# Preserve `prisma db push` path unchanged.
if ProxyExtrasDBManager.spend_logs_is_partitioned():
raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
original_dir = os.getcwd()
os.chdir(migrations_dir)
try:
@ -972,6 +1109,8 @@ class ProxyExtrasDBManager:
)
raise
else:
if ProxyExtrasDBManager.spend_logs_is_partitioned():
raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
# Use prisma db push with increased timeout
subprocess.run(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.89"
version = "0.4.90"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.89"
version = "0.4.90"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

3
litellm/__init__.py
View file

@ -274,7 +274,6 @@ databricks_key: Optional[str] = None
openai_like_key: Optional[str] = None
azure_key: Optional[str] = None
anthropic_key: Optional[str] = None
autorouter_savings_baseline_model: Optional[str] = None
replicate_key: Optional[str] = None
bytez_key: Optional[str] = None
gdc_key: Optional[str] = None
@ -445,6 +444,7 @@ max_ui_session_budget: Optional[float] = (
1.0 # USD budget for each dashboard login session (playground, test connection)
)
internal_user_budget_duration: Optional[str] = None
budget_rollover: bool = False # carry spend beyond max_budget into the next window instead of zeroing it
tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None
max_end_user_budget: Optional[float] = None
max_end_user_budget_id: Optional[str] = None
@ -486,6 +486,7 @@ public_mcp_servers: Optional[List[str]] = None
public_mcp_hub_strict_whitelist: bool = True
public_model_groups: Optional[List[str]] = None
public_agent_groups: Optional[List[str]] = None
agent_search_embedding_model: Optional[str] = None
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
# New format: { "displayName": { "url": "...", "index": 0 } }
# Old format: { "displayName": "url" } (for backward compatibility)

64
litellm/_logging.py
View file

@ -5,7 +5,7 @@ import os
import sys
from datetime import datetime
from logging import Formatter
from typing import Any, Final
from typing import Any, Final, TextIO
import litellm
from litellm.constants import (
@ -234,11 +234,65 @@ class CorrelationContextFilter(logging.Filter):
_correlation_filter: Final = CorrelationContextFilter()
json_logs = bool(os.getenv("JSON_LOGS", False))
_LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s"
_LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s"
_PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX
_COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}"
def _stream_is_tty(stream: TextIO | None) -> bool:
"""True when the stream is an open interactive terminal; never raises.
A stream can be None (pythonw/embedded interpreters), lack isatty entirely
(GUI log-redirect shims), or be closed; import must survive all three.
"""
try:
return stream is not None and stream.isatty()
except (AttributeError, ValueError):
return False
def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
"""The plain-text log format, colorized only when both streams are an interactive terminal.
Honors the NO_COLOR convention from no-color.org: color is disabled when
NO_COLOR is present with a non-empty value.
"""
if os.environ.get("NO_COLOR"):
return _PLAIN_LOG_FORMAT
return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT
class LevelRoutingStreamHandler(logging.StreamHandler):
"""Writes records below WARNING to stdout and WARNING and above to stderr.
Collectors that derive severity from the stream report every stderr line as an error.
"""
def emit(self, record: logging.LogRecord) -> None:
preferred: Final = sys.stdout if record.levelno < logging.WARNING else sys.stderr
if preferred is None or getattr(preferred, "closed", False):
self.stream = sys.stderr # rebind-ok: fall back to the pre-fix stream rather than raising per record
else:
self.stream = preferred # rebind-ok: StreamHandler.emit writes self.stream under the handler lock
super().emit(record)
def _parse_json_logs_env(value: str | None) -> bool:
"""Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs.
Matches the reader in litellm-proxy-extras/_logging.py. The previous
bool(os.getenv(...)) treated any non-empty value, including "false" and "0",
as enabled.
"""
return (value or "").lower() == "true"
json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
# Create a handler for the logger (you may need to adapt this based on your needs)
log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
numeric_level: Final[str] = getattr(logging, log_level.upper())
handler: Final = logging.StreamHandler()
handler: Final = LevelRoutingStreamHandler()
handler.setLevel(numeric_level)
handler.addFilter(_secret_filter)
handler.addFilter(_correlation_filter)
@ -447,7 +501,7 @@ if json_logs:
_setup_json_exception_handlers(JsonFormatter())
else:
formatter: Final = CorrelationPlainFormatter(
"\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(filename)s:%(lineno)s - %(message)s",
_plain_log_format(sys.stdout, sys.stderr),
datefmt="%H:%M:%S",
)
@ -628,7 +682,7 @@ def _turn_on_json():
- Adds a JSON formatter to all loggers
"""
handler: Final = logging.StreamHandler()
handler: Final = LevelRoutingStreamHandler()
handler.setFormatter(JsonFormatter())
_initialize_loggers_with_handler(handler)
# Set up exception handlers

35
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -59,9 +59,11 @@ if TYPE_CHECKING:
from litellm.types.llms.openai import (
ALL_RESPONSES_API_TOOL_PARAMS,
AllMessageValues,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
ChatCompletionToolReferenceObject,
OpenAIMessageContentListBlock,
)
from litellm.types.utils import Choices
@ -175,6 +177,16 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li
return "length"
def _input_file_from_file_value(file_value: object) -> dict[str, object]:
if not isinstance(file_value, dict):
return {"type": "input_file"}
file_dict: Final = cast("dict[str, object]", file_value) # cast-ok: runtime dict checked
return {
"type": "input_file",
**{key: file_dict[key] for key in ("file_id", "file_data", "filename") if key in file_dict},
}
def _incomplete_reason_from_response_payload(response_payload: object) -> str | None:
if not isinstance(response_payload, Mapping):
return None
@ -957,7 +969,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
content: str
| list[object]
| Iterable[
Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock", "ChatCompletionRedactedThinkingBlock"]
Union[
"OpenAIMessageContentListBlock",
"ChatCompletionThinkingBlock",
"ChatCompletionRedactedThinkingBlock",
"ChatCompletionToolReferenceObject",
]
]
| None,
role: str,
@ -1006,17 +1023,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
result.append(converted)
verbose_logger.debug("Chat provider: image -> %s", converted)
elif item_type == "file":
# Map Chat Completion file to Responses API input_file
# {"type": "file", "file": {"file_data": "...", "filename": "..."}}
# -> {"type": "input_file", "file_data": "...", "filename": "..."}
file_data = item.get("file", {})
converted = {"type": "input_file"}
if isinstance(file_data, dict):
for key in ["file_id", "file_data", "filename"]:
if key in file_data:
converted[key] = file_data[key]
converted = _input_file_from_file_value(
cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked
)
result.append(converted)
verbose_logger.debug("Chat provider: file -> %s", converted)
elif item_type == "tool_reference":
verbose_logger.debug(
"Chat provider: tool_reference has no responses API equivalent; skipped"
)
elif item_type in [
"input_text",
"input_image",

19
litellm/constants.py
View file

@ -296,6 +296,9 @@ GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = int(
os.getenv("GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS", 24 * 60 * 60)
)
BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS: Final = 25_000
DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES: Final = 500_000
PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS: Final = 4096
PRESIDIO_ANALYZE_CHUNK_CONCURRENCY: Final = 8
# Aggregation threshold: default to 80% of the asyncio queue maxsize so the check can always trigger.
# Must be < LITELLM_ASYNCIO_QUEUE_MAXSIZE; if set higher the aggregation logic will never fire.
MAX_SIZE_IN_MEMORY_QUEUE: Final = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8)))
@ -626,6 +629,15 @@ LITELLM_CHAT_PROVIDERS: Final = [
"amazon_nova",
]
# Resolving these providers runs an OAuth device flow (their provider info IS the login), so any
# metadata or capability lookup against them can block for minutes waiting on a human.
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
{
"github_copilot",
"chatgpt",
}
)
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
"openai",
"azure",
@ -1473,6 +1485,12 @@ LITELLM_PROXY_MASTER_KEY_ALIAS: Final = "litellm_proxy_master_key"
# ``ProxyLogging._handle_logging_proxy_only_error``.
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: Final = "litellm_no_upstream_llm_call"
# Key/team metadata fields naming the OTel Resource ``service.name``, highest
# precedence first. Shared between the OTel v2 tenant router (which reads them
# out of ``user_api_key_auth_metadata``) and proxy request setup (which re-applies
# the key's values after the team metadata merge so a key outranks its team).
OTEL_SERVICE_NAME_METADATA_KEYS: Final = ("otel_service_name_override", "otel_service_name")
# Key Rotation Constants
LITELLM_KEY_ROTATION_ENABLED: Final = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS: Final = int(
@ -1646,6 +1664,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"enable_anthropic_prompt_caching",
"anthropic_prompt_caching_ttl",
"max_ui_session_budget",
"budget_rollover",
]
SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))

39
litellm/cost_calculator.py
View file

@ -2,7 +2,7 @@
## File for 'response_cost' calculation in Logging
import logging
import time
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, cast
@ -76,7 +76,10 @@ from litellm.llms.perplexity.cost_calculator import (
from litellm.llms.tencent.cost_calculator import (
cost_per_token as tencent_cost_per_token,
)
from litellm.llms.together_ai.cost_calculator import get_model_params_and_category
from litellm.llms.together_ai.cost_calculator import (
get_model_params_and_category,
has_together_registry_pricing,
)
from litellm.llms.vertex_ai.cost_calculator import (
cost_per_character as google_cost_per_character,
)
@ -557,9 +560,10 @@ def cost_per_token(
)
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
return openai_cost_per_token(
return generic_cost_per_token(
model=model_without_prefix,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,
data_residency=data_residency,
)
@ -735,6 +739,13 @@ def _get_provider_for_cost_calc(
return custom_llm_provider
def _get_hidden_str_for_cost_calc(hidden_params: object, key: str) -> str | None:
if not isinstance(hidden_params, Mapping):
return None
value: Final[object] = hidden_params.get(key)
return value if isinstance(value, str) and value else None
def _select_model_name_for_cost_calc(
model: str | None,
completion_response: object | None,
@ -751,7 +762,6 @@ def _select_model_name_for_cost_calc(
"""
return_model: str | None = None
region_name: str | None = None
custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
completion_response_model: str | None = None
@ -761,6 +771,14 @@ def _select_model_name_for_cost_calc(
elif isinstance(completion_response, dict):
completion_response_model = completion_response.get("model", None)
hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
explicit_pricing: Final = custom_pricing is True or base_model is not None
priced_from_response: Final = provider_response_model is not None or completion_response_model is not None
region_name: Final = (
_get_hidden_str_for_cost_calc(hidden_params, "region_name")
if not explicit_pricing and priced_from_response
else None
)
if custom_pricing is True:
if router_model_id is not None and router_model_id in litellm.model_cost:
@ -776,14 +794,12 @@ def _select_model_name_for_cost_calc(
else:
return_model = model
elif base_model is not None:
return_model = base_model
elif base_model is not None or provider_response_model is not None:
return_model = base_model if base_model is not None else provider_response_model
elif completion_response_model is None and hidden_params is not None:
if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
return_model = hidden_params.get("model", model)
elif hidden_params is not None and hidden_params.get("region_name", None) is not None:
region_name = hidden_params.get("region_name", None)
if return_model is None and completion_response_model is not None:
return_model = completion_response_model
@ -1568,10 +1584,9 @@ def completion_cost(
return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj)
# Calculate cost based on prompt_tokens, completion_tokens
if "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai":
# together ai prices based on size of llm
# get_model_params_and_category takes a model name and returns the category of LLM size it is in model_prices_and_context_window.json
if (
"togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
) and not has_together_registry_pricing(model, litellm.model_cost):
model = get_model_params_and_category(model, call_type=CallTypes(call_type))
# replicate llms are calculate based on time for request running

41
litellm/experimental_mcp_client/client.py
View file

@ -56,6 +56,9 @@ from litellm.types.mcp import (
MCPStdioConfig,
MCPTransport,
MCPTransportType,
credential_redirect_hook,
has_header,
without_header,
)
@ -273,6 +276,7 @@ class MCPClient:
transport_type: MCPTransportType = MCPTransport.http,
auth_type: MCPAuthType = None,
auth_value: str | dict[str, str] | None = None,
auth_header_name: str | None = None,
timeout: float | None = None,
stdio_config: MCPStdioConfig | None = None,
extra_headers: dict[str, str] | None = None,
@ -288,6 +292,11 @@ class MCPClient:
self.auth_type: MCPAuthType = auth_type
self.timeout: float = timeout if timeout is not None else MCP_CLIENT_TIMEOUT
self._mcp_auth_value: str | dict[str, str] | None = None
# The one place this client decides which header its credential occupies: the operator's
# configured slot on the v1 path, or the slot the v2 resolver's auth object already owns.
# Every consumer reads this rather than re-deriving it, since each re-derivation so far
# picked up a different bug.
self._credential_slot: str | None = auth_header_name or getattr(resolved_auth, "header_name", None)
self.stdio_config: MCPStdioConfig | None = stdio_config
self.extra_headers: dict[str, str] | None = extra_headers
self.ssl_verify: VerifyTypes | None = ssl_verify
@ -501,26 +510,33 @@ class MCPClient:
else:
self._mcp_auth_value = mcp_auth_value
def _header_slot(self, default: str) -> str:
return self._credential_slot or default
def _get_auth_headers(self) -> dict:
"""Generate authentication headers based on auth type."""
headers: Final = {}
if self._mcp_auth_value:
if isinstance(self._mcp_auth_value, str):
if self.auth_type == MCPAuth.bearer_token:
headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
static_bearer: Final = strip_auth_scheme(self._mcp_auth_value, "Bearer")
headers[self._header_slot("Authorization")] = f"Bearer {static_bearer}"
elif self.auth_type == MCPAuth.basic:
headers["Authorization"] = f"Basic {self._mcp_auth_value}"
headers[self._header_slot("Authorization")] = f"Basic {self._mcp_auth_value}"
elif self.auth_type == MCPAuth.api_key:
headers["X-API-Key"] = self._mcp_auth_value
headers[self._header_slot("X-API-Key")] = self._mcp_auth_value
elif self.auth_type == MCPAuth.authorization:
# This auth type means the caller owns the whole header value.
headers["Authorization"] = self._mcp_auth_value
headers[self._header_slot("Authorization")] = self._mcp_auth_value
elif self.auth_type == MCPAuth.oauth2:
headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
oauth2_bearer: Final = strip_auth_scheme(self._mcp_auth_value, "Bearer")
headers[self._header_slot("Authorization")] = f"Bearer {oauth2_bearer}"
elif self.auth_type == MCPAuth.token:
headers["Authorization"] = f"token {strip_auth_scheme(self._mcp_auth_value, 'token')}"
scheme_token: Final = strip_auth_scheme(self._mcp_auth_value, "token")
headers[self._header_slot("Authorization")] = f"token {scheme_token}"
elif self.auth_type == MCPAuth.oauth2_token_exchange:
headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
exchanged_bearer: Final = strip_auth_scheme(self._mcp_auth_value, "Bearer")
headers[self._header_slot("Authorization")] = f"Bearer {exchanged_bearer}"
elif isinstance(self._mcp_auth_value, dict):
headers.update(self._mcp_auth_value)
# Note: aws_sigv4 auth is not handled here — SigV4 requires per-request
@ -528,7 +544,14 @@ class MCPClient:
# of static headers. See MCPSigV4Auth and _create_httpx_client_factory().
# update the headers with the extra headers
if self.extra_headers:
headers.update(self.extra_headers)
# Mirrors _resolve_v2_auth: when the operator named a slot for the credential the
# gateway resolved, no injected header may shadow it, case-insensitively, since HTTP
# header names are. Without a configured slot the old precedence stands unchanged.
slot: Final = self._credential_slot
injected: Final = (
without_header(self.extra_headers, slot) if slot and has_header(headers, slot) else self.extra_headers
)
headers.update(injected or {})
return _strip_header_whitespace(headers)
def _create_httpx_client_factory(self) -> Callable[..., httpx.AsyncClient]:
@ -556,12 +579,14 @@ class MCPClient:
# SigV4 aws_auth. Both are None for the common case — no behavior change.
fallback_auth: Final = self._resolved_auth if self._resolved_auth is not None else self._aws_auth
effective_auth: Final = auth if auth is not None else fallback_auth
guard: Final = credential_redirect_hook(self.server_url, self._credential_slot)
return httpx.AsyncClient(
headers=headers,
timeout=timeout,
auth=effective_auth,
verify=ssl_config,
follow_redirects=True,
event_hooks={"request": [guard]} if guard else {},
)
return factory

11
litellm/integrations/SlackAlerting/batching_handler.py
View file

@ -10,6 +10,8 @@ from typing import TYPE_CHECKING, Any, Final
from litellm._logging import verbose_proxy_logger
from .ms_teams import MS_TEAMS_ALERTING_DESTINATION, build_ms_teams_payload
if TYPE_CHECKING:
from .slack_alerting import SlackAlerting as _SlackAlerting
@ -62,14 +64,17 @@ async def send_to_webhook(slackAlertingInstance: SlackAlertingType, item, count)
if count > 1:
payload["text"] = f"[Num Alerts: {count}]\n\n{payload['text']}"
request_body: Final = (
build_ms_teams_payload(payload["text"]) if item.get("format") == MS_TEAMS_ALERTING_DESTINATION else payload
)
response: Final = await slackAlertingInstance.async_http_handler.post(
url=item["url"],
headers=item["headers"],
data=json.dumps(payload),
data=json.dumps(request_body),
)
if response.status_code != 200:
verbose_proxy_logger.debug("Error sending slack alert to url=%s. Error=%s", item["url"], response.text)
verbose_proxy_logger.debug("Error sending alert to url=%s. Error=%s", item["url"], response.text)
except Exception as e:
verbose_proxy_logger.debug("Error sending slack alert: %s", e)
verbose_proxy_logger.debug("Error sending alert: %s", e)
finally:
_print_alerting_payload_warning(payload, slackAlertingInstance=slackAlertingInstance)

75
litellm/integrations/SlackAlerting/ms_teams.py Normal file
View file

@ -0,0 +1,75 @@
"""Microsoft Teams alert delivery helpers.
Teams incoming webhooks (Workflows and legacy connectors) accept an Adaptive
Card wrapped in a message attachment, so alert text is delivered as a single
wrapped TextBlock.
"""
import os
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from typing_extensions import ReadOnly, TypedDict
from litellm.types.integrations.slack_alerting import AlertType
MS_TEAMS_WEBHOOK_URL_ENV: Final = "MS_TEAMS_WEBHOOK_URL"
MS_TEAMS_ALERTING_DESTINATION: Final = "ms_teams"
MS_TEAMS_ALERT_HEADERS: Final[Mapping[str, str]] = MappingProxyType({"Content-type": "application/json"})
class MSTeamsTextBlock(TypedDict):
type: ReadOnly[str]
text: ReadOnly[str]
wrap: ReadOnly[bool]
class MSTeamsAdaptiveCard(TypedDict):
type: ReadOnly[str]
version: ReadOnly[str]
body: ReadOnly[tuple[MSTeamsTextBlock, ...]]
class MSTeamsAttachment(TypedDict):
contentType: ReadOnly[str]
content: ReadOnly[MSTeamsAdaptiveCard]
class MSTeamsMessage(TypedDict):
type: ReadOnly[str]
attachments: ReadOnly[tuple[MSTeamsAttachment, ...]]
class MSTeamsAlertText(TypedDict):
text: ReadOnly[str]
class MSTeamsQueueItem(TypedDict):
url: ReadOnly[str]
headers: ReadOnly[Mapping[str, str]]
payload: ReadOnly[MSTeamsAlertText]
alert_type: ReadOnly[AlertType]
format: ReadOnly[str]
def get_ms_teams_webhook_url() -> str | None:
return os.getenv(MS_TEAMS_WEBHOOK_URL_ENV)
def build_ms_teams_payload(text: str) -> MSTeamsMessage:
return MSTeamsMessage(
type="message",
attachments=(
MSTeamsAttachment(
contentType="application/vnd.microsoft.card.adaptive",
content=MSTeamsAdaptiveCard(
type="AdaptiveCard",
version="1.4",
body=(MSTeamsTextBlock(type="TextBlock", text=text, wrap=True),),
),
),
),
)

79
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -57,6 +57,13 @@ from litellm.types.proxy.model_deprecation import (
from ..email_templates.templates import *
from .batching_handler import send_to_webhook, squash_payloads
from .ms_teams import (
MS_TEAMS_ALERT_HEADERS,
MS_TEAMS_ALERTING_DESTINATION,
MSTeamsAlertText,
MSTeamsQueueItem,
get_ms_teams_webhook_url,
)
from .utils import process_slack_alerting_variables
if TYPE_CHECKING:
@ -1431,13 +1438,43 @@ Model Info:
# only send budget alerts over Email
await self.send_email_alert_using_smtp(webhook_event=user_info, alert_type=alert_type)
if "slack" not in self.alerting:
send_to_slack: Final = "slack" in self.alerting
send_to_ms_teams: Final = MS_TEAMS_ALERTING_DESTINATION in self.alerting
if not send_to_slack and not send_to_ms_teams:
return
if alert_type not in self.alert_types:
return
from datetime import datetime
current_time: Final = datetime.now().strftime("%H:%M:%S")
_proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
alert_type_name: Final = getattr(alert_type, "name", alert_type)
alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
if alert_type == "daily_reports" or alert_type == "new_model_added":
formatted_message = alert_type_formatted + message
else:
formatted_message = (
f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
)
if kwargs:
for key, value in kwargs.items():
formatted_message += f"\n\n{key}: `{value}`\n\n"
if alerting_metadata:
for key, value in alerting_metadata.items():
formatted_message += f"\n\n*Alerting Metadata*: \n{key}: `{value}`\n\n"
if _proxy_base_url is not None:
formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`"
if send_to_ms_teams:
self._enqueue_ms_teams_alert(formatted_message=formatted_message, alert_type=alert_type)
if not send_to_slack:
if len(self.log_queue) >= self.batch_size:
await self.flush_queue()
return
# Check if digest mode is enabled for this alert type
alert_type_name_str: Final = getattr(alert_type, "value", str(alert_type))
_atc: Final = self.alert_type_config.get(alert_type_name_str)
@ -1473,28 +1510,6 @@ Model Info:
)
return # Suppress immediate alert; will be emitted by _flush_digest_buckets
# Get the current timestamp
current_time: Final = datetime.now().strftime("%H:%M:%S")
_proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
# Use .name if it's an enum, otherwise use as is
alert_type_name: Final = getattr(alert_type, "name", alert_type)
alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
if alert_type == "daily_reports" or alert_type == "new_model_added":
formatted_message = alert_type_formatted + message
else:
formatted_message = (
f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
)
if kwargs:
for key, value in kwargs.items():
formatted_message += f"\n\n{key}: `{value}`\n\n"
if alerting_metadata:
for key, value in alerting_metadata.items():
formatted_message += f"\n\n*Alerting Metadata*: \n{key}: `{value}`\n\n"
if _proxy_base_url is not None:
formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`"
# check if we find the slack webhook url in self.alert_to_webhook_url
if self.alert_to_webhook_url is not None and alert_type in self.alert_to_webhook_url:
slack_webhook_url: str | list[str] | None = self.alert_to_webhook_url[alert_type]
@ -1531,6 +1546,24 @@ Model Info:
if len(self.log_queue) >= self.batch_size:
await self.flush_queue()
def _enqueue_ms_teams_alert(self, formatted_message: str, alert_type: AlertType) -> None:
ms_teams_webhook_url: Final = get_ms_teams_webhook_url()
if ms_teams_webhook_url is None:
verbose_proxy_logger.error(
"MS Teams alerting is enabled but MS_TEAMS_WEBHOOK_URL is not set. Dropping alert type=%s",
alert_type,
)
return
payload: Final[MSTeamsAlertText] = {"text": formatted_message}
item: Final[MSTeamsQueueItem] = {
"url": ms_teams_webhook_url,
"headers": MS_TEAMS_ALERT_HEADERS,
"payload": payload,
"alert_type": alert_type,
"format": MS_TEAMS_ALERTING_DESTINATION,
}
self.log_queue.append(item)
async def async_send_batch(self):
if not self.log_queue:
return

87
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -24,6 +24,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
with_prompt_cache_breakpoint,
)
from litellm.types.integrations.anthropic_cache_control_hook import (
GATEWAY_INJECTED_CACHE_METADATA_KEY,
GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
CacheControlInjectionPoint,
CacheControlMessageInjectionPoint,
)
@ -185,7 +187,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
reserved_blocks: Final = (
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
processed_messages = self._apply_message_injections(
points=applied_message_points,
messages=processed_messages,
@ -194,7 +196,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
if (
openai_dialect
and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before
and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before
):
non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
@ -236,7 +238,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return provider
@staticmethod
def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
def count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
system_blocks: Final = (
sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0
)
@ -258,7 +260,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
``max_blocks`` is reached. Injection points are honored in config order,
so earlier points win when slots are scarce.
"""
used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages)
used_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
limit_reached = False
for point in points:
@ -376,7 +378,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
# 2. list of objects - only apply to last item per Anthropic spec
elif isinstance(message_content, list):
if len(message_content) > 0 and isinstance(message_content[-1], dict):
message_content[-1]["cache_control"] = control
message_content[-1]["cache_control"] = control # pyright: ignore[reportGeneralTypeIssues] # loose runtime dict
return message
@staticmethod
@ -454,8 +456,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system)
message_blocks: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
system_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints((), processed_system)
if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks:
system_already_has_cc: Final = isinstance(processed_system, list) and any(
@ -589,7 +591,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
carry the mark either at the top level (Anthropic shape) or nested under
``function`` (OpenAI shape); the Anthropic chat transform accepts both.
"""
if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0:
if AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) > 0:
return True
if tools is not None:
return any(
@ -749,6 +751,64 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if points:
non_default_params["cache_control_injection_points"] = points
@staticmethod
def record_gateway_injection(
request_kwargs: Mapping[str, object],
added: int,
) -> None:
"""Name the deployment whose payload the gateway, not the client, put breakpoints on.
Spend accounting only asks whether litellm acted, so what it needs is which
deployment, not a count. Recording that is what makes the mark attempt-scoped: the
metadata bucket is one dict shared by every retry, failover and fallback of a
request, and ``litellm_call_id`` is shared with it, so anything request-scoped
written by one attempt is read by all of them and each boundary would have to
remember to strip it. The deployment is the part that actually changes when the
request moves, so a leg that injected nothing is never credited for one that did.
It also makes a zero delta (hook re-entry) and a negative one (a prompt manager
replacing the messages) harmless, since neither rewrites an earlier mark.
A pass that runs before a deployment is chosen, which is what the proxy does for
prompt templates, injects into the payload every leg goes on to send, so it marks
the request for all of them rather than for one.
Only what this pass actually placed counts. A ``tool_config`` point is placed by
the Bedrock converse transform, and only when the request carries tools, so the
presence of one here says nothing about whether a breakpoint reaches the wire;
claiming it marked three request shapes out of four that inject nothing. Missing
that Bedrock credit is the fail-closed direction, and the alternative is a
provider transform that carries spend-attribution state.
Reads whichever bucket the request actually carries rather than asking the shared
name resolver, which answers on key presence: ``litellm_params`` declares
``litellm_metadata`` as None on every request, so the resolver names a bucket that
is not there and the mark is dropped.
Never CREATES the bucket. The proxy seeds it on every request and is the marker's
only reader, so a request without one is a bare SDK call nothing would consume it
from. Creating it would also add a key to a dict call sites splat as ``**kwargs``,
and on the Responses API ``metadata`` is both this bucket's default name and an
explicit parameter, so the splat collides with the caller's own value.
"""
if added <= 0:
return
bucket: Final = next(
(
candidate
for candidate in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata"))
if isinstance(candidate, dict)
),
None,
)
if bucket is not None:
model_info: Final = request_kwargs.get("model_info")
bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = (
model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT)
if isinstance(model_info, dict)
else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT
)
@staticmethod
def maybe_inject_cache_control(
messages: list[dict],
@ -798,17 +858,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system)
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
messages=messages,
system=system,
injection_points=injection_points,
openai_dialect=openai_dialect,
)
if (
openai_dialect
and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before
):
breakpoints_added: Final = (
AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before
)
AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added)
if openai_dialect and breakpoints_added > 0:
kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
if remaining:
kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)

91
litellm/integrations/custom_guardrail.py
View file

@ -60,6 +60,12 @@ _PRE_CALL_EXECUTED_TOKEN: Final = secrets.token_hex(16)
_GUARDRAIL_BLOCK_STATUS_CODES: Final = frozenset({400, 403, 422})
DEFAULT_ADVISORY_MESSAGE: Final = (
"The user's latest message was flagged for {reason} by a content safety "
"guardrail. This may be a false positive. Use your judgment: respond "
"helpfully if the request is legitimate, or decline if it is not."
)
_guardrail_self_recorded: Final[contextvars.ContextVar[bool]] = contextvars.ContextVar(
"litellm_guardrail_self_recorded", default=False
)
@ -158,6 +164,7 @@ class CustomGuardrail(CustomLogger):
sensitive_data_route_to_model: str | None = None,
sticky_session_routing: bool = True,
run_in_parallel: bool = False,
scan_raw_request: bool = False,
only_scan_new_messages: bool = False,
**kwargs,
):
@ -180,6 +187,13 @@ class CustomGuardrail(CustomLogger):
run_in_parallel: When True, this pre_call or post_call guardrail runs concurrently with
other opted-in guardrails of the same hook. Only safe for block-only guardrails that
do not mutate the request or response.
scan_raw_request: When True, this pre_call guardrail always evaluates the request as it
was before any guardrail in this hook ran, regardless of where it's declared in the
guardrails list -- so an earlier guardrail that masks/rewrites content (e.g. PII
redaction) can never hide a violation from this one. Only safe for block-only
guardrails: any data this guardrail returns is discarded, matching run_in_parallel's
contract, since applying its mutations on top of a stale snapshot would silently
undo whatever later guardrails already did to the live request.
"""
self.guardrail_name = guardrail_name
self.supported_event_hooks = supported_event_hooks
@ -195,6 +209,7 @@ class CustomGuardrail(CustomLogger):
self.sensitive_data_route_to_model: str | None = sensitive_data_route_to_model
self.sticky_session_routing: bool = sticky_session_routing
self.run_in_parallel: bool = run_in_parallel
self.scan_raw_request: bool = scan_raw_request
self.only_scan_new_messages: bool = only_scan_new_messages
if supported_event_hooks:
@ -281,6 +296,82 @@ class CustomGuardrail(CustomLogger):
original_response=original_response,
)
def inject_advisory_message(
self,
data: dict[str, Any], # mutable-ok: caller's dict is mutated in place, matching mark_pre_call_hook_ran
message: str,
) -> bool:
"""
Append an advisory system message to the request in place, so the LLM
itself can weigh a possible false-positive guardrail flag rather than
the request being hard-blocked or silently allowed.
Unlike raise_passthrough_exception, this does NOT short-circuit the LLM
call; the request proceeds normally with the extra message appended.
Guardrails should call this from on_flagged handling analogous to how
passthrough-supporting guardrails call raise_passthrough_exception.
Args:
data: The request data dictionary, mutated in place to append the
advisory message to its "messages" list and/or "input"/
"instructions" text.
message: The formatted advisory message to append as a system message.
Returns:
True if the advisory was actually written somewhere the model will
see it. False if ``data["input"]`` is a structured Responses-API
list (not a plain string) -- the Responses API reads only
``input``, so appending to ``messages`` would be inert regardless
of whether a ``messages`` list also happens to be present, and
there is no field this helper can safely append into. The caller
must treat this like any other case where the mitigation can't
land and degrade to blocking instead of silently letting the
flagged request through unmodified.
"""
advisory_message: Final = {"role": "system", "content": message} # mutable-ok: plain dict for live request
existing_messages: Final = data.get("messages")
existing_input: Final = data.get("input")
existing_instructions: Final = data.get("instructions")
if isinstance(existing_instructions, str):
# Responses API "instructions" is the privileged, developer-set
# system-level field the model treats as authoritative -- unlike
# "input", which the caller controls and could use to tell the
# model to disregard a trailing warning. Prefer it over "input"
# whenever present.
if isinstance(existing_messages, list):
messages_with_instructions_note: Final = [ # mutable-ok: fresh list
*existing_messages,
advisory_message,
]
data["messages"] = messages_with_instructions_note # rebind-ok: mutates caller's dict by design
data["instructions"] = f"{existing_instructions}\n\n{message}" # rebind-ok: mutates caller's dict by design
return True
if isinstance(existing_input, str):
# A plain-string "input" doesn't rule out "messages" also being a
# real, read field (e.g. a chat-completions call carrying a stray
# "input"), so write to both when both are present.
if isinstance(existing_messages, list):
messages_with_input_note: Final = [*existing_messages, advisory_message] # mutable-ok: fresh list
data["messages"] = messages_with_input_note # rebind-ok: mutates caller's dict by design
# The Responses API reads "input", not "messages" -- appending only to
# "messages" would leave the advisory unreachable for that endpoint.
data["input"] = f"{existing_input}\n\n{message}" # rebind-ok: mutates caller's dict by design
return True
if existing_input is not None:
# existing_input is a structured (non-string) Responses-API item
# list. That endpoint reads only "input", so appending to
# "messages" -- even if "messages" also happens to be present --
# would never reach the model. Leave data untouched and report
# non-delivery so the caller degrades to blocking.
return False
if isinstance(existing_messages, list):
messages_without_input_note: Final = [*existing_messages, advisory_message] # mutable-ok: fresh list
data["messages"] = messages_without_input_note # rebind-ok: mutates caller's dict by design
return True
sole_message: Final = [advisory_message] # mutable-ok: plain list for the live JSON request
data["messages"] = sole_message # rebind-ok: mutates caller's dict by design
return True
def raise_sensitive_data_route_exception(
self,
route_to_model: str,

1
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -149,7 +149,6 @@ class PromptManager:
)
self.prompts[template_id] = template
except Exception:
# Optional: print(f"Error loading prompt from JSON: {template_id}")
pass
def _load_prompt_file(self, file_path: str | Path, prompt_id: str) -> PromptTemplate:

33
litellm/integrations/langfuse/langfuse.py
View file

@ -5,6 +5,7 @@ import os
import traceback
from collections.abc import Callable, Iterable, Mapping
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast
@ -137,6 +138,16 @@ def resolve_langfuse_credentials(
return public_key, secret_key, resolved_host
@lru_cache(maxsize=8)
def _warn_invalid_deployment_environment(raw_value: str, error: str) -> None:
verbose_logger.warning(
"Ignoring invalid LANGFUSE_TRACING_ENVIRONMENT=%r for the langfuse callback: %s. "
"Traces will be sent to Langfuse's default environment.",
raw_value,
error,
)
class LangFuseLogger:
# Class variables or attributes
def __init__(
@ -165,9 +176,11 @@ class LangFuseLogger:
# add http:// if unset, assume communicating over private network - e.g. render
self.langfuse_host = "http://" + self.langfuse_host
_env_override: Final = str(langfuse_environment).strip() if langfuse_environment is not None else None
self.langfuse_environment = _env_override or os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
if self.langfuse_environment:
validate_langfuse_environment_value(self.langfuse_environment)
if _env_override:
validate_langfuse_environment_value(_env_override)
self.langfuse_environment: str | None = _env_override
else:
self.langfuse_environment = self.resolve_deployment_environment()
self.langfuse_release = os.getenv("LANGFUSE_RELEASE")
self.langfuse_debug = os.getenv("LANGFUSE_DEBUG")
self.langfuse_flush_interval = LangFuseLogger._get_langfuse_flush_interval(flush_interval)
@ -953,6 +966,20 @@ class LangFuseLogger:
verbose_logger.warning("Failed to apply masking function: %s. Returning original data.", e)
return data
@staticmethod
def resolve_deployment_environment() -> str | None:
"""Resolve LANGFUSE_TRACING_ENVIRONMENT: stripped value, "default" plus a warning when invalid, None when unset."""
raw: Final = os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
if not raw:
return None
value: Final = raw.strip()
try:
validate_langfuse_environment_value(value)
except ValueError as e:
_warn_invalid_deployment_environment(raw, str(e))
return "default"
return value
@staticmethod
def _get_langfuse_flush_interval(flush_interval: int) -> int:
"""

9
litellm/integrations/langfuse/langfuse_handler.py
View file

@ -1,5 +1,3 @@
import os
"""
This file contains the LangFuseHandler class
@ -8,6 +6,7 @@ Used to get the LangFuseLogger for a given request
Handles Key/Team Based Langfuse Logging
"""
import os
from typing import TYPE_CHECKING, Any, Final
from litellm.litellm_core_utils.litellm_logging import StandardCallbackDynamicParams
@ -157,7 +156,11 @@ class LangFuseHandler:
if raw is None:
return None
value = str(raw).strip()
if not value or value == os.getenv("LANGFUSE_TRACING_ENVIRONMENT"):
if (
not value
or value == os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
or value == LangFuseLogger.resolve_deployment_environment()
):
return None
return value

4
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -2,6 +2,7 @@
Call Hook for LiteLLM Proxy which allows Langfuse prompt management.
"""
import inspect
import os
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@ -109,6 +110,9 @@ def langfuse_client_init(
cert=os.getenv("SSL_CERTIFICATE", litellm.ssl_certificate),
)
if "environment" in inspect.signature(Langfuse.__init__).parameters:
parameters["environment"] = LangFuseLogger.resolve_deployment_environment()
client: Final = Langfuse(**parameters)
return client

6
litellm/integrations/otel/emitter.py
View file

@ -146,7 +146,7 @@ class SpanEmitter:
For callers that own and manage their own span lifecycle. ``tracer``
overrides the bound tracer for this span only, used for per-request
multi-tenant credential routing. ``links`` records related-but-not-parent
spans (e.g. the transport span of an MCP message, per MCP semconv).
spans (e.g. the trace context an MCP client propagated in ``params._meta``).
"""
return (tracer or self._tracer).start_span(
name,
@ -196,8 +196,8 @@ class SpanEmitter:
Return the span, or ``None`` if it was deduplicated away. ``tracer``
overrides the bound tracer for this span, used for per-request routing.
``links`` records related-but-not-parent spans (the transport span of an
MCP message).
``links`` records related-but-not-parent spans (e.g. the trace context an
MCP client propagated in ``params._meta``).
"""
# LLM-call and MCP tool-call spans carry a dedup key (their request's
# call id), so a sync+async double-firing coalesces. ``isinstance`` narrows

14
litellm/integrations/otel/logger.py
View file

@ -390,10 +390,10 @@ class OpenTelemetryV2(CustomLogger):
MCP tool calls reach the success/failure callbacks like any other request
(with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have
no ``pre_call`` carrier — so they get their own CLIENT span here. Per the MCP
semconv it parents to the trace context the client propagated in
``params._meta`` (or starts a new root) and links the transport span, rather
than nesting under the HTTP/session span. Returns whether it handled the
no ``pre_call`` carrier — so they get their own CLIENT span here. It nests
under the transport span of the request carrying this message, and trace
context the client propagated in ``params._meta`` is recorded as a span
link (see ``resolve_mcp_span_context``). Returns whether it handled the
event, so the caller skips the LLM-call path. The whole span is emitted at
once (there is no boundary to open it at), deduped on the call id.
"""
@ -436,9 +436,9 @@ class OpenTelemetryV2(CustomLogger):
Like a tool call, listing reaches the success/failure callbacks (here with
``call_type`` ``list_mcp_tools``) with no ``pre_call`` carrier, so it gets its
own CLIENT span. Per the MCP semconv it parents to the ``params._meta`` trace
context (or starts a new root) and links the transport span, rather than
nesting under the HTTP/session span. Returns whether it handled the event so
own CLIENT span, nested under the transport span of the request carrying
this message with any ``params._meta`` trace context recorded as a span
link (see ``resolve_mcp_span_context``). Returns whether it handled the event so
the caller skips the LLM-call path.
"""
raw_payload: Final = kwargs.get("standard_logging_object")

44
litellm/integrations/otel/model/spans.py
View file

@ -10,6 +10,8 @@ Canonical hierarchy::
│ └── DB_CALL (CLIENT) # its key/user/team lookups nest here
├── GUARDRAIL (INTERNAL) # request-lifecycle hook, sibling of LLM_CALL
├── LLM_CALL (CLIENT)
├── MCP_TOOL_CALL (CLIENT) # nests under the POST carrying the message
├── MCP_LIST_TOOLS (CLIENT) # (client-propagated context is a span link)
└── DB_CALL (CLIENT) # e.g. the spend-log write
Guardrails parent to PROXY_REQUEST, not LLM_CALL: pre/during/post-call guardrail
@ -18,14 +20,14 @@ before the LLM call even starts), so a guardrail is a sibling of the LLM call,
not a child of it. The emitter parents every span to the ambient OTel context
(the active server span), which matches this.
MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) have two shapes, chosen at emit
time by :func:`resolve_mcp_span_context`. When the client propagates trace context
in ``params._meta`` MCP and the HTTP transport are independent contexts per the
OTel GenAI MCP semconv, so the span parents to that propagated context and records
the ``PROXY_REQUEST`` transport span as a span *link*, never a parent — the shape
this registry's ``parent=None, links=PROXY_REQUEST`` entry encodes. When nothing is
propagated (the common case) the span nests under the transport span of the request
carrying that message, so the tool call stays in one trace.
MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) are parented at emit time by
:func:`resolve_mcp_span_context`: they nest under the ``PROXY_REQUEST`` transport
span of the request carrying that message, so the tool call stays in one trace.
Trace context the client propagated in ``params._meta`` (SEP-414) is recorded as
a span *link*, never the parent — a remote parent would root the span in a trace
whose root never reaches the gateway's tracing backend. Links always target that
remote client context, never a registry role, so ``SpanSpec`` declares no link
field; the concrete transport parent is resolved per message at emit time.
Not every service call becomes a span — :func:`span_role_for_service` decides:
@ -85,25 +87,19 @@ class SpanSpec:
role: SpanRole
kind: LiteLLMSpanKind
parent: SpanRole | None
links: SpanRole | None = None
SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = {
SpanRole.PROXY_REQUEST: SpanSpec(SpanRole.PROXY_REQUEST, LiteLLMSpanKind.SERVER, parent=None),
SpanRole.LLM_CALL: SpanSpec(SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
# The proxy is an MCP client to the upstream server, so MCP spans are CLIENT
# spans. With trace context propagated in ``params._meta``, MCP and the HTTP
# transport are independent contexts (OTel GenAI MCP semconv): the span parents
# to the propagated context and records the PROXY_REQUEST transport span as a
# span *link*, never a parent — the shape ``parent=None, links=PROXY_REQUEST``
# encodes. With nothing propagated, ``resolve_mcp_span_context`` nests the span
# under that message's transport span instead, keeping the call in one trace.
SpanRole.MCP_TOOL_CALL: SpanSpec(
SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
),
SpanRole.MCP_LIST_TOOLS: SpanSpec(
SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
),
# spans. ``resolve_mcp_span_context`` nests them under the PROXY_REQUEST
# transport span of the request carrying that message (resolved per message at
# emit time), keeping the call in one trace. Trace context the client
# propagated in ``params._meta`` becomes a span *link* to that remote context,
# which is not a registry role, so ``SpanSpec`` has no link field.
SpanRole.MCP_TOOL_CALL: SpanSpec(SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
SpanRole.MCP_LIST_TOOLS: SpanSpec(SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
SpanRole.GUARDRAIL: SpanSpec(SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
SpanRole.DB_CALL: SpanSpec(SpanRole.DB_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
SpanRole.SERVICE: SpanSpec(SpanRole.SERVICE, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
@ -209,8 +205,8 @@ def service_span_name(data: "ServiceSpanData") -> str:
def root_roles() -> list[SpanRole]:
"""Roles with no in-process parent. They start a new trace unless they adopt a
remote parent (e.g. an MCP span joining the client's propagated context)."""
"""Roles with no in-process parent, i.e. they start a new trace (only the
instrumentor-owned ``PROXY_REQUEST`` server span today)."""
return [role for role, spec in SPAN_REGISTRY.items() if spec.parent is None]
@ -227,8 +223,6 @@ def validate_registry(
raise ValueError(f"SPAN_REGISTRY[{role}] has mismatched role {spec.role}")
if spec.parent is not None and spec.parent not in reg:
raise ValueError(f"span role {role} declares unknown parent {spec.parent}")
if spec.links is not None and spec.links not in reg:
raise ValueError(f"span role {role} declares unknown link target {spec.links}")
missing: Final = [role for role in SpanRole if role not in reg]
if missing:
raise ValueError(f"SPAN_REGISTRY is missing roles: {missing}")

63
litellm/integrations/otel/plumbing/context.py
View file

@ -57,8 +57,8 @@ def request_root_span() -> "Span | None":
# The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the
# MCP client propagated in the current request's ``params._meta``. The MCP gateway
# sets it per message so the MCP span can parent to the client's span rather than
# to the transport. A ``ContextVar`` because, like the root-span anchor, it must
# sets it per message so the MCP span can record the client's span as a span
# link. A ``ContextVar`` because, like the root-span anchor, it must
# ride the request task and be readable by the inline success-logging callback.
_mcp_message_trace_carrier: Final["ContextVar[Mapping[str, str] | None]"] = ContextVar(
"litellm_otel_mcp_message_trace_carrier", default=None
@ -148,10 +148,10 @@ def _mcp_transport_span_context() -> "SpanContext | None":
Prefers the transport the gateway published for this specific message; falls
back to the ambient request anchor for paths that emit an MCP span on the
request task itself (the REST MCP endpoints, the SDK). Parenting and linking
only need the immutable context, and unlike ``mcp_message_transport_span`` they
stay correct against a transport that has already finished, so this does not
require the span to still be recording.
request task itself (the REST MCP endpoints). Parenting needs only the
immutable context, and unlike ``mcp_message_transport_span`` it stays correct
against a transport that has already finished, so this does not require the
span to still be recording.
"""
published: Final = _mcp_message_transport_span.get()
if published is not None:
@ -222,25 +222,31 @@ def resolve_mcp_span_context(
) -> "tuple[Context, tuple[Link, ...]]":
"""Parent context + links for an MCP message span.
The span always nests under the transport span of the request carrying this
message, so a tool call and the ``POST`` that carried it stay in one trace.
The transport comes from :func:`_mcp_transport_span_context`, which is the
*current message's* POST rather than whatever request happened to open the
session, so a long-lived session does not glue every message under its first
request.
When the client propagates W3C trace context in the request's ``params._meta``
(SEP-414), MCP and the underlying transport are independent lifecycles — one
streamable-HTTP session multiplexes many messages, and the client's own span is
the truthful parent. So, per the OTel GenAI MCP semconv:
(SEP-414), that remote context is recorded as a span *link*, never the parent.
The OTel GenAI MCP semconv prefers the inverse (remote parent, transport link),
but the gateway's tracing backend only ever receives the gateway's half of such
a trace: parenting into the client's trace id roots the span in a trace whose
root span never reaches the backend, so the span is unreachable from the trace
view and the transport transaction shows a dangling link (observed with
clients that propagate synthetic trace ids). Anchoring to the gateway's own
request and linking the client's context keeps every trace renderable while
preserving the client-side correlation.
* parent to the trace context the client propagated (a *remote* parent), and
* record the transport span as a *link*, never the parent.
Almost no client implements SEP-414 yet, so in practice nothing is propagated.
Rooting the span there splits a single tool call into two disconnected traces
joined only by a link, which is how it surfaces in APM: the ``POST`` transaction
and the ``tools/call`` span share no trace. With no remote parent to honor,
parent to the transport span of the request carrying this message instead, so
the call stays in one trace; no link is added since the transport is now the
real parent. The transport comes from :func:`_mcp_transport_span_context`, which
is the *current message's* POST rather than whatever request happened to open
the session, so a long-lived session does not glue every message under its
first request. With neither a remote parent nor a transport the returned context
carries no span and the span legitimately starts its own root trace.
With no transport at all the span starts its own root trace, still carrying
the link — the client context is only ever a link, so this event keeps one
shape everywhere. Both returned contexts are built on an explicitly empty
base, so ambient (stale session) state can never leak in, and the span
inherits the transport's sampling decision exactly like every other
request-level span — a client's sampled flag neither forces nor suppresses
recording.
Only trace context (``traceparent``/``tracestate``) is extracted, never the
client's W3C Baggage: ``params._meta`` is caller-controlled, and the otel
@ -251,13 +257,12 @@ def resolve_mcp_span_context(
never fall through to the ambient (stale session) span.
"""
source: Final = carrier if carrier is not None else _mcp_message_trace_carrier.get()
parent: Final = _PROPAGATOR.extract(dict(source or {}), context=Context())
propagated: Final = get_current_span(_PROPAGATOR.extract(dict(source or {}), context=Context()))
links: Final = (Link(propagated.get_span_context()),) if is_recordable_span(propagated) else ()
transport: Final = _mcp_transport_span_context()
if is_recordable_span(get_current_span(parent)):
return parent, (Link(transport),) if transport is not None else ()
if transport is not None:
return context_from_span(NonRecordingSpan(transport)), ()
return parent, ()
if transport is None:
return Context(), links
return context_from_span(NonRecordingSpan(transport), context=Context()), links
def is_recordable_span(obj: object) -> bool:

70
litellm/integrations/otel/plumbing/routing.py
View file

@ -2,12 +2,13 @@
When a request carries team/key vendor credentials in
``standard_callback_dynamic_params``, or the key/team config resolved at auth
names a destination project, its spans must export through a
``TracerProvider`` whose OTLP headers carry those credentials / that project.
``TenantTracerCache`` builds and caches one provider per distinct
(credentials, project) pair, and otherwise hands back the logger's default
tracer. This lets a single logger fan requests out to many tenants without
needing a logger per tenant.
names a destination project or a service name, its spans must export through a
``TracerProvider`` whose OTLP headers carry those credentials / that project,
or whose Resource carries that ``service.name``. ``TenantTracerCache`` builds
and caches one provider per distinct (credentials, project, service name)
tuple, and otherwise hands back the logger's default tracer. This lets a
single logger fan requests out to many tenants without needing a logger per
tenant.
"""
import threading
@ -22,6 +23,7 @@ from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import Tracer
from litellm._logging import verbose_logger
from litellm.constants import OTEL_SERVICE_NAME_METADATA_KEYS
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.providers import (
build_tracer_provider,
@ -65,8 +67,30 @@ _MAX_RETIRED_PROVIDERS: Final = 64
_HeaderItems: TypeAlias = tuple[tuple[str, str], ...]
_RouteKey: TypeAlias = tuple[_HeaderItems, _HeaderItems, str | None, str | None]
_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
#: Key/team config fields naming the Resource ``service.name``, highest
#: precedence first. Read only from ``user_api_key_auth_metadata`` (the config
#: the proxy resolved at auth), never from client-supplied request metadata:
#: the service name picks the dataset/service traces land in (Honeycomb routes
#: datasets by it), so a caller must not be able to choose one.
_SERVICE_NAME_KEYS: Final = OTEL_SERVICE_NAME_METADATA_KEYS
def tenant_service_name(auth_metadata: Mapping[str, str] | None) -> str | None:
"""The per-request ``service.name`` override for this key/team, if any.
``None`` keeps the env-configured default (``OTEL_SERVICE_NAME``).
"""
if not auth_metadata:
return None
return next(
(stripped for key in _SERVICE_NAME_KEYS if (stripped := (auth_metadata.get(key) or "").strip())),
None,
)
def _shutdown_provider(provider: TracerProvider) -> None:
"""Flush + stop an evicted provider's processors (reclaims their threads).
@ -116,7 +140,7 @@ class TenantRoute:
class TenantTracerCache:
"""Credential/project-scoped ``TracerProvider`` cache keyed by the routing headers."""
"""Tenant-scoped ``TracerProvider`` cache keyed by routing headers and service name."""
def __init__(
self,
@ -131,7 +155,7 @@ class TenantTracerCache:
# thread-pool workers concurrently with the event loop, so cache
# updates, span counts, and retirement must be atomic.
self._lock: Final = threading.Lock()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems, str | None], TracerProvider] = (
self._providers: OrderedDict[_RouteKey, TracerProvider] = (
OrderedDict() # mutable-ok: bounded LRU; eviction needs in-place ordered mutation
)
self._open_span_counts: dict[TracerProvider, int] = {} # mutable-ok: live refcount state
@ -172,10 +196,11 @@ class TenantTracerCache:
) -> TenantRoute:
"""Return the tracer (and trace-detachment flag) for this request.
Use ``default`` unless the request's dynamic credentials or its key/team
project require a scoped tracer, in which case build (or reuse) one. The
cache is a bounded LRU: the least-recently-used provider is flushed and
shut down on overflow so its exporter threads don't accumulate.
Use ``default`` unless the request's dynamic credentials, its key/team
project, or its key/team service name require a scoped tracer, in
which case build (or reuse) one. The cache is a bounded LRU: the
least-recently-used provider is flushed and shut down on overflow so
its exporter threads don't accumulate.
A routed provider is returned already held — its open-span count is
incremented in the same critical section as the cache update — so a
@ -184,7 +209,8 @@ class TenantTracerCache:
"""
credential_headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) or _NO_HEADERS
project_headers: Final = self._project_headers(auth_metadata)
if not credential_headers and not project_headers:
service_name: Final = tenant_service_name(auth_metadata)
if not credential_headers and not project_headers and service_name is None:
return TenantRoute(tracer=default, detached=False)
# A fixed per-integration region endpoint (New Relic us/eu), never a
# caller-supplied host; ``None`` keeps the preset's own endpoint.
@ -193,9 +219,12 @@ class TenantTracerCache:
tuple(sorted(credential_headers.items())),
tuple(sorted(project_headers.items())),
endpoint,
service_name,
)
with self._lock:
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers, endpoint)
provider: Final = self._cached_provider_locked(
cache_key, credential_headers, project_headers, endpoint, service_name
)
self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
evicted: Final = self._evicted_on_overflow_locked()
if evicted is not None:
@ -208,16 +237,19 @@ class TenantTracerCache:
def _cached_provider_locked(
self,
cache_key: tuple[_HeaderItems, _HeaderItems, str | None],
cache_key: _RouteKey,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None,
service_name: str | None,
) -> TracerProvider:
cached: Final = self._providers.get(cache_key)
if cached is not None:
self._providers.move_to_end(cache_key)
return cached
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers, endpoint))
built: Final = build_tracer_provider(
self._routed_config(credential_headers, project_headers, endpoint, service_name)
)
self._providers[cache_key] = built
return built
@ -267,6 +299,7 @@ class TenantTracerCache:
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
service_name: str | None = None,
) -> OpenTelemetryV2Config:
"""Clone the config, rewriting headers on the callback's own exporter.
@ -285,7 +318,10 @@ class TenantTracerCache:
self._routed_exporter(spec, credential_headers, project_headers, endpoint)
for spec in self._config.exporters
]
return self._config.model_copy(update={"exporters": exporters})
update: Final = (
{"exporters": exporters} if service_name is None else {"exporters": exporters, "service_name": service_name}
)
return self._config.model_copy(update=update)
def _routed_exporter(
self,

170
litellm/integrations/shadow_eval_logger.py
View file

@ -38,6 +38,7 @@ from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalD
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
if TYPE_CHECKING:
from litellm.proxy.db.shadow_eval_funnel import ShadowEvalFunnelStage
from litellm.proxy.utils import PrismaClient
from litellm.router import Router
from litellm.types.utils import StandardLoggingPayload
@ -386,6 +387,13 @@ def _judge_user_prompt(conversation: str, response_a: str, response_b: str) -> s
)
def _leg_eval_spend(sums: Mapping[str, object]) -> float:
return sum(
float(raw) if isinstance(raw := sums.get(column), (int, float)) else 0.0
for column in ("judge_cost", "shadow_cost", "shadow_classifier_cost")
)
def _job_spend_counter_key(job_id: str) -> str:
return f"spend:shadow_eval:{job_id}"
@ -412,6 +420,15 @@ async def _add_job_spend_to_counter(counter_key: str, cost: float) -> None:
verbose_logger.warning("shadow_eval: spend counter increment failed for %s: %s", counter_key, e)
def _record_funnel_event(job_id: str, stage: "ShadowEvalFunnelStage") -> None:
try:
from litellm.proxy.db.shadow_eval_funnel import record_shadow_eval_funnel_event
record_shadow_eval_funnel_event(job_id, stage)
except Exception as e: # noqa: BLE001 # coverage stats are advisory; sampling must proceed
verbose_logger.debug("shadow_eval: funnel increment failed for %s: %s", job_id, e)
async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
"""Whether the shadowed key or its team is over budget, decided by the same owners
the request path uses, so counter keys and thresholds can never drift from auth's.
@ -452,6 +469,14 @@ async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
return False
def _forwarded_team_id(metadata: Mapping[str, object]) -> str | None:
"""The shadowed key's team, the identity the judge call already carries in its metadata
and the router already selects deployments with. Read here too so the arm choice, which
happens before the router sees the call, is made under the same team."""
team_id: Final = metadata.get("user_api_key_team_id")
return team_id if isinstance(team_id, str) and team_id else None
def _routing_decision(metadata: Mapping[str, object]) -> Mapping[str, object]:
"""The routing decision a pre-routing strategy wrote to a call's metadata, empty when
a plain model served it. Read off the sampled request for the control arm, and off the
@ -466,6 +491,13 @@ def _routed_tier(metadata: Mapping[str, object]) -> str | None:
return str(raw) if raw is not None else None
def _decision_classifier_cost(metadata: Mapping[str, object]) -> float:
"""What the arm's own routing decision says its classifier call billed: the money a
completion cost alone omits, and 0 for a plain model that never classifies."""
raw: Final = _routing_decision(metadata).get("classifier_cost")
return float(raw) if isinstance(raw, (int, float)) else 0.0
def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool:
"""Whether the router under evaluation served this request, which is what decides
the direction it belongs to. A forward job skips its own router's traffic, since
@ -481,6 +513,7 @@ class _CallFailure:
error: str
cost: float = 0.0
classifier_cost: float = 0.0
@dataclass(frozen=True, slots=True)
@ -491,6 +524,7 @@ class _ShadowResponse:
model: str
tier: str | None
cost: float
classifier_cost: float
@dataclass(frozen=True, slots=True)
@ -567,6 +601,7 @@ class ShadowEvalLogger(CustomLogger):
jobs_cache: InMemoryCache | None = None,
job_spend_reader: Callable[[str, float, float], Awaitable[float]] | None = None,
job_spend_writer: Callable[[str, float], Awaitable[None]] | None = None,
funnel_recorder: Callable[[str, "ShadowEvalFunnelStage"], None] | None = None,
) -> None:
"""Providers are callables so the proxy's lazily-initialized globals are resolved
at call time, not at logger construction. The spend reader and writer wrap the
@ -576,6 +611,7 @@ class ShadowEvalLogger(CustomLogger):
self._jobs_cache = jobs_cache or _jobs_cache
self._read_job_spend = job_spend_reader or _job_spend_from_counter
self._write_job_spend = job_spend_writer or _add_job_spend_to_counter
self._record_funnel = funnel_recorder or _record_funnel_event
self._inflight_shadow_tasks: int = 0
# Starts per job since the last cache fill, never decremented within a
# generation; the refill absorbs written rows and resets.
@ -602,7 +638,8 @@ class ShadowEvalLogger(CustomLogger):
await prisma.db.litellm_shadowevalattempt.group_by(
by=["job_id"],
count=True,
sum={"judge_cost": True, "shadow_cost": True}, # mutable-ok: Prisma aggregate spec
# mutable-ok: Prisma aggregate spec
sum={"judge_cost": True, "shadow_cost": True, "shadow_classifier_cost": True},
where={"job_id": {"in": [str(record.id) for record in records]}}, # mutable-ok: Prisma filter
)
if records
@ -611,8 +648,7 @@ class ShadowEvalLogger(CustomLogger):
attempt_stats: Final = { # mutable-ok: frozen snapshot of the grouped read
str(row["job_id"]): (
int(row["_count"]["_all"]),
float((row["_sum"] or {}).get("judge_cost") or 0.0)
+ float((row["_sum"] or {}).get("shadow_cost") or 0.0),
_leg_eval_spend(row["_sum"] or _EMPTY_METADATA),
)
for row in grouped or []
}
@ -638,6 +674,32 @@ class ShadowEvalLogger(CustomLogger):
#### hook ####
def _sampled_jobs(
self,
active_jobs: Sequence[ActiveShadowEvalJob],
request_metadata: Mapping[str, object],
request_id: str,
) -> tuple[ActiveShadowEvalJob, ...]:
"""The jobs that sample this request. A key can hold one job per direction, and a
request routed by one job's router while bypassing the other's qualifies for both;
each is separately budgeted, so both fire. An admitting job that loses the sampling
dice is counted, so results can weigh judged rows against the traffic they stand for."""
eligible: list[ActiveShadowEvalJob] = [] # mutable-ok: bucketed per-job admission
now: Final = datetime.now(timezone.utc)
for job in active_jobs:
if (
now >= job.ends_at
or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns
or (job.max_budget is not None and job.spend >= job.max_budget)
or _request_was_routed_by(request_metadata, job.router_name) != (job.direction == "reverse")
):
continue
if not _sample_hits(request_id, job.id, job.shadow_percentage):
self._record_funnel(job.id, "not_sampled")
continue
eligible.append(job)
return tuple(eligible)
async def async_log_success_event(
self,
kwargs: Mapping[str, object],
@ -669,18 +731,8 @@ class ShadowEvalLogger(CustomLogger):
return # only surfaces this table can normalize are comparable; unknown types fail closed
if ops.wire_params and _request_mutating_guardrail_ran(request_metadata):
return # the wire-body snapshot predates the rewrite; replaying it would resurrect stripped content
# A key can hold one job per direction, and a request routed by one job's
# router while bypassing the other's qualifies for both. Each is separately
# budgeted, so both fire; the request is normalized once, and only when at
# least one job sampled it.
eligible: Final = tuple(
job
for job in (await self._active_jobs()).get(str(api_key_hash), ())
if datetime.now(timezone.utc) < job.ends_at
and job.attempts + self._job_starts.get(job.id, 0) < job.max_turns
and (job.max_budget is None or job.spend < job.max_budget)
and _sample_hits(request_id, job.id, job.shadow_percentage)
and _request_was_routed_by(request_metadata, job.router_name) == (job.direction == "reverse")
eligible: Final = self._sampled_jobs(
(await self._active_jobs()).get(str(api_key_hash), ()), request_metadata, request_id
)
if not eligible:
return
@ -691,12 +743,18 @@ class ShadowEvalLogger(CustomLogger):
response_obj,
)
if sample is None:
for job in eligible:
self._record_funnel(job.id, "unjudgeable")
return
messages, shadow_params, real_text = sample
control_tier: Final = _routed_tier(request_metadata)
real_cost: Final = float(payload.get("response_cost") or 0.0)
real_cache_hit: Final = payload.get("cache_hit") is True
real_classifier_cost: Final = _decision_classifier_cost(request_metadata)
for job in eligible:
if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS:
return
self._record_funnel(job.id, "shed")
continue
self._job_starts[job.id] = self._job_starts.get(job.id, 0) + 1
self._inflight_shadow_tasks += 1
asyncio.create_task(
@ -706,6 +764,9 @@ class ShadowEvalLogger(CustomLogger):
messages=messages,
real_text=real_text,
real_model=payload.get("model") or "",
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
control_tier=control_tier,
shadow_params=shadow_params,
parent_metadata=MappingProxyType(dict(request_metadata)), # mutable-ok: frozen snapshot
@ -726,37 +787,66 @@ class ShadowEvalLogger(CustomLogger):
messages: Sequence[Mapping[str, object]],
real_text: str,
real_model: str,
real_cost: float,
real_classifier_cost: float,
real_cache_hit: bool,
control_tier: str | None,
shadow_params: Mapping[str, object],
parent_metadata: Mapping[str, object],
) -> None:
"""Budget gate -> shadow call -> blind judge -> one attempt row. The prisma gate
sits above the dispatch so no provider spend happens without a place to record
the outcome, and the budget read lives here rather than in the success hook."""
"""Budget gate -> shadow call -> blind judge -> one attempt row, and every exit
in exactly one coverage bucket: the gates that decline to spend on an admitted
sample (no DB to record into, an over-budget key, an unverifiable or exhausted
eval budget) count it withheld, so eligible traffic still reconciles as
not_sampled + unjudgeable + shed + withheld + attempt rows. The prisma gate sits
above the dispatch so no provider spend happens without a place to record the
outcome, and the budget read lives here rather than in the success hook."""
prisma: Final = self._prisma_provider()
try:
if prisma is None:
self._record_funnel(job.id, "withheld")
return
if await _key_or_team_is_over_budget(parent_metadata):
self._record_funnel(job.id, "withheld")
return
if job.max_budget is not None:
try:
spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget)
except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it
verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e)
self._record_funnel(job.id, "withheld")
return
if spend >= job.max_budget:
self._record_funnel(job.id, "withheld")
return
shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata)
except Exception as e: # noqa: BLE001 # detached task: nothing billed yet, record and never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
await self._record_attempt(
prisma, job, request_id, control_tier, outcome="error", error=f"pipeline error: {e}"
prisma,
job,
request_id,
control_tier,
outcome="error",
error=f"pipeline error: {e}",
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
return
if isinstance(shadow, _CallFailure):
await self._record_attempt(
prisma, job, request_id, control_tier, outcome="error", error=shadow.error, shadow_cost=shadow.cost
prisma,
job,
request_id,
control_tier,
outcome="error",
error=shadow.error,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
return
# From here the shadow call has billed, so every exit records its cost.
@ -779,6 +869,10 @@ class ShadowEvalLogger(CustomLogger):
shadow=shadow,
judge_cost=verdict.cost,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
return
await self._record_attempt(
@ -792,6 +886,10 @@ class ShadowEvalLogger(CustomLogger):
confidence=verdict.confidence,
judge_cost=verdict.cost,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
except Exception as e: # noqa: BLE001 # detached task: the shadow call billed, record its cost, never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
@ -804,6 +902,10 @@ class ShadowEvalLogger(CustomLogger):
error=f"pipeline error: {e}",
shadow=shadow,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
async def _record_attempt(
@ -814,15 +916,20 @@ class ShadowEvalLogger(CustomLogger):
control_tier: str | None,
*,
outcome: str,
real_cost: float,
real_classifier_cost: float,
real_cache_hit: bool,
shadow: _ShadowResponse | None = None,
real_model: str = "",
confidence: float | None = None,
judge_cost: float = 0.0,
shadow_cost: float = 0.0,
shadow_classifier_cost: float = 0.0,
error: str | None = None,
) -> None:
if judge_cost + shadow_cost > 0:
await self._write_job_spend(_job_spend_counter_key(job.id), judge_cost + shadow_cost)
eval_spend: Final = judge_cost + shadow_cost + shadow_classifier_cost
if eval_spend > 0:
await self._write_job_spend(_job_spend_counter_key(job.id), eval_spend)
if prisma is None:
return
try:
@ -837,6 +944,10 @@ class ShadowEvalLogger(CustomLogger):
"confidence": confidence,
"judge_cost": judge_cost,
"shadow_cost": shadow_cost,
"shadow_classifier_cost": shadow_classifier_cost,
"real_cost": real_cost,
"real_classifier_cost": real_classifier_cost,
"real_cache_hit": real_cache_hit,
"error": error[:_MAX_ERROR_CHARS] if error else None,
}
)
@ -873,15 +984,23 @@ class ShadowEvalLogger(CustomLogger):
)
except Exception as e: # noqa: BLE001 # provider errors become error rows, not crashes
verbose_logger.debug("shadow_eval: router call failed: %s", e)
return _CallFailure(f"shadow router call failed: {_failure_detail(e)}")
return _CallFailure(
f"shadow router call failed: {_failure_detail(e)}",
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
text: Final = _chat_final_text(response)
if not text:
return _CallFailure("shadow router returned an empty response", cost=_call_cost(response))
return _CallFailure(
"shadow router returned an empty response",
cost=_call_cost(response),
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
return _ShadowResponse(
text=text,
model=str(getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or ""),
tier=_routed_tier(shadow_metadata),
cost=_call_cost(response),
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
async def _call_judge(
@ -915,6 +1034,7 @@ class ShadowEvalLogger(CustomLogger):
self._router_provider(),
judge_model,
judge_messages, # pyright: ignore[reportArgumentType] # plain SDK message dicts
team_id=_forwarded_team_id(parent_metadata),
temperature=0,
max_tokens=JUDGE_MAX_OUTPUT_TOKENS,
response_format=PAIRWISE_JUDGE_RESPONSE_FORMAT,

193
litellm/litellm_core_utils/audio_utils/subtitle_utils.py Normal file
View file

@ -0,0 +1,193 @@
"""Provider-agnostic SRT/WebVTT subtitle synthesis from timestamped transcription tokens."""
from collections.abc import Sequence
from dataclasses import dataclass
from itertools import accumulate, chain
from typing import Final
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
CUE_MAX_TOKENS: Final = 15
CUE_MAX_DURATION_MS: Final = 5000
SRT_RESPONSE_FORMAT: Final = "srt"
VTT_RESPONSE_FORMAT: Final = "vtt"
SUBTITLE_RESPONSE_FORMATS: Final = frozenset((SRT_RESPONSE_FORMAT, VTT_RESPONSE_FORMAT))
@dataclass(frozen=True, slots=True)
class SubtitleToken:
text: str
start_ms: int | None = None
end_ms: int | None = None
speaker: str | int | None = None
@dataclass(frozen=True, slots=True)
class SubtitleCue:
start_ms: int
end_ms: int
text: str
@dataclass(frozen=True, slots=True)
class _CueAccumulator:
texts: tuple[str, ...] = ()
start_ms: int | None = None
end_ms: int | None = None
speaker: str | int | None = None
def _completed_cue(accumulator: _CueAccumulator) -> tuple[SubtitleCue, ...]:
if not accumulator.texts or accumulator.start_ms is None:
return ()
text: Final = "".join(accumulator.texts).strip()
if not text:
return ()
end_ms: Final = accumulator.end_ms if accumulator.end_ms is not None else accumulator.start_ms
return (SubtitleCue(start_ms=accumulator.start_ms, end_ms=end_ms, text=text),)
def _cue_break_reached(accumulator: _CueAccumulator, token: SubtitleToken) -> bool:
if len(accumulator.texts) >= CUE_MAX_TOKENS:
return True
return (
accumulator.start_ms is not None
and token.start_ms is not None
and token.start_ms - accumulator.start_ms >= CUE_MAX_DURATION_MS
)
_AbsorbStep = tuple[tuple[SubtitleCue, ...], _CueAccumulator]
def _absorb_token(accumulator: _CueAccumulator, token: SubtitleToken) -> _AbsorbStep:
if token.start_ms is None and accumulator.start_ms is None:
return (), accumulator
if token.speaker is not None and token.speaker != accumulator.speaker:
return _completed_cue(accumulator), _CueAccumulator(
texts=(token.text,),
start_ms=token.start_ms,
end_ms=token.end_ms,
speaker=token.speaker,
)
if _cue_break_reached(accumulator, token):
return _completed_cue(accumulator), _CueAccumulator(
texts=(token.text,),
start_ms=token.start_ms,
end_ms=token.end_ms,
speaker=accumulator.speaker,
)
return (), _CueAccumulator(
texts=(*accumulator.texts, token.text),
start_ms=accumulator.start_ms if accumulator.start_ms is not None else token.start_ms,
end_ms=token.end_ms if token.end_ms is not None else accumulator.end_ms,
speaker=accumulator.speaker,
)
def _absorb_step(carry: _AbsorbStep, token: SubtitleToken) -> _AbsorbStep:
return _absorb_token(carry[1], token)
def group_subtitle_tokens_into_cues(tokens: Sequence[SubtitleToken]) -> tuple[SubtitleCue, ...]:
steps: Final = tuple(accumulate(tokens, _absorb_step, initial=((), _CueAccumulator())))
completed: Final = chain.from_iterable(emitted for emitted, _ in steps)
return (*completed, *_completed_cue(steps[-1][1]))
def _format_timestamp(total_ms: int, millis_separator: str) -> str:
clamped: Final = max(total_ms, 0)
hours, hour_remainder = divmod(clamped, 3_600_000)
minutes, minute_remainder = divmod(hour_remainder, 60_000)
seconds, millis = divmod(minute_remainder, 1_000)
return f"{hours:02d}:{minutes:02d}:{seconds:02d}{millis_separator}{millis:03d}"
def _render_srt(cues: Sequence[SubtitleCue]) -> str:
lines: Final = tuple(
line
for index, cue in enumerate(cues, start=1)
for line in (
str(index),
f"{_format_timestamp(cue.start_ms, ',')} --> {_format_timestamp(cue.end_ms, ',')}",
cue.text,
"",
)
)
return "\n".join(lines)
def _render_vtt(cues: Sequence[SubtitleCue]) -> str:
cue_lines: Final = tuple(
line
for cue in cues
for line in (
f"{_format_timestamp(cue.start_ms, '.')} --> {_format_timestamp(cue.end_ms, '.')}",
cue.text,
"",
)
)
return "\n".join(("WEBVTT", "", *cue_lines))
def render_subtitle_tokens_as_srt(tokens: Sequence[SubtitleToken]) -> str:
"""Render tokens as an SRT document; empty string when no token has timestamp data."""
cues: Final = group_subtitle_tokens_into_cues(tokens)
if not cues:
return ""
return _render_srt(cues)
def render_subtitle_tokens_as_vtt(tokens: Sequence[SubtitleToken]) -> str:
"""Render tokens as a WebVTT document; the WEBVTT header is emitted even without cues."""
return _render_vtt(group_subtitle_tokens_into_cues(tokens))
class TranscriptionWordTiming(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
word: str = ""
start: float | None = None
end: float | None = None
speaker: str | None = None
_WORD_TIMINGS_ADAPTER: Final = TypeAdapter(tuple[TranscriptionWordTiming, ...])
def _seconds_to_ms(seconds: float | None) -> int | None:
if seconds is None:
return None
return round(seconds * 1000)
def _word_to_subtitle_token(word: TranscriptionWordTiming) -> SubtitleToken:
return SubtitleToken(
text=f"{word.word} ",
start_ms=_seconds_to_ms(word.start),
end_ms=_seconds_to_ms(word.end),
speaker=word.speaker,
)
def _parse_word_timings(words: object) -> tuple[TranscriptionWordTiming, ...]:
try:
return _WORD_TIMINGS_ADAPTER.validate_python(words)
except ValidationError:
return ()
def synthesize_subtitle_document(words: object, response_format: str) -> str | None:
"""
Build an SRT/VTT document from OpenAI verbose_json-style word dicts
(word/start/end in float seconds, optional speaker). Returns None when the
format is not a subtitle format or the words carry no usable timestamps.
"""
if response_format not in SUBTITLE_RESPONSE_FORMATS:
return None
tokens: Final = tuple(_word_to_subtitle_token(word) for word in _parse_word_timings(words))
cues: Final = group_subtitle_tokens_into_cues(tokens)
if not cues:
return None
return _render_srt(cues) if response_format == SRT_RESPONSE_FORMAT else _render_vtt(cues)

56
litellm/litellm_core_utils/core_helpers.py
View file

@ -454,6 +454,62 @@ def safe_deep_copy(data):
return new_data
def independent_snapshot(
data: dict, # mutable-ok: caller-defined request-payload shape
) -> dict: # mutable-ok: caller-defined request-payload shape
"""
A copy of ``data`` whose top-level keys are deep-copied independently
where possible -- always attempted, regardless of
``litellm.safe_memory_mode``. Unlike ``safe_deep_copy``, which can return
the *original* object outright under that mode (defeating any isolation
guarantee for every key, not just the ones that need it), this never
skips copying wholesale.
Real proxy requests carry ``data["litellm_logging_obj"]`` (a ``Logging``
instance nesting a live OTel span with a real lock) by the time
``pre_call_hook`` runs, which can never be deep-copied. Any individual
key that fails to deep-copy falls back to sharing its original
reference, same crash tolerance as ``safe_deep_copy``'s own per-key
fallback; callers needing true isolation (e.g. a guardrail's
``scan_raw_request`` snapshot) only depend on the keys that are plain,
cleanly-copyable structures (``messages``/``input``,
``metadata``/``litellm_metadata``).
"""
sanitized: Final = {
key: (
{ # mutable-ok: same request-payload shape as data
inner_key: ("placeholder" if inner_key == "litellm_parent_otel_span" else inner_value)
for inner_key, inner_value in value.items()
}
if key in ("metadata", "litellm_metadata") and isinstance(value, dict)
else value
)
for key, value in data.items()
}
def _copied_value(key: str, sanitized_value: object) -> object:
try:
copied_value: Final = copy.deepcopy(sanitized_value)
except Exception: # noqa: BLE001 # any unpicklable value falls back to the original reference for this key only
return data.get(key)
original_value: Final = data.get(key)
if (
key in ("metadata", "litellm_metadata")
and isinstance(copied_value, dict)
and isinstance(original_value, dict)
and "litellm_parent_otel_span" in original_value
):
return { # mutable-ok: same request-payload shape as data
**copied_value,
"litellm_parent_otel_span": original_value["litellm_parent_otel_span"],
}
return copied_value
return { # mutable-ok: same request-payload shape as data
key: _copied_value(key, value) for key, value in sanitized.items()
}
def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
"""
Recursively filter out Exception objects and callable objects from dicts/lists.

4
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -2222,6 +2222,8 @@ def _map_exception_by_status(
status_code: Final = original_exception.status_code if hasattr(original_exception, "status_code") else None
if not isinstance(status_code, int) or status_code < 400:
return
if getattr(original_exception, "status_code_is_synthesized", False):
return
message: Final = f"{exception_provider} - {error_str}"
response: Final = original_exception.response if hasattr(original_exception, "response") else None
match status_code:
@ -2341,6 +2343,7 @@ def exception_type(
litellm_response_headers: Final = _get_response_headers(original_exception=original_exception)
try:
error_str = redact_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception)
extra_information = ""
if model or custom_llm_provider:
if hasattr(original_exception, "message"):
error_str = (
@ -2357,7 +2360,6 @@ def exception_type(
# Common Extra information needed for all providers
# We pass num retries, api_base, vertex_deployment etc to the exception here
################################################################################
extra_information = ""
try:
_api_base: Final = litellm.get_api_base(model=model, optional_params=extra_kwargs)
messages: Final = litellm.get_first_chars_messages(kwargs=completion_kwargs)

14
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -2,7 +2,7 @@ from typing import Final, cast
from urllib.parse import urlparse
import litellm
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.litellm_core_utils.fallback_generalizations import (
match_routing_generalization,
)
@ -127,6 +127,18 @@ def handle_anthropic_text_model_custom_llm_provider(
return model, custom_llm_provider
def declared_authenticating_provider(model: str, custom_llm_provider: str | None = None) -> str | None:
"""The authenticating provider this pair already names, or None.
get_llm_provider runs the OAuth device flow for github_copilot and chatgpt, because their
provider info includes the key it unlocks. For a metadata question that flow is pure hazard,
and for a declared pair the resolver's answer is the declaration itself, so metadata callers
adopt the declaration instead of resolving.
"""
declared: Final = custom_llm_provider or model.split("/", 1)[0]
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
def get_llm_provider(
model: str,
custom_llm_provider: str | None = None,

5
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -2,6 +2,7 @@ from typing import Final, Literal
import litellm
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.types.utils import LlmProviders, LlmProvidersSet
@ -30,6 +31,10 @@ def get_supported_openai_params(
- List if custom_llm_provider is mapped
- None if unmapped
"""
if not custom_llm_provider:
custom_llm_provider = declared_authenticating_provider(
model
) # rebind-ok: resolving would run the provider's OAuth flow
if not custom_llm_provider:
try:
custom_llm_provider = litellm.get_llm_provider(model=model)[1]

20
litellm/litellm_core_utils/litellm_logging.py
View file

@ -888,7 +888,10 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_management_logger: CustomLogger | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs
) -> tuple[str, list[AllMessageValues], dict]:
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
model=model,
non_default_params=non_default_params,
@ -898,6 +901,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if custom_logger:
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
(
model,
messages,
@ -913,6 +917,11 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_label=prompt_label,
prompt_version=prompt_version,
)
if request_kwargs is not None:
AnthropicCacheControlHook.record_gateway_injection(
request_kwargs,
AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
)
self.messages = messages
return model, messages, non_default_params
@ -928,7 +937,10 @@ class Logging(LiteLLMLoggingBaseClass):
tools: list[dict] | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs
) -> tuple[str, list[AllMessageValues], dict]:
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
model=model,
tools=tools,
@ -939,6 +951,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if custom_logger:
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
(
model,
messages,
@ -956,6 +969,11 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_label=prompt_label,
prompt_version=prompt_version,
)
if request_kwargs is not None:
AnthropicCacheControlHook.record_gateway_injection(
request_kwargs,
AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
)
self.messages = messages
return model, messages, non_default_params
@ -6040,7 +6058,7 @@ def get_standard_logging_object_payload(
prompt_tokens=usage_dict.get("prompt_tokens", 0),
completion_tokens=usage_dict.get("completion_tokens", 0),
request_tags=request_tags,
end_user=end_user_id or "",
end_user=end_user_id,
api_base=StandardLoggingPayloadSetup.strip_trailing_slash(litellm_params.get("api_base", "")) or "",
model_group=_model_group,
model_id=_model_id,

24
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -7,7 +7,9 @@ from typing import Any, Final, Literal
import litellm
from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests_from_usage,
)
from litellm.types.llms.openai import (
FileSearchTool,
ResponsesAPIResponse,
@ -368,7 +370,7 @@ class StandardBuiltInToolCostTracking:
get_anthropic_web_search_requests_from_response,
)
if usage is not None and (get_web_search_requests(getattr(usage, "server_tool_use", None)) is not None):
if usage is not None and (get_web_search_requests_from_usage(usage) is not None):
return usage
web_search_requests: Final = get_anthropic_web_search_requests_from_response(response_object)
if web_search_requests is None:
@ -416,7 +418,7 @@ class StandardBuiltInToolCostTracking:
# Anthropic Claude (direct API and Vertex AI) uses server_tool_use.web_search_requests.
# Without this check, Claude ModelResponse always falls through to return False
# and _handle_web_search_cost() is never called.
if hasattr(usage, "server_tool_use") and get_web_search_requests(usage.server_tool_use) is not None:
if get_web_search_requests_from_usage(usage) is not None:
return True
# xAI reports usage.server_side_tool_usage_details.web_search_calls; a searched
# answer with no url_citation annotations has no other chat-path signal
@ -429,16 +431,12 @@ class StandardBuiltInToolCostTracking:
response_object=response_object, output_type="web_search_call"
)
elif usage is not None:
if (
hasattr(usage, "server_tool_use")
and get_web_search_requests(usage.server_tool_use) is not None
or (
hasattr(usage, "prompt_tokens_details")
and usage.prompt_tokens_details is not None
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
and hasattr(usage.prompt_tokens_details, "web_search_requests")
and usage.prompt_tokens_details.web_search_requests is not None
)
if get_web_search_requests_from_usage(usage) is not None or (
hasattr(usage, "prompt_tokens_details")
and usage.prompt_tokens_details is not None
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
and hasattr(usage.prompt_tokens_details, "web_search_requests")
and usage.prompt_tokens_details.web_search_requests is not None
):
return True
if _usage_reports_server_side_web_search_calls(usage):

51
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1,6 +1,7 @@
# What is this?
## Helper utilities for cost_per_token()
import re
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
@ -72,6 +73,19 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
return value if isinstance(value, int) else None
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+")
def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None:
value: Final = None if optional_params is None else optional_params.get(key)
return value if isinstance(value, str) else None
def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None:
value: Final = _requested_image_param(optional_params, "size")
return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
def get_web_search_requests(server_tool_use: Any) -> int | None:
"""
Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value
@ -92,6 +106,16 @@ def get_web_search_requests(server_tool_use: Any) -> int | None:
return getattr(server_tool_use, "web_search_requests", None)
def get_web_search_requests_from_usage(usage: Usage) -> int | None:
"""Read ``web_search_requests`` from a ``Usage``'s ``server_tool_use``.
``Usage`` deletes unset optional fields from ``__dict__`` (see
``SafeAttributeModel``), so direct attribute access can raise
``AttributeError``; ``getattr`` with a default is required here.
"""
return get_web_search_requests(getattr(usage, "server_tool_use", None))
def _is_above_128k(tokens: float) -> bool:
if tokens > 128000:
return True
@ -1301,12 +1325,13 @@ class CostCalculatorUtils:
cost_calculator as vertex_ai_image_cost_calculator,
)
if size is None:
size = completion_response.size or "1024-x-1024"
if quality is None:
quality = completion_response.quality or "standard"
if n is None:
n = len(completion_response.data) if completion_response.data else 0
resolved_size: Final = (
size or completion_response.size or _requested_image_size(optional_params) or "1024-x-1024"
)
resolved_quality: Final = (
quality or completion_response.quality or _requested_image_param(optional_params, "quality") or "standard"
)
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
if isinstance(completion_response, ImageResponse):
@ -1318,7 +1343,7 @@ class CostCalculatorUtils:
if isinstance(completion_response, ImageResponse):
return bedrock_image_cost_calculator(
model=model,
size=size,
size=resolved_size,
image_response=completion_response,
optional_params=optional_params,
)
@ -1414,19 +1439,19 @@ class CostCalculatorUtils:
# Fall through to default for DALL-E models
return default_image_cost_calculator(
model=model,
quality=quality,
quality=resolved_quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
n=resolved_n,
size=resolved_size,
optional_params=optional_params,
)
else:
return default_image_cost_calculator(
model=model,
quality=quality,
quality=resolved_quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
n=resolved_n,
size=resolved_size,
optional_params=optional_params,
)
return 0.0

69
litellm/litellm_core_utils/llm_judge.py
View file

@ -4,7 +4,9 @@ from __future__ import annotations
import json
import re
from typing import TYPE_CHECKING, Final
from dataclasses import dataclass
from functools import lru_cache
from typing import TYPE_CHECKING, Final, Literal
import litellm
@ -56,17 +58,62 @@ def extract_text_from_content(content: object) -> str:
return ""
def router_resolves_model(router: Router | None, model: str) -> bool:
"""Whether the model name resolves through the proxy's router (configured deployment
or model-group alias), the same check the judge dispatch itself makes, so start-time
validation cannot accept a name the call path then fails on."""
return router is not None and bool(model in router.model_group_alias or router.get_model_list(model_name=model))
@lru_cache(maxsize=512)
def _provider_qualified(model: str) -> str | None:
"""`model` in the one spelling litellm itself resolves it to, or None if it maps to no
provider.
A deployment may be configured as `openai/gpt-4o` and a judge given as `gpt-4o`; both
reach the same model, so an identity that keeps them apart reports two models where
there is one. None is a different answer from "unchanged": a name that is already
provider-qualified normalises to itself, and reading that as a failure would call every
correctly-spelled public model unresolvable.
"""
try:
stripped, provider, _, _ = litellm.get_llm_provider(model=model)
except Exception: # noqa: BLE001 # an unmapped name has no provider, which is the answer
return None
return f"{provider}/{stripped}" if provider and stripped else None
@dataclass(frozen=True, slots=True)
class JudgeTarget:
"""Where a call to one model name goes for one caller, and what answers it.
The single answer to that question: the resolvability gate, the judge-vs-candidate
gate and the dispatch all read it, so none of them can decide it differently. Splitting
it is what let start-time validation accept a team's own model while dispatch sent the
literal name to the SDK.
"""
via: Literal["router", "sdk", "nothing"]
models: frozenset[str]
def judge_target(router: Router | None, model: str, team_id: str | None = None) -> JudgeTarget:
"""Resolve `model` the way a call from `team_id` would be.
Three outcomes and no others: the router serves it (a deployment, a team-public name,
an alias, a routing group or a wildcard, exactly the channels `get_model_list`
composes); the SDK serves it because litellm recognises the provider; or nothing does,
which is the only case a caller may refuse on.
`team_id` is part of the question, not a refinement of it. A team-public name resolves
only for its own team and a team's own deployment resolves for nobody else, so asking
without it answers for a caller who does not exist.
"""
served: Final = router.resolved_litellm_models(model, team_id=team_id) if router is not None else ()
if served:
return JudgeTarget("router", frozenset(_provider_qualified(m) or m for m in served))
qualified: Final = _provider_qualified(model)
return JudgeTarget("sdk", frozenset({qualified})) if qualified is not None else JudgeTarget("nothing", frozenset())
async def judge_acompletion(
router: Router | None,
judge_model: str,
messages: list[AllMessageValues], # mutable-ok: the SDK acompletion signature takes a list
team_id: str | None = None,
**params: object,
) -> ModelResponse:
"""Dispatch a judge call through the proxy's router when the judge model is a
@ -74,9 +121,13 @@ async def judge_acompletion(
provider-qualified public names. The router path never retries or falls back:
a failed judge call is the caller's counted failure, not a spend multiplier.
Sampling preferences are advisory: models that removed sampling params (e.g.
claude-sonnet-5) drop them instead of rejecting the judge call."""
if router_resolves_model(router, judge_model):
return await router.acompletion( # pyright: ignore[reportOptionalMemberAccess] # router_resolves_model implies router is not None
claude-sonnet-5) drop them instead of rejecting the judge call.
The arm is chosen by `judge_target` under the caller's own team, the same call
start-time validation makes, so a judge a team can reach cannot be validated as a
deployment and then dispatched as a public name the SDK has never heard of."""
if judge_target(router, judge_model, team_id).via == "router":
return await router.acompletion( # pyright: ignore[reportOptionalMemberAccess] # a router target implies router is not None
model=judge_model,
messages=messages,
num_retries=0,

40
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -1747,6 +1747,46 @@ def hoist_images_from_tool_messages(
]
def _is_tool_reference_part(part: object) -> bool:
return isinstance(part, dict) and part.get("type") == "tool_reference"
def _tool_message_carries_tool_reference(message: AllMessageValues) -> bool:
if message.get("role") != "tool":
return False
content = message.get("content")
return isinstance(content, list) and any(_is_tool_reference_part(part) for part in content)
def _drop_tool_reference_parts(message: AllMessageValues) -> AllMessageValues:
if not _tool_message_carries_tool_reference(message):
return message
content = cast(list, message.get("content")) # cast-ok: shape checked by _tool_message_carries_tool_reference
remaining_parts = [ # mutable-ok: tool message content must stay a json list
part for part in content if not _is_tool_reference_part(part)
]
new_content = remaining_parts if remaining_parts else ""
rewritten = {**message, "content": new_content} # mutable-ok: chat messages are plain json dicts
return cast(AllMessageValues, rewritten) # cast-ok: dict spread keeps keys like cache_control
def drop_tool_reference_parts_from_tool_messages(
messages: list[AllMessageValues], # mutable-ok: message pipelines type messages as mutable lists
) -> list[AllMessageValues]: # mutable-ok: message pipelines type messages as mutable lists
"""
Remove tool_reference content parts from role:"tool" messages.
The OpenAI chat spec only accepts text in tool messages, so a tool_reference
part carried through the Anthropic adapter makes strict providers reject the
request. The reference names an already-declared tool rather than carrying
content, so it is dropped; a reference-only result keeps its tool message with
empty text so the preceding tool_call stays answered.
"""
if not any(_tool_message_carries_tool_reference(message) for message in messages):
return messages
return [_drop_tool_reference_parts(message) for message in messages] # mutable-ok: pipelines mutate message lists
def _attempt_json_repair(s: str) -> Any | None:
"""
Attempt to repair truncated JSON produced by LLM tool calls.

17
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1412,7 +1412,7 @@ def convert_to_gemini_tool_call_result(
)
except Exception as e:
verbose_logger.warning("Failed to process image in tool response: %s", e)
elif content_type in ("file", "input_file"):
elif content_type in ("file", "input_file"): # pyright: ignore[reportUnnecessaryContains] # loose runtime dict
# Extract file for inline_data (for tool results with PDF, audio, video, etc.)
file_data = content.get("file_data", "")
if not file_data:
@ -1564,14 +1564,23 @@ def convert_to_anthropic_tool_result(
}
"""
anthropic_content: (
str | list[AnthropicMessagesToolResultContent | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam]
str
| list[
AnthropicMessagesToolResultContent
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
| ToolReference
]
) = ""
if isinstance(message["content"], str):
anthropic_content = message["content"]
elif isinstance(message["content"], list):
content_list: Final = message["content"]
anthropic_content_list: list[
AnthropicMessagesToolResultContent | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam
AnthropicMessagesToolResultContent
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
| ToolReference
] = []
for content in content_list:
if content["type"] == "text":
@ -1614,6 +1623,8 @@ def convert_to_anthropic_tool_result(
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
elif content["type"] == "tool_reference":
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":
file_content = cast(ChatCompletionFileObject, content)
_file_block = anthropic_process_openai_file_message(file_content)

20
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -330,6 +330,24 @@ class RealTimeStreaming:
except (AttributeError, TypeError):
pass
def _flush_unbilled_transcription_usage(self) -> None:
if self.provider_config is None:
return
usage: Final = self.provider_config.unbilled_usage_on_session_close(self.model)
if usage is None:
return
flush_event: Final = (
cast( # cast-ok: usage-only partial event, the same shape _capture_transcription_usage logs
OpenAIRealtimeEvents,
{
"type": "conversation.item.input_audio_transcription.completed",
"usage": usage,
},
)
)
self.store_message(flush_event)
self._capture_transcription_usage(flush_event)
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
"""Extract function_call items from response.done events for spend logging."""
try:
@ -955,6 +973,7 @@ class RealTimeStreaming:
transcript = event.get("transcript", "")
self._collect_user_input_from_backend_event(cast(dict, event))
self.store_message(event_str)
self._capture_transcription_usage(event)
await self._send_event_to_client(event, event_str)
blocked = await self.run_realtime_guardrails(
cast(str, transcript),
@ -1068,6 +1087,7 @@ class RealTimeStreaming:
except Exception as e:
verbose_logger.exception("Error in backend to client send messages: %s", e)
finally:
self._flush_unbilled_transcription_usage()
await self.log_messages()
@staticmethod

25
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -239,6 +239,22 @@ class ChunkProcessor:
model_response._hidden_params = chunk.get("_hidden_params", {})
return model_response
@staticmethod
def _get_provider_response_model(
chunks: Sequence["_BaseChunk"],
first_chunk_model: str,
) -> str | None:
models: Final = tuple(
model
for chunk in chunks
if isinstance((hidden_params := chunk.get("_hidden_params")), Mapping)
if isinstance((model := hidden_params.get("provider_response_model")), str) and model
)
return next(
(model for model in models if model != first_chunk_model),
models[0] if models else None,
)
@staticmethod
def apply_provider_assembled_streaming_metadata(
response: ModelResponse,
@ -360,6 +376,15 @@ class ChunkProcessor:
)
response = self.update_model_response_with_hidden_params(model_response=response, chunk=chunk)
provider_response_model: Final = self._get_provider_response_model(
chunks,
first_chunk_model,
)
if provider_response_model is not None:
response._hidden_params = dict( # pyright: ignore[reportPrivateUsage] # ModelResponse exposes no public hidden-params setter
response._hidden_params, # pyright: ignore[reportPrivateUsage] # ModelResponse exposes no public hidden-params getter
provider_response_model=provider_response_model,
)
return response
@staticmethod

58
litellm/litellm_core_utils/streaming_handler.py
View file

@ -8,11 +8,12 @@ import time
import traceback
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final, NoReturn, Protocol, TypeVar, cast
import anyio
import httpx
from pydantic import BaseModel
from pydantic import BaseModel, ValidationError
from typing_extensions import NotRequired, TypedDict
import litellm
@ -182,6 +183,48 @@ class _VertexChunkLike(Protocol):
candidates: Sequence[_VertexCandidateLike]
class _ParsedChunkHiddenParams(BaseModel):
provider_specific_fields: Mapping[str, object] | None = None
def _provider_response_model(chunk: object) -> str | None:
model: Final[object] = chunk.get("model") if isinstance(chunk, Mapping) else getattr(chunk, "model", None)
return model if isinstance(model, str) and model else None
def _parsed_provider_hidden_params(hidden: object) -> _ParsedChunkHiddenParams | None:
if not isinstance(hidden, dict):
return None
try:
return _ParsedChunkHiddenParams.model_validate(hidden)
except ValidationError:
return None
def _provider_hidden_params(
chunk: object,
provider_response_model: str | None,
) -> Mapping[str, object] | None:
hidden: Final[object] = getattr(chunk, "_hidden_params", None)
parsed: Final = _parsed_provider_hidden_params(hidden)
provider_specific_fields: Final[object | None] = (
dict(parsed.provider_specific_fields) # mutable-ok: stream assembly merges provider metadata into this dict
if parsed is not None and parsed.provider_specific_fields
else None
)
params: Final[Mapping[str, object]] = MappingProxyType(
{
key: value
for key, value in (
("provider_response_model", provider_response_model),
("provider_specific_fields", provider_specific_fields),
)
if value is not None
}
)
return params or None
class CustomStreamWrapper:
def __init__(
self,
@ -211,6 +254,7 @@ class CustomStreamWrapper:
self.thinking_content = ""
self.system_fingerprint: str | None = None
self._provider_response_model: str | None = None
self.received_finish_reason: str | None = None
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
self.special_tokens = [
@ -801,7 +845,9 @@ class CustomStreamWrapper:
except Exception as e:
raise e
def model_response_creator(self, chunk: dict | None = None, hidden_params: dict | None = None):
def model_response_creator(
self, chunk: dict | None = None, hidden_params: Mapping[str, object] | None = None
) -> ModelResponseStream:
_model: Final = self._cached_model_name
_logging_obj_llm_provider: Final = self._cached_logging_llm_provider
@ -1504,7 +1550,12 @@ class CustomStreamWrapper:
def chunk_creator(self, chunk: Any):
if hasattr(chunk, "id"):
self.response_id = chunk.id
model_response = self.model_response_creator()
provider_response_model: Final = _provider_response_model(chunk)
if provider_response_model is not None:
self._provider_response_model = provider_response_model
model_response = self.model_response_creator(
hidden_params=_provider_hidden_params(chunk, self._provider_response_model)
)
response_obj: dict[str, Any] = {}
try:
# return this for all models
@ -2318,6 +2369,7 @@ class CustomStreamWrapper:
partial_response: Final = litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
logging_obj=self.logging_obj,
)
if partial_response is None:
return

81
litellm/litellm_core_utils/token_counter.py
View file

@ -3,7 +3,7 @@
import base64
import io
import struct
from collections.abc import Callable, Mapping
from collections.abc import Callable, Iterable, Mapping, Sequence
from typing import Any, Final, Literal, cast
import tiktoken
@ -25,14 +25,21 @@ from litellm.litellm_core_utils.default_encoding import encoding as default_enco
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,
AnthropicContentParamSourceFileId,
AnthropicContentParamSourceUrl,
AnthropicMessagesDocumentParam,
AnthropicMessagesImageParam,
AnthropicMessagesTextParam,
AnthropicMessagesToolResultParam,
AnthropicMessagesToolUseParam,
)
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionDocumentObject,
ChatCompletionNamedToolChoiceParam,
ChatCompletionToolParam,
OpenAIMessageContent,
OpenAIMessageContentListBlock,
)
from litellm.types.utils import Message, SelectTokenizerResponse
@ -346,7 +353,7 @@ def token_counter(
model="",
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
text: str | list[str] | None = None,
messages: list[AllMessageValues | Message] | None = None,
messages: Sequence[AllMessageValues | Message] | None = None,
count_response_tokens: bool | None = False,
tools: list[ChatCompletionToolParam] | None = None,
tool_choice: ChatCompletionNamedToolChoiceParam | None = None,
@ -646,6 +653,46 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type:
return expected_cls
def _anthropic_image_source_data(
source: AnthropicContentParamSource | AnthropicContentParamSourceUrl | AnthropicContentParamSourceFileId,
) -> str:
if source["type"] == "base64":
data: Final = source.get("data")
if not data:
return ""
media_type: Final = source.get("media_type") or "image/png"
return f"data:{media_type};base64,{data}"
if source["type"] == "url":
return source.get("url") or ""
return ""
def _count_document_tokens(
document: ChatCompletionDocumentObject | AnthropicMessagesDocumentParam,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
) -> int:
source: Final = document["source"]
metadata_tokens: Final = sum(
count_function(text) for text in (document.get("title"), document.get("context")) if text
)
if source["type"] == "text":
return metadata_tokens + count_function(source["data"])
if source["type"] == "content":
content: Final = source["content"]
if isinstance(content, str):
return metadata_tokens + count_function(content)
return metadata_tokens + _count_content_list(
count_function, content, use_default_image_token_count, default_token_count
)
return metadata_tokens + calculate_img_tokens(
data=_anthropic_image_source_data(source),
mode="auto",
use_default_image_token_count=use_default_image_token_count,
)
def _count_anthropic_content(
content: Mapping[str, Any],
count_function: TokenCounterFunction,
@ -697,13 +744,17 @@ def _count_anthropic_content(
def _count_content_list(
count_function: TokenCounterFunction,
content_list: OpenAIMessageContent,
content_list: str
| Iterable[
OpenAIMessageContentListBlock
| AnthropicMessagesTextParam
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
],
use_default_image_token_count: bool,
default_token_count: int | None,
) -> int:
"""
Recursively count tokens from a list of content blocks.
"""
"""Recursively count tokens from a list of content blocks."""
try:
num_tokens = 0
for c in content_list:
@ -714,6 +765,19 @@ def _count_content_list(
elif c["type"] == "image_url":
image_url = c.get("image_url")
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
elif c["type"] == "image":
num_tokens += calculate_img_tokens(
data=_anthropic_image_source_data(c["source"]),
mode="auto",
use_default_image_token_count=use_default_image_token_count,
)
elif c["type"] == "document":
num_tokens += _count_document_tokens(
c,
count_function,
use_default_image_token_count,
default_token_count,
)
elif c["type"] in ("tool_use", "tool_result"):
num_tokens += _count_anthropic_content(
c,
@ -742,7 +806,8 @@ def _count_content_list(
content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__
raise ValueError(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)."
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)."
)
return num_tokens
except Exception as e:

24
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -24,10 +24,12 @@ from litellm._logging import verbose_proxy_logger
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
is_provider_native_tool_dict,
)
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.utils import (
anthropic_tool_name,
anthropic_tool_names,
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
@ -360,7 +362,13 @@ class AnthropicMessagesHandler(BaseTranslation):
structured_messages: Final = [full_structured_messages[index] for index in scoped_message_indices]
tools_to_check: Final[list[ChatCompletionToolParam]] = (
[] if scan_only_tool_results else chat_completion_compatible_request.get("tools", [])
[]
if scan_only_tool_results
else [
tool
for tool in chat_completion_compatible_request.get("tools", [])
if not is_provider_native_tool_dict(tool)
]
)
# Step 1: Extract all text content and images
@ -419,7 +427,10 @@ class AnthropicMessagesHandler(BaseTranslation):
tool_name=anthropic_tool_name,
)
if scan_only_tool_results
else anthropic_tools
else [
*(tool for tool in data.get("tools") or [] if is_provider_native_tool_dict(tool)),
*anthropic_tools,
]
)
guardrailed_structured_messages: Final = guardrailed_inputs.get("structured_messages")
@ -677,12 +688,9 @@ class AnthropicMessagesHandler(BaseTranslation):
)
def extract_request_tool_names(self, data: dict) -> list[str]:
"""Extract tool names from Anthropic messages request (tools[].name)."""
names: Final[list[str]] = []
for tool in data.get("tools") or []:
if isinstance(tool, dict) and tool.get("name"):
names.append(str(tool["name"]))
return names
"""Extract every tool name in an Anthropic messages request: tools[].name, plus
tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
@classmethod
def _extract_input_text_and_images(

46
litellm/llms/anthropic/common_utils.py
View file

@ -974,19 +974,25 @@ def strip_advisor_blocks_from_messages(messages: list[Any], replace_with_text: b
return messages
def is_anthropic_invalid_thinking_signature_error(error_text: str) -> bool:
def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool:
"""
Detect Anthropic 400 errors caused by missing or invalid thinking signatures.
Detect Anthropic 400 errors caused by invalid thinking blocks in replayed
history: a missing or invalid signature, or a block with empty thinking text.
Known error formats:
{"message":"messages.2.content.0.thinking.signature.str: Input should be a valid string"}
messages.N.content.M.thinking.signature.str: Input should be a valid string
messages.N.content.M: Invalid `signature` in `thinking` block
messages.N.content.M.thinking: each thinking block must contain thinking
"""
if not error_text:
return False
lower: Final = error_text.lower()
return "thinking" in lower and "signature" in lower and ("invalid" in lower or "valid string" in lower)
if "thinking" not in lower:
return False
if "signature" in lower and ("invalid" in lower or "valid string" in lower):
return True
return "must contain thinking" in lower
def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[Any]:
@ -1028,22 +1034,29 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
data.pop("thinking", None)
def strip_empty_text_blocks_from_anthropic_messages(
def strip_empty_content_blocks_from_anthropic_messages(
messages: list[Any],
) -> list[Any]:
"""
Return a new message list with empty or whitespace-only ``{"type": "text"}``
content blocks removed.
and ``{"type": "thinking"}`` content blocks removed.
Anthropic's API rejects requests containing such blocks with
``"messages: text content blocks must be non-empty"``, but assistant
messages from Anthropic routinely arrive with ``{"type": "text", "text": ""}``
alongside ``tool_use`` blocks (see anthropics/anthropic-sdk-python#461).
``"messages: text content blocks must be non-empty"`` and
``"messages.N.content.M.thinking: each thinking block must contain
thinking"`` respectively. Assistant messages routinely arrive with
``{"type": "text", "text": ""}`` alongside ``tool_use`` blocks (see
anthropics/anthropic-sdk-python#461), and a turn served by a
non-Anthropic reasoning model through the /v1/messages bridge can carry
``{"type": "thinking", "thinking": ""}`` when the model produced no
reasoning text (e.g. it went straight to parallel tool calls).
Multi-turn tool-use clients (e.g. Claude Code) loop these prior responses
back as conversation history, which then causes the next request to 400
on the unified ``/v1/messages`` path. ``/v1/chat/completions`` already
handles this in ``anthropic_messages_pt``; this helper provides the
equivalent guarantee for the native Anthropic Messages path.
``redacted_thinking`` blocks are never touched: they carry opaque
``data`` instead of thinking text.
Messages whose content is a list and becomes empty after stripping are
omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`.
@ -1056,7 +1069,7 @@ def strip_empty_text_blocks_from_anthropic_messages(
out.append(m)
continue
content = m["content"]
filtered = [b for b in content if not _is_empty_text_block(b)]
filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_thinking_block(b)]
if len(filtered) == len(content):
out.append(m)
elif filtered:
@ -1071,6 +1084,21 @@ def _is_empty_text_block(block: Any) -> bool:
return not isinstance(text, str) or not text.strip()
def is_empty_thinking_block(block: object) -> bool:
"""
True for a ``{"type": "thinking"}`` content block whose thinking text is
missing, not a string, or empty/whitespace-only after ``.strip()``.
Anthropic rejects such blocks with ``"each thinking block must contain
thinking"`` (whitespace-only included, verified live), regardless of any
signature they carry. ``redacted_thinking`` blocks are a different type
and always return False.
"""
if not isinstance(block, dict) or block.get("type") != "thinking":
return False
thinking: Final = block.get("thinking")
return not isinstance(thinking, str) or not thinking.strip()
def normalize_anthropic_tool_use_id(raw_id: str) -> str:
"""
Normalize a tool_use / tool_result id for Anthropic's ``^[a-zA-Z0-9_-]+$``

4
litellm/llms/anthropic/cost_calculation.py
View file

@ -10,7 +10,7 @@ from pydantic import BaseModel, ValidationError
from litellm.litellm_core_utils.llm_cost_calc.utils import (
generic_cost_per_token,
get_provider_specific_geo_multiplier,
get_web_search_requests,
get_web_search_requests_from_usage,
)
if TYPE_CHECKING:
@ -104,7 +104,7 @@ def get_cost_for_anthropic_web_search(
if usage is None:
return 0.0
web_search_requests: Final = get_web_search_requests(getattr(usage, "server_tool_use", None))
web_search_requests: Final = get_web_search_requests_from_usage(usage)
if web_search_requests is None:
return 0.0

8
litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file

@ -1029,6 +1029,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
@staticmethod
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
from litellm.llms.anthropic.common_utils import is_empty_thinking_block
choice: Final = chunk.choices[0]
if choice.finish_reason is not None:
return False
@ -1039,7 +1041,11 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
return False
if getattr(delta, "reasoning_content", None):
return False
if getattr(delta, "thinking_blocks", None):
# thinking_blocks whose entries are all empty (even if signed) must not
# open a block: the emitted {"type": "thinking", "thinking": ""} gets
# replayed as history and Anthropic rejects it (LIT-6357).
thinking_blocks: Final = getattr(delta, "thinking_blocks", None)
if thinking_blocks and any(isinstance(b, dict) and not is_empty_thinking_block(b) for b in thinking_blocks):
return False
return True

264
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -1,8 +1,8 @@
import copy
import hashlib
import json
from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
import litellm
from litellm.llms.anthropic.experimental_pass_through.utils import (
@ -18,6 +18,22 @@ TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LE
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
_ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset(
{"name", "type", "input_schema", "description", "cache_control", "strict"}
)
def _is_openai_function_tool(tool: Mapping[str, object]) -> bool:
return tool.get("type") == "function" and "function" in tool
def is_provider_native_tool_dict(tool: Mapping[str, object]) -> bool:
if len(tool) != 1:
return False
key, value = next(iter(tool.items()))
return key not in _ANTHROPIC_TOOL_SCHEMA_KEYS and isinstance(value, dict)
def truncate_tool_name(name: str) -> str:
"""
Truncate tool names that exceed OpenAI's 64-character limit.
@ -73,7 +89,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking_budget,
)
from litellm.llms.anthropic.common_utils import normalize_anthropic_tool_use_id
from litellm.llms.anthropic.common_utils import (
is_empty_thinking_block,
normalize_anthropic_tool_use_id,
)
from litellm.llms.anthropic.experimental_pass_through.context_management import (
PolyfillResult,
)
@ -126,7 +145,9 @@ from litellm.types.llms.openai import (
ChatCompletionToolMessage,
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
ChatCompletionToolReferenceObject,
ChatCompletionUserMessage,
ToolMessageContentPart,
)
from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
@ -135,6 +156,8 @@ from .streaming_iterator import AnthropicStreamWrapper
if TYPE_CHECKING:
from litellm.types.llms.anthropic import ContentBlockContentBlockDict
ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]
class AnthropicAdapter:
def __init__(self) -> None:
@ -412,90 +435,13 @@ class LiteLLMAnthropicMessagesAdapter:
self._add_cache_control_if_applicable(content, doc_obj, model)
new_user_content_list.append(doc_obj)
elif content.get("type") == "tool_result":
if "content" not in content:
tool_result = ChatCompletionToolMessage(
role="tool",
tool_call_id=content.get("tool_use_id", ""),
content="",
)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result)
elif isinstance(content.get("content"), str):
tool_result = ChatCompletionToolMessage(
role="tool",
tool_call_id=content.get("tool_use_id", ""),
content=str(content.get("content", "")),
)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result)
elif isinstance(content.get("content"), list):
# Combine all content items into a single tool message
# to avoid creating multiple tool_result blocks with the same ID
# (each tool_use must have exactly one tool_result)
content_items = list(content.get("content", []))
# Single-item text keeps the backward-compatible string format; a single
# image or document becomes a structured image_url part
if len(content_items) == 1:
c = content_items[0]
if isinstance(c, str):
tool_result = ChatCompletionToolMessage(
role="tool",
tool_call_id=content.get("tool_use_id", ""),
content=c,
)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result)
elif isinstance(c, dict):
if c.get("type") == "text":
tool_result = ChatCompletionToolMessage(
role="tool",
tool_call_id=content.get("tool_use_id", ""),
content=c.get("text", ""),
)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result)
elif c.get("type") in ("image", "document"):
image_part = self._tool_result_image_part(c.get("source"))
tool_result = ChatCompletionToolMessage(
role="tool",
tool_call_id=content.get("tool_use_id", ""),
content=[image_part] # mutable-ok: content must be a json list
if image_part
else "",
)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result)
else:
# For multiple content items, combine into a single tool message
# with list content to preserve all items while having one tool_use_id
combined_content_parts: list[
ChatCompletionTextObject | ChatCompletionImageObject
] = []
for c in content_items:
if isinstance(c, str):
combined_content_parts.append(ChatCompletionTextObject(type="text", text=c))
elif isinstance(c, dict):
if c.get("type") == "text":
combined_content_parts.append(
ChatCompletionTextObject(
type="text",
text=c.get("text", ""),
)
)
elif c.get("type") in ("image", "document"):
image_part = self._tool_result_image_part(c.get("source"))
if image_part:
combined_content_parts.append(image_part)
# Create a single tool message with combined content
if combined_content_parts:
tool_result = ChatCompletionToolMessage(
role="tool",
tool_call_id=content.get("tool_use_id", ""),
content=combined_content_parts,
)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result)
tool_result = ChatCompletionToolMessage(
role="tool",
tool_call_id=content.get("tool_use_id", ""),
content=self._tool_result_content(content.get("content")),
)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result)
if len(tool_message_list) > 0:
new_messages.extend(tool_message_list)
@ -771,6 +717,10 @@ class LiteLLMAnthropicMessagesAdapter:
new_tools.append(tool)
continue
if _is_openai_function_tool(tool) or is_provider_native_tool_dict(tool):
new_tools.append(cast(ChatCompletionToolParam, tool)) # cast-ok: passed through verbatim to provider
continue
raw_name = tool.get("name")
if raw_name is None or (isinstance(raw_name, str) and not str(raw_name).strip()):
original_name = f"litellm_unnamed_tool_{idx}"
@ -943,6 +893,31 @@ class LiteLLMAnthropicMessagesAdapter:
)
return "prompt_cache_key" in (supported_params or ())
@staticmethod
def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool:
"""Whether the target declares ``reasoning_effort`` among its supported params.
A Claude-family target is recognized by name, which says nothing about the carrier the
provider serving it accepts: Snowflake serves Claude over the Anthropic dialect and
declares ``thinking`` alone, so storing the tier there raises before the request reaches
the wire.
Without a resolved provider the tier stays behind, which is what this bridge sent before
it carried one at all. Reading the declaration from the model's own prefix instead would
resolve the provider through a lookup that runs an OAuth device flow for two of them, and
this runs inside a logging callback as well as on the request path.
Unlike ``_supports_prompt_cache_key`` this does not exclude a provider that proxies an
unknown backend, because that provider declares this param and forwards it to a proxy
that resolves the real target itself, where a derived cache key has no such guarantee.
"""
if not model or not custom_llm_provider:
return False
supported_params: Final = litellm.get_supported_openai_params(
model=model, custom_llm_provider=custom_llm_provider
)
return "reasoning_effort" in (supported_params or ())
def _translate_metadata_to_openai(
self,
anthropic_message_request: AnthropicMessagesRequest,
@ -1031,8 +1006,32 @@ class LiteLLMAnthropicMessagesAdapter:
self,
anthropic_message_request: AnthropicMessagesRequest,
new_kwargs: ChatCompletionRequest,
*,
custom_llm_provider: str | None = None,
) -> None:
"""Translate Anthropic thinking to either thinking or reasoning_effort."""
"""Translate Anthropic thinking to either thinking or reasoning_effort.
A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one
speaks that param. Carrying its adaptive effort tier alongside takes two different params,
because the two are not interchangeable at the provider mapping below.
Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking``
alone. Another bridged Claude target takes ``reasoning_effort`` if it declares that param,
and used to be sent no tier at all, so an adaptive request arrived byte-identical whichever
effort the caller asked for. That tier stays a plain string there, since the summary it
would otherwise be wrapped with already travels inside the forwarded ``thinking`` block,
and the wrapped dict is rejected outright by some of these providers.
A target declaring neither carrier keeps its bare ``thinking`` block. Being Claude-family
is a fact about the model, not about the params the provider in front of it accepts, so
the tier is offered only where the target says it is taken.
``reasoning_effort`` is not a substitute for ``output_config`` on the Bedrock side: an
application inference profile ARN resolves to neither, so the tier is dropped, and providers
that rebuild ``output_config`` from it overwrite a caller-set ``thinking.display`` doing so.
An adaptive request with no tier stays untouched either way, so the provider's own default
still applies.
"""
if "thinking" not in anthropic_message_request:
return
@ -1041,35 +1040,40 @@ class LiteLLMAnthropicMessagesAdapter:
return
model: Final = new_kwargs.get("model", "")
if self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model):
is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model(
model
)
is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
output_config: Final = anthropic_message_request.get("output_config")
if is_claude_target:
new_kwargs["thinking"] = thinking
# Adaptive thinking without its effort tier makes Bedrock Converse
# return zero reasoning blocks, so forward output_config (minus
# `format`, already translated to response_format) for Bedrock
# targets only: other bridged providers reject the raw param, and
# get_llm_provider strips the `bedrock/` prefix before this runs.
if model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model(model):
claude_output_config: Final = anthropic_message_request.get("output_config")
if isinstance(claude_output_config, dict):
effort_config: Final = {k: v for k, v in claude_output_config.items() if k != "format"}
if is_bedrock_target:
if isinstance(output_config, dict):
effort_config: Final = {k: v for k, v in output_config.items() if k != "format"}
if effort_config:
new_kwargs["output_config"] = effort_config # rebind-ok: out-param store like thinking above
return
if not self._target_declares_reasoning_effort(model, custom_llm_provider):
return
thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
declared_effort: Final = (
output_config.get("effort") if thinking_type == "adaptive" and isinstance(output_config, dict) else None
)
if is_claude_target and not declared_effort:
return
reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(cast(AnthropicThinkingParam, thinking))
reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort(
cast(AnthropicThinkingParam, thinking)
)
if not reasoning_effort:
return
thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
# For adaptive thinking, override with output_config.effort if available
if thinking_type == "adaptive":
output_config: Final = anthropic_message_request.get("output_config")
if isinstance(output_config, dict) and output_config.get("effort"):
reasoning_effort = output_config["effort"]
new_kwargs["reasoning_effort"] = self._apply_reasoning_summary_wrapping(
reasoning_effort, cast(dict[str, object], thinking)
new_kwargs["reasoning_effort"] = (
reasoning_effort
if is_claude_target
else self._apply_reasoning_summary_wrapping(reasoning_effort, cast(dict[str, object], thinking))
)
def _translate_output_format_to_openai(
@ -1165,6 +1169,7 @@ class LiteLLMAnthropicMessagesAdapter:
self._translate_thinking_to_openai(
anthropic_message_request=anthropic_message_request,
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
)
## CONVERT STOP_SEQUENCES
self._translate_stop_sequences_to_openai(
@ -1210,6 +1215,39 @@ class LiteLLMAnthropicMessagesAdapter:
return None
def _tool_result_content(self, raw_content: object) -> ToolResultContent:
if isinstance(raw_content, str):
return raw_content
if not isinstance(raw_content, list):
return ""
items: Final = cast(Sequence[object], raw_content) # cast-ok: untrusted client payload
parts: Final = tuple(part for part in (self._tool_result_part(item) for item in items) if part is not None)
match parts:
case ():
return ""
case ({"type": "text", "text": str(text)},):
return text
case _:
return list(parts) # mutable-ok: content must be a json list
def _tool_result_part(self, item: object) -> ToolMessageContentPart | None:
if isinstance(item, str):
return ChatCompletionTextObject(type="text", text=item)
if not isinstance(item, dict):
return None
block: Final = cast(Mapping[str, object], item) # cast-ok: untrusted client payload
match block.get("type"):
case "text":
return ChatCompletionTextObject(type="text", text=str(block.get("text") or ""))
case "image" | "document":
return self._tool_result_image_part(block.get("source"))
case "tool_reference":
return ChatCompletionToolReferenceObject(
type="tool_reference", tool_name=str(block.get("tool_name") or "")
)
case _:
return None
def _tool_result_image_part(self, image_source: object) -> ChatCompletionImageObject | None:
if not isinstance(image_source, dict):
return None
@ -1229,6 +1267,8 @@ class LiteLLMAnthropicMessagesAdapter:
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
for thinking_block in choice.message.thinking_blocks:
if thinking_block.get("type") == "thinking":
if is_empty_thinking_block(thinking_block):
continue
thinking_value = thinking_block.get("thinking", "")
signature_value = thinking_block.get("signature", "")
new_content.append(
@ -1358,12 +1398,10 @@ class LiteLLMAnthropicMessagesAdapter:
@classmethod
def _get_web_search_request_count(cls, usage: Usage) -> int:
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests,
get_web_search_requests_from_usage,
)
from_server_tool_use: Final = cls._positive_int(
get_web_search_requests(getattr(usage, "server_tool_use", None))
)
from_server_tool_use: Final = cls._positive_int(get_web_search_requests_from_usage(usage))
if from_server_tool_use > 0:
return from_server_tool_use
return cls._first_positive_prompt_tokens_detail_value(usage, ("web_search_requests",))

27
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -17,7 +17,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.anthropic.common_utils import (
flatten_unencrypted_web_search_results_in_anthropic_messages,
sanitize_tool_use_ids_in_anthropic_messages,
strip_empty_text_blocks_from_anthropic_messages,
strip_empty_content_blocks_from_anthropic_messages,
)
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
@ -242,17 +242,20 @@ async def anthropic_messages(
"""
Async: Make llm api request in Anthropic /messages API spec.
Runs the empty-text-block sanitizer before any backend dispatch.
Runs the empty-content-block sanitizer before any backend dispatch.
"""
# Anthropic's API rejects requests containing empty / whitespace-only
# text content blocks with "messages: text content blocks must be
# non-empty". Multi-turn tool-use clients (e.g. Claude Code) routinely
# loop assistant responses that contain {"type": "text", "text": ""}
# alongside tool_use blocks back as conversation history, which then
# causes the next /v1/messages call to 400. /v1/chat/completions
# already handles this in anthropic_messages_pt; sanitize the native
# Anthropic Messages path here for the same guarantee. See #22930.
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
# text content blocks ("messages: text content blocks must be
# non-empty") and empty thinking blocks ("each thinking block must
# contain thinking"). Multi-turn tool-use clients (e.g. Claude Code)
# routinely loop assistant responses that contain such blocks — an empty
# text block alongside tool_use, or an empty thinking block from a turn
# a non-Anthropic reasoning model served through the bridge — back as
# conversation history, which then causes the next /v1/messages call to
# 400. /v1/chat/completions already handles this in
# anthropic_messages_pt; sanitize the native Anthropic Messages path
# here for the same guarantee. See #22930.
messages = strip_empty_content_blocks_from_anthropic_messages(messages)
# Replay of cross-provider tool history (e.g. kimi -> Anthropic) may carry
# ids like ``functions.Bash:0`` that violate Anthropic's id pattern.
messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
@ -374,7 +377,7 @@ async def anthropic_messages(
api_base=api_base,
client=client,
custom_llm_provider=custom_llm_provider,
# messages were already empty-text-block sanitized at the top of this
# messages were already empty-content-block sanitized at the top of this
# function and are NOT reassigned before this dispatch, so the handler
# can skip its (otherwise redundant) second full-messages scan. Passed
# explicitly (not via **kwargs) so it only affects this direct
@ -451,7 +454,7 @@ def anthropic_messages_handler(
# ``_litellm_messages_presanitized`` to skip this redundant second
# full-messages scan. Pop it so it never leaks into provider params.
if not kwargs.pop("_litellm_messages_presanitized", False):
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
messages = strip_empty_content_blocks_from_anthropic_messages(messages)
messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages)

33
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -342,7 +342,7 @@ class BaseAnthropicMessagesStreamingIterator:
self.start_time = datetime.now()
self.completion_start_time: datetime | None = None
async def _handle_streaming_logging(self, collected_chunks: list[bytes]):
async def _handle_streaming_logging(self, collected_chunks: list[bytes], *, stream_teardown: bool = False):
"""Handle the logging after all chunks have been collected."""
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamingHandler,
@ -354,21 +354,26 @@ class BaseAnthropicMessagesStreamingIterator:
if self.completion_start_time is not None:
self.litellm_logging_obj.completion_start_time = self.completion_start_time
self.litellm_logging_obj.model_call_details["completion_start_time"] = self.completion_start_time
logging_coroutine: Final = PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=self.litellm_logging_obj,
passthrough_success_handler_obj=GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
url_route="/v1/messages",
request_body=self.request_body or {},
endpoint_type=EndpointType.ANTHROPIC,
start_time=self.start_time,
raw_bytes=collected_chunks,
end_time=end_time,
)
deferred_dispatch_armed: Final = (
getattr(self.litellm_logging_obj, "_on_deferred_stream_complete", None) is not None
)
if deferred_dispatch_armed and not stream_teardown:
self.litellm_logging_obj._deferred_stream_complete_args = (logging_coroutine,)
return
# Enqueue on the rooted logging worker rather than asyncio.create_task:
# this also runs during generator teardown after a client disconnect,
# where an unrooted task could be garbage-collected before it bills.
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
async_coroutine=PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=self.litellm_logging_obj,
passthrough_success_handler_obj=GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
url_route="/v1/messages",
request_body=self.request_body or {},
endpoint_type=EndpointType.ANTHROPIC,
start_time=self.start_time,
raw_bytes=collected_chunks,
end_time=end_time,
)
)
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
def get_async_streaming_response_iterator(
self,
@ -433,7 +438,7 @@ class BaseAnthropicMessagesStreamingIterator:
# post-loop logging below never runs and the tokens already streamed
# (and billed by the provider) would never reach spend tracking. See LIT-5839.
if collected_chunks:
await self._handle_streaming_logging(collected_chunks)
await self._handle_streaming_logging(collected_chunks, stream_teardown=True)
raise
if not saw_terminal_event:

58
litellm/llms/anthropic/experimental_pass_through/utils.py
View file

@ -1,4 +1,6 @@
import os
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import litellm
@ -6,6 +8,15 @@ from litellm.types.utils import ModelInfo
OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH: Final = 64
_EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
{
"max": ("max", "xhigh", "high"),
"xhigh": ("xhigh", "high"),
"minimal": ("minimal", "low"),
}
)
_THINKING_OFF: Final = "none"
def prompt_cache_key_from_user_id(user_id: object) -> str | None:
if user_id is None:
@ -28,38 +39,33 @@ def normalize_reasoning_effort_value(
model: str,
custom_llm_provider: str | None = None,
) -> str:
"""
Normalize a reasoning effort value based on model capabilities.
"""Lower a tier the deployment does not accept to the nearest one it does, leaving others alone.
Degradation chains:
- "max" → max / xhigh / high
- "xhigh" → xhigh / high
- "minimal" → minimal / low
- other values pass through unchanged
The accepted set is resolved by the same owner that answers ``/model_group/info``, so a level
the proxy advertises is a level this path forwards.
A deployment that refuses every step of a chain falls back to an accepted level read off that
same set rather than to an assumed one, since an entry naming its levels outright can exclude
the tiers the per-level flags treat as unconditional. ``none`` is never that fallback and is
never degraded to, being an off switch rather than a tier; an always-on-thinking model is
handled where the thinking block is built. A deployment accepting no tier at all keeps the
chain's floor, which is what every deployment degraded to before there was anything to ask.
"""
if effort not in ("max", "xhigh", "minimal"):
chain: Final = _EFFORT_DEGRADATION_CHAIN.get(effort)
if chain is None:
return effort
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
from litellm.utils import get_model_info
model_info: ModelInfo | None = None
try:
model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
model_info: Final[ModelInfo] = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception:
model_info = None
return chain[-1]
if effort == "max":
if model_info and model_info.get("supports_max_reasoning_effort"):
return "max"
if model_info and model_info.get("supports_xhigh_reasoning_effort"):
return "xhigh"
return "high"
elif effort == "xhigh":
if model_info and model_info.get("supports_xhigh_reasoning_effort"):
return "xhigh"
return "high"
elif effort == "minimal":
if model_info and model_info.get("supports_minimal_reasoning_effort"):
return "minimal"
return "low"
return "medium"
supported: Final = resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True)
if not supported:
return chain[-1]
accepted_tiers: Final = tuple(level for level in supported if level != _THINKING_OFF)
return next((level for level in (*chain, *accepted_tiers) if level in supported), chain[-1])

20
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -19,19 +19,19 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
GPT5_SERIES_ROUTE = "gpt5_series/"
@classmethod
def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool:
"""Override to handle gpt5_series/ prefix used for Azure routing.
def _model_map_lookup_name(cls, model: str) -> str:
"""Normalise an Azure routing name to its cost-map key.
The parent class calls ``_supports_factory(model, custom_llm_provider=None)``
which fails to resolve ``gpt5_series/gpt-5.1`` to the correct Azure model
entry. Strip the prefix and prepend ``azure/`` so the lookup finds
``azure/gpt-5.1`` in model_prices_and_context_window.json.
Neither ``gpt5_series/gpt-5.1`` nor a bare ``gpt-5.1`` is a key in
model_prices_and_context_window.json; ``azure/gpt-5.1`` is. Overriding the shared
resolver rather than one lookup means the supports, explicitly-disabled and
default-effort answers all read the same entry.
"""
if model.startswith(cls.GPT5_SERIES_ROUTE):
model = "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :]
elif not model.startswith("azure/"):
model = "azure/" + model
return super()._supports_reasoning_effort_level(model, level)
return "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :]
if model.startswith("azure/"):
return model
return "azure/" + model
@classmethod
def is_model_gpt_5_model(cls, model: str) -> bool:

4
litellm/llms/azure/chat/gpt_transformation.py
View file

@ -4,6 +4,7 @@ from httpx._models import Headers, Response
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_tool_reference_parts_from_tool_messages,
hoist_images_from_tool_messages,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
@ -254,7 +255,8 @@ class AzureOpenAIConfig(BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(messages))
stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(messages)
azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages))
return {
"model": model,
"messages": azure_messages,

8
litellm/llms/base_llm/anthropic_messages/transformation.py
View file

@ -159,20 +159,20 @@ class BaseAnthropicMessagesConfig(ABC):
and issue one more attempt (bounded by max_retry_on_anthropic_messages_http_error).
"""
from litellm.llms.anthropic.common_utils import (
is_anthropic_invalid_thinking_signature_error,
is_anthropic_invalid_thinking_block_error,
)
return e.response.status_code == 400 and is_anthropic_invalid_thinking_signature_error(e.response.text)
return e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text)
def transform_anthropic_messages_request_on_http_error(self, e: httpx.HTTPStatusError, request_data: dict) -> dict:
"""
Mutates request_data in place when retrying after a recoverable HTTP error.
"""
from litellm.llms.anthropic.common_utils import (
is_anthropic_invalid_thinking_signature_error,
is_anthropic_invalid_thinking_block_error,
strip_thinking_blocks_from_anthropic_messages_request_dict,
)
if e.response.status_code == 400 and is_anthropic_invalid_thinking_signature_error(e.response.text):
if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
return request_data

10
litellm/llms/base_llm/audio_transcription/transformation.py
View file

@ -42,6 +42,16 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]:
pass
@property
def supports_subtitle_synthesis(self) -> bool:
"""
Opt-in for providers without a native srt/vtt response body: when True
and the user asked for response_format srt/vtt, the http handler
synthesizes the subtitle document from the word timestamps the
provider's TranscriptionResponse carries in `words`.
"""
return False
def get_complete_url(
self,
api_base: str | None,

2
litellm/llms/base_llm/chat/transformation.py
View file

@ -48,8 +48,10 @@ class BaseLLMException(Exception):
request: httpx.Request | None = None,
response: httpx.Response | None = None,
body: dict | None = None,
status_code_is_synthesized: bool = False,
):
self.status_code = status_code
self.status_code_is_synthesized = status_code_is_synthesized
self.message: str = message
self.headers = headers
if request:

31
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -158,6 +158,22 @@ def openai_messages_without_tool(
return tuple(m for m in messages if _message_role(m) != "tool")
def filter_messages_by_skip_flags(
guardrail_to_apply: object, messages: Sequence[AllMessageValues]
) -> tuple[tuple[AllMessageValues, ...], bool]:
system_filtered = (
openai_messages_without_system(messages)
if effective_skip_system_message_for_guardrail(guardrail_to_apply)
else tuple(messages)
)
fully_filtered = (
openai_messages_without_tool(system_filtered)
if effective_skip_tool_message_for_guardrail(guardrail_to_apply)
else system_filtered
)
return fully_filtered, len(fully_filtered) != len(messages)
def effective_scan_only_tool_results_for_guardrail(guardrail_to_apply: object) -> bool:
return getattr(guardrail_to_apply, "scan_only_tool_results", None) is True
@ -209,9 +225,20 @@ def openai_tool_name(tool: object) -> str | None:
return flat_name if isinstance(flat_name, str) else None
def anthropic_tool_names(tool: object) -> tuple[str, ...]:
"""Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus
``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks
must see both, or a decoy flat name could smuggle a disallowed ``function.name`` through."""
if not isinstance(tool, dict):
return ()
function: Final = tool.get("function") if tool.get("type") == "function" else None
function_name: Final = function.get("name") if isinstance(function, dict) else None
return tuple(name for name in (tool.get("name"), function_name) if isinstance(name, str) and name)
def anthropic_tool_name(tool: object) -> str | None:
name: Final = tool.get("name") if isinstance(tool, dict) else None
return name if isinstance(name, str) else None
names: Final = anthropic_tool_names(tool)
return names[0] if names else None
def merge_returned_tools_into_request_tools(

4
litellm/llms/base_llm/realtime/transformation.py
View file

@ -5,6 +5,7 @@ import httpx
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
from litellm.types.realtime import (
RealtimeInputAudioTranscriptionUsage,
RealtimeResponseTransformInput,
RealtimeResponseTypedDict,
)
@ -70,6 +71,9 @@ class BaseRealtimeConfig(ABC):
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
return None
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
return None
def transform_session_created_event(
self,
model: str,

7
litellm/llms/bedrock/base_aws_llm.py
View file

@ -1434,9 +1434,12 @@ class BaseAWSLLM:
data: str | bytes,
headers: dict,
api_key: str | None = None,
supports_bearer_token: bool = True,
) -> AWSPreparedRequest:
if api_key is not None:
aws_bearer_token: str | None = api_key
if not supports_bearer_token:
aws_bearer_token: str | None = None
elif api_key is not None:
aws_bearer_token = api_key
else:
aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")

26
litellm/llms/bedrock/batches/handler.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, cast
@ -68,6 +69,19 @@ def _predict_output_file_uri(output_prefix: str, input_uri: str, job_id: str | N
return f"{output_prefix}{job_id}/{input_basename}.out"
def _record_counts_from_response(response: Mapping[str, object]) -> BatchRequestCounts | None:
total_records: Final = response.get("totalRecordCount")
success_records: Final = response.get("successRecordCount")
if not isinstance(total_records, int) or not isinstance(success_records, int):
return None
error_records: Final = response.get("errorRecordCount")
return BatchRequestCounts(
total=total_records,
completed=success_records,
failed=error_records if isinstance(error_records, int) else 0,
)
def _to_epoch(value: Any) -> int | None:
if value is None:
return None
@ -271,11 +285,11 @@ class BedrockBatchesHandler:
``aws_external_id``). Unknown keys are ignored.
Returns:
``LiteLLMBatch`` shaped like an OpenAI Batch resource. Note that
``request_counts`` is always ``(0, 0, 0)`` because
``GetModelInvocationJob`` does not surface per-record counts;
callers that need accurate counts should parse
``manifest.json.out`` from the output S3 prefix.
``LiteLLMBatch`` shaped like an OpenAI Batch resource.
``request_counts`` maps ``GetModelInvocationJob``'s
``totalRecordCount`` / ``successRecordCount`` / ``errorRecordCount``
when the provider reports them, and is ``None`` when it does not
(older botocore, or a status that omits counts).
"""
try:
import boto3
@ -386,7 +400,7 @@ class BedrockBatchesHandler:
failed_at=completed_at if openai_status == "failed" else None,
cancelled_at=completed_at if openai_status == "cancelled" else None,
expired_at=completed_at if openai_status == "expired" else None,
request_counts=BatchRequestCounts(total=0, completed=0, failed=0),
request_counts=_record_counts_from_response(response),
metadata=openai_batch_metadata,
completion_window="24h",
endpoint="/v1/chat/completions",

13
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -1546,6 +1546,7 @@ class AmazonConverseConfig(BaseConfig):
messages: list[AllMessageValues] | None = None,
headers: dict | None = None,
drop_params: bool = False,
litellm_params: Mapping[str, object] | None = None,
) -> CommonRequestObject:
## VALIDATE REQUEST
"""
@ -1608,6 +1609,16 @@ class AmazonConverseConfig(BaseConfig):
if point.get("location") == "tool_config":
cache_point = self._build_cache_point_block(point.get("control"), model)
bedrock_tools.append(ToolBlock(cachePoint=cache_point))
# Spend attribution credits the gateway only for breakpoints it placed, and
# this is the one place a tool_config point becomes one. The hook that reads
# the configuration cannot record it: whether a cachePoint lands depends on
# this provider and on the request carrying tools, neither of which it sees.
if litellm_params is not None:
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
)
AnthropicCacheControlHook.record_gateway_injection(litellm_params, 1)
break
bedrock_tool_config: ToolConfigBlock | None = None
@ -1670,6 +1681,7 @@ class AmazonConverseConfig(BaseConfig):
messages=messages,
headers=headers,
drop_params=litellm_params.get("drop_params") is True,
litellm_params=litellm_params,
)
bedrock_messages: Final = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
@ -1729,6 +1741,7 @@ class AmazonConverseConfig(BaseConfig):
messages=messages,
headers=headers,
drop_params=litellm_params.get("drop_params") is True,
litellm_params=litellm_params,
)
## TRANSFORMATION ##

30
litellm/llms/bedrock/rerank/handler.py
View file

@ -138,11 +138,6 @@ class BedrockRerankHandler(BaseAWSLLM):
data: dict,
optional_params: dict,
) -> BedrockPreparedRequest:
try:
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(optional_params, model)
### SET RUNTIME ENDPOINT ###
@ -153,24 +148,21 @@ class BedrockRerankHandler(BaseAWSLLM):
)
proxy_endpoint_url = proxy_endpoint_url.replace("bedrock-runtime", "bedrock-agent-runtime")
proxy_endpoint_url = f"{proxy_endpoint_url}/rerank"
sigv4: Final = SigV4Auth(
boto3_credentials_info.credentials,
"bedrock",
boto3_credentials_info.aws_region_name,
)
# Make POST Request
body: Final = json.dumps(data).encode("utf-8")
body: Final = json.dumps(data).encode("utf-8")
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
request: Final = AWSRequest(method="POST", url=proxy_endpoint_url, data=body, headers=headers)
sigv4.add_auth(request)
if (
extra_headers is not None and "Authorization" in extra_headers
): # prevent sigv4 from overwriting the auth header
request.headers["Authorization"] = extra_headers["Authorization"]
prepped: Final = request.prepare()
prepped: Final = self.get_request_headers(
credentials=boto3_credentials_info.credentials,
aws_region_name=boto3_credentials_info.aws_region_name,
extra_headers=extra_headers,
endpoint_url=proxy_endpoint_url,
data=body,
headers=headers,
supports_bearer_token=False,
)
return BedrockPreparedRequest(
endpoint_url=proxy_endpoint_url,

32
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -25,6 +25,10 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
SUBTITLE_RESPONSE_FORMATS,
synthesize_subtitle_document,
)
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
@ -1297,9 +1301,23 @@ class BaseLLMHTTPHandler:
api_key: str | None,
) -> TranscriptionResponse:
"""Shared logic for transforming audio transcription responses."""
return provider_config.transform_audio_transcription_response(
transformed: Final = provider_config.transform_audio_transcription_response(
raw_response=response,
)
if not provider_config.supports_subtitle_synthesis:
return transformed
requested_format: Final = optional_params.get("response_format")
if not isinstance(requested_format, str) or requested_format not in SUBTITLE_RESPONSE_FORMATS:
return transformed
document: Final = synthesize_subtitle_document(
words=transformed.get("words"),
response_format=requested_format,
)
if document is not None:
transformed.text = document
if "words" in transformed:
delattr(transformed, "words")
return transformed
def audio_transcriptions(
self,
@ -5930,11 +5948,13 @@ class BaseLLMHTTPHandler:
BaseEvalsAPIConfig,
],
):
status_code = getattr(e, "status_code", 500)
received_status_code: Final = (
e.response.status_code if isinstance(e, httpx.HTTPStatusError) else getattr(e, "status_code", None)
)
status_code = received_status_code if isinstance(received_status_code, int) else 500
error_headers = getattr(e, "headers", None)
if isinstance(e, httpx.HTTPStatusError):
error_text = e.response.text
status_code = e.response.status_code
else:
error_text = getattr(e, "text", str(e))
error_response: Final = getattr(e, "response", None)
@ -5954,13 +5974,17 @@ class BaseLLMHTTPHandler:
status_code=status_code,
message=error_text,
headers=error_headers,
status_code_is_synthesized=not isinstance(received_status_code, int),
)
raise provider_config.get_error_class(
provider_error: Final = provider_config.get_error_class(
error_message=error_text,
status_code=status_code,
headers=error_headers,
)
if not isinstance(received_status_code, int):
provider_error.status_code_is_synthesized = True
raise provider_error
@staticmethod
def _append_query_params(url: str, query_params: RealtimeQueryParams | None) -> str:

18
litellm/llms/dashscope/image_generation/transformation.py
View file

@ -11,7 +11,7 @@ Request format:
"input": {
"messages": [{"role": "user", "content": [{"text": "<prompt>"}]}]
},
"parameters": {"size": "1024*1024", ...}
"parameters": {"size": "1024*1024", "n": 1, ...}
}
Response format:
@ -19,7 +19,7 @@ Response format:
"output": {
"choices": [{"message": {"content": [{"image": "<url>"}]}}]
},
"usage": {"input_tokens": 0, "output_tokens": 0, "width": 1024, "height": 1024, "image_count": 1}
"usage": {"output_width": 1024, "output_height": 1024, "output_image_count": 1}
}
"""
@ -48,6 +48,8 @@ else:
DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
CHAT_COMPATIBLE_MODE_PATH: Final = "/compatible-mode/v1"
# Maps OpenAI size strings (WxH) to DashScope size strings (W*H)
OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
"256x256": "256*256",
@ -61,7 +63,8 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
"""
Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro).
Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro,
qwen-image-3.0, qwen-image-3.0-pro).
"""
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
@ -84,8 +87,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
if k == "size":
# Convert "WxH" → "W*H"
mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*"))
elif k == "n":
mapped["image_count"] = v
else:
mapped[k] = v
return mapped
def get_complete_url(
@ -97,7 +100,10 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
litellm_params: dict,
stream: bool | None = None,
) -> str:
return api_base or get_secret_str("DASHSCOPE_API_BASE_IMAGE") or DEFAULT_API_BASE
image_api_base: Final = (
api_base if api_base and not api_base.rstrip("/").endswith(CHAT_COMPATIBLE_MODE_PATH) else None
)
return image_api_base or get_secret_str("DASHSCOPE_API_BASE_IMAGE") or DEFAULT_API_BASE
def validate_environment(
self,

0
litellm/llms/gemini/audio_transcription/__init__.py Normal file
View file

256
litellm/llms/gemini/audio_transcription/transformation.py Normal file
View file

@ -0,0 +1,256 @@
import base64
from collections.abc import Mapping, Sequence
from typing import Final
from httpx import Headers, Response
from litellm.litellm_core_utils.audio_utils.subtitle_utils import SUBTITLE_RESPONSE_FORMATS
from litellm.litellm_core_utils.audio_utils.utils import (
normalize_transcription_language_to_bcp47,
process_audio_file,
)
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
BaseAudioTranscriptionConfig,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
from litellm.types.llms.gemini_audio_transcription import (
GeminiTranscriptionAudioInput,
GeminiTranscriptionConfig,
GeminiTranscriptionInteractionRequest,
GeminiTranscriptionInteractionResponse,
GeminiTranscriptionWordAnnotation,
)
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIAudioTranscriptionOptionalParams,
)
from litellm.types.utils import (
FileTypes,
TranscriptionResponse,
TranscriptionUsageInputTokenDetailsObject,
TranscriptionUsageTokensObject,
)
INTERACTIONS_API_REVISION: Final = "2026-05-20"
WORD_INFO_ANNOTATION_TYPE: Final = "word_info"
class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
"""
Maps OpenAI /v1/audio/transcriptions onto the Gemini Interactions API
(POST /v1beta/interactions) for transcription models like
gemini-3.5-transcribe. https://ai.google.dev/gemini-api/docs/transcribe
"""
def get_supported_openai_params(
self, model: str
) -> list[OpenAIAudioTranscriptionOptionalParams]: # mutable-ok: BaseAudioTranscriptionConfig signature
return ["language", "response_format", "timestamp_granularities"] # mutable-ok: base contract returns a list
@property
def supports_subtitle_synthesis(self) -> bool:
return True
def map_openai_params(
self,
non_default_params: Mapping[str, object],
optional_params: Mapping[str, object],
model: str,
drop_params: bool,
) -> dict: # mutable-ok: BaseAudioTranscriptionConfig signature
supported_params: Final = frozenset(self.get_supported_openai_params(model))
accepted: Final = tuple((k, v) for k, v in non_default_params.items() if k in supported_params)
return dict((*optional_params.items(), *accepted)) # mutable-ok: base contract returns a plain dict
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict | Headers, # mutable-ok: base signature and BaseLLMException take dict | Headers
) -> BaseLLMException:
return GeminiError(status_code=status_code, message=error_message, headers=headers)
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict: # mutable-ok: BaseAudioTranscriptionConfig signature
resolved_api_key: Final = GeminiModelInfo.get_api_key(api_key)
if not resolved_api_key:
raise GeminiError(
status_code=401,
message="Google API key is required. Set GOOGLE_API_KEY or GEMINI_API_KEY environment variable.",
)
return { # mutable-ok: the http handler passes these headers straight to httpx
**headers,
"Content-Type": "application/json",
"x-goog-api-key": resolved_api_key,
"Api-Revision": INTERACTIONS_API_REVISION,
}
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
stream: bool | None = None,
) -> str:
resolved_api_base: Final = GeminiModelInfo.get_api_base(api_base)
return f"{resolved_api_base}/v1beta/interactions"
def transform_audio_transcription_request(
self,
model: str,
audio_file: FileTypes,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> AudioTranscriptionRequestData:
processed_audio: Final = process_audio_file(audio_file)
audio_input: Final = GeminiTranscriptionAudioInput(
type="audio",
data=base64.b64encode(processed_audio.file_content).decode("utf-8"),
mime_type=processed_audio.content_type,
)
request: Final = _build_interaction_request(
model=model,
audio_input=audio_input,
transcription_config=_build_transcription_config(optional_params),
)
return AudioTranscriptionRequestData(data=dict(request)) # mutable-ok: AudioTranscriptionRequestData wants dict
def transform_audio_transcription_response(
self,
raw_response: Response,
) -> TranscriptionResponse:
try:
response_json: Final = raw_response.json()
except ValueError:
raise GeminiError(
status_code=raw_response.status_code,
message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}",
)
parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json)
if parsed.status != "completed":
raise GeminiError(
status_code=raw_response.status_code,
message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}",
)
text_contents: Final = tuple(
content
for step in parsed.steps
for content in step.content
if content.type == "text" and content.text is not None
)
response: Final = TranscriptionResponse(text=" ".join(content.text or "" for content in text_contents))
response["task"] = "transcribe"
words: Final = tuple(
word
for content in text_contents
for annotation in content.annotations
if (word := _annotation_to_word(annotation)) is not None
)
if words:
response["words"] = list(words) # mutable-ok: verbose_json words is a JSON array
last_word_end: Final = words[-1].get("end")
if last_word_end is not None:
response["duration"] = last_word_end
if parsed.usage is not None:
audio_tokens: Final = sum(
by_modality.tokens
for by_modality in parsed.usage.input_tokens_by_modality
if by_modality.modality == "audio"
)
response.usage = TranscriptionUsageTokensObject(
type="tokens",
input_tokens=parsed.usage.total_input_tokens,
output_tokens=parsed.usage.total_output_tokens,
total_tokens=parsed.usage.total_tokens,
input_token_details=TranscriptionUsageInputTokenDetailsObject(
audio_tokens=audio_tokens,
text_tokens=parsed.usage.total_input_tokens - audio_tokens,
),
)
return response
_EMPTY_TRANSCRIPTION_CONFIG: Final[GeminiTranscriptionConfig] = {}
_WORD_TIMESTAMP_CONFIG: Final[GeminiTranscriptionConfig] = {
"mode": {
"type": "verbatim",
"timestamp_granularities": ("word",),
"diarization_mode": "speaker",
},
}
def _build_interaction_request(
model: str,
audio_input: GeminiTranscriptionAudioInput,
transcription_config: GeminiTranscriptionConfig,
) -> GeminiTranscriptionInteractionRequest:
if not transcription_config:
bare_request: Final[GeminiTranscriptionInteractionRequest] = {
"model": model.removeprefix("gemini/"),
"input": (audio_input,),
}
return bare_request
configured_request: Final[GeminiTranscriptionInteractionRequest] = {
"model": model.removeprefix("gemini/"),
"input": (audio_input,),
"generation_config": {"transcription_config": transcription_config},
}
return configured_request
def _language_config(language: object) -> GeminiTranscriptionConfig:
if not isinstance(language, str) or not language:
return _EMPTY_TRANSCRIPTION_CONFIG
language_config: Final[GeminiTranscriptionConfig] = {
"language_codes": (normalize_transcription_language_to_bcp47(language),),
}
return language_config
def _timestamp_config(timestamp_granularities: object, response_format: object) -> GeminiTranscriptionConfig:
wants_word_timestamps: Final = (
isinstance(timestamp_granularities, list) and "word" in timestamp_granularities
) or (isinstance(response_format, str) and response_format in SUBTITLE_RESPONSE_FORMATS)
return _WORD_TIMESTAMP_CONFIG if wants_word_timestamps else _EMPTY_TRANSCRIPTION_CONFIG
def _build_transcription_config(optional_params: Mapping[str, object]) -> GeminiTranscriptionConfig:
transcription_config: Final[GeminiTranscriptionConfig] = {
**_language_config(optional_params.get("language")),
**_timestamp_config(optional_params.get("timestamp_granularities"), optional_params.get("response_format")),
}
return transcription_config
def _annotation_to_word(annotation: GeminiTranscriptionWordAnnotation) -> Mapping[str, str | float] | None:
if annotation.type != WORD_INFO_ANNOTATION_TYPE or annotation.text is None:
return None
entries: Final = (
("word", annotation.text),
("start", _parse_offset_seconds(annotation.start_offset)),
("end", _parse_offset_seconds(annotation.end_offset)),
("speaker", annotation.speaker),
)
return {key: value for key, value in entries if value is not None} # mutable-ok: word entries serialize to JSON
def _parse_offset_seconds(offset: str | None) -> float | None:
if offset is None or not offset.endswith("s"):
return None
try:
return float(offset[:-1])
except ValueError:
return None

21
litellm/llms/gemini/chat/transformation.py
View file

@ -8,7 +8,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
from litellm.litellm_core_utils.prompt_templates.image_handling import (
convert_url_to_base64,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionFileObject
from litellm.types.llms.openai import AllMessageValues, ChatCompletionFileObject, ChatCompletionImageObject
from litellm.types.llms.vertex_ai import ContentType, PartType
from litellm.utils import supports_reasoning
@ -16,6 +16,13 @@ from ...vertex_ai.gemini.transformation import _gemini_convert_messages_with_his
from ...vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
def _image_url_fields(img_element: ChatCompletionImageObject) -> tuple[str | None, str | None, str | None]:
image_value: Final = img_element.get("image_url")
if isinstance(image_value, dict):
return image_value.get("url"), image_value.get("format"), image_value.get("detail")
return image_value, None, None
class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
"""
Reference: https://ai.google.dev/api/rest/v1beta/GenerationConfig
@ -118,16 +125,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
_parts: list[PartType] = []
for element in _message_content:
if element.get("type") == "image_url":
img_element = element
_image_url: str | None = None
format: str | None = None
detail: str | None = None
if isinstance(img_element.get("image_url"), dict):
_image_url = img_element["image_url"].get("url")
format = img_element["image_url"].get("format")
detail = img_element["image_url"].get("detail")
else:
_image_url = img_element.get("image_url")
img_element = cast(ChatCompletionImageObject, element) # cast-ok: runtime type tag checked
_image_url, format, detail = _image_url_fields(img_element)
if _image_url and "https://" in _image_url:
image_obj = convert_to_anthropic_image_obj(_image_url, format=format)
converted_image_url = convert_generic_image_chunk_to_openai_image_obj(image_obj)

6
litellm/llms/gemini/cost_calculator.py
View file

@ -39,7 +39,9 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa
``model_info`` when available, falling back to $0.035 for models not
yet updated in the pricing JSON.
"""
from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests_from_usage,
)
from litellm.types.utils import PromptTokensDetailsWrapper
_DEFAULT_COST: Final = 35e-3
@ -57,7 +59,7 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa
)
else None
)
requests_from_server_tool_use: Final = get_web_search_requests(getattr(usage, "server_tool_use", None))
requests_from_server_tool_use: Final = get_web_search_requests_from_usage(usage)
number_of_web_search_requests: Final = requests_from_prompt_details or requests_from_server_tool_use or 0
billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt"

85
litellm/llms/gemini/realtime/transformation.py
View file

@ -4,7 +4,7 @@ This file contains the transformation logic for the Gemini realtime API.
import json
from collections import OrderedDict
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Any, Final, cast
import litellm
@ -53,6 +53,7 @@ from litellm.types.llms.vertex_ai import (
)
from litellm.types.realtime import (
ALL_DELTA_TYPES,
RealtimeInputAudioTranscriptionUsage,
RealtimeModalityResponseTransformOutput,
RealtimeResponseTransformInput,
RealtimeResponseTypedDict,
@ -95,6 +96,18 @@ def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
return VertexGeminiConfig()._map_audio_params({"voice": voice})
# Google bills Live transcription at an estimated 25 audio tokens/sec of input and
# 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing).
GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25
GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE: Final = 175
PCM16_INPUT_AUDIO_BYTES_PER_SECOND: Final = 48000
def _base64_decoded_byte_count(data: str) -> int:
padding: Final = 2 if data.endswith("==") else 1 if data.endswith("=") else 0
return max(len(data) * 3 // 4 - padding, 0)
class GeminiRealtimeConfig(BaseRealtimeConfig):
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
@ -104,6 +117,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# Gemini Live sometimes emits usageMetadata in a standalone frame between
# turns; buffer it here so the next response.done carries the token counts.
self._pending_usage_metadata: dict | None = None
self._unbilled_input_audio_bytes: int = 0
def is_setup_message(self, msg_obj: dict) -> bool:
return "setup" in msg_obj
@ -384,17 +398,25 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return bool(entry.get("gemini_native_audio") or entry.get("gemini_audio_only_live"))
@staticmethod
def _coerce_response_modalities(model: str, modalities: list[Any]) -> list[str]:
"""Map unsupported TEXT responseModalities to AUDIO for audio-only Live models."""
normalized: Final = [
def _is_text_only_live_model(model: str) -> bool:
return GeminiRealtimeConfig._model_cost_entry(model).get("mode") == "audio_transcription"
@staticmethod
def _default_response_modality(model: str) -> GeminiResponseModalities:
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[Any]) -> tuple[str, ...]:
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
normalized: Final = tuple(
modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
]
if not GeminiRealtimeConfig._is_audio_only_live_model(model):
return normalized
if "TEXT" not in normalized:
return normalized
without_text: Final = [modality for modality in normalized if modality != "TEXT"]
return without_text if without_text else ["AUDIO"]
)
if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
return normalized
@staticmethod
def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]:
@ -436,7 +458,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if session_configuration_request is None:
generation_config: Final = new_overrides.setdefault("generationConfig", {})
generation_config.setdefault("responseModalities", ["AUDIO"])
generation_config.setdefault("responseModalities", [GeminiRealtimeConfig._default_response_modality(model)])
new_overrides.setdefault("inputAudioTranscription", {})
new_overrides["model"] = f"models/{model}"
verbose_logger.debug("Gemini Realtime: Sending initial setup with tools to backend")
@ -558,9 +580,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return self._handle_conversation_item(json_message)
if msg_type == "input_audio_buffer.append":
realtime_input_dict["audio"] = HttpxBlobType(
mimeType=self.get_audio_mime_type(), data=json_message["audio"]
)
audio_b64: Final = json_message["audio"]
if isinstance(audio_b64, str):
self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64)
realtime_input_dict["audio"] = HttpxBlobType(mimeType=self.get_audio_mime_type(), data=audio_b64)
realtime_input_dict = cast(
BidiGenerateContentRealtimeInput,
@ -1151,6 +1174,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
raise ValueError(f"Unknown openai event: {key}, value: {value}")
return openai_event
def _consume_input_transcription_usage_estimate(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
"""Gemini Live sends no usageMetadata for transcribe sessions; estimate billing from streamed audio duration."""
if self._unbilled_input_audio_bytes <= 0 or not self._is_text_only_live_model(model):
return None
audio_seconds: Final = self._unbilled_input_audio_bytes / PCM16_INPUT_AUDIO_BYTES_PER_SECOND
self._unbilled_input_audio_bytes = 0
audio_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND)
output_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE / 60)
usage: Final[RealtimeInputAudioTranscriptionUsage] = {
"type": "tokens",
"input_tokens": audio_tokens,
"output_tokens": output_tokens,
"total_tokens": audio_tokens + output_tokens,
"input_token_details": {"text_tokens": 0, "audio_tokens": audio_tokens},
}
return usage
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
return self._consume_input_transcription_usage_estimate(model)
def transform_realtime_response(
self,
message: str | bytes,
@ -1190,6 +1233,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if isinstance(server_content, dict):
input_tx: Final = server_content.get("inputTranscription")
if isinstance(input_tx, dict) and input_tx.get("text"):
transcription_usage: Final = self._consume_input_transcription_usage_estimate(model)
returned_message.append(
cast(
OpenAIRealtimeEvents,
@ -1199,6 +1243,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"transcript": input_tx["text"],
"item_id": f"item_{uuid.uuid4()}",
"content_index": 0,
**({} if transcription_usage is None else {"usage": transcription_usage}),
},
)
)
@ -1235,6 +1280,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
)
)
# Transcription-only models emit generationComplete with no prior
# modelTurn delta; there is no started OpenAI response to close, so
# drop it and let siblings (turnComplete, usageMetadata) process.
if current_delta_type is None and "modelTurn" not in server_content:
server_content.pop("generationComplete", None)
# Mark transcription-only serverContent as handled so the main loop
# skips it; sibling keys like toolCall are still processed below.
_model_content_keys: Final = {
@ -1583,7 +1634,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
```
"""
response_modalities: Final[list[GeminiResponseModalities]] = ["AUDIO"]
response_modalities: Final[list[GeminiResponseModalities]] = [
GeminiRealtimeConfig._default_response_modality(model)
]
output_audio_transcription: Final = False
# if "audio" in model: ## UNCOMMENT THIS WHEN AUDIO IS SUPPORTED
# output_audio_transcription = True

2
litellm/llms/mistral/chat/transformation.py
View file

@ -295,7 +295,7 @@ class MistralConfig(OpenAIGPTConfig):
file_id = file_content.get("file", {}).get("file_id")
if file_id:
# Replace 'file' with 'file_id'
file_content["file_id"] = file_id
file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape
file_content.pop("file", None)
return messages

40
litellm/llms/moonshot/chat/transformation.py
View file

@ -2,7 +2,7 @@
Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions`
"""
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from typing import Any, Final, Literal, cast, overload
import litellm
@ -16,6 +16,15 @@ from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
def _reasoning_effort_string(value: object) -> str | None:
"""The /v1/messages and /v1/responses bridges wrap the level as {"effort", "summary"} for
providers with a reasoning-summary surface. Moonshot's API takes only the bare string and 400s
on an object, so the level is unwrapped and the summary, which has no Moonshot equivalent, is
dropped."""
effort: Final = value.get("effort") if isinstance(value, Mapping) else value
return effort if isinstance(effort, str) else None
class MoonshotChatConfig(OpenAIGPTConfig):
@overload
def _transform_messages(
@ -93,20 +102,18 @@ class MoonshotChatConfig(OpenAIGPTConfig):
- functions parameter is not supported (use tools instead)
- tool_choice doesn't support "required" value
- kimi-thinking-preview doesn't support tool calls at all
A reasoning model additionally takes `reasoning_effort`, which the OpenAI base list this
subtracts from does not carry, so it has to be added back rather than merely kept.
"""
excluded_params: Final[list[str]] = ["functions"]
# kimi-thinking-preview has additional limitations
if "kimi-thinking-preview" in model:
excluded_params.extend(["tools", "tool_choice"])
excluded_params: Final = frozenset(
("functions", "tools", "tool_choice") if "kimi-thinking-preview" in model else ("functions",)
)
base_openai_params: Final = super().get_supported_openai_params(model=model)
final_params: Final[list[str]] = []
for param in base_openai_params:
if param not in excluded_params:
final_params.append(param)
return final_params
supported: Final = [param for param in base_openai_params if param not in excluded_params]
if supports_reasoning(model=model, custom_llm_provider="moonshot"):
return [*supported, "reasoning_effort"]
return supported
def map_openai_params(
self,
@ -126,7 +133,12 @@ class MoonshotChatConfig(OpenAIGPTConfig):
for param, value in non_default_params.items():
if param == "max_completion_tokens":
optional_params["max_tokens"] = value
elif param in supported_openai_params:
elif param not in supported_openai_params:
continue
elif param == "reasoning_effort":
if (effort := _reasoning_effort_string(value)) is not None:
optional_params["reasoning_effort"] = effort
else:
optional_params[param] = value
##########################################

80
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -6,11 +6,28 @@ import litellm
from litellm.utils import (
_is_explicitly_disabled_factory,
_supports_factory,
declared_value_factory,
)
from .gpt_transformation import OpenAIGPTConfig
def _catalogue_declares_default_effort() -> bool:
"""Whether the loaded cost map carries default_reasoning_effort for ANY entry.
The map is fetched from the published branch at import time, so it can be OLDER than the
code reading it. On such a map every model looks undeclared, and treating that as "reasoning
is active" would silently strip temperature from the gpt-5.1/5.2/5.4 deployments that accept
it - a regression caused purely by data lag rather than by anything about the model.
So the absence of the key is only meaningful once the catalogue is known to carry it at all.
A map that has never heard of the key predates the feature, and the honest answer there is
the one litellm gave before it existed. Scanning costs ~80us on the largest published map and
only on the fallback path, which is noise beside the request it precedes.
"""
return any(isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in litellm.model_cost.values())
def _normalize_reasoning_effort_for_chat_completion(
value: str | dict | None,
) -> str | None:
@ -114,6 +131,17 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
except (ValueError, IndexError):
return False
@classmethod
def _model_map_lookup_name(cls, model: str) -> str:
"""The name this model is looked up by in the cost map.
Identity here, because an OpenAI model name is already its map key. Azure overrides
it: its routing prefixes are not map keys, so every capability lookup has to
normalise the name the same way, and doing that in ONE place is what keeps the
supports/disabled/default answers from disagreeing about which entry they read.
"""
return model
@classmethod
def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool:
"""Check if the model supports a specific reasoning_effort level.
@ -123,11 +151,40 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
Returns False for unknown models (safe fallback).
"""
return _supports_factory(
model=model,
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key=f"supports_{level}_reasoning_effort",
)
@classmethod
def effort_resolves_to_none(cls, model: str, effective_effort: str | None) -> bool:
"""Whether this request's reasoning effort ends up as "none", which is the single
condition under which the provider accepts a non-default temperature or the
top_p/logprobs sampling params.
An explicit reasoning_effort answers outright. When the request omits it the answer
is the model's DEFAULT effort, which only the map can state: supporting "none" is a
different fact from defaulting to it, and reading the former as the latter is what
forwarded temperature=0 to every gpt-5.5/5.6 deployment.
An undeclared default resolves to False. The map not saying is not the model
saying no, so the gate takes the conservative branch: a param the provider would
have rejected gets dropped or refused with an actionable error, and a model
released before its map entry declares a default needs no code change to be safe.
"""
if effective_effort is not None:
return effective_effort == "none"
declared: Final = declared_value_factory(
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key="default_reasoning_effort",
)
if declared is not None:
return declared == "none"
if not _catalogue_declares_default_effort():
return cls._supports_reasoning_effort_level(model, "none")
return False
@classmethod
def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool:
"""Return True only when the model map explicitly sets the capability to False.
@ -140,7 +197,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
Use this for opt-out checks where unknown models should be allowed through.
"""
return _is_explicitly_disabled_factory(
model=model,
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key=f"supports_{level}_reasoning_effort",
)
@ -260,15 +317,16 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
if supports_none:
sampling_params: Final = ["logprobs", "top_logprobs", "top_p"]
has_sampling: Final = any(p in non_default_params for p in sampling_params)
if has_sampling and effective_effort not in (None, "none"):
if has_sampling and not self.effort_resolves_to_none(model, effective_effort):
if litellm.drop_params or drop_params:
for p in sampling_params:
non_default_params.pop(p, None)
else:
raise litellm.utils.UnsupportedParamsError(
message=(
"gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when "
f"reasoning_effort='none'. Current reasoning_effort='{effective_effort}'. "
f"{model} only supports logprobs, top_p, top_logprobs when reasoning_effort "
"resolves to 'none', either set explicitly on the request or declared as the "
f"model's default_reasoning_effort. Current reasoning_effort={effective_effort!r}. "
"To drop unsupported params set `litellm.drop_params = True`"
),
status_code=400,
@ -277,17 +335,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
if "temperature" in non_default_params:
temperature_value: Final[float | None] = non_default_params.pop("temperature")
if temperature_value is not None:
# models supporting reasoning_effort="none" also support flexible temperature
if supports_none and (effective_effort == "none" or effective_effort is None) or temperature_value == 1:
# a non-default temperature rides on the effort resolving to "none", not on
# the model merely supporting it
if (supports_none and self.effort_resolves_to_none(model, effective_effort)) or temperature_value == 1:
optional_params["temperature"] = temperature_value
elif litellm.drop_params or drop_params:
pass
else:
raise litellm.utils.UnsupportedParamsError(
message=(
f"gpt-5 models (including gpt-5-codex) don't support temperature={temperature_value}. "
"Only temperature=1 is supported. "
"For gpt-5.1, temperature is supported when reasoning_effort='none' (or not specified, as it defaults to 'none'). "
f"{model} doesn't support temperature={temperature_value} while reasoning is "
"active. Only temperature=1 is supported unless reasoning_effort resolves to "
"'none', either set explicitly on the request or declared as the model's "
"default_reasoning_effort. "
"To drop unsupported params set `litellm.drop_params = True`"
),
status_code=400,

4
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
_should_convert_tool_call_to_json_mode,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_tool_reference_parts_from_tool_messages,
get_tool_call_names,
hoist_images_from_tool_messages,
)
@ -338,7 +339,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
self, messages: list[AllMessageValues], model: str, is_async: bool = False
) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]:
"""OpenAI no longer supports image_url as a string, so we need to convert it to a dict"""
hoisted_messages: Final = hoist_images_from_tool_messages(messages)
stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(messages)
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
async def _async_transform():
for message in hoisted_messages:

24
litellm/llms/openai/responses/transformation.py
View file

@ -61,6 +61,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
key="supports_none_reasoning_effort",
)
@staticmethod
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
"""Whether this request's reasoning effort ends up as "none", the one condition
under which a non-default temperature is accepted.
Delegates to the chat-completions gpt-5 config so both surfaces answer from one
rule: the Responses API reaches the same models over a different wire, and a second
copy of the rule here is what let this surface keep forwarding temperature after the
chat surface stopped.
"""
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
return OpenAIGPT5Config.effort_resolves_to_none(model, effort)
@staticmethod
def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None":
"""Raise sub-minimum max_output_tokens up to the OpenAI Responses API minimum.
@ -116,17 +130,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
reasoning: Final = params.get("reasoning") or {}
effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None
supports_none: Final = self._supports_reasoning_effort_none(model=model)
if supports_none and (effort == "none" or effort is None):
if supports_none and self._effort_resolves_to_none(model, effort):
pass # flexible temperature allowed
elif drop_params or litellm.drop_params:
params.pop("temperature", None)
else:
raise litellm.UnsupportedParamsError(
message=(
f"gpt-5 models don't support temperature={temperature}. "
"Only temperature=1 is supported. "
"For models like gpt-5.1/5.4, temperature is supported "
"when reasoning.effort='none' (or not specified). "
f"{model} doesn't support temperature={temperature} while reasoning is "
"active. Only temperature=1 is supported unless reasoning.effort resolves "
"to 'none', either set explicitly on the request or declared as the "
"model's default_reasoning_effort. "
"To drop unsupported params set `litellm.drop_params = True`"
),
status_code=400,

162
litellm/llms/soniox/common_utils.py
View file

@ -3,9 +3,13 @@ Shared utilities for the Soniox provider (https://soniox.com).
"""
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Final, TypeAlias
from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
SubtitleToken,
render_subtitle_tokens_as_srt,
render_subtitle_tokens_as_vtt,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
SonioxToken: TypeAlias = Mapping[str, object]
@ -121,128 +125,17 @@ def render_soniox_tokens(tokens: Sequence[SonioxToken]) -> str:
return "".join(text_parts)
# ---------------------------------------------------------------------------
# SRT / VTT subtitle rendering
# ---------------------------------------------------------------------------
# Maximum number of tokens to group into a single subtitle cue.
_CUE_MAX_TOKENS: Final[int] = 15
# Maximum duration (in ms) for a single cue before forcing a break.
_CUE_MAX_DURATION_MS: Final[int] = 5000
def _token_speaker(value: object) -> str | int | None:
return value if isinstance(value, str | int) else None
def _format_timestamp_srt(ms: int) -> str:
"""Format milliseconds as SRT timestamp: HH:MM:SS,mmm"""
ms = max(ms, 0)
hours: Final = ms // 3_600_000
ms %= 3_600_000
minutes: Final = ms // 60_000
ms %= 60_000
seconds: Final = ms // 1_000
millis: Final = ms % 1_000
return f"{hours:02d}:{minutes:02d}:{seconds:02d},{millis:03d}"
def _format_timestamp_vtt(ms: int) -> str:
"""Format milliseconds as VTT timestamp: HH:MM:SS.mmm"""
ms = max(ms, 0)
hours: Final = ms // 3_600_000
ms %= 3_600_000
minutes: Final = ms // 60_000
ms %= 60_000
seconds: Final = ms // 1_000
millis: Final = ms % 1_000
return f"{hours:02d}:{minutes:02d}:{seconds:02d}.{millis:03d}"
@dataclass(frozen=True, slots=True)
class _SubtitleCue:
start_ms: int
end_ms: int
text: str
def _group_tokens_into_cues(
tokens: Sequence[SonioxToken],
) -> list[_SubtitleCue]:
"""
Group Soniox tokens into subtitle cues.
Each cue has:
- start_ms: int
- end_ms: int
- text: str
Grouping heuristics:
- A new cue starts when token count exceeds _CUE_MAX_TOKENS.
- A new cue starts when duration exceeds _CUE_MAX_DURATION_MS.
- A new cue starts when the speaker changes (if diarization is on).
- Tokens without timestamps are appended to the current cue.
"""
cues: Final[list[_SubtitleCue]] = []
current_tokens: list[str] = []
current_start: int | None = None
current_end: int | None = None
current_speaker: object = None
def _flush() -> None:
if current_tokens and current_start is not None:
text: Final = "".join(current_tokens).strip()
if text:
cues.append(
_SubtitleCue(
start_ms=current_start,
end_ms=(current_end if current_end is not None else current_start),
text=text,
)
)
for token in tokens:
start_ms = _token_milliseconds(token.get("start_ms"))
end_ms = _token_milliseconds(token.get("end_ms"))
text = _token_text(token.get("text", ""))
speaker = token.get("speaker")
# Skip tokens with no timestamp data entirely if we have no cue started
if start_ms is None and current_start is None:
continue
# Speaker change forces a new cue
if speaker is not None and speaker != current_speaker:
_flush()
current_tokens = []
current_start = start_ms
current_end = end_ms
current_speaker = speaker
current_tokens.append(text)
continue
# Duration or token count exceeded -> flush
should_break = False
if (
len(current_tokens) >= _CUE_MAX_TOKENS
or current_start is not None
and start_ms is not None
and (start_ms - current_start) >= _CUE_MAX_DURATION_MS
):
should_break = True
if should_break:
_flush()
current_tokens = []
current_start = start_ms
current_end = end_ms
current_tokens.append(text)
else:
if current_start is None:
current_start = start_ms
if end_ms is not None:
current_end = end_ms
current_tokens.append(text)
_flush()
return cues
def _soniox_token_to_subtitle_token(token: SonioxToken) -> SubtitleToken:
return SubtitleToken(
text=_token_text(token.get("text", "")),
start_ms=_token_milliseconds(token.get("start_ms")),
end_ms=_token_milliseconds(token.get("end_ms")),
speaker=_token_speaker(token.get("speaker")),
)
def render_soniox_tokens_as_srt(tokens: Sequence[SonioxToken]) -> str:
@ -251,20 +144,7 @@ def render_soniox_tokens_as_srt(tokens: Sequence[SonioxToken]) -> str:
Returns an empty string if no tokens have timestamp data.
"""
cues: Final = _group_tokens_into_cues(tokens)
if not cues:
return ""
lines: Final[list[str]] = []
for idx, cue in enumerate(cues, start=1):
start = _format_timestamp_srt(cue.start_ms)
end = _format_timestamp_srt(cue.end_ms)
lines.append(str(idx))
lines.append(f"{start} --> {end}")
lines.append(cue.text)
lines.append("") # blank line between cues
return "\n".join(lines)
return render_subtitle_tokens_as_srt(tuple(_soniox_token_to_subtitle_token(token) for token in tokens))
def render_soniox_tokens_as_vtt(tokens: Sequence[SonioxToken]) -> str:
@ -273,14 +153,4 @@ def render_soniox_tokens_as_vtt(tokens: Sequence[SonioxToken]) -> str:
Returns the VTT header even if no cues are present.
"""
cues: Final = _group_tokens_into_cues(tokens)
lines: Final[list[str]] = ["WEBVTT", ""]
for cue in cues:
start = _format_timestamp_vtt(cue.start_ms)
end = _format_timestamp_vtt(cue.end_ms)
lines.append(f"{start} --> {end}")
lines.append(cue.text)
lines.append("") # blank line between cues
return "\n".join(lines)
return render_subtitle_tokens_as_vtt(tuple(_soniox_token_to_subtitle_token(token) for token in tokens))

95
litellm/llms/tencent/chat/transformation.py
View file

@ -3,14 +3,36 @@ Translates from OpenAI's `/v1/chat/completions` to Tencent TokenHub's
OpenAI-compatible endpoint.
"""
from typing import Final
from collections.abc import Mapping
from typing import Final, TypedDict
from typing_extensions import ReadOnly
import litellm
from litellm.secret_managers.main import get_secret_str
from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class ThinkingPayload(TypedDict, total=False):
"""Tencent TokenHub `thinking` object.
`type` ("enabled"/"disabled"/"adaptive") is required by TokenHub when the
object is passed; `budget_tokens` is auto-filled server-side when omitted.
Ref: https://www.tencentcloud.com/document/product/1300/82345
"""
type: ReadOnly[str]
budget_tokens: ReadOnly[int]
class ThinkingExtraBody(TypedDict, total=False):
"""`extra_body` payload carrying TokenHub's `thinking` object."""
thinking: ReadOnly[Mapping[str, object]]
class TencentChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
params: Final = super().get_supported_openai_params(model)
@ -25,18 +47,71 @@ class TencentChatConfig(OpenAIGPTConfig):
model: str,
drop_params: bool,
) -> dict:
optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params)
mapped_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
thinking_value: Final = optional_params.pop("thinking", None)
reasoning_effort: Final = optional_params.pop("reasoning_effort", None)
thinking_value: Final = mapped_params.pop("thinking", None)
reasoning_effort: Final = mapped_params.pop("reasoning_effort", None)
if thinking_value is not None:
if isinstance(thinking_value, dict):
optional_params["thinking"] = thinking_value
elif reasoning_effort is not None and reasoning_effort != "none":
optional_params["thinking"] = {"type": "enabled"}
thinking: Final = self._resolve_thinking_payload(
model=model,
thinking_value=thinking_value, # pyright: ignore[reportUnknownArgumentType] # value popped from the untyped provider params dict
reasoning_effort=reasoning_effort, # pyright: ignore[reportUnknownArgumentType] # value popped from the untyped provider params dict
)
if thinking is not None:
# TokenHub expects `thinking` in the request JSON body, but the
# OpenAI SDK's chat.completions.create() rejects unknown top-level
# kwargs, so it travels via `extra_body`, which the SDK merges into
# the payload. A plain assignment is merge-safe: get_optional_params
# spreads this dict into its own extra_body assembly downstream.
extra_body: Final[ThinkingExtraBody] = {"thinking": thinking}
mapped_params["extra_body"] = extra_body
return mapped_params
return optional_params
@classmethod
def _resolve_thinking_payload(
cls,
model: str,
thinking_value: object,
reasoning_effort: object,
) -> Mapping[str, object] | None:
if isinstance(thinking_value, dict):
return cls._coerce_thinking_type_for_model(model=model, thinking=thinking_value) # pyright: ignore[reportUnknownArgumentType] # isinstance narrows to dict[Unknown, Unknown] out of the untyped provider params dict
if isinstance(reasoning_effort, str):
# TokenHub recommends explicitly disabling thinking rather than
# relying on per-model defaults (deepseek-v4-* default to enabled).
payload: Final[ThinkingPayload] = {"type": "disabled" if reasoning_effort == "none" else "enabled"}
return cls._coerce_thinking_type_for_model(model=model, thinking=payload)
return None
@staticmethod
def _coerce_thinking_type_for_model(model: str, thinking: Mapping[str, object]) -> Mapping[str, object]:
"""Coerce `thinking.type` to a value the model accepts.
MiniMax models on TokenHub only accept "adaptive"/"disabled" and reject
"enabled" with a 400; "adaptive" (the model decides when to think) is
the closest semantic, so "enabled" is coerced for them. The capability
is read from the model map's `supports_adaptive_thinking` flag, so
aliases and newly onboarded adaptive-only models need no code change.
Ref: https://www.tencentcloud.com/document/product/1300/82345
"""
if thinking.get("type") != "enabled" or not TencentChatConfig._is_adaptive_thinking_model(model):
return thinking
budget: Final[object] = thinking.get("budget_tokens")
if isinstance(budget, int):
coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget}
return coerced_with_budget
coerced: Final[ThinkingPayload] = {"type": "adaptive"}
return coerced
@staticmethod
def _is_adaptive_thinking_model(model: str) -> bool:
"""Read `supports_adaptive_thinking` from the model map under tencent."""
try:
model_info: Final[Mapping[str, object]] = litellm.get_model_info(model=model, custom_llm_provider="tencent")
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for unmapped models
return False
return model_info.get("supports_adaptive_thinking") is True
def _get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None

83
litellm/llms/together_ai/chat/transformation.py
View file

@ -4,7 +4,8 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
Docs: https://docs.together.ai/docs/chat-overview
"""
from collections.abc import Callable, Container, Coroutine
from collections.abc import Callable, Container, Coroutine, Mapping
from types import MappingProxyType
from typing import (
Final,
Literal,
@ -12,11 +13,14 @@ from typing import (
overload,
)
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_function_calling, supports_response_schema
from litellm.utils import supports_function_calling, supports_reasoning, supports_response_schema
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@ -38,6 +42,34 @@ def _registry_verdict(model: str, flag: str, check: Callable[[str], bool]) -> bo
return None
ADJUSTABLE_EFFORT_REASONING_MODELS: Final = frozenset(
{
"openai/gpt-oss-120b",
"openai/gpt-oss-20b",
}
)
HYBRID_REASONING_MODELS: Final = frozenset(
{
"MiniMaxAI/MiniMax-M3",
"Qwen/Qwen3.5-9B",
"Qwen/Qwen3.6-Plus",
"deepseek-ai/DeepSeek-V4-Pro",
"moonshotai/Kimi-K3",
"nvidia/nemotron-3-ultra-550b-a55b",
"zai-org/GLM-5.2",
}
)
HIGH_MAX_EFFORT_MODEL_PREFIX: Final = "deepseek-ai/DeepSeek-V4-Pro"
EFFORT_TRANSLATION: Final = MappingProxyType({"minimal": "low", "xhigh": "high", "max": "high"})
HIGH_MAX_EFFORT_TRANSLATION: Final = MappingProxyType(
{"minimal": "high", "low": "high", "medium": "high", "xhigh": "max"}
)
class TogetherReasoningToggle(TypedDict):
enabled: ReadOnly[bool]
def _function_calling_verdict(model: str) -> bool | None:
return _registry_verdict(
model,
@ -83,6 +115,38 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params:
)
def _supports_together_reasoning(model: str) -> bool:
if model in ADJUSTABLE_EFFORT_REASONING_MODELS or model in HYBRID_REASONING_MODELS:
return True
if model.startswith(HIGH_MAX_EFFORT_MODEL_PREFIX):
return True
return supports_reasoning(model, custom_llm_provider="together_ai")
def _adjustable_effort(effort: str, model: str) -> str:
if effort == "none":
verbose_logger.debug(
"together_ai model %s cannot disable reasoning; mapping reasoning_effort=none to low", model
)
return "low"
return EFFORT_TRANSLATION.get(effort, effort)
def _reasoning_effort_payload(effort: str, model: str) -> Mapping[str, object]:
if effort == "default":
return MappingProxyType({})
if model in ADJUSTABLE_EFFORT_REASONING_MODELS:
return MappingProxyType({"reasoning_effort": _adjustable_effort(effort, model)})
if effort == "none":
disable_reasoning: Final[TogetherReasoningToggle] = {"enabled": False}
return MappingProxyType({"reasoning": disable_reasoning})
if effort in (declared_reasoning_efforts_for_model(model, "together_ai") or ()):
return MappingProxyType({"reasoning_effort": effort})
if model.startswith(HIGH_MAX_EFFORT_MODEL_PREFIX):
return MappingProxyType({"reasoning_effort": HIGH_MAX_EFFORT_TRANSLATION.get(effort, effort)})
return MappingProxyType({"reasoning_effort": EFFORT_TRANSLATION.get(effort, effort)})
def _drop_response_format(passed_params: Container[str], model: str, drop_params: bool) -> bool:
if "response_format" not in passed_params:
return False
@ -153,6 +217,15 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
return super()._transform_messages(stripped, model, is_async=True)
return super()._transform_messages(stripped, model, is_async=False)
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: inherited contract
supported_params: Final = super().get_supported_openai_params(model)
if not _supports_together_reasoning(model):
return supported_params
return [ # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
*supported_params,
"reasoning_effort",
]
def map_openai_params(
self,
non_default_params: dict,
@ -165,4 +238,10 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
mapped_openai_params.pop(param)
if _drop_response_format(mapped_openai_params, model, drop_params):
mapped_openai_params.pop("response_format")
effort: Final = mapped_openai_params.get("reasoning_effort")
if not isinstance(effort, str):
return mapped_openai_params
mapped_openai_params.pop("reasoning_effort")
for key, value in _reasoning_effort_payload(effort, model).items():
mapped_openai_params.setdefault(key, value)
return mapped_openai_params

7
litellm/llms/together_ai/cost_calculator.py
View file

@ -3,6 +3,7 @@ Handles calculating cost for together ai models
"""
import re
from collections.abc import Mapping
from typing import Final
from litellm.constants import (
@ -18,6 +19,12 @@ from litellm.constants import (
from litellm.types.utils import CallTypes
def has_together_registry_pricing(model: str, cost_map: Mapping[str, object]) -> bool:
stripped: Final = model.removeprefix("together_ai/")
entry: Final = cost_map.get(f"together_ai/{stripped}")
return isinstance(entry, Mapping) and "input_cost_per_token" in entry
# Extract the number of billion parameters from the model name
# only used for together_computer LLMs
def get_model_params_and_category(model_name, call_type: CallTypes) -> str:

80
litellm/main.py
View file

@ -531,6 +531,7 @@ async def acompletion(
tools=tools,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
#########################################################
# if the chat completion logging hook removed all tools,
@ -1219,6 +1220,7 @@ def _register_custom_pricing_for_request(
shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry),
},
persist_across_reloads=False,
warning_display_name=shared_key,
)
@ -5245,6 +5247,7 @@ def completion(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
### LITELLM SYSTEM PROMPT ###
@ -8587,6 +8590,47 @@ def stream_chunk_builder_text_completion(chunks: list, messages: list | None = N
return TextCompletionResponse(**response)
def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None:
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)):
return float(usage_cost)
if logging_obj is not None:
return None
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
"custom_llm_provider"
)
try:
return litellm.completion_cost(completion_response=response, custom_llm_provider=provider_hint)
except Exception:
return _stream_builder_model_map_cost(response)
def _joined_streamed_citations(streamed_citations: "tuple[object, ...]") -> "list[object]":
if all(isinstance(citation, list) for citation in streamed_citations):
return list(streamed_citations) # mutable-ok: JSON list field
return [list(streamed_citations)] # mutable-ok: JSON list field
def _stream_builder_model_map_cost(response: ModelResponse) -> float | None:
model_name: Final = getattr(response, "model", None)
usage: Final = getattr(response, "usage", None)
if not isinstance(model_name, str) or not model_name or not isinstance(usage, Usage):
return None
try:
prompt_cost, completion_tokens_cost = litellm.cost_per_token(model=model_name, usage_object=usage)
return prompt_cost + completion_tokens_cost
except Exception: # noqa: BLE001 # cost_per_token raises bare Exception for unpriceable models
return None
def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
response_cost: Final = _stream_builder_response_cost(response, logging_obj)
if response_cost is None:
return
hidden_params: Final = response._hidden_params # pyright: ignore[reportPrivateUsage] # no public accessor
hidden_params["response_cost"] = response_cost
def stream_chunk_builder(
chunks: list,
messages: list | None = None,
@ -8687,6 +8731,8 @@ def stream_chunk_builder(
"cost",
logging_obj._response_cost_calculator(result=response),
)
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
@ -8811,18 +8857,26 @@ def stream_chunk_builder(
]
if len(provider_specific_chunks) > 0:
combined_provider_fields: Final[dict[str, object]] = {}
for chunk in provider_specific_chunks:
fields = chunk["choices"][0]["delta"]["provider_specific_fields"]
if isinstance(fields, dict):
for key, value in fields.items():
if key not in combined_provider_fields:
combined_provider_fields[key] = value
elif isinstance(value, list) and isinstance(combined_provider_fields[key], list):
# For lists like web_search_results, take the last (most complete) one
combined_provider_fields[key] = value
else:
combined_provider_fields[key] = value
provider_field_dicts: Final = tuple(
fields
for chunk in provider_specific_chunks
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
if isinstance(fields, dict)
)
streamed_citations: Final = tuple(
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
)
citation_fields: Final = (
{"citations": _joined_streamed_citations(streamed_citations)} # mutable-ok: JSON dict field
if streamed_citations
else {} # mutable-ok: JSON dict field
)
combined_provider_fields: Final = { # mutable-ok: Message.provider_specific_fields is a plain dict field
key: value
for fields in (citation_fields, *provider_field_dicts)
for key, value in fields.items()
if key != "citation"
}
if combined_provider_fields:
_choice = cast(Choices, response.choices[0])
@ -8859,6 +8913,8 @@ def stream_chunk_builder(
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
setattr(usage, "cost", logging_obj._response_cost_calculator(result=response))
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
except Exception as e:

3442
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

152
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1,5 +1,5 @@
import re
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
@ -67,6 +67,9 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
_EMPTY_TOOLSET_GRANTS: Final[Mapping[str, Sequence[str]]] = MappingProxyType({})
def _as_list(values: Sequence[str] | None) -> list[str] | None: # mutable-ok: resolver returns a list
"""Widen a read-only allowlist back to the mutable list the resolver's own contract returns,
preserving the ``None`` that means "no restriction"."""
@ -1497,7 +1500,11 @@ class MCPRequestHandler:
team_set: Final = set(allowed_mcp_servers_for_team)
grants_set: Final = set(key_access_group_grants)
has_lower_level_mcp_restrictions = bool(key_set or team_set or grants_set)
# A DECLARED toolset restricts even when it resolves to no servers: the org
# ceiling below may only cap it, never substitute the org's full server list.
has_lower_level_mcp_restrictions = bool(key_set or team_set or grants_set) or (
await MCPRequestHandler._key_or_team_declares_toolsets(user_api_key_auth)
)
# 1. Key/team ceiling. An empty set means "this level does not restrict".
if not team_set:
@ -1941,6 +1948,105 @@ class MCPRequestHandler:
return team_obj.object_permission
@staticmethod
async def _toolset_tool_permissions(
object_permission: LiteLLM_ObjectPermissionTable | None,
) -> Mapping[str, Sequence[str]]:
"""The ``server_id -> tool names`` grants of this permission row's toolsets, empty when it
declares none. The shared resolver for the team, org, and internal-user levels, so a toolset
behaves identically wherever it is attached.
RAISES ``UnloadableEntitlementError`` when the row DECLARES toolsets but resolution yields
nothing (deleted or unknown ids, a swallowed DB fault, or a toolset with no tools): that is a
KNOWN restriction with unknown contents, and every caller already turns this error into deny
rather than letting the level read as unrestricted."""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
if object_permission is None or not object_permission.mcp_toolsets:
return _EMPTY_TOOLSET_GRANTS
resolved: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions(
toolset_ids=object_permission.mcp_toolsets
)
if not resolved:
raise UnloadableEntitlementError(
f"declared mcp_toolsets {object_permission.mcp_toolsets!r} resolved to no grants"
)
return resolved
@staticmethod
async def _toolset_tools_for_server(
object_permission: LiteLLM_ObjectPermissionTable | None,
server_id: str,
) -> Sequence[str] | None:
"""Tool names this row's toolsets grant on ``server_id``, ``None`` when its toolsets place
no restriction on that server (it declares no toolsets, or none of them name it)."""
return (await MCPRequestHandler._toolset_tool_permissions(object_permission)).get(server_id)
@staticmethod
def _union_tool_grants(
direct: Sequence[str] | None,
via_toolsets: Sequence[str] | None,
) -> Sequence[str] | None:
"""Union of one level's direct tool grants and its toolset-granted tools on one server,
``None`` when neither source restricts (allow-all from this level)."""
if direct is None and via_toolsets is None:
return None
return tuple({*(direct or ()), *(via_toolsets or ())})
@staticmethod
async def _key_object_permission_hydrated(
user_api_key_auth: UserAPIKeyAuth,
) -> LiteLLM_ObjectPermissionTable | None:
"""The key's object_permission, loading it by ``object_permission_id`` when the main auth
flow cached the key with the relation unhydrated (its loader swallows a failed read and
caches the partial object)."""
loaded: Final = MCPRequestHandler._get_key_object_permission(user_api_key_auth)
if loaded is not None or not user_api_key_auth.object_permission_id:
return loaded
from litellm.proxy.auth.auth_checks import get_object_permission
from litellm.proxy.proxy_server import (
prisma_client,
proxy_logging_obj,
user_api_key_cache,
)
if prisma_client is None:
return None
return await get_object_permission(
object_permission_id=user_api_key_auth.object_permission_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=user_api_key_auth.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
@staticmethod
async def _key_or_team_declares_toolsets(user_api_key_auth: UserAPIKeyAuth | None) -> bool:
"""Whether the key or its team GRANTS any toolset, resolvable or not. A declared toolset is
a lower-level restriction even when it resolves to no servers (deleted or unknown ids), so the
org ceiling may only cap it; reading an empty resolution as "no restriction" would substitute
the org's entire server list for the narrowest grant an operator can write.
Falls back to the DB when the auth object carries ``object_permission_id`` unhydrated (the
main auth flow swallows a failed load and caches the partial object). An INDETERMINATE fault
answers False — no gate, org substitution as before the fault — mirroring how the org ceiling
keeps key auth open on a fault it cannot classify."""
if user_api_key_auth is None:
return False
try:
key_obj_perm: Final = await MCPRequestHandler._key_object_permission_hydrated(user_api_key_auth)
if key_obj_perm is not None and key_obj_perm.mcp_toolsets:
return True
if not user_api_key_auth.team_id:
return False
team_obj_perm: Final = await MCPRequestHandler._get_team_object_permission(user_api_key_auth)
return bool(team_obj_perm is not None and team_obj_perm.mcp_toolsets)
except Exception as e: # noqa: BLE001 # indeterminate fault: no gate, as before this level existed
verbose_logger.warning("Failed to check declared MCP toolsets, org ceiling unchanged: %s", e)
return False
@staticmethod
async def get_allowed_tools_for_server(
server_id: str,
@ -2004,12 +2110,17 @@ class MCPRequestHandler:
if key_direct_tools is not None or key_toolset_tools is not None
else None
)
team_tools: Final = (
team_direct_tools: Final = (
global_mcp_server_manager.expand_tool_permissions(team_obj_perm.mcp_tool_permissions).get(server_id)
if team_obj_perm
else None
)
# Tools granted through the team's toolsets restrict this server exactly
# as the team's direct tool permissions do, mirroring the key path above
team_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server(team_obj_perm, server_id)
team_tools: Final = MCPRequestHandler._union_tool_grants(team_direct_tools, team_toolset_tools)
# Apply same inheritance logic as get_allowed_mcp_servers
if team_tools:
if key_tools:
@ -2094,11 +2205,13 @@ class MCPRequestHandler:
e,
)
return allowed_tools
org_tools: Final = (
org_direct_tools: Final = (
global_mcp_server_manager.expand_tool_permissions(org_obj_perm.mcp_tool_permissions).get(server_id)
if org_obj_perm and org_obj_perm.mcp_tool_permissions
else None
)
org_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server(org_obj_perm, server_id)
org_tools: Final = MCPRequestHandler._union_tool_grants(org_direct_tools, org_toolset_tools)
if org_tools is not None:
allowed_tools = (
list(set(allowed_tools) & set(org_tools)) if allowed_tools is not None else list(org_tools)
@ -2340,7 +2453,8 @@ class MCPRequestHandler:
async def _team_granted_servers(team_obj: LiteLLM_TeamTable, team_access_group_servers: list[str]) -> set[str]:
"""The raw MCP-server set a team grants (before any org ceiling): its object_permission (direct
``mcp_servers``, the ``all_proxy_servers`` sentinel → the full registry, legacy access groups,
tool-perm-referenced servers) unioned with its unified ``access_group_ids`` servers."""
tool-perm-referenced servers, toolset-referenced servers) unioned with its unified
``access_group_ids`` servers."""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
@ -2357,6 +2471,7 @@ class MCPRequestHandler:
set(global_mcp_server_manager.expand_permission_list(object_permissions.mcp_servers or []))
| set(legacy_access_group_servers)
| set(global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).keys())
| (await MCPRequestHandler._toolset_tool_permissions(object_permissions)).keys()
| set(team_access_group_servers)
)
@ -2415,6 +2530,8 @@ class MCPRequestHandler:
servers: Final = await MCPRequestHandler._team_granted_servers(team_obj, team_access_group_servers)
return list(servers)
except Exception as e:
if isinstance(e, UnloadableEntitlementError):
raise
verbose_logger.warning("Failed to get allowed MCP servers for team: %s", e)
return []
@ -2546,7 +2663,13 @@ class MCPRequestHandler:
global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).keys()
)
all_servers: Final = direct_mcp_servers + access_group_servers + tool_perm_servers
# servers referenced by the org's toolset grants are part of the org ceiling,
# exactly as servers referenced by its inline tool permissions are
toolset_grants: Final = await MCPRequestHandler._toolset_tool_permissions(object_permissions)
all_servers: Final = tuple(
{*direct_mcp_servers, *access_group_servers, *tool_perm_servers, *toolset_grants}
)
return list(set(all_servers))
except Exception as e:
# None = ceiling UNRESOLVED, distinct from [] = org places no restriction. Collapsing them
@ -2740,8 +2863,8 @@ class MCPRequestHandler:
``[]`` means this human places no restriction (allow-all from this level); ``None`` means the
ceiling is UNRESOLVED, which the caller denies on. Servers named only under
``mcp_tool_permissions`` count as entitled, exactly as they do for a key or a team, so
granting one tool never requires naming its server twice.
``mcp_tool_permissions`` or reached through ``mcp_toolsets`` count as entitled, exactly as
they do for a key or a team, so granting one tool never requires naming its server twice.
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
@ -2759,7 +2882,8 @@ class MCPRequestHandler:
tool_perm_servers: Final = list(
global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).keys()
)
return list(set(direct_mcp_servers + access_group_servers + tool_perm_servers))
toolset_grants: Final = await MCPRequestHandler._toolset_tool_permissions(object_permissions)
return tuple({*direct_mcp_servers, *access_group_servers, *tool_perm_servers, *toolset_grants})
except Exception as e: # noqa: BLE001 # any resolution fault is an unresolved ceiling, never "no ceiling"
verbose_logger.warning("Failed to get allowed MCP servers for user: %s", e)
return None
@ -2860,12 +2984,14 @@ class MCPRequestHandler:
verbose_logger.warning("MCP user tool ceiling unresolvable, denying tools on %r: %s", server_id, e)
return []
if object_permissions is None or not object_permissions.mcp_tool_permissions:
if object_permissions is None:
return allowed_tools
user_tools = global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).get(
server_id
)
user_direct_tools: Final = global_mcp_server_manager.expand_tool_permissions(
object_permissions.mcp_tool_permissions
).get(server_id)
user_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server(object_permissions, server_id)
user_tools: Final = MCPRequestHandler._union_tool_grants(user_direct_tools, user_toolset_tools)
if user_tools is None:
return allowed_tools
if allowed_tools is None:

91
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -34,6 +34,7 @@ from mcp.types import (
)
from mcp.types import Tool as MCPTool
from pydantic import AnyUrl, BaseModel
from typing_extensions import ReadOnly
import litellm
from litellm._logging import verbose_logger
@ -72,6 +73,7 @@ from litellm.proxy._experimental.mcp_server.oauth2_token_cache import (
MCPPerUserTokenCache,
mcp_per_user_token_cache,
resolve_mcp_auth,
resolved_token_header,
)
from litellm.proxy._experimental.mcp_server.oauth_utils import (
_redact_mcp_resource_url,
@ -99,6 +101,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_
build_token_exchanger,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
DEFAULT_CREDENTIAL_HEADER,
AuthorizationCodeConfig,
ClientCredentialsConfig,
CredError,
@ -153,6 +156,8 @@ from litellm.types.mcp import (
MCPAuth,
MCPStdioConfig,
MCPTokenEndpointAuthMethod,
has_header,
without_header,
)
from litellm.types.mcp_server.mcp_server_manager import (
MCPInfo,
@ -349,6 +354,7 @@ class MCPServerConfig(TypedDict, total=False):
audience: str
subject_token_type: str
upstream_resource: str
upstream_token_header: ReadOnly[str]
id_jag_resource_token_endpoint: str
id_jag_resource: str
client_private_key: str
@ -828,18 +834,6 @@ def _should_strip_caller_authorization(
)
def _without_authorization(
headers: dict[str, str] | None,
) -> dict[str, str] | None:
"""A copy of ``headers`` with any ``Authorization`` key removed (case-insensitive), or
None if nothing remains. Drops only the credential, keeping other forwarded headers.
"""
if not headers:
return None
filtered: Final = {k: v for k, v in headers.items() if k.lower() != "authorization"}
return filtered or None
def _format_byok_openapi_auth_header(mcp_server: MCPServer, mcp_auth_header: str) -> str:
"""Format a raw BYOK credential for OpenAPI tool ``Authorization`` injection.
@ -914,7 +908,9 @@ def _resolve_openapi_tool_auth(
if isinstance(per_server, dict):
authorization: Final = next((v for k, v in per_server.items() if k.lower() == "authorization"), None)
merged: Final = merge_mcp_headers(extra_headers=forwarded, static_headers=_without_authorization(per_server))
merged: Final = merge_mcp_headers(
extra_headers=forwarded, static_headers=without_header(per_server, DEFAULT_CREDENTIAL_HEADER)
)
if authorization is None:
byok: Final = _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None
return byok, merged, mcp_auth_header
@ -981,7 +977,7 @@ def _client_forwarded_authorization_headers(
raw_headers=raw_headers,
user_api_key_auth=user_api_key_auth,
):
return _without_authorization(extra_headers)
return without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER)
return extra_headers
@ -994,7 +990,7 @@ def _take_forwarded_authorization(
if not headers:
return None, headers
value: Final = next((v for k, v in headers.items() if k.lower() == "authorization"), None)
return value, _without_authorization(headers)
return value, without_header(headers, DEFAULT_CREDENTIAL_HEADER)
def _passthrough_token_from_mcp_auth_header(
@ -2166,6 +2162,7 @@ class MCPServerManager:
DEFAULT_SUBJECT_TOKEN_TYPE,
),
upstream_resource=server_config.get("upstream_resource", None),
upstream_token_header=server_config.get("upstream_token_header", None),
# ID-JAG fields
id_jag_resource_token_endpoint=server_config.get("id_jag_resource_token_endpoint", None),
id_jag_resource=server_config.get("id_jag_resource", None),
@ -2698,6 +2695,7 @@ class MCPServerManager:
or (credentials_dict.get("subject_token_type") if credentials_dict else None)
or DEFAULT_SUBJECT_TOKEN_TYPE,
upstream_resource=(credentials_dict.get("upstream_resource") if credentials_dict else None),
upstream_token_header=(credentials_dict.get("upstream_token_header") if credentials_dict else None),
# ID-JAG fields — read from credentials JSON blob
id_jag_resource_token_endpoint=(
credentials_dict.get("id_jag_resource_token_endpoint") if credentials_dict else None
@ -3525,10 +3523,9 @@ class MCPServerManager:
case Ok(auth):
# NoOpAuth has no header_name and so never conflicts.
header_name: Final[str | None] = getattr(auth, "header_name", None)
conflicts: Final = bool(
header_name and extra_headers and any(key.lower() == header_name.lower() for key in extra_headers)
)
if not conflicts:
if header_name is None or not extra_headers:
return auth, extra_headers
if not has_header(extra_headers, header_name):
return auth, extra_headers
if isinstance(
spec.config,
@ -3540,9 +3537,10 @@ class MCPServerManager:
# guardrail such as MCPJWTSigner, static_headers, or any other injected
# Authorization must NOT shadow it (otherwise the upstream gets e.g. the
# signer's JWT instead of the minted token and rejects it, and for M2M the
# one-shot 401 refetch is lost with it). Drop the conflicting header so the
# resolved token reaches upstream.
return auth, _without_authorization(extra_headers)
# one-shot 401 refetch is lost with it). Drop only the header the resolved
# credential is about to occupy, so a static credential the operator aimed at a
# DIFFERENT header still reaches upstream.
return auth, without_header(extra_headers, header_name)
# Other modes: an Authorization already supplied via extra_headers (a forwarded caller
# header or static_headers) is intentional and wins; v1 applies those last.
return None, extra_headers
@ -3650,6 +3648,7 @@ class MCPServerManager:
):
spec = None
auth_value: Final = await resolve_mcp_auth(resolved_server, mcp_auth_header) if spec is None else None
auth_header_name: Final = resolved_token_header(resolved_server, mcp_auth_header) if spec is None else None
# Create sampling and elicitation callbacks for this client
sampling_cb = (
@ -3758,6 +3757,7 @@ class MCPServerManager:
transport_type=transport,
auth_type=resolved_server.auth_type,
auth_value=auth_value,
auth_header_name=auth_header_name,
timeout=(resolved_server.timeout if resolved_server.timeout is not None else MCP_CLIENT_TIMEOUT),
extra_headers=extra_headers,
aws_auth=aws_auth,
@ -5256,7 +5256,9 @@ class MCPServerManager:
proxy_logging_obj: Optional ProxyLogging object for hook integration
host_progress_callback: Optional callback for progress updates
hook_extra_headers: Optional headers injected by pre_mcp_call guardrail
hooks. Merged last (highest priority) into outbound request headers.
hooks. Merged last into outbound request headers, except a hook
Authorization header is dropped when an upstream credential already
occupies the Authorization slot.
Returns:
CallToolResult from the MCP server
@ -5304,7 +5306,7 @@ class MCPServerManager:
raw_headers=raw_headers,
user_api_key_auth=user_api_key_auth,
):
extra_headers = _without_authorization(extra_headers)
extra_headers = without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER)
elif mcp_server.is_client_forwarded_token:
extra_headers = _client_forwarded_authorization_headers(
mcp_server=mcp_server,
@ -5347,27 +5349,26 @@ class MCPServerManager:
if hook_extra_headers:
if extra_headers is None:
extra_headers = {}
if "Authorization" in hook_extra_headers:
if "Authorization" in extra_headers:
verbose_logger.warning(
"MCPServerManager: hook_extra_headers 'Authorization' will overwrite "
"the existing Authorization header from static_headers. "
"The hook JWT will take precedence."
)
elif server_auth_header is not None:
# server_auth_header is passed separately to _create_mcp_client as
# auth_value. Both will reach the upstream server — warn so admins
# know two Authorization credentials are being sent.
verbose_logger.warning(
"MCPServerManager: hook_extra_headers injects 'Authorization' while "
"server '%s' already has a configured authentication_token. "
"Both credentials will be sent; the hook header is in extra_headers "
"and the server token is in auth_value — the upstream server decides "
"which one wins. Consider unsetting authentication_token if you want "
"the hook JWT to be the sole credential.",
mcp_server.server_name or mcp_server.name,
)
extra_headers.update(hook_extra_headers)
hook_has_authorization: Final = any(k.lower() == "authorization" for k in hook_extra_headers)
existing_has_authorization: Final = any(k.lower() == "authorization" for k in extra_headers)
server_auth_occupies_authorization: Final = (
any(k.lower() == "authorization" for k in server_auth_header)
if isinstance(server_auth_header, dict)
else server_auth_header is not None and mcp_server.auth_type != MCPAuth.api_key
)
if hook_has_authorization and (existing_has_authorization or server_auth_occupies_authorization):
# Mirror the tools/list signer guard: an upstream credential (user OAuth,
# static header, or configured authentication_token) already occupies the
# Authorization slot, so the hook must not replace it.
verbose_logger.warning(
"MCPServerManager: dropping hook-injected 'Authorization' header for "
"server '%s' because an upstream credential already occupies the "
"Authorization slot; the existing credential is kept.",
mcp_server.server_name or mcp_server.name,
)
extra_headers.update({k: v for k, v in hook_extra_headers.items() if k.lower() != "authorization"})
else:
extra_headers.update(hook_extra_headers)
# Reset to None if no headers were actually added
if extra_headers is not None and len(extra_headers) == 0:

18
litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py
View file

@ -7,6 +7,7 @@ with ``client_id``, ``client_secret``, and ``token_url``.
import asyncio
import hashlib
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
import httpx
@ -313,9 +314,26 @@ async def resolve_mcp_auth(
1. ``mcp_auth_header`` — per-request/per-user override
2. OAuth2 client_credentials token — auto-fetched and cached
3. ``server.authentication_token`` — static token from config/DB
``resolved_token_header`` answers, for the same two inputs, which header the value belongs in.
"""
if mcp_auth_header:
return mcp_auth_header
if server.has_client_credentials:
return await mcp_oauth2_token_cache.async_get_token(server)
return server.authentication_token
def resolved_token_header(
server: "MCPServer",
mcp_auth_header: str | Mapping[str, str] | None = None,
) -> str | None:
"""Which upstream header the value ``resolve_mcp_auth`` just returned belongs in.
``None`` means keep the auth_type default. A caller-supplied ``mcp_auth_header`` is the caller's
own credential aimed at the slot the upstream normally uses, so it never moves; only the values
the gateway resolved from its own config (the minted M2M token, the static token) follow
``upstream_token_header``. Same inputs and same branch order as ``resolve_mcp_auth``, so the two
cannot disagree about which case they are in.
"""
return None if mcp_auth_header else server.upstream_token_header

40
litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py
View file

@ -47,12 +47,14 @@ def sanitize_openapi_tool_name(raw_name: str) -> str:
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import async_safe_get
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._experimental.mcp_server.tool_registry import (
global_mcp_tool_registry,
)
from litellm.types.mcp import credential_redirect_hook, custom_credential_slot
class _OpenAPIJSONSchema(TypedDict, total=False):
@ -119,6 +121,10 @@ _request_resolved_auth_headers: Final[contextvars.ContextVar[dict[str, str] | No
"_request_resolved_auth_headers", default=None
)
_request_upstream_url: Final[contextvars.ContextVar[str | None]] = contextvars.ContextVar(
"_request_upstream_url", default=None
)
def _sanitize_path_parameter_value(param_value: object, param_name: str) -> str:
"""Ensure path params cannot introduce directory traversal."""
@ -349,6 +355,35 @@ def build_input_schema(operation: _OpenAPIOperation) -> dict[str, object]:
}
async def _drop_credential_across_origin(request: httpx.Request) -> None:
"""Apply this request's cross-origin credential guard, if it needs one.
Reads the per-request context rather than closing over it so the hook is one stable object, which
keeps the guarded client cacheable. A closure would key a new entry per call, and the handler it
built would never be closed.
"""
guard: Final = credential_redirect_hook(
_request_upstream_url.get() or "", custom_credential_slot(_request_resolved_auth_headers.get())
)
if guard is not None:
await guard(request)
def _upstream_client() -> AsyncHTTPHandler:
"""The HTTP client for one upstream call, guarded when a credential rides a custom slot.
A resolved credential outside ``Authorization`` is not stripped across origins by the client
itself, so this arm installs the same hook the MCP client uses. Both variants come from the
shared cache, so a guarded call reuses its connection pool like any other.
"""
if custom_credential_slot(_request_resolved_auth_headers.get()) is None:
return get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
return get_async_httpx_client(
llm_provider=httpxSpecialProvider.MCP,
params={"event_hooks": {"request": [_drop_credential_across_origin]}},
)
def _merge_openapi_tool_request_headers(
static_headers: dict[str, str],
) -> dict[str, str]:
@ -510,8 +545,9 @@ def create_tool_function(
except (json.JSONDecodeError, TypeError):
json_body = {"data": body_value}
client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
client: Final = _upstream_client()
upstream: Final = server_label or f"{original_method.upper()} {path}"
url_token: Final = _request_upstream_url.set(url)
try:
if original_method == "get":
@ -529,6 +565,8 @@ def create_tool_function(
except MaskedHTTPStatusError as e:
_raise_for_upstream_failure(e.response, upstream, relays_upstream_auth)
raise
finally:
_request_upstream_url.reset(url_token)
_raise_for_upstream_failure(response, upstream, relays_upstream_auth)
return response.text

6
litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py
View file

@ -21,6 +21,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
Result,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
DEFAULT_CREDENTIAL_HEADER,
Ambient,
ApiKeyConfig,
ApiKeySource,
@ -35,6 +36,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
ClientCredentialsConfig,
ClientSecretAuth,
CredError,
HeaderCarrier,
IdJagConfig,
NoneConfig,
PassthroughConfig,
@ -45,9 +47,11 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
Subject,
TokenExchangeConfig,
parse_auth_spec_kind,
validate_header_name,
)
__all__ = [
"DEFAULT_CREDENTIAL_HEADER",
"Ambient",
"ApiKeyConfig",
"ApiKeySource",
@ -63,6 +67,7 @@ __all__ = [
"ClientSecretAuth",
"CredError",
"Error",
"HeaderCarrier",
"IdJagConfig",
"NoOpAuth",
"NoneConfig",
@ -78,4 +83,5 @@ __all__ = [
"TokenExchangeConfig",
"UpstreamCredentialProvider",
"parse_auth_spec_kind",
"validate_header_name",
]

17
litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py
View file

@ -20,6 +20,7 @@ from typing_extensions import assert_never
from litellm.proxy._experimental.mcp_server.oauth_utils import resolve_upstream_resource
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
DEFAULT_CREDENTIAL_HEADER,
ApiKeyConfig,
AuthorizationCodeConfig,
ClientAuth,
@ -45,6 +46,15 @@ _TOKEN_EXCHANGE_SUBJECT_TOKEN_DEFAULT: Final = "urn:ietf:params:oauth:token-type
_ID_JAG_SUBJECT_TOKEN_DEFAULT: Final = "urn:ietf:params:oauth:token-type:id_token"
def token_header(server: MCPServer) -> str:
"""The upstream header this server's resolved credential occupies.
One owner for every arm, so no spec builder spells the default itself and a server can never
hand two arms different answers.
"""
return server.upstream_token_header or DEFAULT_CREDENTIAL_HEADER
def to_subject(user_api_key_auth: UserAPIKeyAuth | None, subject_token: str | None) -> Subject:
"""Map v1's authenticated principal onto the resolver's Subject.
@ -122,7 +132,7 @@ def _oauth2_spec(server: MCPServer, resource: str) -> ServerSpec | None:
return ServerSpec(
server_id=server.server_id,
resource=resource,
config=AuthorizationCodeConfig(),
config=AuthorizationCodeConfig(header_name=token_header(server)),
)
return None
@ -140,6 +150,7 @@ def _client_credentials_spec(server: MCPServer, resource: str) -> ServerSpec:
server_id=server.server_id,
resource=resource,
config=ClientCredentialsConfig(
header_name=token_header(server),
client_id=server.client_id,
client_secret=SecretStr(server.client_secret) if server.client_secret else None,
token_url=server.effective_token_url,
@ -173,6 +184,7 @@ def _token_exchange_spec(server: MCPServer, resource: str) -> ServerSpec | None:
server_id=server.server_id,
resource=resource,
config=TokenExchangeConfig(
header_name=token_header(server),
profile=profile,
subject_token_type=server.subject_token_type or DEFAULT_SUBJECT_TOKEN_TYPE,
token_exchange_endpoint=endpoint,
@ -206,7 +218,7 @@ def _shared_key_spec(
server_id=server.server_id,
resource=resource,
config=ApiKeyConfig(
header_name=header_name,
header_name=server.upstream_token_header or header_name,
value_prefix=value_prefix,
key_source=SharedKey(value=SecretStr(value)),
),
@ -231,6 +243,7 @@ def _id_jag_spec(server: MCPServer, resource: str) -> ServerSpec | None:
server_id=server.server_id,
resource=resource,
config=IdJagConfig(
header_name=token_header(server),
org_token_endpoint=org_token_endpoint,
resource_token_endpoint=resource_token_endpoint,
client_id=client_id,

3
litellm/proxy/_experimental/mcp_server/outbound_credentials/bridge_credentials.py
View file

@ -239,5 +239,6 @@ def resolve_bridge_envelope(
if opened.identity.server_id != expected_server_id:
return BridgeEnvelopeInvalid()
grant: Final = opened.grant
upstream_authorization: Final = f"{grant.token_type} {grant.access_token.get_secret_value()}"
authorization_scheme: Final = "Bearer" if grant.token_type.lower() == "bearer" else grant.token_type
upstream_authorization: Final = f"{authorization_scheme} {grant.access_token.get_secret_value()}"
return BridgeEnvelopeAdmitted(identity=opened.identity, upstream_authorization=SecretStr(upstream_authorization))

17
litellm/proxy/_experimental/mcp_server/outbound_credentials/client_credentials.py
View file

@ -50,6 +50,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
ClientCredentialsConfig,
CredError,
HeaderCarrier,
)
@ -328,14 +329,21 @@ class ClientCredentialsBearerAuth(httpx.Auth):
refetch fails, or the retried request 401s again, the upstream's response stands.
"""
def __init__(self, access_token: str, refetch: Callable[[str], Awaitable[str | None]]) -> None:
self.header_name = "Authorization"
def __init__(
self,
access_token: str,
refetch: Callable[[str], Awaitable[str | None]],
carrier: HeaderCarrier,
) -> None:
self._carrier = carrier
self.header_name = carrier.header_name
self._access_token = SecretStr(access_token)
self._refetch = refetch
async def async_auth_flow(self, request: httpx.Request) -> AsyncGenerator[httpx.Request, httpx.Response]:
token: Final = self._access_token.get_secret_value()
request.headers[self.header_name] = f"Bearer {token}"
name, value = self._carrier.header(token)
request.headers[name] = value
response: Final = yield request
if response.status_code != 401:
return
@ -343,7 +351,8 @@ class ClientCredentialsBearerAuth(httpx.Auth):
if fresh is None:
return
self._access_token = SecretStr(fresh)
request.headers[self.header_name] = f"Bearer {fresh}"
fresh_name, fresh_value = self._carrier.header(fresh)
request.headers[fresh_name] = fresh_value
yield request
def sync_auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:

19
litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py
View file

@ -145,8 +145,8 @@ class UpstreamCredentialProvider:
return await self._token_exchange(subject, server, config)
case IdJagConfig() as config:
return await self._id_jag(subject, server, config)
case AuthorizationCodeConfig():
return await self._authorization_code(subject, server)
case AuthorizationCodeConfig() as config:
return await self._authorization_code(subject, server, config)
case AwsSigV4Config():
return _not_implemented(AuthSpecKind.aws_sigv4)
assert_never(server.config)
@ -284,15 +284,19 @@ class UpstreamCredentialProvider:
match await self._exchanged_tokens.get_or_compute(slot, _exchange, fingerprint=fingerprint):
case Ok(access_token):
return Ok(StaticHeaderAuth(f"Bearer {access_token}"))
header_name, header_value = config.header(access_token)
return Ok(StaticHeaderAuth(header_value, header_name=header_name))
case Error(err):
return Error(err)
async def _authorization_code(self, subject: Subject, server: ServerSpec) -> Result[StaticHeaderAuth, CredError]:
async def _authorization_code(
self, subject: Subject, server: ServerSpec, config: AuthorizationCodeConfig
) -> Result[StaticHeaderAuth, CredError]:
token: Final = await self._authz_token(subject, server)
if token is None:
return Error(CredError.of_unauthorized("Authorization required: complete the OAuth flow for this server."))
return Ok(StaticHeaderAuth(f"Bearer {token.access_token}", header_name="Authorization"))
header_name, header_value = config.header(token.access_token)
return Ok(StaticHeaderAuth(header_value, header_name=header_name))
async def _client_credentials(
self, server_id: str, config: ClientCredentialsConfig
@ -307,7 +311,7 @@ class UpstreamCredentialProvider:
match await self._client_credentials_source.get(server_id, config):
case Ok(token):
refetch: Final = partial(self._client_credentials_source.refetch, server_id, config)
return Ok(ClientCredentialsBearerAuth(token.access_token, refetch))
return Ok(ClientCredentialsBearerAuth(token.access_token, refetch, config))
case Error(err):
return Error(err)
@ -332,7 +336,8 @@ class UpstreamCredentialProvider:
inbound.get_secret_value(), server, config, tenant_id=subject.tenant_id
):
case Ok(token):
return Ok(StaticHeaderAuth(f"Bearer {token.access_token}", header_name="Authorization"))
header_name, header_value = config.header(token.access_token)
return Ok(StaticHeaderAuth(header_value, header_name=header_name))
case Error(err):
return Error(err)

74
litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py
View file

@ -31,7 +31,7 @@ from enum import Enum
from typing import Annotated, Final, Literal
from expression import case, tag, tagged_union
from pydantic import BaseModel, ConfigDict, Field, SecretStr
from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator
from typing_extensions import assert_never
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
@ -39,7 +39,11 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
Ok,
Result,
)
from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE
from litellm.types.mcp import (
DEFAULT_CREDENTIAL_HEADER,
DEFAULT_SUBJECT_TOKEN_TYPE,
normalize_upstream_header_name,
)
class AuthSpecKind(str, Enum):
@ -161,7 +165,52 @@ class CredError:
assert_never(self.tag)
class AuthorizationCodeConfig(BaseModel):
def validate_header_name(raw: str) -> Result[str, CredError]:
"""``normalize_upstream_header_name`` with this package's error-as-value policy.
The grammar itself lives in ``litellm.types.mcp`` so the v1 model, the management endpoint and
this vocabulary all judge a header name the same way while each keeps its own failure shape.
"""
normalized: Final = normalize_upstream_header_name(raw)
if normalized is None:
return Error(CredError.of_misconfigured(f"invalid upstream header name: {raw!r}"))
return Ok(normalized)
class HeaderCarrier(BaseModel):
"""Where a resolved credential is written upstream, and how its value is formatted.
``Authorization: Bearer`` is only OAuth's *default* conveyance (RFC 6750 section 2.1), not its
only one: an ESB or API gateway commonly terminates its own credential in a private header while
a second credential passes through to the origin, so a credential has to be able to say which
slot it owns. Modeled like OpenAPI's apiKey scheme, so any upstream convention is expressible
(Authorization + Bearer, a raw value on X-API-Key, Ocp-Apim-Subscription-Key, esb-oauth, ...).
Every config whose credential the gateway mints or holds inherits this, so no resolver arm names
a header itself and the conflict rule in ``_resolve_v2_auth`` can always ask the auth object
which slot it is about to occupy. ``passthrough`` deliberately does not: it forwards the
caller's own credential into the slot the caller used, and mints nothing to place.
"""
model_config = ConfigDict(frozen=True)
header_name: str = DEFAULT_CREDENTIAL_HEADER
value_prefix: str = "Bearer"
@field_validator("header_name")
@classmethod
def _check_header_name(cls, value: str) -> str:
match validate_header_name(value):
case Ok(name):
return name
case Error(err):
raise ValueError(err.summary)
def header(self, value: str) -> tuple[str, str]:
formatted: Final = f"{self.value_prefix} {value}" if self.value_prefix else value
return self.header_name, formatted
class AuthorizationCodeConfig(HeaderCarrier):
"""Per-user 3LO; the gateway is the OAuth client and stores the user's token.
Endpoints are discovered (RFC 9728 -> RFC 8414) and the client is registered via DCR
@ -179,7 +228,7 @@ class AuthorizationCodeConfig(BaseModel):
token_url: str | None = None
class ClientCredentialsConfig(BaseModel):
class ClientCredentialsConfig(HeaderCarrier):
"""M2M service account; one upstream identity for every user.
Fields are optional so the config can be built incomplete: a value may be supplied at
@ -203,7 +252,7 @@ class ClientCredentialsConfig(BaseModel):
token_endpoint_auth_method: Literal["client_secret_post", "client_secret_basic"] | None = None
class TokenExchangeConfig(BaseModel):
class TokenExchangeConfig(HeaderCarrier):
"""OBO: swap the caller's live inbound token for a token bound to the upstream's audience. The
gateway authenticates to the exchange endpoint as an OAuth client (`client_id`/`client_secret`);
the inbound token is sent only to that endpoint, never to the upstream.
@ -255,7 +304,7 @@ class ClientSecretAuth(BaseModel):
ClientAuth = Annotated[PrivateKeyJwtAuth | ClientSecretAuth, Field(discriminator="source")]
class IdJagConfig(BaseModel):
class IdJagConfig(HeaderCarrier):
"""draft-ietf-oauth-identity-assertion-authz-grant (Okta "AI agent token exchange").
Two legs: leg 1 is an RFC 8693 token exchange at the IdP org AS (`org_token_endpoint`) that
@ -297,23 +346,16 @@ class Byok(BaseModel):
ApiKeySource = Annotated[SharedKey | Byok, Field(discriminator="source")]
class ApiKeyConfig(BaseModel):
class ApiKeyConfig(HeaderCarrier):
"""A fixed credential injected as a header. The value is shared (in config) or seeded
per-user (pulled from the store); `header_name` and `value_prefix` say where and how it is
written, modeled like OpenAPI's apiKey scheme so any upstream convention is expressible
(Authorization + Bearer, a raw value on X-API-Key, Ocp-Apim-Subscription-Key, etc.).
per-user (pulled from the store); the inherited `header_name` and `value_prefix` say where
and how it is written.
"""
model_config = ConfigDict(frozen=True)
kind: Literal[AuthSpecKind.api_key] = AuthSpecKind.api_key
header_name: str = "Authorization"
value_prefix: str = "Bearer"
key_source: ApiKeySource
def header(self, value: str) -> tuple[str, str]:
formatted: Final = f"{self.value_prefix} {value}" if self.value_prefix else value
return self.header_name, formatted
class PassthroughConfig(BaseModel):
"""Client-driven upstream OAuth; the gateway forwards the client's upstream token."""

18
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -168,8 +168,11 @@ if MCP_AVAILABLE:
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.tool_search import (
AGENT_SEARCH_TOOL_NAME,
DEFAULT_AGENT_SEARCH_TOP_K,
MCP_TOOL_SEARCH_TOOL_NAME,
coerce_top_k,
handle_agent_search,
handle_mcp_tool_call,
handle_mcp_tool_search,
)
@ -182,6 +185,14 @@ if MCP_AVAILABLE:
detail={"error": "forbidden", "message": f"{tool_name} requires mcp_tool_search_enabled on the key"},
)
tool_arguments: Final = data.get("arguments") or {}
if tool_name == AGENT_SEARCH_TOOL_NAME:
return await handle_agent_search(
query=str(tool_arguments.get("query", "")),
top_k=coerce_top_k(
tool_arguments.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K
),
user_api_key_dict=user_api_key_dict,
)
rest_client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
(
virtual_mcp_auth_header,
@ -939,12 +950,9 @@ if MCP_AVAILABLE:
tool_name: Final[str | None] = data.get("name")
tool_arguments: Final[dict[str, object]] = data.get("arguments") or {}
from litellm.proxy._experimental.mcp_server.tool_search import (
MCP_TOOL_CALL_TOOL_NAME,
MCP_TOOL_SEARCH_TOOL_NAME,
)
from litellm.proxy._experimental.mcp_server.tool_search import VIRTUAL_TOOL_NAMES
if tool_name in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME):
if tool_name in VIRTUAL_TOOL_NAMES:
return await _handle_virtual_mcp_tool(request, data, tool_name, user_api_key_dict)
# Validate required parameters early

28
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -246,11 +246,12 @@ def _mcp_meta_trace_carrier(req_ctx: object) -> dict[str, str] | None:
"""The W3C trace context (``traceparent``/``tracestate``) the MCP client
propagated in the request's ``params._meta`` (SEP-414), or ``None``.
When present, per the OTel MCP semconv the MCP span parents to this propagated
context rather than to the HTTP transport (which is recorded as a link instead).
When absent, the span nests under the transport span of the request carrying
this specific message, so a streamable-HTTP session that multiplexes many
messages still does not glue every message under the session's first request;
When present, the MCP span records this propagated context as a span *link*,
never the parent — a remote parent would root the span in a trace whose root
never reaches the gateway's tracing backend. The span itself nests under the
transport span of the request carrying this specific message, so a
streamable-HTTP session that multiplexes many messages still does not glue
every message under the session's first request;
see ``resolve_mcp_span_context``. The client's W3C Baggage is
deliberately excluded: it is caller-controlled, and the otel baggage processor
stamps allowlisted baggage keys (``litellm.team.id``, ``litellm.metadata.*``,
@ -432,7 +433,6 @@ if MCP_AVAILABLE:
_client_forwarded_authorization_headers,
_resolve_openapi_tool_auth,
_should_strip_caller_authorization,
_without_authorization,
global_mcp_server_manager,
)
from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
@ -451,6 +451,7 @@ if MCP_AVAILABLE:
split_server_prefix_from_name,
strip_known_server_prefix,
)
from litellm.types.mcp import DEFAULT_CREDENTIAL_HEADER, without_header
######################################################
############ MCP Tools List REST API Response Object #
@ -911,14 +912,17 @@ if MCP_AVAILABLE:
the caller falls through to normal tool routing.
"""
from litellm.proxy._experimental.mcp_server.tool_search import (
MCP_TOOL_CALL_TOOL_NAME,
AGENT_SEARCH_TOOL_NAME,
DEFAULT_AGENT_SEARCH_TOP_K,
MCP_TOOL_SEARCH_TOOL_NAME,
VIRTUAL_TOOL_NAMES,
coerce_top_k,
handle_agent_search,
handle_mcp_tool_call,
handle_mcp_tool_search,
)
if name not in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME):
if name not in VIRTUAL_TOOL_NAMES:
return None
if not getattr(
@ -951,6 +955,12 @@ if MCP_AVAILABLE:
)
assert user_api_key_auth is not None # guaranteed by the flag check above
if name == AGENT_SEARCH_TOOL_NAME:
return await handle_agent_search(
query=str(args.get("query", "")),
top_k=coerce_top_k(args.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K),
user_api_key_dict=user_api_key_auth,
)
virtual_logging_obj: Final = await _build_virtual_call_logging_obj(
name=name,
arguments=args,
@ -1732,7 +1742,7 @@ if MCP_AVAILABLE:
raw_headers=raw_headers,
user_api_key_auth=user_api_key_auth,
):
extra_headers = _without_authorization(extra_headers)
extra_headers = without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER)
elif is_client_forwarded_mode:
if not withhold_forwarded_authorization:
extra_headers = _client_forwarded_authorization_headers(

Some files were not shown because too many files have changed in this diff Show more