scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

Merge remote-tracking branch 'upstream/litellm_internal_staging' into litellm_bedrock_mantle_native_web_search

Browse source 
# Conflicts:
#	litellm/model_prices_and_context_window_backup.json
#	model_prices_and_context_window.json
This commit is contained in:
longwind48 2026-08-29 10:49:09 +08:00
commit a632e02920
408 changed files with 36008 additions and 2618 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
.circleci/config.yml
View file

@ -2421,45 +2421,6 @@ jobs:
- wait_for_service:
url: http://localhost:4000
timeout: "300"
# Add Ruby installation and testing before the existing Node.js and Python tests
- run:
name: Install Ruby and Bundler
command: |
# Clone RVM at pinned tag and verify the commit SHA matches the
# published tag before running its install script.
RVM_VERSION="1.29.12"
RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81"
git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm
RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)"
if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then
echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2
exit 1
fi
# Import RVM signing keys (used by `rvm install` to verify Ruby tarballs)
gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
# Install RVM from the verified checkout. The install script
# sources `scripts/functions/installer` using paths relative to
# its own working directory, so it must be run from /tmp/rvm.
(cd /tmp/rvm && ./install --path "$HOME/.rvm")
source "$HOME/.rvm/scripts/rvm"
# Install Ruby 3.2.2 (RVM verifies the tarball PGP signature)
rvm install 3.2.2
rvm use 3.2.2 --default
# Install latest Bundler
gem install bundler
- run:
name: Run Ruby tests
command: |
source $HOME/.rvm/scripts/rvm
cd tests/pass_through_tests/ruby_passthrough_tests
bundle install
bundle exec rspec
no_output_timeout: 30m
# Install Node.js directly from nodejs.org with SHA256 verification,
# instead of piping NodeSource's setup_24.x apt-repo installer into
# sudo bash (which runs a mutable upstream script unattended).

68
.github/workflows/sync-together-ai-models.yml vendored Normal file
View file

@ -0,0 +1,68 @@
name: Sync Together AI model registry
on:
schedule:
- cron: "30 6 * * *"
workflow_dispatch:
permissions:
contents: write
pull-requests: write
jobs:
sync_together_ai_models:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: litellm_internal_staging
persist-credentials: false
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Look for an already-open sync PR
id: existing
run: |
open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
--jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
if [ -n "$open_pr" ]; then
echo "An open sync PR already exists on branch $open_pr; skipping this run."
fi
env:
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
- name: Run the sync
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
env:
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
- name: Regenerate the JSON schema
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create a pull request when the registry changed
if: steps.existing.outputs.open_pr == ''
run: |
if git diff --quiet; then
echo "Registry already in sync; no PR needed."
exit 0
fi
branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b "$branch"
git add model_prices_and_context_window.json \
litellm/model_prices_and_context_window_backup.json \
model_prices_and_context_window.schema.json
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
gh auth setup-git
git push origin "$branch"
gh pr create --title "feat(models): sync together_ai model registry" \
--body-file "$RUNNER_TEMP/pr_body.md" \
--head "$branch" \
--base litellm_internal_staging
env:
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}

2
.github/workflows/test-terraform-provider.yml vendored
View file

@ -114,4 +114,4 @@ jobs:
- name: Audit provider endpoints against the schema
working-directory: terraform/provider
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json" -coverage-allowlist ./tools/endpointaudit/coverage_allowlist.txt

1
.github/workflows/test-unit.yml vendored
View file

@ -141,6 +141,7 @@ jobs:
test-path: >-
tests/test_litellm/proxy/analytics_endpoints
tests/test_litellm/proxy/management_endpoints
tests/test_litellm/proxy/list_api
tests/test_litellm/proxy/memory
tests/test_litellm/proxy/guardrails
tests/test_litellm/proxy/management_helpers

10
basedpyright-code-budget.json
View file

@ -3,7 +3,7 @@
"limit": 18483
},
"reportArgumentType": {
"limit": 2564
"limit": 2557
},
"reportAssignmentType": {
"limit": 319
@ -57,7 +57,7 @@
"limit": 5659
},
"reportMissingTypeArgument": {
"limit": 15484
"limit": 15482
},
"reportMissingTypeStubs": {
"limit": 40
@ -105,13 +105,13 @@
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38782
"limit": 38779
},
"reportUnknownParameterType": {
"limit": 19829
"limit": 19827
},
"reportUnknownVariableType": {
"limit": 30349
"limit": 30348
},
"reportUnnecessaryCast": {
"limit": 117

9
ci_cd/generate_model_prices_schema.py
View file

@ -220,6 +220,15 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
"enum": ["low", "medium", "high", "max", "xhigh"],
},
"default_reasoning_effort": {
"type": "string",
"description": (
"Reasoning effort the provider applies when the request omits reasoning_effort. "
"Gates whether a non-default temperature or the top_p/logprobs sampling params are "
"accepted, which hold only when the effort resolves to 'none'."
),
"enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
},
"comment": STRING,
"audio_transcription_config": STRING,
}

126
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -12,7 +12,7 @@ Endpoints for /project operations
import json
from collections.abc import Sequence
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Final
from fastapi import APIRouter, Depends, HTTPException, Request
@ -35,6 +35,8 @@ if TYPE_CHECKING:
LiteLLM_VerificationTokenActions,
)
from litellm import Router
router = APIRouter()
@ -205,6 +207,114 @@ def _check_team_project_limits(
)
def _project_models_missing_positive_quota(
models: list[str] | None,
rpm_limits: Mapping[str, object] | None,
tpm_limits: Mapping[str, object] | None,
) -> list[str]:
"""Return the models that lack a positive `rpm` AND `tpm` quota.
A valid quota is a positive integer; null, zero, and negative are rejected
because downstream rate limiters treat a non-positive limit as immediately
exhausted (every request blocked).
"""
def _is_positive(value: object) -> bool:
return isinstance(value, int) and not isinstance(value, bool) and value > 0
rpm = rpm_limits or {}
tpm = tpm_limits or {}
return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))]
def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
def _project_models_expanding_at_request_time(
models: Sequence[str] | None, access_group_names: frozenset[str]
) -> tuple[str, ...]:
"""Entries project auth expands to many concrete models (`all-proxy-models`, `*` patterns,
access groups). The rate limiter looks quotas up by the exact requested model name, so a
quota keyed on one of these entries is never applied."""
return tuple(
model
for model in (models or ())
if model == SpecialModelNames.all_proxy_models.value or "*" in model or model in access_group_names
)
def _raise_on_project_models_expanding_at_request_time(
models: Sequence[str] | None, access_group_names: frozenset[str]
) -> None:
expanding: Final = _project_models_expanding_at_request_time(models, access_group_names)
if not expanding:
return
raise HTTPException(
status_code=400,
detail={
"error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead."
},
)
def _raise_on_missing_project_model_quota(
data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset()
) -> None:
"""Require a positive `rpm`/`tpm` quota for every model on project CREATE.
`model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request
model's `set_model_info` validator, so they are read from there.
Only invoked when `general_settings.enforce_project_model_quota` is enabled
(default off), so it is opt-in and does not change behavior for existing users.
"""
_raise_on_project_models_expanding_at_request_time(data.models, access_group_names)
metadata = data.metadata or {}
missing = _project_models_missing_positive_quota(
data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit")
)
if not missing:
return
raise HTTPException(
status_code=400,
detail={
"error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
},
)
def _raise_on_missing_project_model_quota_on_update(
data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset()
) -> None:
"""Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE.
`/project/update` replaces `models` and `metadata` when they are provided, so the
check runs on what the project WILL look like: a partial update that doesn't touch
models/quota keeps the existing values, while one that adds a model or clears a
model's quota must leave every resulting model with a positive limit.
Only invoked when `general_settings.enforce_project_model_quota` is enabled
(default off), so it is opt-in and does not change behavior for existing users.
"""
resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or [])
resulting_metadata = (
data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {})
)
_raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names)
missing = _project_models_missing_positive_quota(
resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit")
)
if not missing:
return
raise HTTPException(
status_code=400,
detail={
"error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
},
)
async def _create_budget_for_project(
data: NewProjectRequest,
user_id: str | None,
@ -352,7 +462,9 @@ async def new_project(
```
"""
from litellm.proxy.proxy_server import (
general_settings,
litellm_proxy_admin_name,
llm_router,
premium_user,
prisma_client,
)
@ -399,6 +511,10 @@ async def new_project(
data=data,
)
# Opt-in (default off): require rpm/tpm for every model added to the project.
if general_settings.get("enforce_project_model_quota", False):
_raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router))
# Check if user has permission to create projects for this team
# only team admins can create projects for their team
has_permission = await _check_user_permission_for_project(
@ -538,7 +654,9 @@ async def update_project(
```
"""
from litellm.proxy.proxy_server import (
general_settings,
litellm_proxy_admin_name,
llm_router,
premium_user,
prisma_client,
user_api_key_cache,
@ -642,6 +760,12 @@ async def update_project(
data=data,
)
# Opt-in (default off): require rpm/tpm for every model the update would leave on the project.
if general_settings.get("enforce_project_model_quota", False):
_raise_on_missing_project_model_quota_on_update(
data, existing_project, _router_access_group_names(llm_router)
)
# Prepare update data
update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name

18
litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

22
litellm-proxy-extras/litellm_proxy_extras/migrations/20260828000000_shadow_eval_cost_comparison/migration.sql Normal file
View file

@ -0,0 +1,22 @@
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cost" DOUBLE PRECISION;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "shadow_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cache_hit" BOOLEAN NOT NULL DEFAULT false;
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalFunnel" (
"job_id" TEXT NOT NULL,
"not_sampled" INTEGER NOT NULL DEFAULT 0,
"unjudgeable" INTEGER NOT NULL DEFAULT 0,
"shed" INTEGER NOT NULL DEFAULT 0,
"withheld" INTEGER NOT NULL DEFAULT 0,
CONSTRAINT "LiteLLM_ShadowEvalFunnel_pkey" PRIMARY KEY ("job_id")
);

21
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -1527,12 +1533,27 @@ model LiteLLM_ShadowEvalAttempt {
confidence Float?
judge_cost Float @default(0)
shadow_cost Float @default(0)
real_cost Float? // NULL = row predates cost measurement; comparisons read only measured rows
real_classifier_cost Float @default(0)
shadow_classifier_cost Float @default(0)
real_cache_hit Boolean @default(false)
error String?
created_at DateTime @default(now())
@@index([job_id])
}
// Per-leg sampling funnel counters the attempt rows cannot derive: requests an
// admitting job saw but did not judge. attempted = the leg's attempt rows; the
// leg's eligible traffic = not_sampled + unjudgeable + shed + withheld + attempted.
model LiteLLM_ShadowEvalFunnel {
job_id String @id
not_sampled Int @default(0)
unjudgeable Int @default(0)
shed Int @default(0)
withheld Int @default(0)
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

2
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -470,7 +470,7 @@ class ProxyExtrasDBManager:
ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
@staticmethod
def _mark_migrations_applied(migrations_dir: str):
def _mark_migrations_applied(migrations_dir: str) -> None:
migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
logger.info(f"Resolving {len(migration_names)} migrations")
for migration_name in migration_names:

2
litellm/__init__.py
View file

@ -274,7 +274,6 @@ databricks_key: Optional[str] = None
openai_like_key: Optional[str] = None
azure_key: Optional[str] = None
anthropic_key: Optional[str] = None
autorouter_savings_baseline_model: Optional[str] = None
replicate_key: Optional[str] = None
bytez_key: Optional[str] = None
gdc_key: Optional[str] = None
@ -487,6 +486,7 @@ public_mcp_servers: Optional[List[str]] = None
public_mcp_hub_strict_whitelist: bool = True
public_model_groups: Optional[List[str]] = None
public_agent_groups: Optional[List[str]] = None
agent_search_embedding_model: Optional[str] = None
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
# New format: { "displayName": { "url": "...", "index": 0 } }
# Old format: { "displayName": "url" } (for backward compatibility)

9
litellm/constants.py
View file

@ -629,6 +629,15 @@ LITELLM_CHAT_PROVIDERS: Final = [
"amazon_nova",
]
# Resolving these providers runs an OAuth device flow (their provider info IS the login), so any
# metadata or capability lookup against them can block for minutes waiting on a human.
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
{
"github_copilot",
"chatgpt",
}
)
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
"openai",
"azure",

24
litellm/cost_calculator.py
View file

@ -2,7 +2,7 @@
## File for 'response_cost' calculation in Logging
import logging
import time
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, cast
@ -739,6 +739,13 @@ def _get_provider_for_cost_calc(
return custom_llm_provider
def _get_hidden_str_for_cost_calc(hidden_params: object, key: str) -> str | None:
if not isinstance(hidden_params, Mapping):
return None
value: Final[object] = hidden_params.get(key)
return value if isinstance(value, str) and value else None
def _select_model_name_for_cost_calc(
model: str | None,
completion_response: object | None,
@ -755,7 +762,6 @@ def _select_model_name_for_cost_calc(
"""
return_model: str | None = None
region_name: str | None = None
custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
completion_response_model: str | None = None
@ -765,6 +771,14 @@ def _select_model_name_for_cost_calc(
elif isinstance(completion_response, dict):
completion_response_model = completion_response.get("model", None)
hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
explicit_pricing: Final = custom_pricing is True or base_model is not None
priced_from_response: Final = provider_response_model is not None or completion_response_model is not None
region_name: Final = (
_get_hidden_str_for_cost_calc(hidden_params, "region_name")
if not explicit_pricing and priced_from_response
else None
)
if custom_pricing is True:
if router_model_id is not None and router_model_id in litellm.model_cost:
@ -780,14 +794,12 @@ def _select_model_name_for_cost_calc(
else:
return_model = model
elif base_model is not None:
return_model = base_model
elif base_model is not None or provider_response_model is not None:
return_model = base_model if base_model is not None else provider_response_model
elif completion_response_model is None and hidden_params is not None:
if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
return_model = hidden_params.get("model", model)
elif hidden_params is not None and hidden_params.get("region_name", None) is not None:
region_name = hidden_params.get("region_name", None)
if return_model is None and completion_response_model is not None:
return_model = completion_response_model

2
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -1447,10 +1447,8 @@ Model Info:
from datetime import datetime
# Get the current timestamp
current_time: Final = datetime.now().strftime("%H:%M:%S")
_proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
# Use .name if it's an enum, otherwise use as is
alert_type_name: Final = getattr(alert_type, "name", alert_type)
alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
if alert_type == "daily_reports" or alert_type == "new_model_added":

85
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -24,6 +24,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
with_prompt_cache_breakpoint,
)
from litellm.types.integrations.anthropic_cache_control_hook import (
GATEWAY_INJECTED_CACHE_METADATA_KEY,
GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
CacheControlInjectionPoint,
CacheControlMessageInjectionPoint,
)
@ -185,7 +187,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
reserved_blocks: Final = (
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
processed_messages = self._apply_message_injections(
points=applied_message_points,
messages=processed_messages,
@ -194,7 +196,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
if (
openai_dialect
and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before
and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before
):
non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
@ -236,7 +238,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return provider
@staticmethod
def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
def count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
system_blocks: Final = (
sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0
)
@ -258,7 +260,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
``max_blocks`` is reached. Injection points are honored in config order,
so earlier points win when slots are scarce.
"""
used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages)
used_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
limit_reached = False
for point in points:
@ -454,8 +456,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system)
message_blocks: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
system_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints((), processed_system)
if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks:
system_already_has_cc: Final = isinstance(processed_system, list) and any(
@ -589,7 +591,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
carry the mark either at the top level (Anthropic shape) or nested under
``function`` (OpenAI shape); the Anthropic chat transform accepts both.
"""
if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0:
if AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) > 0:
return True
if tools is not None:
return any(
@ -749,6 +751,64 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if points:
non_default_params["cache_control_injection_points"] = points
@staticmethod
def record_gateway_injection(
request_kwargs: Mapping[str, object],
added: int,
) -> None:
"""Name the deployment whose payload the gateway, not the client, put breakpoints on.
Spend accounting only asks whether litellm acted, so what it needs is which
deployment, not a count. Recording that is what makes the mark attempt-scoped: the
metadata bucket is one dict shared by every retry, failover and fallback of a
request, and ``litellm_call_id`` is shared with it, so anything request-scoped
written by one attempt is read by all of them and each boundary would have to
remember to strip it. The deployment is the part that actually changes when the
request moves, so a leg that injected nothing is never credited for one that did.
It also makes a zero delta (hook re-entry) and a negative one (a prompt manager
replacing the messages) harmless, since neither rewrites an earlier mark.
A pass that runs before a deployment is chosen, which is what the proxy does for
prompt templates, injects into the payload every leg goes on to send, so it marks
the request for all of them rather than for one.
Only what this pass actually placed counts. A ``tool_config`` point is placed by
the Bedrock converse transform, and only when the request carries tools, so the
presence of one here says nothing about whether a breakpoint reaches the wire;
claiming it marked three request shapes out of four that inject nothing. Missing
that Bedrock credit is the fail-closed direction, and the alternative is a
provider transform that carries spend-attribution state.
Reads whichever bucket the request actually carries rather than asking the shared
name resolver, which answers on key presence: ``litellm_params`` declares
``litellm_metadata`` as None on every request, so the resolver names a bucket that
is not there and the mark is dropped.
Never CREATES the bucket. The proxy seeds it on every request and is the marker's
only reader, so a request without one is a bare SDK call nothing would consume it
from. Creating it would also add a key to a dict call sites splat as ``**kwargs``,
and on the Responses API ``metadata`` is both this bucket's default name and an
explicit parameter, so the splat collides with the caller's own value.
"""
if added <= 0:
return
bucket: Final = next(
(
candidate
for candidate in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata"))
if isinstance(candidate, dict)
),
None,
)
if bucket is not None:
model_info: Final = request_kwargs.get("model_info")
bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = (
model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT)
if isinstance(model_info, dict)
else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT
)
@staticmethod
def maybe_inject_cache_control(
messages: list[dict],
@ -798,17 +858,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system)
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
messages=messages,
system=system,
injection_points=injection_points,
openai_dialect=openai_dialect,
)
if (
openai_dialect
and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before
):
breakpoints_added: Final = (
AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before
)
AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added)
if openai_dialect and breakpoints_added > 0:
kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
if remaining:
kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)

91
litellm/integrations/custom_guardrail.py
View file

@ -60,6 +60,12 @@ _PRE_CALL_EXECUTED_TOKEN: Final = secrets.token_hex(16)
_GUARDRAIL_BLOCK_STATUS_CODES: Final = frozenset({400, 403, 422})
DEFAULT_ADVISORY_MESSAGE: Final = (
"The user's latest message was flagged for {reason} by a content safety "
"guardrail. This may be a false positive. Use your judgment: respond "
"helpfully if the request is legitimate, or decline if it is not."
)
_guardrail_self_recorded: Final[contextvars.ContextVar[bool]] = contextvars.ContextVar(
"litellm_guardrail_self_recorded", default=False
)
@ -158,6 +164,7 @@ class CustomGuardrail(CustomLogger):
sensitive_data_route_to_model: str | None = None,
sticky_session_routing: bool = True,
run_in_parallel: bool = False,
scan_raw_request: bool = False,
only_scan_new_messages: bool = False,
**kwargs,
):
@ -180,6 +187,13 @@ class CustomGuardrail(CustomLogger):
run_in_parallel: When True, this pre_call or post_call guardrail runs concurrently with
other opted-in guardrails of the same hook. Only safe for block-only guardrails that
do not mutate the request or response.
scan_raw_request: When True, this pre_call guardrail always evaluates the request as it
was before any guardrail in this hook ran, regardless of where it's declared in the
guardrails list -- so an earlier guardrail that masks/rewrites content (e.g. PII
redaction) can never hide a violation from this one. Only safe for block-only
guardrails: any data this guardrail returns is discarded, matching run_in_parallel's
contract, since applying its mutations on top of a stale snapshot would silently
undo whatever later guardrails already did to the live request.
"""
self.guardrail_name = guardrail_name
self.supported_event_hooks = supported_event_hooks
@ -195,6 +209,7 @@ class CustomGuardrail(CustomLogger):
self.sensitive_data_route_to_model: str | None = sensitive_data_route_to_model
self.sticky_session_routing: bool = sticky_session_routing
self.run_in_parallel: bool = run_in_parallel
self.scan_raw_request: bool = scan_raw_request
self.only_scan_new_messages: bool = only_scan_new_messages
if supported_event_hooks:
@ -281,6 +296,82 @@ class CustomGuardrail(CustomLogger):
original_response=original_response,
)
def inject_advisory_message(
self,
data: dict[str, Any], # mutable-ok: caller's dict is mutated in place, matching mark_pre_call_hook_ran
message: str,
) -> bool:
"""
Append an advisory system message to the request in place, so the LLM
itself can weigh a possible false-positive guardrail flag rather than
the request being hard-blocked or silently allowed.
Unlike raise_passthrough_exception, this does NOT short-circuit the LLM
call; the request proceeds normally with the extra message appended.
Guardrails should call this from on_flagged handling analogous to how
passthrough-supporting guardrails call raise_passthrough_exception.
Args:
data: The request data dictionary, mutated in place to append the
advisory message to its "messages" list and/or "input"/
"instructions" text.
message: The formatted advisory message to append as a system message.
Returns:
True if the advisory was actually written somewhere the model will
see it. False if ``data["input"]`` is a structured Responses-API
list (not a plain string) -- the Responses API reads only
``input``, so appending to ``messages`` would be inert regardless
of whether a ``messages`` list also happens to be present, and
there is no field this helper can safely append into. The caller
must treat this like any other case where the mitigation can't
land and degrade to blocking instead of silently letting the
flagged request through unmodified.
"""
advisory_message: Final = {"role": "system", "content": message} # mutable-ok: plain dict for live request
existing_messages: Final = data.get("messages")
existing_input: Final = data.get("input")
existing_instructions: Final = data.get("instructions")
if isinstance(existing_instructions, str):
# Responses API "instructions" is the privileged, developer-set
# system-level field the model treats as authoritative -- unlike
# "input", which the caller controls and could use to tell the
# model to disregard a trailing warning. Prefer it over "input"
# whenever present.
if isinstance(existing_messages, list):
messages_with_instructions_note: Final = [ # mutable-ok: fresh list
*existing_messages,
advisory_message,
]
data["messages"] = messages_with_instructions_note # rebind-ok: mutates caller's dict by design
data["instructions"] = f"{existing_instructions}\n\n{message}" # rebind-ok: mutates caller's dict by design
return True
if isinstance(existing_input, str):
# A plain-string "input" doesn't rule out "messages" also being a
# real, read field (e.g. a chat-completions call carrying a stray
# "input"), so write to both when both are present.
if isinstance(existing_messages, list):
messages_with_input_note: Final = [*existing_messages, advisory_message] # mutable-ok: fresh list
data["messages"] = messages_with_input_note # rebind-ok: mutates caller's dict by design
# The Responses API reads "input", not "messages" -- appending only to
# "messages" would leave the advisory unreachable for that endpoint.
data["input"] = f"{existing_input}\n\n{message}" # rebind-ok: mutates caller's dict by design
return True
if existing_input is not None:
# existing_input is a structured (non-string) Responses-API item
# list. That endpoint reads only "input", so appending to
# "messages" -- even if "messages" also happens to be present --
# would never reach the model. Leave data untouched and report
# non-delivery so the caller degrades to blocking.
return False
if isinstance(existing_messages, list):
messages_without_input_note: Final = [*existing_messages, advisory_message] # mutable-ok: fresh list
data["messages"] = messages_without_input_note # rebind-ok: mutates caller's dict by design
return True
sole_message: Final = [advisory_message] # mutable-ok: plain list for the live JSON request
data["messages"] = sole_message # rebind-ok: mutates caller's dict by design
return True
def raise_sensitive_data_route_exception(
self,
route_to_model: str,

161
litellm/integrations/shadow_eval_logger.py
View file

@ -38,6 +38,7 @@ from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalD
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
if TYPE_CHECKING:
from litellm.proxy.db.shadow_eval_funnel import ShadowEvalFunnelStage
from litellm.proxy.utils import PrismaClient
from litellm.router import Router
from litellm.types.utils import StandardLoggingPayload
@ -386,6 +387,13 @@ def _judge_user_prompt(conversation: str, response_a: str, response_b: str) -> s
)
def _leg_eval_spend(sums: Mapping[str, object]) -> float:
return sum(
float(raw) if isinstance(raw := sums.get(column), (int, float)) else 0.0
for column in ("judge_cost", "shadow_cost", "shadow_classifier_cost")
)
def _job_spend_counter_key(job_id: str) -> str:
return f"spend:shadow_eval:{job_id}"
@ -412,6 +420,15 @@ async def _add_job_spend_to_counter(counter_key: str, cost: float) -> None:
verbose_logger.warning("shadow_eval: spend counter increment failed for %s: %s", counter_key, e)
def _record_funnel_event(job_id: str, stage: "ShadowEvalFunnelStage") -> None:
try:
from litellm.proxy.db.shadow_eval_funnel import record_shadow_eval_funnel_event
record_shadow_eval_funnel_event(job_id, stage)
except Exception as e: # noqa: BLE001 # coverage stats are advisory; sampling must proceed
verbose_logger.debug("shadow_eval: funnel increment failed for %s: %s", job_id, e)
async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
"""Whether the shadowed key or its team is over budget, decided by the same owners
the request path uses, so counter keys and thresholds can never drift from auth's.
@ -474,6 +491,13 @@ def _routed_tier(metadata: Mapping[str, object]) -> str | None:
return str(raw) if raw is not None else None
def _decision_classifier_cost(metadata: Mapping[str, object]) -> float:
"""What the arm's own routing decision says its classifier call billed: the money a
completion cost alone omits, and 0 for a plain model that never classifies."""
raw: Final = _routing_decision(metadata).get("classifier_cost")
return float(raw) if isinstance(raw, (int, float)) else 0.0
def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool:
"""Whether the router under evaluation served this request, which is what decides
the direction it belongs to. A forward job skips its own router's traffic, since
@ -489,6 +513,7 @@ class _CallFailure:
error: str
cost: float = 0.0
classifier_cost: float = 0.0
@dataclass(frozen=True, slots=True)
@ -499,6 +524,7 @@ class _ShadowResponse:
model: str
tier: str | None
cost: float
classifier_cost: float
@dataclass(frozen=True, slots=True)
@ -575,6 +601,7 @@ class ShadowEvalLogger(CustomLogger):
jobs_cache: InMemoryCache | None = None,
job_spend_reader: Callable[[str, float, float], Awaitable[float]] | None = None,
job_spend_writer: Callable[[str, float], Awaitable[None]] | None = None,
funnel_recorder: Callable[[str, "ShadowEvalFunnelStage"], None] | None = None,
) -> None:
"""Providers are callables so the proxy's lazily-initialized globals are resolved
at call time, not at logger construction. The spend reader and writer wrap the
@ -584,6 +611,7 @@ class ShadowEvalLogger(CustomLogger):
self._jobs_cache = jobs_cache or _jobs_cache
self._read_job_spend = job_spend_reader or _job_spend_from_counter
self._write_job_spend = job_spend_writer or _add_job_spend_to_counter
self._record_funnel = funnel_recorder or _record_funnel_event
self._inflight_shadow_tasks: int = 0
# Starts per job since the last cache fill, never decremented within a
# generation; the refill absorbs written rows and resets.
@ -610,7 +638,8 @@ class ShadowEvalLogger(CustomLogger):
await prisma.db.litellm_shadowevalattempt.group_by(
by=["job_id"],
count=True,
sum={"judge_cost": True, "shadow_cost": True}, # mutable-ok: Prisma aggregate spec
# mutable-ok: Prisma aggregate spec
sum={"judge_cost": True, "shadow_cost": True, "shadow_classifier_cost": True},
where={"job_id": {"in": [str(record.id) for record in records]}}, # mutable-ok: Prisma filter
)
if records
@ -619,8 +648,7 @@ class ShadowEvalLogger(CustomLogger):
attempt_stats: Final = { # mutable-ok: frozen snapshot of the grouped read
str(row["job_id"]): (
int(row["_count"]["_all"]),
float((row["_sum"] or {}).get("judge_cost") or 0.0)
+ float((row["_sum"] or {}).get("shadow_cost") or 0.0),
_leg_eval_spend(row["_sum"] or _EMPTY_METADATA),
)
for row in grouped or []
}
@ -646,6 +674,32 @@ class ShadowEvalLogger(CustomLogger):
#### hook ####
def _sampled_jobs(
self,
active_jobs: Sequence[ActiveShadowEvalJob],
request_metadata: Mapping[str, object],
request_id: str,
) -> tuple[ActiveShadowEvalJob, ...]:
"""The jobs that sample this request. A key can hold one job per direction, and a
request routed by one job's router while bypassing the other's qualifies for both;
each is separately budgeted, so both fire. An admitting job that loses the sampling
dice is counted, so results can weigh judged rows against the traffic they stand for."""
eligible: list[ActiveShadowEvalJob] = [] # mutable-ok: bucketed per-job admission
now: Final = datetime.now(timezone.utc)
for job in active_jobs:
if (
now >= job.ends_at
or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns
or (job.max_budget is not None and job.spend >= job.max_budget)
or _request_was_routed_by(request_metadata, job.router_name) != (job.direction == "reverse")
):
continue
if not _sample_hits(request_id, job.id, job.shadow_percentage):
self._record_funnel(job.id, "not_sampled")
continue
eligible.append(job)
return tuple(eligible)
async def async_log_success_event(
self,
kwargs: Mapping[str, object],
@ -677,18 +731,8 @@ class ShadowEvalLogger(CustomLogger):
return # only surfaces this table can normalize are comparable; unknown types fail closed
if ops.wire_params and _request_mutating_guardrail_ran(request_metadata):
return # the wire-body snapshot predates the rewrite; replaying it would resurrect stripped content
# A key can hold one job per direction, and a request routed by one job's
# router while bypassing the other's qualifies for both. Each is separately
# budgeted, so both fire; the request is normalized once, and only when at
# least one job sampled it.
eligible: Final = tuple(
job
for job in (await self._active_jobs()).get(str(api_key_hash), ())
if datetime.now(timezone.utc) < job.ends_at
and job.attempts + self._job_starts.get(job.id, 0) < job.max_turns
and (job.max_budget is None or job.spend < job.max_budget)
and _sample_hits(request_id, job.id, job.shadow_percentage)
and _request_was_routed_by(request_metadata, job.router_name) == (job.direction == "reverse")
eligible: Final = self._sampled_jobs(
(await self._active_jobs()).get(str(api_key_hash), ()), request_metadata, request_id
)
if not eligible:
return
@ -699,12 +743,18 @@ class ShadowEvalLogger(CustomLogger):
response_obj,
)
if sample is None:
for job in eligible:
self._record_funnel(job.id, "unjudgeable")
return
messages, shadow_params, real_text = sample
control_tier: Final = _routed_tier(request_metadata)
real_cost: Final = float(payload.get("response_cost") or 0.0)
real_cache_hit: Final = payload.get("cache_hit") is True
real_classifier_cost: Final = _decision_classifier_cost(request_metadata)
for job in eligible:
if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS:
return
self._record_funnel(job.id, "shed")
continue
self._job_starts[job.id] = self._job_starts.get(job.id, 0) + 1
self._inflight_shadow_tasks += 1
asyncio.create_task(
@ -714,6 +764,9 @@ class ShadowEvalLogger(CustomLogger):
messages=messages,
real_text=real_text,
real_model=payload.get("model") or "",
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
control_tier=control_tier,
shadow_params=shadow_params,
parent_metadata=MappingProxyType(dict(request_metadata)), # mutable-ok: frozen snapshot
@ -734,37 +787,66 @@ class ShadowEvalLogger(CustomLogger):
messages: Sequence[Mapping[str, object]],
real_text: str,
real_model: str,
real_cost: float,
real_classifier_cost: float,
real_cache_hit: bool,
control_tier: str | None,
shadow_params: Mapping[str, object],
parent_metadata: Mapping[str, object],
) -> None:
"""Budget gate -> shadow call -> blind judge -> one attempt row. The prisma gate
sits above the dispatch so no provider spend happens without a place to record
the outcome, and the budget read lives here rather than in the success hook."""
"""Budget gate -> shadow call -> blind judge -> one attempt row, and every exit
in exactly one coverage bucket: the gates that decline to spend on an admitted
sample (no DB to record into, an over-budget key, an unverifiable or exhausted
eval budget) count it withheld, so eligible traffic still reconciles as
not_sampled + unjudgeable + shed + withheld + attempt rows. The prisma gate sits
above the dispatch so no provider spend happens without a place to record the
outcome, and the budget read lives here rather than in the success hook."""
prisma: Final = self._prisma_provider()
try:
if prisma is None:
self._record_funnel(job.id, "withheld")
return
if await _key_or_team_is_over_budget(parent_metadata):
self._record_funnel(job.id, "withheld")
return
if job.max_budget is not None:
try:
spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget)
except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it
verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e)
self._record_funnel(job.id, "withheld")
return
if spend >= job.max_budget:
self._record_funnel(job.id, "withheld")
return
shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata)
except Exception as e: # noqa: BLE001 # detached task: nothing billed yet, record and never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
await self._record_attempt(
prisma, job, request_id, control_tier, outcome="error", error=f"pipeline error: {e}"
prisma,
job,
request_id,
control_tier,
outcome="error",
error=f"pipeline error: {e}",
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
return
if isinstance(shadow, _CallFailure):
await self._record_attempt(
prisma, job, request_id, control_tier, outcome="error", error=shadow.error, shadow_cost=shadow.cost
prisma,
job,
request_id,
control_tier,
outcome="error",
error=shadow.error,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
return
# From here the shadow call has billed, so every exit records its cost.
@ -787,6 +869,10 @@ class ShadowEvalLogger(CustomLogger):
shadow=shadow,
judge_cost=verdict.cost,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
return
await self._record_attempt(
@ -800,6 +886,10 @@ class ShadowEvalLogger(CustomLogger):
confidence=verdict.confidence,
judge_cost=verdict.cost,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
except Exception as e: # noqa: BLE001 # detached task: the shadow call billed, record its cost, never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
@ -812,6 +902,10 @@ class ShadowEvalLogger(CustomLogger):
error=f"pipeline error: {e}",
shadow=shadow,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
async def _record_attempt(
@ -822,15 +916,20 @@ class ShadowEvalLogger(CustomLogger):
control_tier: str | None,
*,
outcome: str,
real_cost: float,
real_classifier_cost: float,
real_cache_hit: bool,
shadow: _ShadowResponse | None = None,
real_model: str = "",
confidence: float | None = None,
judge_cost: float = 0.0,
shadow_cost: float = 0.0,
shadow_classifier_cost: float = 0.0,
error: str | None = None,
) -> None:
if judge_cost + shadow_cost > 0:
await self._write_job_spend(_job_spend_counter_key(job.id), judge_cost + shadow_cost)
eval_spend: Final = judge_cost + shadow_cost + shadow_classifier_cost
if eval_spend > 0:
await self._write_job_spend(_job_spend_counter_key(job.id), eval_spend)
if prisma is None:
return
try:
@ -845,6 +944,10 @@ class ShadowEvalLogger(CustomLogger):
"confidence": confidence,
"judge_cost": judge_cost,
"shadow_cost": shadow_cost,
"shadow_classifier_cost": shadow_classifier_cost,
"real_cost": real_cost,
"real_classifier_cost": real_classifier_cost,
"real_cache_hit": real_cache_hit,
"error": error[:_MAX_ERROR_CHARS] if error else None,
}
)
@ -881,15 +984,23 @@ class ShadowEvalLogger(CustomLogger):
)
except Exception as e: # noqa: BLE001 # provider errors become error rows, not crashes
verbose_logger.debug("shadow_eval: router call failed: %s", e)
return _CallFailure(f"shadow router call failed: {_failure_detail(e)}")
return _CallFailure(
f"shadow router call failed: {_failure_detail(e)}",
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
text: Final = _chat_final_text(response)
if not text:
return _CallFailure("shadow router returned an empty response", cost=_call_cost(response))
return _CallFailure(
"shadow router returned an empty response",
cost=_call_cost(response),
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
return _ShadowResponse(
text=text,
model=str(getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or ""),
tier=_routed_tier(shadow_metadata),
cost=_call_cost(response),
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
async def _call_judge(

56
litellm/litellm_core_utils/core_helpers.py
View file

@ -454,6 +454,62 @@ def safe_deep_copy(data):
return new_data
def independent_snapshot(
data: dict, # mutable-ok: caller-defined request-payload shape
) -> dict: # mutable-ok: caller-defined request-payload shape
"""
A copy of ``data`` whose top-level keys are deep-copied independently
where possible -- always attempted, regardless of
``litellm.safe_memory_mode``. Unlike ``safe_deep_copy``, which can return
the *original* object outright under that mode (defeating any isolation
guarantee for every key, not just the ones that need it), this never
skips copying wholesale.
Real proxy requests carry ``data["litellm_logging_obj"]`` (a ``Logging``
instance nesting a live OTel span with a real lock) by the time
``pre_call_hook`` runs, which can never be deep-copied. Any individual
key that fails to deep-copy falls back to sharing its original
reference, same crash tolerance as ``safe_deep_copy``'s own per-key
fallback; callers needing true isolation (e.g. a guardrail's
``scan_raw_request`` snapshot) only depend on the keys that are plain,
cleanly-copyable structures (``messages``/``input``,
``metadata``/``litellm_metadata``).
"""
sanitized: Final = {
key: (
{ # mutable-ok: same request-payload shape as data
inner_key: ("placeholder" if inner_key == "litellm_parent_otel_span" else inner_value)
for inner_key, inner_value in value.items()
}
if key in ("metadata", "litellm_metadata") and isinstance(value, dict)
else value
)
for key, value in data.items()
}
def _copied_value(key: str, sanitized_value: object) -> object:
try:
copied_value: Final = copy.deepcopy(sanitized_value)
except Exception: # noqa: BLE001 # any unpicklable value falls back to the original reference for this key only
return data.get(key)
original_value: Final = data.get(key)
if (
key in ("metadata", "litellm_metadata")
and isinstance(copied_value, dict)
and isinstance(original_value, dict)
and "litellm_parent_otel_span" in original_value
):
return { # mutable-ok: same request-payload shape as data
**copied_value,
"litellm_parent_otel_span": original_value["litellm_parent_otel_span"],
}
return copied_value
return { # mutable-ok: same request-payload shape as data
key: _copied_value(key, value) for key, value in sanitized.items()
}
def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
"""
Recursively filter out Exception objects and callable objects from dicts/lists.

2
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -2222,6 +2222,8 @@ def _map_exception_by_status(
status_code: Final = original_exception.status_code if hasattr(original_exception, "status_code") else None
if not isinstance(status_code, int) or status_code < 400:
return
if getattr(original_exception, "status_code_is_synthesized", False):
return
message: Final = f"{exception_provider} - {error_str}"
response: Final = original_exception.response if hasattr(original_exception, "response") else None
match status_code:

14
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -2,7 +2,7 @@ from typing import Final, cast
from urllib.parse import urlparse
import litellm
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.litellm_core_utils.fallback_generalizations import (
match_routing_generalization,
)
@ -127,6 +127,18 @@ def handle_anthropic_text_model_custom_llm_provider(
return model, custom_llm_provider
def declared_authenticating_provider(model: str, custom_llm_provider: str | None = None) -> str | None:
"""The authenticating provider this pair already names, or None.
get_llm_provider runs the OAuth device flow for github_copilot and chatgpt, because their
provider info includes the key it unlocks. For a metadata question that flow is pure hazard,
and for a declared pair the resolver's answer is the declaration itself, so metadata callers
adopt the declaration instead of resolving.
"""
declared: Final = custom_llm_provider or model.split("/", 1)[0]
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
def get_llm_provider(
model: str,
custom_llm_provider: str | None = None,

5
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -2,6 +2,7 @@ from typing import Final, Literal
import litellm
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.types.utils import LlmProviders, LlmProvidersSet
@ -30,6 +31,10 @@ def get_supported_openai_params(
- List if custom_llm_provider is mapped
- None if unmapped
"""
if not custom_llm_provider:
custom_llm_provider = declared_authenticating_provider(
model
) # rebind-ok: resolving would run the provider's OAuth flow
if not custom_llm_provider:
try:
custom_llm_provider = litellm.get_llm_provider(model=model)[1]

20
litellm/litellm_core_utils/litellm_logging.py
View file

@ -888,7 +888,10 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_management_logger: CustomLogger | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs
) -> tuple[str, list[AllMessageValues], dict]:
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
model=model,
non_default_params=non_default_params,
@ -898,6 +901,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if custom_logger:
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
(
model,
messages,
@ -913,6 +917,11 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_label=prompt_label,
prompt_version=prompt_version,
)
if request_kwargs is not None:
AnthropicCacheControlHook.record_gateway_injection(
request_kwargs,
AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
)
self.messages = messages
return model, messages, non_default_params
@ -928,7 +937,10 @@ class Logging(LiteLLMLoggingBaseClass):
tools: list[dict] | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs
) -> tuple[str, list[AllMessageValues], dict]:
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
model=model,
tools=tools,
@ -939,6 +951,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if custom_logger:
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
(
model,
messages,
@ -956,6 +969,11 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_label=prompt_label,
prompt_version=prompt_version,
)
if request_kwargs is not None:
AnthropicCacheControlHook.record_gateway_injection(
request_kwargs,
AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
)
self.messages = messages
return model, messages, non_default_params
@ -6040,7 +6058,7 @@ def get_standard_logging_object_payload(
prompt_tokens=usage_dict.get("prompt_tokens", 0),
completion_tokens=usage_dict.get("completion_tokens", 0),
request_tags=request_tags,
end_user=end_user_id or "",
end_user=end_user_id,
api_base=StandardLoggingPayloadSetup.strip_trailing_slash(litellm_params.get("api_base", "")) or "",
model_group=_model_group,
model_id=_model_id,

41
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1,6 +1,7 @@
# What is this?
## Helper utilities for cost_per_token()
import re
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
@ -72,6 +73,19 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
return value if isinstance(value, int) else None
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+")
def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None:
value: Final = None if optional_params is None else optional_params.get(key)
return value if isinstance(value, str) else None
def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None:
value: Final = _requested_image_param(optional_params, "size")
return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
def get_web_search_requests(server_tool_use: Any) -> int | None:
"""
Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value
@ -1311,12 +1325,13 @@ class CostCalculatorUtils:
cost_calculator as vertex_ai_image_cost_calculator,
)
if size is None:
size = completion_response.size or "1024-x-1024"
if quality is None:
quality = completion_response.quality or "standard"
if n is None:
n = len(completion_response.data) if completion_response.data else 0
resolved_size: Final = (
size or completion_response.size or _requested_image_size(optional_params) or "1024-x-1024"
)
resolved_quality: Final = (
quality or completion_response.quality or _requested_image_param(optional_params, "quality") or "standard"
)
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
if isinstance(completion_response, ImageResponse):
@ -1328,7 +1343,7 @@ class CostCalculatorUtils:
if isinstance(completion_response, ImageResponse):
return bedrock_image_cost_calculator(
model=model,
size=size,
size=resolved_size,
image_response=completion_response,
optional_params=optional_params,
)
@ -1424,19 +1439,19 @@ class CostCalculatorUtils:
# Fall through to default for DALL-E models
return default_image_cost_calculator(
model=model,
quality=quality,
quality=resolved_quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
n=resolved_n,
size=resolved_size,
optional_params=optional_params,
)
else:
return default_image_cost_calculator(
model=model,
quality=quality,
quality=resolved_quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
n=resolved_n,
size=resolved_size,
optional_params=optional_params,
)
return 0.0

25
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -239,6 +239,22 @@ class ChunkProcessor:
model_response._hidden_params = chunk.get("_hidden_params", {})
return model_response
@staticmethod
def _get_provider_response_model(
chunks: Sequence["_BaseChunk"],
first_chunk_model: str,
) -> str | None:
models: Final = tuple(
model
for chunk in chunks
if isinstance((hidden_params := chunk.get("_hidden_params")), Mapping)
if isinstance((model := hidden_params.get("provider_response_model")), str) and model
)
return next(
(model for model in models if model != first_chunk_model),
models[0] if models else None,
)
@staticmethod
def apply_provider_assembled_streaming_metadata(
response: ModelResponse,
@ -360,6 +376,15 @@ class ChunkProcessor:
)
response = self.update_model_response_with_hidden_params(model_response=response, chunk=chunk)
provider_response_model: Final = self._get_provider_response_model(
chunks,
first_chunk_model,
)
if provider_response_model is not None:
response._hidden_params = dict( # pyright: ignore[reportPrivateUsage] # ModelResponse exposes no public hidden-params setter
response._hidden_params, # pyright: ignore[reportPrivateUsage] # ModelResponse exposes no public hidden-params getter
provider_response_model=provider_response_model,
)
return response
@staticmethod

50
litellm/litellm_core_utils/streaming_handler.py
View file

@ -187,17 +187,42 @@ class _ParsedChunkHiddenParams(BaseModel):
provider_specific_fields: Mapping[str, object] | None = None
def _provider_hidden_params(chunk: object) -> Mapping[str, object] | None:
hidden: Final[object] = getattr(chunk, "_hidden_params", None)
def _provider_response_model(chunk: object) -> str | None:
model: Final[object] = chunk.get("model") if isinstance(chunk, Mapping) else getattr(chunk, "model", None)
return model if isinstance(model, str) and model else None
def _parsed_provider_hidden_params(hidden: object) -> _ParsedChunkHiddenParams | None:
if not isinstance(hidden, dict):
return None
try:
parsed: Final = _ParsedChunkHiddenParams.model_validate(hidden)
return _ParsedChunkHiddenParams.model_validate(hidden)
except ValidationError:
return None
if not parsed.provider_specific_fields:
return None
return MappingProxyType({"provider_specific_fields": dict(parsed.provider_specific_fields)})
def _provider_hidden_params(
chunk: object,
provider_response_model: str | None,
) -> Mapping[str, object] | None:
hidden: Final[object] = getattr(chunk, "_hidden_params", None)
parsed: Final = _parsed_provider_hidden_params(hidden)
provider_specific_fields: Final[object | None] = (
dict(parsed.provider_specific_fields) # mutable-ok: stream assembly merges provider metadata into this dict
if parsed is not None and parsed.provider_specific_fields
else None
)
params: Final[Mapping[str, object]] = MappingProxyType(
{
key: value
for key, value in (
("provider_response_model", provider_response_model),
("provider_specific_fields", provider_specific_fields),
)
if value is not None
}
)
return params or None
class CustomStreamWrapper:
@ -229,6 +254,7 @@ class CustomStreamWrapper:
self.thinking_content = ""
self.system_fingerprint: str | None = None
self._provider_response_model: str | None = None
self.received_finish_reason: str | None = None
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
self.special_tokens = [
@ -819,7 +845,9 @@ class CustomStreamWrapper:
except Exception as e:
raise e
def model_response_creator(self, chunk: dict | None = None, hidden_params: Mapping[str, object] | None = None):
def model_response_creator(
self, chunk: dict | None = None, hidden_params: Mapping[str, object] | None = None
) -> ModelResponseStream:
_model: Final = self._cached_model_name
_logging_obj_llm_provider: Final = self._cached_logging_llm_provider
@ -1522,7 +1550,12 @@ class CustomStreamWrapper:
def chunk_creator(self, chunk: Any):
if hasattr(chunk, "id"):
self.response_id = chunk.id
model_response = self.model_response_creator(hidden_params=_provider_hidden_params(chunk))
provider_response_model: Final = _provider_response_model(chunk)
if provider_response_model is not None:
self._provider_response_model = provider_response_model
model_response = self.model_response_creator(
hidden_params=_provider_hidden_params(chunk, self._provider_response_model)
)
response_obj: dict[str, Any] = {}
try:
# return this for all models
@ -2336,6 +2369,7 @@ class CustomStreamWrapper:
partial_response: Final = litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
logging_obj=self.logging_obj,
)
if partial_response is None:
return

81
litellm/litellm_core_utils/token_counter.py
View file

@ -3,7 +3,7 @@
import base64
import io
import struct
from collections.abc import Callable, Mapping
from collections.abc import Callable, Iterable, Mapping, Sequence
from typing import Any, Final, Literal, cast
import tiktoken
@ -25,14 +25,21 @@ from litellm.litellm_core_utils.default_encoding import encoding as default_enco
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,
AnthropicContentParamSourceFileId,
AnthropicContentParamSourceUrl,
AnthropicMessagesDocumentParam,
AnthropicMessagesImageParam,
AnthropicMessagesTextParam,
AnthropicMessagesToolResultParam,
AnthropicMessagesToolUseParam,
)
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionDocumentObject,
ChatCompletionNamedToolChoiceParam,
ChatCompletionToolParam,
OpenAIMessageContent,
OpenAIMessageContentListBlock,
)
from litellm.types.utils import Message, SelectTokenizerResponse
@ -346,7 +353,7 @@ def token_counter(
model="",
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
text: str | list[str] | None = None,
messages: list[AllMessageValues | Message] | None = None,
messages: Sequence[AllMessageValues | Message] | None = None,
count_response_tokens: bool | None = False,
tools: list[ChatCompletionToolParam] | None = None,
tool_choice: ChatCompletionNamedToolChoiceParam | None = None,
@ -646,6 +653,46 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type:
return expected_cls
def _anthropic_image_source_data(
source: AnthropicContentParamSource | AnthropicContentParamSourceUrl | AnthropicContentParamSourceFileId,
) -> str:
if source["type"] == "base64":
data: Final = source.get("data")
if not data:
return ""
media_type: Final = source.get("media_type") or "image/png"
return f"data:{media_type};base64,{data}"
if source["type"] == "url":
return source.get("url") or ""
return ""
def _count_document_tokens(
document: ChatCompletionDocumentObject | AnthropicMessagesDocumentParam,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
) -> int:
source: Final = document["source"]
metadata_tokens: Final = sum(
count_function(text) for text in (document.get("title"), document.get("context")) if text
)
if source["type"] == "text":
return metadata_tokens + count_function(source["data"])
if source["type"] == "content":
content: Final = source["content"]
if isinstance(content, str):
return metadata_tokens + count_function(content)
return metadata_tokens + _count_content_list(
count_function, content, use_default_image_token_count, default_token_count
)
return metadata_tokens + calculate_img_tokens(
data=_anthropic_image_source_data(source),
mode="auto",
use_default_image_token_count=use_default_image_token_count,
)
def _count_anthropic_content(
content: Mapping[str, Any],
count_function: TokenCounterFunction,
@ -697,13 +744,17 @@ def _count_anthropic_content(
def _count_content_list(
count_function: TokenCounterFunction,
content_list: OpenAIMessageContent,
content_list: str
| Iterable[
OpenAIMessageContentListBlock
| AnthropicMessagesTextParam
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
],
use_default_image_token_count: bool,
default_token_count: int | None,
) -> int:
"""
Recursively count tokens from a list of content blocks.
"""
"""Recursively count tokens from a list of content blocks."""
try:
num_tokens = 0
for c in content_list:
@ -714,6 +765,19 @@ def _count_content_list(
elif c["type"] == "image_url":
image_url = c.get("image_url")
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
elif c["type"] == "image":
num_tokens += calculate_img_tokens(
data=_anthropic_image_source_data(c["source"]),
mode="auto",
use_default_image_token_count=use_default_image_token_count,
)
elif c["type"] == "document":
num_tokens += _count_document_tokens(
c,
count_function,
use_default_image_token_count,
default_token_count,
)
elif c["type"] in ("tool_use", "tool_result"):
num_tokens += _count_anthropic_content(
c,
@ -742,7 +806,8 @@ def _count_content_list(
content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__
raise ValueError(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)."
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)."
)
return num_tokens
except Exception as e:

46
litellm/llms/anthropic/common_utils.py
View file

@ -974,19 +974,25 @@ def strip_advisor_blocks_from_messages(messages: list[Any], replace_with_text: b
return messages
def is_anthropic_invalid_thinking_signature_error(error_text: str) -> bool:
def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool:
"""
Detect Anthropic 400 errors caused by missing or invalid thinking signatures.
Detect Anthropic 400 errors caused by invalid thinking blocks in replayed
history: a missing or invalid signature, or a block with empty thinking text.
Known error formats:
{"message":"messages.2.content.0.thinking.signature.str: Input should be a valid string"}
messages.N.content.M.thinking.signature.str: Input should be a valid string
messages.N.content.M: Invalid `signature` in `thinking` block
messages.N.content.M.thinking: each thinking block must contain thinking
"""
if not error_text:
return False
lower: Final = error_text.lower()
return "thinking" in lower and "signature" in lower and ("invalid" in lower or "valid string" in lower)
if "thinking" not in lower:
return False
if "signature" in lower and ("invalid" in lower or "valid string" in lower):
return True
return "must contain thinking" in lower
def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[Any]:
@ -1028,22 +1034,29 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
data.pop("thinking", None)
def strip_empty_text_blocks_from_anthropic_messages(
def strip_empty_content_blocks_from_anthropic_messages(
messages: list[Any],
) -> list[Any]:
"""
Return a new message list with empty or whitespace-only ``{"type": "text"}``
content blocks removed.
and ``{"type": "thinking"}`` content blocks removed.
Anthropic's API rejects requests containing such blocks with
``"messages: text content blocks must be non-empty"``, but assistant
messages from Anthropic routinely arrive with ``{"type": "text", "text": ""}``
alongside ``tool_use`` blocks (see anthropics/anthropic-sdk-python#461).
``"messages: text content blocks must be non-empty"`` and
``"messages.N.content.M.thinking: each thinking block must contain
thinking"`` respectively. Assistant messages routinely arrive with
``{"type": "text", "text": ""}`` alongside ``tool_use`` blocks (see
anthropics/anthropic-sdk-python#461), and a turn served by a
non-Anthropic reasoning model through the /v1/messages bridge can carry
``{"type": "thinking", "thinking": ""}`` when the model produced no
reasoning text (e.g. it went straight to parallel tool calls).
Multi-turn tool-use clients (e.g. Claude Code) loop these prior responses
back as conversation history, which then causes the next request to 400
on the unified ``/v1/messages`` path. ``/v1/chat/completions`` already
handles this in ``anthropic_messages_pt``; this helper provides the
equivalent guarantee for the native Anthropic Messages path.
``redacted_thinking`` blocks are never touched: they carry opaque
``data`` instead of thinking text.
Messages whose content is a list and becomes empty after stripping are
omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`.
@ -1056,7 +1069,7 @@ def strip_empty_text_blocks_from_anthropic_messages(
out.append(m)
continue
content = m["content"]
filtered = [b for b in content if not _is_empty_text_block(b)]
filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_thinking_block(b)]
if len(filtered) == len(content):
out.append(m)
elif filtered:
@ -1071,6 +1084,21 @@ def _is_empty_text_block(block: Any) -> bool:
return not isinstance(text, str) or not text.strip()
def is_empty_thinking_block(block: object) -> bool:
"""
True for a ``{"type": "thinking"}`` content block whose thinking text is
missing, not a string, or empty/whitespace-only after ``.strip()``.
Anthropic rejects such blocks with ``"each thinking block must contain
thinking"`` (whitespace-only included, verified live), regardless of any
signature they carry. ``redacted_thinking`` blocks are a different type
and always return False.
"""
if not isinstance(block, dict) or block.get("type") != "thinking":
return False
thinking: Final = block.get("thinking")
return not isinstance(thinking, str) or not thinking.strip()
def normalize_anthropic_tool_use_id(raw_id: str) -> str:
"""
Normalize a tool_use / tool_result id for Anthropic's ``^[a-zA-Z0-9_-]+$``

8
litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file

@ -1029,6 +1029,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
@staticmethod
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
from litellm.llms.anthropic.common_utils import is_empty_thinking_block
choice: Final = chunk.choices[0]
if choice.finish_reason is not None:
return False
@ -1039,7 +1041,11 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
return False
if getattr(delta, "reasoning_content", None):
return False
if getattr(delta, "thinking_blocks", None):
# thinking_blocks whose entries are all empty (even if signed) must not
# open a block: the emitted {"type": "thinking", "thinking": ""} gets
# replayed as history and Anthropic rejects it (LIT-6357).
thinking_blocks: Final = getattr(delta, "thinking_blocks", None)
if thinking_blocks and any(isinstance(b, dict) and not is_empty_thinking_block(b) for b in thinking_blocks):
return False
return True

7
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -89,7 +89,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking_budget,
)
from litellm.llms.anthropic.common_utils import normalize_anthropic_tool_use_id
from litellm.llms.anthropic.common_utils import (
is_empty_thinking_block,
normalize_anthropic_tool_use_id,
)
from litellm.llms.anthropic.experimental_pass_through.context_management import (
PolyfillResult,
)
@ -1264,6 +1267,8 @@ class LiteLLMAnthropicMessagesAdapter:
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
for thinking_block in choice.message.thinking_blocks:
if thinking_block.get("type") == "thinking":
if is_empty_thinking_block(thinking_block):
continue
thinking_value = thinking_block.get("thinking", "")
signature_value = thinking_block.get("signature", "")
new_content.append(

27
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -17,7 +17,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.anthropic.common_utils import (
flatten_unencrypted_web_search_results_in_anthropic_messages,
sanitize_tool_use_ids_in_anthropic_messages,
strip_empty_text_blocks_from_anthropic_messages,
strip_empty_content_blocks_from_anthropic_messages,
)
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
@ -242,17 +242,20 @@ async def anthropic_messages(
"""
Async: Make llm api request in Anthropic /messages API spec.
Runs the empty-text-block sanitizer before any backend dispatch.
Runs the empty-content-block sanitizer before any backend dispatch.
"""
# Anthropic's API rejects requests containing empty / whitespace-only
# text content blocks with "messages: text content blocks must be
# non-empty". Multi-turn tool-use clients (e.g. Claude Code) routinely
# loop assistant responses that contain {"type": "text", "text": ""}
# alongside tool_use blocks back as conversation history, which then
# causes the next /v1/messages call to 400. /v1/chat/completions
# already handles this in anthropic_messages_pt; sanitize the native
# Anthropic Messages path here for the same guarantee. See #22930.
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
# text content blocks ("messages: text content blocks must be
# non-empty") and empty thinking blocks ("each thinking block must
# contain thinking"). Multi-turn tool-use clients (e.g. Claude Code)
# routinely loop assistant responses that contain such blocks — an empty
# text block alongside tool_use, or an empty thinking block from a turn
# a non-Anthropic reasoning model served through the bridge — back as
# conversation history, which then causes the next /v1/messages call to
# 400. /v1/chat/completions already handles this in
# anthropic_messages_pt; sanitize the native Anthropic Messages path
# here for the same guarantee. See #22930.
messages = strip_empty_content_blocks_from_anthropic_messages(messages)
# Replay of cross-provider tool history (e.g. kimi -> Anthropic) may carry
# ids like ``functions.Bash:0`` that violate Anthropic's id pattern.
messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
@ -374,7 +377,7 @@ async def anthropic_messages(
api_base=api_base,
client=client,
custom_llm_provider=custom_llm_provider,
# messages were already empty-text-block sanitized at the top of this
# messages were already empty-content-block sanitized at the top of this
# function and are NOT reassigned before this dispatch, so the handler
# can skip its (otherwise redundant) second full-messages scan. Passed
# explicitly (not via **kwargs) so it only affects this direct
@ -451,7 +454,7 @@ def anthropic_messages_handler(
# ``_litellm_messages_presanitized`` to skip this redundant second
# full-messages scan. Pop it so it never leaks into provider params.
if not kwargs.pop("_litellm_messages_presanitized", False):
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
messages = strip_empty_content_blocks_from_anthropic_messages(messages)
messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages)

33
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -342,7 +342,7 @@ class BaseAnthropicMessagesStreamingIterator:
self.start_time = datetime.now()
self.completion_start_time: datetime | None = None
async def _handle_streaming_logging(self, collected_chunks: list[bytes]):
async def _handle_streaming_logging(self, collected_chunks: list[bytes], *, stream_teardown: bool = False):
"""Handle the logging after all chunks have been collected."""
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamingHandler,
@ -354,21 +354,26 @@ class BaseAnthropicMessagesStreamingIterator:
if self.completion_start_time is not None:
self.litellm_logging_obj.completion_start_time = self.completion_start_time
self.litellm_logging_obj.model_call_details["completion_start_time"] = self.completion_start_time
logging_coroutine: Final = PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=self.litellm_logging_obj,
passthrough_success_handler_obj=GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
url_route="/v1/messages",
request_body=self.request_body or {},
endpoint_type=EndpointType.ANTHROPIC,
start_time=self.start_time,
raw_bytes=collected_chunks,
end_time=end_time,
)
deferred_dispatch_armed: Final = (
getattr(self.litellm_logging_obj, "_on_deferred_stream_complete", None) is not None
)
if deferred_dispatch_armed and not stream_teardown:
self.litellm_logging_obj._deferred_stream_complete_args = (logging_coroutine,)
return
# Enqueue on the rooted logging worker rather than asyncio.create_task:
# this also runs during generator teardown after a client disconnect,
# where an unrooted task could be garbage-collected before it bills.
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
async_coroutine=PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=self.litellm_logging_obj,
passthrough_success_handler_obj=GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
url_route="/v1/messages",
request_body=self.request_body or {},
endpoint_type=EndpointType.ANTHROPIC,
start_time=self.start_time,
raw_bytes=collected_chunks,
end_time=end_time,
)
)
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
def get_async_streaming_response_iterator(
self,
@ -433,7 +438,7 @@ class BaseAnthropicMessagesStreamingIterator:
# post-loop logging below never runs and the tokens already streamed
# (and billed by the provider) would never reach spend tracking. See LIT-5839.
if collected_chunks:
await self._handle_streaming_logging(collected_chunks)
await self._handle_streaming_logging(collected_chunks, stream_teardown=True)
raise
if not saw_terminal_event:

20
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -19,19 +19,19 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
GPT5_SERIES_ROUTE = "gpt5_series/"
@classmethod
def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool:
"""Override to handle gpt5_series/ prefix used for Azure routing.
def _model_map_lookup_name(cls, model: str) -> str:
"""Normalise an Azure routing name to its cost-map key.
The parent class calls ``_supports_factory(model, custom_llm_provider=None)``
which fails to resolve ``gpt5_series/gpt-5.1`` to the correct Azure model
entry. Strip the prefix and prepend ``azure/`` so the lookup finds
``azure/gpt-5.1`` in model_prices_and_context_window.json.
Neither ``gpt5_series/gpt-5.1`` nor a bare ``gpt-5.1`` is a key in
model_prices_and_context_window.json; ``azure/gpt-5.1`` is. Overriding the shared
resolver rather than one lookup means the supports, explicitly-disabled and
default-effort answers all read the same entry.
"""
if model.startswith(cls.GPT5_SERIES_ROUTE):
model = "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :]
elif not model.startswith("azure/"):
model = "azure/" + model
return super()._supports_reasoning_effort_level(model, level)
return "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :]
if model.startswith("azure/"):
return model
return "azure/" + model
@classmethod
def is_model_gpt_5_model(cls, model: str) -> bool:

8
litellm/llms/base_llm/anthropic_messages/transformation.py
View file

@ -159,20 +159,20 @@ class BaseAnthropicMessagesConfig(ABC):
and issue one more attempt (bounded by max_retry_on_anthropic_messages_http_error).
"""
from litellm.llms.anthropic.common_utils import (
is_anthropic_invalid_thinking_signature_error,
is_anthropic_invalid_thinking_block_error,
)
return e.response.status_code == 400 and is_anthropic_invalid_thinking_signature_error(e.response.text)
return e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text)
def transform_anthropic_messages_request_on_http_error(self, e: httpx.HTTPStatusError, request_data: dict) -> dict:
"""
Mutates request_data in place when retrying after a recoverable HTTP error.
"""
from litellm.llms.anthropic.common_utils import (
is_anthropic_invalid_thinking_signature_error,
is_anthropic_invalid_thinking_block_error,
strip_thinking_blocks_from_anthropic_messages_request_dict,
)
if e.response.status_code == 400 and is_anthropic_invalid_thinking_signature_error(e.response.text):
if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
return request_data

2
litellm/llms/base_llm/chat/transformation.py
View file

@ -46,8 +46,10 @@ class BaseLLMException(Exception):
request: httpx.Request | None = None,
response: httpx.Response | None = None,
body: dict | None = None,
status_code_is_synthesized: bool = False,
):
self.status_code = status_code
self.status_code_is_synthesized = status_code_is_synthesized
self.message: str = message
self.headers = headers
if request:

16
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -158,6 +158,22 @@ def openai_messages_without_tool(
return tuple(m for m in messages if _message_role(m) != "tool")
def filter_messages_by_skip_flags(
guardrail_to_apply: object, messages: Sequence[AllMessageValues]
) -> tuple[tuple[AllMessageValues, ...], bool]:
system_filtered = (
openai_messages_without_system(messages)
if effective_skip_system_message_for_guardrail(guardrail_to_apply)
else tuple(messages)
)
fully_filtered = (
openai_messages_without_tool(system_filtered)
if effective_skip_tool_message_for_guardrail(guardrail_to_apply)
else system_filtered
)
return fully_filtered, len(fully_filtered) != len(messages)
def effective_scan_only_tool_results_for_guardrail(guardrail_to_apply: object) -> bool:
return getattr(guardrail_to_apply, "scan_only_tool_results", None) is True

13
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -1543,6 +1543,7 @@ class AmazonConverseConfig(BaseConfig):
messages: list[AllMessageValues] | None = None,
headers: dict | None = None,
drop_params: bool = False,
litellm_params: Mapping[str, object] | None = None,
) -> CommonRequestObject:
## VALIDATE REQUEST
"""
@ -1605,6 +1606,16 @@ class AmazonConverseConfig(BaseConfig):
if point.get("location") == "tool_config":
cache_point = self._build_cache_point_block(point.get("control"), model)
bedrock_tools.append(ToolBlock(cachePoint=cache_point))
# Spend attribution credits the gateway only for breakpoints it placed, and
# this is the one place a tool_config point becomes one. The hook that reads
# the configuration cannot record it: whether a cachePoint lands depends on
# this provider and on the request carrying tools, neither of which it sees.
if litellm_params is not None:
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
)
AnthropicCacheControlHook.record_gateway_injection(litellm_params, 1)
break
bedrock_tool_config: ToolConfigBlock | None = None
@ -1667,6 +1678,7 @@ class AmazonConverseConfig(BaseConfig):
messages=messages,
headers=headers,
drop_params=litellm_params.get("drop_params") is True,
litellm_params=litellm_params,
)
bedrock_messages: Final = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
@ -1726,6 +1738,7 @@ class AmazonConverseConfig(BaseConfig):
messages=messages,
headers=headers,
drop_params=litellm_params.get("drop_params") is True,
litellm_params=litellm_params,
)
## TRANSFORMATION ##

12
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -5947,11 +5947,13 @@ class BaseLLMHTTPHandler:
BaseEvalsAPIConfig,
],
):
status_code = getattr(e, "status_code", 500)
received_status_code: Final = (
e.response.status_code if isinstance(e, httpx.HTTPStatusError) else getattr(e, "status_code", None)
)
status_code = received_status_code if isinstance(received_status_code, int) else 500
error_headers = getattr(e, "headers", None)
if isinstance(e, httpx.HTTPStatusError):
error_text = e.response.text
status_code = e.response.status_code
else:
error_text = getattr(e, "text", str(e))
error_response: Final = getattr(e, "response", None)
@ -5971,13 +5973,17 @@ class BaseLLMHTTPHandler:
status_code=status_code,
message=error_text,
headers=error_headers,
status_code_is_synthesized=not isinstance(received_status_code, int),
)
raise provider_config.get_error_class(
provider_error: Final = provider_config.get_error_class(
error_message=error_text,
status_code=status_code,
headers=error_headers,
)
if not isinstance(received_status_code, int):
provider_error.status_code_is_synthesized = True
raise provider_error
@staticmethod
def _append_query_params(url: str, query_params: RealtimeQueryParams | None) -> str:

40
litellm/llms/moonshot/chat/transformation.py
View file

@ -2,7 +2,7 @@
Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions`
"""
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from typing import Any, Final, Literal, cast, overload
import litellm
@ -16,6 +16,15 @@ from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
def _reasoning_effort_string(value: object) -> str | None:
"""The /v1/messages and /v1/responses bridges wrap the level as {"effort", "summary"} for
providers with a reasoning-summary surface. Moonshot's API takes only the bare string and 400s
on an object, so the level is unwrapped and the summary, which has no Moonshot equivalent, is
dropped."""
effort: Final = value.get("effort") if isinstance(value, Mapping) else value
return effort if isinstance(effort, str) else None
class MoonshotChatConfig(OpenAIGPTConfig):
@overload
def _transform_messages(
@ -93,20 +102,18 @@ class MoonshotChatConfig(OpenAIGPTConfig):
- functions parameter is not supported (use tools instead)
- tool_choice doesn't support "required" value
- kimi-thinking-preview doesn't support tool calls at all
A reasoning model additionally takes `reasoning_effort`, which the OpenAI base list this
subtracts from does not carry, so it has to be added back rather than merely kept.
"""
excluded_params: Final[list[str]] = ["functions"]
# kimi-thinking-preview has additional limitations
if "kimi-thinking-preview" in model:
excluded_params.extend(["tools", "tool_choice"])
excluded_params: Final = frozenset(
("functions", "tools", "tool_choice") if "kimi-thinking-preview" in model else ("functions",)
)
base_openai_params: Final = super().get_supported_openai_params(model=model)
final_params: Final[list[str]] = []
for param in base_openai_params:
if param not in excluded_params:
final_params.append(param)
return final_params
supported: Final = [param for param in base_openai_params if param not in excluded_params]
if supports_reasoning(model=model, custom_llm_provider="moonshot"):
return [*supported, "reasoning_effort"]
return supported
def map_openai_params(
self,
@ -126,7 +133,12 @@ class MoonshotChatConfig(OpenAIGPTConfig):
for param, value in non_default_params.items():
if param == "max_completion_tokens":
optional_params["max_tokens"] = value
elif param in supported_openai_params:
elif param not in supported_openai_params:
continue
elif param == "reasoning_effort":
if (effort := _reasoning_effort_string(value)) is not None:
optional_params["reasoning_effort"] = effort
else:
optional_params[param] = value
##########################################

80
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -6,11 +6,28 @@ import litellm
from litellm.utils import (
_is_explicitly_disabled_factory,
_supports_factory,
declared_value_factory,
)
from .gpt_transformation import OpenAIGPTConfig
def _catalogue_declares_default_effort() -> bool:
"""Whether the loaded cost map carries default_reasoning_effort for ANY entry.
The map is fetched from the published branch at import time, so it can be OLDER than the
code reading it. On such a map every model looks undeclared, and treating that as "reasoning
is active" would silently strip temperature from the gpt-5.1/5.2/5.4 deployments that accept
it - a regression caused purely by data lag rather than by anything about the model.
So the absence of the key is only meaningful once the catalogue is known to carry it at all.
A map that has never heard of the key predates the feature, and the honest answer there is
the one litellm gave before it existed. Scanning costs ~80us on the largest published map and
only on the fallback path, which is noise beside the request it precedes.
"""
return any(isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in litellm.model_cost.values())
def _normalize_reasoning_effort_for_chat_completion(
value: str | dict | None,
) -> str | None:
@ -114,6 +131,17 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
except (ValueError, IndexError):
return False
@classmethod
def _model_map_lookup_name(cls, model: str) -> str:
"""The name this model is looked up by in the cost map.
Identity here, because an OpenAI model name is already its map key. Azure overrides
it: its routing prefixes are not map keys, so every capability lookup has to
normalise the name the same way, and doing that in ONE place is what keeps the
supports/disabled/default answers from disagreeing about which entry they read.
"""
return model
@classmethod
def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool:
"""Check if the model supports a specific reasoning_effort level.
@ -123,11 +151,40 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
Returns False for unknown models (safe fallback).
"""
return _supports_factory(
model=model,
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key=f"supports_{level}_reasoning_effort",
)
@classmethod
def effort_resolves_to_none(cls, model: str, effective_effort: str | None) -> bool:
"""Whether this request's reasoning effort ends up as "none", which is the single
condition under which the provider accepts a non-default temperature or the
top_p/logprobs sampling params.
An explicit reasoning_effort answers outright. When the request omits it the answer
is the model's DEFAULT effort, which only the map can state: supporting "none" is a
different fact from defaulting to it, and reading the former as the latter is what
forwarded temperature=0 to every gpt-5.5/5.6 deployment.
An undeclared default resolves to False. The map not saying is not the model
saying no, so the gate takes the conservative branch: a param the provider would
have rejected gets dropped or refused with an actionable error, and a model
released before its map entry declares a default needs no code change to be safe.
"""
if effective_effort is not None:
return effective_effort == "none"
declared: Final = declared_value_factory(
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key="default_reasoning_effort",
)
if declared is not None:
return declared == "none"
if not _catalogue_declares_default_effort():
return cls._supports_reasoning_effort_level(model, "none")
return False
@classmethod
def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool:
"""Return True only when the model map explicitly sets the capability to False.
@ -140,7 +197,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
Use this for opt-out checks where unknown models should be allowed through.
"""
return _is_explicitly_disabled_factory(
model=model,
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key=f"supports_{level}_reasoning_effort",
)
@ -260,15 +317,16 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
if supports_none:
sampling_params: Final = ["logprobs", "top_logprobs", "top_p"]
has_sampling: Final = any(p in non_default_params for p in sampling_params)
if has_sampling and effective_effort not in (None, "none"):
if has_sampling and not self.effort_resolves_to_none(model, effective_effort):
if litellm.drop_params or drop_params:
for p in sampling_params:
non_default_params.pop(p, None)
else:
raise litellm.utils.UnsupportedParamsError(
message=(
"gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when "
f"reasoning_effort='none'. Current reasoning_effort='{effective_effort}'. "
f"{model} only supports logprobs, top_p, top_logprobs when reasoning_effort "
"resolves to 'none', either set explicitly on the request or declared as the "
f"model's default_reasoning_effort. Current reasoning_effort={effective_effort!r}. "
"To drop unsupported params set `litellm.drop_params = True`"
),
status_code=400,
@ -277,17 +335,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
if "temperature" in non_default_params:
temperature_value: Final[float | None] = non_default_params.pop("temperature")
if temperature_value is not None:
# models supporting reasoning_effort="none" also support flexible temperature
if supports_none and (effective_effort == "none" or effective_effort is None) or temperature_value == 1:
# a non-default temperature rides on the effort resolving to "none", not on
# the model merely supporting it
if (supports_none and self.effort_resolves_to_none(model, effective_effort)) or temperature_value == 1:
optional_params["temperature"] = temperature_value
elif litellm.drop_params or drop_params:
pass
else:
raise litellm.utils.UnsupportedParamsError(
message=(
f"gpt-5 models (including gpt-5-codex) don't support temperature={temperature_value}. "
"Only temperature=1 is supported. "
"For gpt-5.1, temperature is supported when reasoning_effort='none' (or not specified, as it defaults to 'none'). "
f"{model} doesn't support temperature={temperature_value} while reasoning is "
"active. Only temperature=1 is supported unless reasoning_effort resolves to "
"'none', either set explicitly on the request or declared as the model's "
"default_reasoning_effort. "
"To drop unsupported params set `litellm.drop_params = True`"
),
status_code=400,

24
litellm/llms/openai/responses/transformation.py
View file

@ -61,6 +61,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
key="supports_none_reasoning_effort",
)
@staticmethod
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
"""Whether this request's reasoning effort ends up as "none", the one condition
under which a non-default temperature is accepted.
Delegates to the chat-completions gpt-5 config so both surfaces answer from one
rule: the Responses API reaches the same models over a different wire, and a second
copy of the rule here is what let this surface keep forwarding temperature after the
chat surface stopped.
"""
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
return OpenAIGPT5Config.effort_resolves_to_none(model, effort)
@staticmethod
def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None":
"""Raise sub-minimum max_output_tokens up to the OpenAI Responses API minimum.
@ -116,17 +130,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
reasoning: Final = params.get("reasoning") or {}
effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None
supports_none: Final = self._supports_reasoning_effort_none(model=model)
if supports_none and (effort == "none" or effort is None):
if supports_none and self._effort_resolves_to_none(model, effort):
pass # flexible temperature allowed
elif drop_params or litellm.drop_params:
params.pop("temperature", None)
else:
raise litellm.UnsupportedParamsError(
message=(
f"gpt-5 models don't support temperature={temperature}. "
"Only temperature=1 is supported. "
"For models like gpt-5.1/5.4, temperature is supported "
"when reasoning.effort='none' (or not specified). "
f"{model} doesn't support temperature={temperature} while reasoning is "
"active. Only temperature=1 is supported unless reasoning.effort resolves "
"to 'none', either set explicitly on the request or declared as the "
"model's default_reasoning_effort. "
"To drop unsupported params set `litellm.drop_params = True`"
),
status_code=400,

3
litellm/llms/together_ai/chat/transformation.py
View file

@ -18,6 +18,7 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_function_calling, supports_reasoning, supports_response_schema
@ -139,6 +140,8 @@ def _reasoning_effort_payload(effort: str, model: str) -> Mapping[str, object]:
if effort == "none":
disable_reasoning: Final[TogetherReasoningToggle] = {"enabled": False}
return MappingProxyType({"reasoning": disable_reasoning})
if effort in (declared_reasoning_efforts_for_model(model, "together_ai") or ()):
return MappingProxyType({"reasoning_effort": effort})
if model.startswith(HIGH_MAX_EFFORT_MODEL_PREFIX):
return MappingProxyType({"reasoning_effort": HIGH_MAX_EFFORT_TRANSLATION.get(effort, effort)})
return MappingProxyType({"reasoning_effort": EFFORT_TRANSLATION.get(effort, effort)})

79
litellm/main.py
View file

@ -531,6 +531,7 @@ async def acompletion(
tools=tools,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
#########################################################
# if the chat completion logging hook removed all tools,
@ -5246,6 +5247,7 @@ def completion(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
### LITELLM SYSTEM PROMPT ###
@ -8588,6 +8590,47 @@ def stream_chunk_builder_text_completion(chunks: list, messages: list | None = N
return TextCompletionResponse(**response)
def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None:
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)):
return float(usage_cost)
if logging_obj is not None:
return None
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
"custom_llm_provider"
)
try:
return litellm.completion_cost(completion_response=response, custom_llm_provider=provider_hint)
except Exception:
return _stream_builder_model_map_cost(response)
def _joined_streamed_citations(streamed_citations: "tuple[object, ...]") -> "list[object]":
if all(isinstance(citation, list) for citation in streamed_citations):
return list(streamed_citations) # mutable-ok: JSON list field
return [list(streamed_citations)] # mutable-ok: JSON list field
def _stream_builder_model_map_cost(response: ModelResponse) -> float | None:
model_name: Final = getattr(response, "model", None)
usage: Final = getattr(response, "usage", None)
if not isinstance(model_name, str) or not model_name or not isinstance(usage, Usage):
return None
try:
prompt_cost, completion_tokens_cost = litellm.cost_per_token(model=model_name, usage_object=usage)
return prompt_cost + completion_tokens_cost
except Exception: # noqa: BLE001 # cost_per_token raises bare Exception for unpriceable models
return None
def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
response_cost: Final = _stream_builder_response_cost(response, logging_obj)
if response_cost is None:
return
hidden_params: Final = response._hidden_params # pyright: ignore[reportPrivateUsage] # no public accessor
hidden_params["response_cost"] = response_cost
def stream_chunk_builder(
chunks: list,
messages: list | None = None,
@ -8688,6 +8731,8 @@ def stream_chunk_builder(
"cost",
logging_obj._response_cost_calculator(result=response),
)
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
@ -8812,18 +8857,26 @@ def stream_chunk_builder(
]
if len(provider_specific_chunks) > 0:
combined_provider_fields: Final[dict[str, object]] = {}
for chunk in provider_specific_chunks:
fields = chunk["choices"][0]["delta"]["provider_specific_fields"]
if isinstance(fields, dict):
for key, value in fields.items():
if key not in combined_provider_fields:
combined_provider_fields[key] = value
elif isinstance(value, list) and isinstance(combined_provider_fields[key], list):
# For lists like web_search_results, take the last (most complete) one
combined_provider_fields[key] = value
else:
combined_provider_fields[key] = value
provider_field_dicts: Final = tuple(
fields
for chunk in provider_specific_chunks
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
if isinstance(fields, dict)
)
streamed_citations: Final = tuple(
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
)
citation_fields: Final = (
{"citations": _joined_streamed_citations(streamed_citations)} # mutable-ok: JSON dict field
if streamed_citations
else {} # mutable-ok: JSON dict field
)
combined_provider_fields: Final = { # mutable-ok: Message.provider_specific_fields is a plain dict field
key: value
for fields in (citation_fields, *provider_field_dicts)
for key, value in fields.items()
if key != "citation"
}
if combined_provider_fields:
_choice = cast(Choices, response.choices[0])
@ -8860,6 +8913,8 @@ def stream_chunk_builder(
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
setattr(usage, "cost", logging_obj._response_cost_calculator(result=response))
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
except Exception as e:

494
litellm/model_prices_and_context_window_backup.json
View file

@ -3409,6 +3409,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -3456,6 +3457,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -3589,6 +3591,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3630,6 +3633,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3671,6 +3675,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3712,6 +3717,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3937,7 +3943,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/eu/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.4e-07,
@ -3972,7 +3979,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/eu/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -4247,7 +4255,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/global/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.25e-07,
@ -4282,7 +4291,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/global/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -5367,6 +5377,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_minimal_reasoning_effort": true
},
"azure/gpt-5.1-chat-2025-11-13": {
@ -5404,7 +5415,8 @@
"supports_system_messages": true,
"supports_tool_choice": false,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/gpt-5.1-codex-2025-11-13": {
"cache_read_input_token_cost": 1.25e-07,
@ -5833,7 +5845,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.25e-07,
@ -5868,7 +5881,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -6315,6 +6329,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6354,6 +6369,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6393,6 +6409,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6438,6 +6455,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6477,6 +6495,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6516,6 +6535,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -7663,6 +7683,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-5.4-mini-2026-03-17": {
@ -7704,6 +7725,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-5.4-nano": {
@ -7745,6 +7767,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-5.4-nano-2026-03-17": {
@ -7786,6 +7809,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-image-1": {
@ -8856,7 +8880,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/us/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.4e-07,
@ -8891,7 +8916,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/us/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -15216,6 +15242,29 @@
"supports_tool_choice": true,
"supports_vision": false
},
"databricks/databricks-glm-5-3-flash": {
"litellm_provider": "databricks",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"metadata": {
"notes": "Databricks has not published pay-per-token DBU rates for this model yet (not on the foundation-model-serving pricing page as of 2026-08-27), so cost fields are omitted until rates are published."
},
"mode": "chat",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"databricks/databricks-gpt-5": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.2502e-07,
@ -22971,9 +23020,9 @@
"input_cost_per_audio_token": 1.5e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
@ -22981,7 +23030,7 @@
"rpm": 2000,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions"
"/v1beta/interactions"
],
"supported_modalities": [
"text",
@ -26292,6 +26341,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true
},
@ -26336,6 +26386,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true
},
@ -26381,6 +26432,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true
},
@ -26426,6 +26478,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -26471,6 +26524,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -27295,6 +27349,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -27343,6 +27398,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -27492,6 +27548,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -27543,6 +27600,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -27591,6 +27649,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -27639,6 +27698,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -30807,6 +30867,7 @@
"supports_tool_choice": true
},
"mistral/codestral-2508": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "mistral",
"max_input_tokens": 128000,
@ -30821,6 +30882,7 @@
"supports_tool_choice": true
},
"mistral/codestral-latest": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "mistral",
"max_input_tokens": 128000,
@ -30828,11 +30890,11 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 9e-07,
"supports_assistant_prefill": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"source": "https://docs.mistral.ai/models/model-cards/codestral-25-08",
"supports_function_calling": true
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"mistral/codestral-mamba-latest": {
"input_cost_per_token": 2.5e-07,
@ -31312,6 +31374,7 @@
"mode": "embedding"
},
"mistral/codestral-embed": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 8192,
@ -31319,6 +31382,7 @@
"mode": "embedding"
},
"mistral/codestral-embed-2505": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 8192,
@ -31368,6 +31432,7 @@
"supports_tool_choice": true
},
"mistral/mistral-large-latest": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31383,6 +31448,7 @@
"supports_vision": true
},
"mistral/mistral-large-3": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31398,6 +31464,7 @@
"supports_vision": true
},
"mistral/mistral-large-2512": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31468,6 +31535,7 @@
"supports_vision": true
},
"mistral/mistral-medium-2604": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31484,6 +31552,7 @@
"supports_vision": true
},
"mistral/mistral-medium-latest": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31516,6 +31585,7 @@
"supports_vision": true
},
"mistral/mistral-medium-3-5": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31545,6 +31615,7 @@
"supports_tool_choice": true
},
"mistral/mistral-small-latest": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31555,9 +31626,9 @@
"source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"mistral/mistral-small-3-2-2506": {
@ -31577,6 +31648,7 @@
"supports_vision": true
},
"mistral/ministral-3-3b-2512": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "mistral",
"max_input_tokens": 131072,
@ -31592,6 +31664,7 @@
"supports_vision": true
},
"mistral/ministral-3-8b-2512": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31607,6 +31680,7 @@
"supports_vision": true
},
"mistral/ministral-3-14b-2512": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 2e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31622,6 +31696,7 @@
"supports_vision": true
},
"mistral/ministral-8b-2512": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31637,6 +31712,7 @@
"supports_vision": true
},
"mistral/ministral-8b-latest": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -31844,6 +31920,24 @@
"supports_tool_choice": true,
"supports_web_search": true
},
"moonshot/kimi-k2.7-code": {
"cache_read_input_token_cost": 1.9e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "moonshot",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://platform.kimi.ai/docs/pricing/chat-k27-code",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true
},
"moonshot/kimi-k2-turbo-preview": {
"cache_read_input_token_cost": 1.5e-07,
"deprecation_date": "2026-05-25",
@ -38625,7 +38719,7 @@
"together_ai/openai/gpt-oss-20b": {
"input_cost_per_token": 5e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 128000,
"max_input_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2e-07,
"source": "https://www.together.ai/models/gpt-oss-20b",
@ -38847,14 +38941,14 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/Qwen/Qwen3.8-2.4T-A95B": {
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1010000,
"max_output_tokens": 1010000,
"max_tokens": 1010000,
"mode": "chat",
"output_cost_per_token": 6.25e-06,
"output_cost_per_token": 6e-06,
"source": "https://docs.together.ai/docs/serverless-models",
"supports_prompt_caching": true
},
@ -44611,6 +44705,22 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"zai/glm-5.3-flash": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
"litellm_provider": "zai",
"max_input_tokens": 1048576,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing",
"supports_vision": true
},
"zai/glm-5.1": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2.6e-07,
@ -49829,14 +49939,14 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-5.6-sol": {
"input_cost_per_token": 4.4e-06,
"input_cost_per_token_above_272k_tokens": 8.8e-06,
"cache_creation_input_token_cost": 5.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.1e-05,
"cache_read_input_token_cost": 4.4e-07,
"cache_read_input_token_cost_above_272k_tokens": 8.8e-07,
"output_cost_per_token": 2.2e-05,
"output_cost_per_token_above_272k_tokens": 3.3e-05,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.012,
"search_context_size_low": 0.012,
@ -50131,8 +50241,11 @@
},
"bedrock_mantle/openai.gpt-5.5": {
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.012,
"search_context_size_low": 0.012,
@ -50164,8 +50277,11 @@
},
"bedrock_mantle/openai.gpt-5.4": {
"input_cost_per_token": 2.75e-06,
"input_cost_per_token_above_272k_tokens": 5.5e-06,
"cache_read_input_token_cost": 2.75e-07,
"cache_read_input_token_cost_above_272k_tokens": 5.5e-07,
"output_cost_per_token": 1.65e-05,
"output_cost_per_token_above_272k_tokens": 2.475e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.012,
"search_context_size_low": 0.012,
@ -51324,10 +51440,10 @@
"mode": "responses",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_tool_choice": false,
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
@ -51521,6 +51637,7 @@
"web_search_billing_unit": "per_query"
},
"mistral/mistral-small-2603": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
@ -54196,5 +54313,318 @@
"supports_reasoning": true,
"supports_vision": true,
"source": "https://deepinfra.com/pricing"
},
"gemini/gemini-omni-1.1-flash": {
"input_cost_per_audio_token": 1.5e-06,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_reasoning_token": 9e-06,
"output_cost_per_token": 9e-06,
"output_cost_per_video_token": 1.75e-05,
"rpm": 2000,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1beta/interactions"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text",
"video"
],
"supports_audio_input": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_video_input": true,
"supports_vision": true,
"tpm": 800000
},
"xai/grok-4.20": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"supports_prompt_caching": true,
"supports_response_schema": true
},
"xai/grok-4.20-reasoning": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"supports_prompt_caching": true,
"supports_response_schema": true
},
"xai/grok-4.20-reasoning-latest": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"supports_prompt_caching": true,
"supports_response_schema": true
},
"xai/grok-imagine-image": {
"input_cost_per_image": 0.02,
"litellm_provider": "xai",
"mode": "image_generation",
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/images/generations"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
]
},
"xai/grok-imagine-image-2026-03-02": {
"input_cost_per_image": 0.02,
"litellm_provider": "xai",
"mode": "image_generation",
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/images/generations"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
]
},
"xai/grok-imagine-image-quality": {
"input_cost_per_image": 0.05,
"litellm_provider": "xai",
"mode": "image_generation",
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/images/generations"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
]
},
"xai/grok-imagine-image-quality-20260403": {
"input_cost_per_image": 0.05,
"litellm_provider": "xai",
"mode": "image_generation",
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/images/generations"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
]
},
"xai/grok-imagine-image-quality-latest": {
"input_cost_per_image": 0.05,
"litellm_provider": "xai",
"mode": "image_generation",
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/images/generations"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
]
},
"xai/grok-imagine-image-pro": {
"input_cost_per_image": 0.05,
"litellm_provider": "xai",
"mode": "image_generation",
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/images/generations"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
],
"deprecation_date": "2026-05-15"
},
"xai/grok-imagine-image-2.0": {
"input_cost_per_image": 0.06,
"litellm_provider": "xai",
"mode": "image_generation",
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/images/generations"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
]
},
"low/1024-x-1024/grok-imagine-image-2.0": {
"input_cost_per_image": 0.04,
"litellm_provider": "xai",
"mode": "image_generation",
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/images/generations"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"image"
]
},
"xai/grok-4.20-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"supports_response_schema": true
},
"xai/grok-4.20-non-reasoning-latest": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"supports_response_schema": true
},
"xai/grok-4.20-multi-agent": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "responses",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/responses"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": false,
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"supports_response_schema": true
},
"xai/grok-4.20-multi-agent-latest": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "responses",
"output_cost_per_token": 2.5e-06,
"source": "https://docs.x.ai/docs/models",
"supported_endpoints": [
"/v1/responses"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": false,
"supports_vision": true,
"supports_web_search": true,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"supports_response_schema": true
}
}

152
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1,5 +1,5 @@
import re
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
@ -67,6 +67,9 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
_EMPTY_TOOLSET_GRANTS: Final[Mapping[str, Sequence[str]]] = MappingProxyType({})
def _as_list(values: Sequence[str] | None) -> list[str] | None: # mutable-ok: resolver returns a list
"""Widen a read-only allowlist back to the mutable list the resolver's own contract returns,
preserving the ``None`` that means "no restriction"."""
@ -1497,7 +1500,11 @@ class MCPRequestHandler:
team_set: Final = set(allowed_mcp_servers_for_team)
grants_set: Final = set(key_access_group_grants)
has_lower_level_mcp_restrictions = bool(key_set or team_set or grants_set)
# A DECLARED toolset restricts even when it resolves to no servers: the org
# ceiling below may only cap it, never substitute the org's full server list.
has_lower_level_mcp_restrictions = bool(key_set or team_set or grants_set) or (
await MCPRequestHandler._key_or_team_declares_toolsets(user_api_key_auth)
)
# 1. Key/team ceiling. An empty set means "this level does not restrict".
if not team_set:
@ -1941,6 +1948,105 @@ class MCPRequestHandler:
return team_obj.object_permission
@staticmethod
async def _toolset_tool_permissions(
object_permission: LiteLLM_ObjectPermissionTable | None,
) -> Mapping[str, Sequence[str]]:
"""The ``server_id -> tool names`` grants of this permission row's toolsets, empty when it
declares none. The shared resolver for the team, org, and internal-user levels, so a toolset
behaves identically wherever it is attached.
RAISES ``UnloadableEntitlementError`` when the row DECLARES toolsets but resolution yields
nothing (deleted or unknown ids, a swallowed DB fault, or a toolset with no tools): that is a
KNOWN restriction with unknown contents, and every caller already turns this error into deny
rather than letting the level read as unrestricted."""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
if object_permission is None or not object_permission.mcp_toolsets:
return _EMPTY_TOOLSET_GRANTS
resolved: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions(
toolset_ids=object_permission.mcp_toolsets
)
if not resolved:
raise UnloadableEntitlementError(
f"declared mcp_toolsets {object_permission.mcp_toolsets!r} resolved to no grants"
)
return resolved
@staticmethod
async def _toolset_tools_for_server(
object_permission: LiteLLM_ObjectPermissionTable | None,
server_id: str,
) -> Sequence[str] | None:
"""Tool names this row's toolsets grant on ``server_id``, ``None`` when its toolsets place
no restriction on that server (it declares no toolsets, or none of them name it)."""
return (await MCPRequestHandler._toolset_tool_permissions(object_permission)).get(server_id)
@staticmethod
def _union_tool_grants(
direct: Sequence[str] | None,
via_toolsets: Sequence[str] | None,
) -> Sequence[str] | None:
"""Union of one level's direct tool grants and its toolset-granted tools on one server,
``None`` when neither source restricts (allow-all from this level)."""
if direct is None and via_toolsets is None:
return None
return tuple({*(direct or ()), *(via_toolsets or ())})
@staticmethod
async def _key_object_permission_hydrated(
user_api_key_auth: UserAPIKeyAuth,
) -> LiteLLM_ObjectPermissionTable | None:
"""The key's object_permission, loading it by ``object_permission_id`` when the main auth
flow cached the key with the relation unhydrated (its loader swallows a failed read and
caches the partial object)."""
loaded: Final = MCPRequestHandler._get_key_object_permission(user_api_key_auth)
if loaded is not None or not user_api_key_auth.object_permission_id:
return loaded
from litellm.proxy.auth.auth_checks import get_object_permission
from litellm.proxy.proxy_server import (
prisma_client,
proxy_logging_obj,
user_api_key_cache,
)
if prisma_client is None:
return None
return await get_object_permission(
object_permission_id=user_api_key_auth.object_permission_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=user_api_key_auth.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
@staticmethod
async def _key_or_team_declares_toolsets(user_api_key_auth: UserAPIKeyAuth | None) -> bool:
"""Whether the key or its team GRANTS any toolset, resolvable or not. A declared toolset is
a lower-level restriction even when it resolves to no servers (deleted or unknown ids), so the
org ceiling may only cap it; reading an empty resolution as "no restriction" would substitute
the org's entire server list for the narrowest grant an operator can write.
Falls back to the DB when the auth object carries ``object_permission_id`` unhydrated (the
main auth flow swallows a failed load and caches the partial object). An INDETERMINATE fault
answers False — no gate, org substitution as before the fault — mirroring how the org ceiling
keeps key auth open on a fault it cannot classify."""
if user_api_key_auth is None:
return False
try:
key_obj_perm: Final = await MCPRequestHandler._key_object_permission_hydrated(user_api_key_auth)
if key_obj_perm is not None and key_obj_perm.mcp_toolsets:
return True
if not user_api_key_auth.team_id:
return False
team_obj_perm: Final = await MCPRequestHandler._get_team_object_permission(user_api_key_auth)
return bool(team_obj_perm is not None and team_obj_perm.mcp_toolsets)
except Exception as e: # noqa: BLE001 # indeterminate fault: no gate, as before this level existed
verbose_logger.warning("Failed to check declared MCP toolsets, org ceiling unchanged: %s", e)
return False
@staticmethod
async def get_allowed_tools_for_server(
server_id: str,
@ -2004,12 +2110,17 @@ class MCPRequestHandler:
if key_direct_tools is not None or key_toolset_tools is not None
else None
)
team_tools: Final = (
team_direct_tools: Final = (
global_mcp_server_manager.expand_tool_permissions(team_obj_perm.mcp_tool_permissions).get(server_id)
if team_obj_perm
else None
)
# Tools granted through the team's toolsets restrict this server exactly
# as the team's direct tool permissions do, mirroring the key path above
team_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server(team_obj_perm, server_id)
team_tools: Final = MCPRequestHandler._union_tool_grants(team_direct_tools, team_toolset_tools)
# Apply same inheritance logic as get_allowed_mcp_servers
if team_tools:
if key_tools:
@ -2094,11 +2205,13 @@ class MCPRequestHandler:
e,
)
return allowed_tools
org_tools: Final = (
org_direct_tools: Final = (
global_mcp_server_manager.expand_tool_permissions(org_obj_perm.mcp_tool_permissions).get(server_id)
if org_obj_perm and org_obj_perm.mcp_tool_permissions
else None
)
org_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server(org_obj_perm, server_id)
org_tools: Final = MCPRequestHandler._union_tool_grants(org_direct_tools, org_toolset_tools)
if org_tools is not None:
allowed_tools = (
list(set(allowed_tools) & set(org_tools)) if allowed_tools is not None else list(org_tools)
@ -2340,7 +2453,8 @@ class MCPRequestHandler:
async def _team_granted_servers(team_obj: LiteLLM_TeamTable, team_access_group_servers: list[str]) -> set[str]:
"""The raw MCP-server set a team grants (before any org ceiling): its object_permission (direct
``mcp_servers``, the ``all_proxy_servers`` sentinel → the full registry, legacy access groups,
tool-perm-referenced servers) unioned with its unified ``access_group_ids`` servers."""
tool-perm-referenced servers, toolset-referenced servers) unioned with its unified
``access_group_ids`` servers."""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
@ -2357,6 +2471,7 @@ class MCPRequestHandler:
set(global_mcp_server_manager.expand_permission_list(object_permissions.mcp_servers or []))
| set(legacy_access_group_servers)
| set(global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).keys())
| (await MCPRequestHandler._toolset_tool_permissions(object_permissions)).keys()
| set(team_access_group_servers)
)
@ -2415,6 +2530,8 @@ class MCPRequestHandler:
servers: Final = await MCPRequestHandler._team_granted_servers(team_obj, team_access_group_servers)
return list(servers)
except Exception as e:
if isinstance(e, UnloadableEntitlementError):
raise
verbose_logger.warning("Failed to get allowed MCP servers for team: %s", e)
return []
@ -2546,7 +2663,13 @@ class MCPRequestHandler:
global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).keys()
)
all_servers: Final = direct_mcp_servers + access_group_servers + tool_perm_servers
# servers referenced by the org's toolset grants are part of the org ceiling,
# exactly as servers referenced by its inline tool permissions are
toolset_grants: Final = await MCPRequestHandler._toolset_tool_permissions(object_permissions)
all_servers: Final = tuple(
{*direct_mcp_servers, *access_group_servers, *tool_perm_servers, *toolset_grants}
)
return list(set(all_servers))
except Exception as e:
# None = ceiling UNRESOLVED, distinct from [] = org places no restriction. Collapsing them
@ -2740,8 +2863,8 @@ class MCPRequestHandler:
``[]`` means this human places no restriction (allow-all from this level); ``None`` means the
ceiling is UNRESOLVED, which the caller denies on. Servers named only under
``mcp_tool_permissions`` count as entitled, exactly as they do for a key or a team, so
granting one tool never requires naming its server twice.
``mcp_tool_permissions`` or reached through ``mcp_toolsets`` count as entitled, exactly as
they do for a key or a team, so granting one tool never requires naming its server twice.
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
@ -2759,7 +2882,8 @@ class MCPRequestHandler:
tool_perm_servers: Final = list(
global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).keys()
)
return list(set(direct_mcp_servers + access_group_servers + tool_perm_servers))
toolset_grants: Final = await MCPRequestHandler._toolset_tool_permissions(object_permissions)
return tuple({*direct_mcp_servers, *access_group_servers, *tool_perm_servers, *toolset_grants})
except Exception as e: # noqa: BLE001 # any resolution fault is an unresolved ceiling, never "no ceiling"
verbose_logger.warning("Failed to get allowed MCP servers for user: %s", e)
return None
@ -2860,12 +2984,14 @@ class MCPRequestHandler:
verbose_logger.warning("MCP user tool ceiling unresolvable, denying tools on %r: %s", server_id, e)
return []
if object_permissions is None or not object_permissions.mcp_tool_permissions:
if object_permissions is None:
return allowed_tools
user_tools = global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).get(
server_id
)
user_direct_tools: Final = global_mcp_server_manager.expand_tool_permissions(
object_permissions.mcp_tool_permissions
).get(server_id)
user_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server(object_permissions, server_id)
user_tools: Final = MCPRequestHandler._union_tool_grants(user_direct_tools, user_toolset_tools)
if user_tools is None:
return allowed_tools
if allowed_tools is None:

18
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -168,8 +168,11 @@ if MCP_AVAILABLE:
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.tool_search import (
AGENT_SEARCH_TOOL_NAME,
DEFAULT_AGENT_SEARCH_TOP_K,
MCP_TOOL_SEARCH_TOOL_NAME,
coerce_top_k,
handle_agent_search,
handle_mcp_tool_call,
handle_mcp_tool_search,
)
@ -182,6 +185,14 @@ if MCP_AVAILABLE:
detail={"error": "forbidden", "message": f"{tool_name} requires mcp_tool_search_enabled on the key"},
)
tool_arguments: Final = data.get("arguments") or {}
if tool_name == AGENT_SEARCH_TOOL_NAME:
return await handle_agent_search(
query=str(tool_arguments.get("query", "")),
top_k=coerce_top_k(
tool_arguments.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K
),
user_api_key_dict=user_api_key_dict,
)
rest_client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
(
virtual_mcp_auth_header,
@ -939,12 +950,9 @@ if MCP_AVAILABLE:
tool_name: Final[str | None] = data.get("name")
tool_arguments: Final[dict[str, object]] = data.get("arguments") or {}
from litellm.proxy._experimental.mcp_server.tool_search import (
MCP_TOOL_CALL_TOOL_NAME,
MCP_TOOL_SEARCH_TOOL_NAME,
)
from litellm.proxy._experimental.mcp_server.tool_search import VIRTUAL_TOOL_NAMES
if tool_name in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME):
if tool_name in VIRTUAL_TOOL_NAMES:
return await _handle_virtual_mcp_tool(request, data, tool_name, user_api_key_dict)
# Validate required parameters early

13
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -912,14 +912,17 @@ if MCP_AVAILABLE:
the caller falls through to normal tool routing.
"""
from litellm.proxy._experimental.mcp_server.tool_search import (
MCP_TOOL_CALL_TOOL_NAME,
AGENT_SEARCH_TOOL_NAME,
DEFAULT_AGENT_SEARCH_TOP_K,
MCP_TOOL_SEARCH_TOOL_NAME,
VIRTUAL_TOOL_NAMES,
coerce_top_k,
handle_agent_search,
handle_mcp_tool_call,
handle_mcp_tool_search,
)
if name not in (MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME):
if name not in VIRTUAL_TOOL_NAMES:
return None
if not getattr(
@ -952,6 +955,12 @@ if MCP_AVAILABLE:
)
assert user_api_key_auth is not None # guaranteed by the flag check above
if name == AGENT_SEARCH_TOOL_NAME:
return await handle_agent_search(
query=str(args.get("query", "")),
top_k=coerce_top_k(args.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K),
user_api_key_dict=user_api_key_auth,
)
virtual_logging_obj: Final = await _build_virtual_call_logging_obj(
name=name,
arguments=args,

156
litellm/proxy/_experimental/mcp_server/tool_search.py
View file

@ -1,8 +1,14 @@
from __future__ import annotations
import json
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, TypedDict, assert_never
from typing_extensions import ReadOnly, Required
import litellm
from litellm.proxy.agent_endpoints.agent_search import DEFAULT_AGENT_SEARCH_TOP_K
if TYPE_CHECKING:
from mcp.types import CallToolResult
@ -12,6 +18,8 @@ if TYPE_CHECKING:
MCP_TOOL_SEARCH_TOOL_NAME: Final[str] = "mcp_tool_search"
MCP_TOOL_CALL_TOOL_NAME: Final[str] = "mcp_tool_call"
AGENT_SEARCH_TOOL_NAME: Final[str] = "agent_search"
VIRTUAL_TOOL_NAMES: Final = frozenset((MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, AGENT_SEARCH_TOOL_NAME))
def coerce_top_k(value: Any, default: int = 5) -> int:
@ -34,46 +42,116 @@ def search_tools(query: str, tools: list[dict[str, Any]], top_k: int = 5) -> lis
return [tool for _, tool in sorted(scored, key=lambda x: x[0], reverse=True)[:top_k]]
def get_virtual_tool_definitions() -> list[dict[str, Any]]:
return [
{
"name": MCP_TOOL_SEARCH_TOOL_NAME,
"description": "Search for MCP tools by keyword. Returns top matching tools with names, descriptions, and input schemas.",
"inputSchema": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Keywords to search for in tool names and descriptions.",
},
"top_k": {
"type": "integer",
"description": "Maximum number of results to return.",
"default": 5,
},
},
"required": ["query"],
class _ToolParamSchema(TypedDict, total=False):
type: Required[ReadOnly[str]]
description: Required[ReadOnly[str]]
default: ReadOnly[int]
class _ToolInputSchema(TypedDict):
type: ReadOnly[str]
properties: ReadOnly[Mapping[str, _ToolParamSchema]]
required: ReadOnly[Sequence[str]]
class VirtualToolDefinition(TypedDict):
name: ReadOnly[str]
description: ReadOnly[str]
inputSchema: ReadOnly[_ToolInputSchema]
def _json_array(*items: str) -> Sequence[str]:
return list(items) # mutable-ok: jsonschema's metaschema only accepts a JSON array for required
_MCP_TOOL_SEARCH_DEFINITION: Final[VirtualToolDefinition] = {
"name": MCP_TOOL_SEARCH_TOOL_NAME,
"description": "Search for MCP tools by keyword. Returns top matching tools with names, descriptions, and input schemas.",
"inputSchema": {
"type": "object",
"properties": {
"query": {"type": "string", "description": "Keywords to search for in tool names and descriptions."},
"top_k": {"type": "integer", "description": "Maximum number of results to return.", "default": 5},
},
"required": _json_array("query"),
},
}
_MCP_TOOL_CALL_DEFINITION: Final[VirtualToolDefinition] = {
"name": MCP_TOOL_CALL_TOOL_NAME,
"description": "Call an MCP tool by name with the given arguments.",
"inputSchema": {
"type": "object",
"properties": {
"tool_name": {"type": "string", "description": "The exact name of the MCP tool to call."},
"arguments": {"type": "object", "description": "Arguments to pass to the tool."},
},
"required": _json_array("tool_name"),
},
}
_AGENT_SEARCH_DEFINITION: Final[VirtualToolDefinition] = {
"name": AGENT_SEARCH_TOOL_NAME,
"description": "Find A2A agents by describing the task in natural language. Returns the best matching agents you can access, ranked by semantic similarity, each with its agent_id, name, description, skills, and score.",
"inputSchema": {
"type": "object",
"properties": {
"query": {"type": "string", "description": "The task the agent should be able to do, in natural language."},
"top_k": {
"type": "integer",
"description": "Maximum number of agents to return.",
"default": DEFAULT_AGENT_SEARCH_TOP_K,
},
},
{
"name": MCP_TOOL_CALL_TOOL_NAME,
"description": "Call an MCP tool by name with the given arguments.",
"inputSchema": {
"type": "object",
"properties": {
"tool_name": {
"type": "string",
"description": "The exact name of the MCP tool to call.",
},
"arguments": {
"type": "object",
"description": "Arguments to pass to the tool.",
},
},
"required": ["tool_name"],
},
},
]
"required": _json_array("query"),
},
}
def get_virtual_tool_definitions() -> tuple[VirtualToolDefinition, ...]:
return (_MCP_TOOL_SEARCH_DEFINITION, _MCP_TOOL_CALL_DEFINITION, _AGENT_SEARCH_DEFINITION)
def _text_tool_result(text: str, is_error: bool) -> CallToolResult:
from mcp.types import CallToolResult, TextContent
return CallToolResult(
content=[TextContent(type="text", text=text)], # mutable-ok: CallToolResult accepts only list content
isError=is_error,
)
async def handle_agent_search(query: str, top_k: int, user_api_key_dict: UserAPIKeyAuth) -> CallToolResult:
from litellm.proxy.agent_endpoints.agent_search import (
AgentSearchEmbeddingFailed,
AgentSearchHits,
AgentSearchNotConfigured,
agent_search_result,
global_agent_search_index,
search_agents,
)
from litellm.proxy.agent_endpoints.auth.agent_permission_handler import accessible_agents
from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user
from litellm.proxy.proxy_server import llm_router
await check_feature_access_for_user(user_api_key_dict, "agents")
outcome: Final = await search_agents(
query=query,
agents=await accessible_agents(user_api_key_dict),
top_k=max(top_k, 1),
router=llm_router,
embedding_model=litellm.agent_search_embedding_model,
index=global_agent_search_index,
user_api_key_dict=user_api_key_dict,
)
match outcome:
case AgentSearchHits(hits):
results: Final = tuple(agent_search_result(hit).model_dump() for hit in hits)
return _text_tool_result(json.dumps(results), is_error=False)
case AgentSearchNotConfigured(reason) | AgentSearchEmbeddingFailed(reason):
return _text_tool_result(reason, is_error=True)
case _:
assert_never(outcome)
async def handle_mcp_tool_search(

97
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -2377,6 +2377,17 @@
],
"title": "Rpm Limit"
},
"search_score": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Search Score"
},
"session_rpm_limit": {
"anyOf": [
{
@ -2689,6 +2700,11 @@
"title": "Total Flat Cost",
"type": "number"
},
"total_gateway_injected_caching_savings_spend": {
"default": 0.0,
"title": "Total Gateway Injected Caching Savings Spend",
"type": "number"
},
"total_pages": {
"default": 1,
"title": "Total Pages",
@ -3175,6 +3191,11 @@
"title": "Flat Cost",
"type": "number"
},
"gateway_injected_caching_savings_spend": {
"default": 0.0,
"title": "Gateway Injected Caching Savings Spend",
"type": "number"
},
"prompt_caching_savings_spend": {
"default": 0.0,
"title": "Prompt Caching Savings Spend",
@ -3404,7 +3425,7 @@
},
"/v1/agents": {
"get": {
"description": "Example usage:\n```\ncurl -X GET \"http://localhost:4000/v1/agents\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nPass `?health_check=true` to filter out agents whose URL is unreachable:\n```\ncurl -X GET \"http://localhost:4000/v1/agents?health_check=true\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nReturns: List[AgentResponse]",
"description": "Example usage:\n```\ncurl -X GET \"http://localhost:4000/v1/agents\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nPass `?health_check=true` to filter out agents whose URL is unreachable:\n```\ncurl -X GET \"http://localhost:4000/v1/agents?health_check=true\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nPass `?query=<task>` to get the best matching agents ranked by semantic similarity:\n```\ncurl -X GET \"http://localhost:4000/v1/agents?query=translate+a+PDF+document&top_k=5\" -H \"Content-Type: application/json\" -H \"Authorization: Bearer your-key\" ```\n\nReturns: List[AgentResponse]",
"operationId": "get_agents_v1_agents_get",
"parameters": [
{
@ -3418,6 +3439,39 @@
"title": "Health Check",
"type": "boolean"
}
},
{
"description": "Describe the task in natural language to rank the agents you can reach by semantic similarity over their name, description, and skills. Each result carries a search_score. Requires litellm_settings.agent_search_embedding_model.",
"in": "query",
"name": "query",
"required": false,
"schema": {
"anyOf": [
{
"minLength": 1,
"type": "string"
},
{
"type": "null"
}
],
"description": "Describe the task in natural language to rank the agents you can reach by semantic similarity over their name, description, and skills. Each result carries a search_score. Requires litellm_settings.agent_search_embedding_model.",
"title": "Query"
}
},
{
"description": "With query: the maximum number of ranked agents to return.",
"in": "query",
"name": "top_k",
"required": false,
"schema": {
"default": 5,
"description": "With query: the maximum number of ranked agents to return.",
"maximum": 100,
"minimum": 1,
"title": "Top K",
"type": "integer"
}
}
],
"responses": {
@ -8964,6 +9018,18 @@
"description": "When True, unified guardrails only evaluate tool results, the untrusted data an agent feeds back into the model, and skip system, user, and assistant content. Intended for agent harnesses whose own prompt scaffolding is trusted but often trips prompt-attack detectors.",
"title": "Scan Only Tool Results"
},
"scan_raw_request": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"description": "When True, this pre_call guardrail always evaluates the request as it was before any guardrail in this hook ran, regardless of its position in the guardrails list -- so the YAML order of guardrails can never change whether this one blocks. Use only for block-only guardrails: any data this guardrail returns is discarded, same contract as run_in_parallel, since an earlier guardrail's masking must not be undone by this one.",
"title": "Scan Raw Request"
},
"sensitive_data_route_to_model": {
"anyOf": [
{
@ -10014,6 +10080,18 @@
"description": "Additional provider-specific parameters for generic guardrail APIs",
"title": "Additional Provider Specific Params"
},
"advisory_system_message": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "Custom advisory message template used when on_flagged='inject_system_message'. Must contain a {reason} placeholder. Defaults to a generic advisory message if unset.",
"title": "Advisory System Message"
},
"akto_account_id": {
"anyOf": [
{
@ -11068,7 +11146,8 @@
{
"enum": [
"block",
"monitor"
"monitor",
"inject_system_message"
],
"type": "string"
},
@ -11077,7 +11156,7 @@
}
],
"default": "block",
"description": "Action to take when content is flagged: 'block' (raise exception) or 'monitor' (log only)",
"description": "Action to take when content is flagged: 'block' (raise exception), 'monitor' (log only), or 'inject_system_message' (append an advisory system message and let the LLM decide)",
"title": "On Flagged"
},
"on_flagged_action": {
@ -11587,6 +11666,18 @@
"description": "When True, unified guardrails only evaluate tool results, the untrusted data an agent feeds back into the model, and skip system, user, and assistant content. Intended for agent harnesses whose own prompt scaffolding is trusted but often trips prompt-attack detectors.",
"title": "Scan Only Tool Results"
},
"scan_raw_request": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"description": "When True, this pre_call guardrail always evaluates the request as it was before any guardrail in this hook ran, regardless of its position in the guardrails list -- so the YAML order of guardrails can never change whether this one blocks. Use only for block-only guardrails: any data this guardrail returns is discarded, same contract as run_in_parallel, since an earlier guardrail's masking must not be undone by this one.",
"title": "Scan Raw Request"
},
"send_user_api_key_alias": {
"anyOf": [
{

1
litellm/proxy/_lazy_openapi_snapshot.py
View file

@ -126,7 +126,6 @@ def _feature_fragment(app: "FastAPI", feat: "LazyFeature", used_operation_ids: s
full: Final = get_openapi(title=app.title, version=app.version, routes=feat_routes)
paths: Final = full.get("paths", {})
_normalize_operation_ids(paths)
# Group all of a feature's routes under one tag.
for path_ops in paths.values():
for method, op in path_ops.items():
if isinstance(op, dict):

6
litellm/proxy/_types.py
View file

@ -566,6 +566,7 @@ class LiteLLMRoutes(enum.Enum):
model_info_routes = [
"/model/info",
"/v1/model/info",
"/model_group/info",
]
llm_api_routes = (
@ -729,6 +730,7 @@ class LiteLLMRoutes(enum.Enum):
"/litellm/.well-known/litellm-ui-config",
"/.well-known/litellm-ui-config",
"/public/model_hub",
"/public/v1/model_hub",
"/public/model_hub/info",
"/public/agent_hub",
"/public/mcp_hub",
@ -938,6 +940,8 @@ class LiteLLMRoutes(enum.Enum):
# Model cost map maintenance views (read-only status / source).
"/schedule/model_cost_map_reload/status",
"/model/cost_map/source",
# A pure read; POST only so the prompt does not ride in a URL.
"/auto_router/classifier/default_prompt",
]
# Spend tracking reads (/spend/logs, /spend/logs/ui, /spend/keys,
# /spend/users, /spend/tags, /spend/calculate, /cost/estimate). Admin
@ -3581,6 +3585,7 @@ class SpendLogsMetadata(TypedDict):
cost_breakdown: CostBreakdown | None # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
compression_savings: CompressionSavingsMetadata | None
autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed
litellm_gateway_injected_cache: ReadOnly[str | None]
class SpendLogsPayload(TypedDict):
@ -4861,6 +4866,7 @@ class BaseDailySpendTransaction(TypedDict):
# cost-savings metrics (dollars, priced per request before aggregation)
compression_savings_spend: float
prompt_caching_savings_spend: float
gateway_injected_caching_savings_spend: float # writable-ok: the rollup queue accumulates into this key in place, as it does for every sibling spend field
# Not required: rows queued by a pod running the previous release, or replayed from
# the Redis buffer across an upgrade, carry no such key. Every reader coalesces a
# missing value to zero, so requiring it here would describe a shape the aggregation

250
litellm/proxy/agent_endpoints/agent_search.py Normal file
View file

@ -0,0 +1,250 @@
"""Semantic ranking over the in-memory A2A agent registry, shared by GET /v1/agents?query= and the agent_search MCP tool."""
from __future__ import annotations
import math
from collections.abc import Awaitable, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
from openai import OpenAIError
from pydantic import BaseModel, ConfigDict, ValidationError
from litellm.exceptions import BudgetExceededError
from litellm.types.agents import AgentResponse
if TYPE_CHECKING:
from litellm.proxy._types import UserAPIKeyAuth
from litellm.router import Router
DEFAULT_AGENT_SEARCH_TOP_K: Final = 5
Vector: TypeAlias = tuple[float, ...]
class Embedder(Protocol):
def __call__(self, texts: Sequence[str]) -> Awaitable[Sequence[Vector]]: ...
@dataclass(frozen=True, slots=True)
class AgentSearchHit:
agent: AgentResponse
score: float
@dataclass(frozen=True, slots=True)
class AgentSearchHits:
hits: tuple[AgentSearchHit, ...]
@dataclass(frozen=True, slots=True)
class AgentSearchNotConfigured:
reason: str
@dataclass(frozen=True, slots=True)
class AgentSearchEmbeddingFailed:
reason: str
AgentSearchOutcome: TypeAlias = AgentSearchHits | AgentSearchNotConfigured | AgentSearchEmbeddingFailed
class _SearchableSkill(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
name: str = ""
description: str = ""
tags: tuple[str, ...] = ()
class _SearchableCard(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
description: str = ""
skills: tuple[_SearchableSkill, ...] = ()
class _EmbeddingItem(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
embedding: tuple[float, ...]
class _EmbeddingData(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
data: tuple[_EmbeddingItem, ...]
class AgentSearchResult(BaseModel):
model_config = ConfigDict(frozen=True)
agent_id: str
agent_name: str
description: str
skills: tuple[_SearchableSkill, ...]
score: float
def _searchable_card(agent: AgentResponse) -> _SearchableCard:
try:
return _SearchableCard.model_validate(agent.agent_card_params)
except ValidationError:
return _SearchableCard()
def _skill_text(skill: _SearchableSkill) -> str:
return " ".join(part for part in (skill.name, skill.description, " ".join(skill.tags)) if part)
def agent_search_text(agent: AgentResponse) -> str:
card: Final = _searchable_card(agent)
skill_lines: Final = tuple(_skill_text(skill) for skill in card.skills)
return "\n".join(part for part in (agent.agent_name, card.description, *skill_lines) if part)
def agent_search_result(hit: AgentSearchHit) -> AgentSearchResult:
card: Final = _searchable_card(hit.agent)
return AgentSearchResult(
agent_id=hit.agent.agent_id,
agent_name=hit.agent.agent_name,
description=card.description,
skills=card.skills,
score=hit.score,
)
def cosine_similarity(left: Vector, right: Vector) -> float:
dot: Final = sum(a * b for a, b in zip(left, right, strict=True))
norms: Final = math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right))
return dot / norms if norms else 0.0
def embedding_spend_metadata(user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]:
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
return { # mutable-ok: the router mutates the metadata dict it is handed
**LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict),
"user_api_key": user_api_key_dict.api_key,
}
def router_embedder(router: Router, embedding_model: str, user_api_key_dict: UserAPIKeyAuth) -> Embedder:
async def embed(texts: Sequence[str]) -> Sequence[Vector]:
batch: Final = list(texts) # mutable-ok: Router.aembedding accepts only str | list input
response: Final = await router.aembedding(
model=embedding_model, input=batch, metadata=embedding_spend_metadata(user_api_key_dict)
)
return tuple(item.embedding for item in _EmbeddingData.model_validate(response.model_dump()).data)
return embed
_NO_VECTORS: Final[Mapping[str, Vector]] = MappingProxyType({})
async def _embed_all(embed: Embedder, texts: Sequence[str]) -> tuple[Vector, ...] | AgentSearchEmbeddingFailed:
try:
vectors: Final = tuple(await embed(texts))
except (OpenAIError, ValueError, BudgetExceededError) as exc:
return AgentSearchEmbeddingFailed(reason=f"embedding the search query failed: {exc}")
if len(vectors) != len(texts):
return AgentSearchEmbeddingFailed(
reason=f"embedding model returned {len(vectors)} vectors for {len(texts)} inputs"
)
return vectors
@dataclass(frozen=True, slots=True)
class _Embedded:
query_vector: Vector
vectors: Mapping[str, Vector]
def _same_dimension(query_vector: Vector, vectors: Mapping[str, Vector], texts: Sequence[str]) -> bool:
return all(len(vectors[text]) == len(query_vector) for text in texts)
async def _embed_query_and_agents(
embed: Embedder, query: str, texts: Sequence[str], cached: Mapping[str, Vector]
) -> _Embedded | AgentSearchEmbeddingFailed:
missing: Final = tuple(dict.fromkeys(text for text in texts if text not in cached))
embedded: Final = await _embed_all(embed, (query, *missing))
if isinstance(embedded, AgentSearchEmbeddingFailed):
return embedded
vectors: Final = MappingProxyType(dict(chain(cached.items(), zip(missing, embedded[1:], strict=True))))
if _same_dimension(embedded[0], vectors, texts):
return _Embedded(query_vector=embedded[0], vectors=vectors)
unique: Final = tuple(dict.fromkeys(texts))
reembedded: Final = await _embed_all(embed, (query, *unique))
if isinstance(reembedded, AgentSearchEmbeddingFailed):
return reembedded
return _Embedded(
query_vector=reembedded[0], vectors=MappingProxyType(dict(zip(unique, reembedded[1:], strict=True)))
)
class AgentSearchIndex:
"""Caches one vector per distinct agent text per embedding model, so repeat searches only embed the query."""
def __init__(self) -> None:
self._vectors: Mapping[str, Mapping[str, Vector]] = MappingProxyType({})
def _merged(self, embedding_model: str, embedded: _Embedded) -> Mapping[str, Vector]:
kept: Final = {
text: vector
for text, vector in self._vectors.get(embedding_model, _NO_VECTORS).items()
if len(vector) == len(embedded.query_vector)
}
return MappingProxyType({**kept, **embedded.vectors})
async def search(
self, query: str, agents: Sequence[AgentResponse], top_k: int, embed: Embedder, embedding_model: str
) -> AgentSearchHits | AgentSearchEmbeddingFailed:
if not agents:
return AgentSearchHits(hits=())
texts: Final = tuple(agent_search_text(agent) for agent in agents)
cached: Final = self._vectors.get(embedding_model, _NO_VECTORS)
embedded: Final = await _embed_query_and_agents(embed, query, texts, cached)
if isinstance(embedded, AgentSearchEmbeddingFailed):
return embedded
if not _same_dimension(embedded.query_vector, embedded.vectors, texts):
return AgentSearchEmbeddingFailed(
reason=f"embedding model {embedding_model} returned vectors of mixed dimensions"
)
self._vectors = MappingProxyType({**self._vectors, embedding_model: self._merged(embedding_model, embedded)})
ranked: Final = sorted(
(
AgentSearchHit(agent=agent, score=cosine_similarity(embedded.query_vector, embedded.vectors[text]))
for agent, text in zip(agents, texts, strict=True)
),
key=lambda hit: hit.score,
reverse=True,
)
return AgentSearchHits(hits=tuple(ranked[:top_k]))
global_agent_search_index: Final = AgentSearchIndex()
async def search_agents(
query: str,
agents: Sequence[AgentResponse],
top_k: int,
router: Router | None,
embedding_model: str | None,
index: AgentSearchIndex,
user_api_key_dict: UserAPIKeyAuth,
) -> AgentSearchOutcome:
if embedding_model is None:
return AgentSearchNotConfigured(
reason="agent search needs litellm_settings.agent_search_embedding_model set to an embedding model from model_list"
)
if router is None:
return AgentSearchNotConfigured(reason="agent search needs a model_list so the embedding model can be called")
return await index.search(
query, agents, top_k, router_embedder(router, embedding_model, user_api_key_dict), embedding_model
)

16
litellm/proxy/agent_endpoints/auth/agent_permission_handler.py
View file

@ -13,9 +13,11 @@ from litellm.proxy._types import (
UI_TEAM_ID,
LiteLLM_ObjectPermissionTable,
LiteLLM_TeamTable,
LitellmUserRoles,
UserAPIKeyAuth,
)
from litellm.repositories.table_repositories import AgentsRepository
from litellm.types.agents import AgentResponse
@dataclass(frozen=True, slots=True)
@ -439,3 +441,17 @@ class AgentRequestHandler:
except Exception as e:
verbose_logger.warning("Failed to get agent access groups for team: %s", e)
return []
async def accessible_agents(user_api_key_auth: UserAPIKeyAuth) -> tuple[AgentResponse, ...]:
"""Every registry agent for proxy admins, else the agents the key's and team's grants reach."""
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
all_agents: Final = global_agent_registry.get_agent_list()
if user_api_key_auth.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN.value):
return all_agents
match await AgentRequestHandler.resolve_agent_access(user_api_key_auth=user_api_key_auth):
case UnrestrictedAgentAccess():
return all_agents
case RestrictedAgentAccess(allowed_agent_ids):
return tuple(agent for agent in all_agents if agent.agent_id in allowed_agent_ids)

95
litellm/proxy/agent_endpoints/endpoints.py
View file

@ -12,10 +12,11 @@ import asyncio
import os
import uuid
from collections.abc import Mapping, Sequence
from typing import Final, TypedDict
from types import MappingProxyType
from typing import Annotated, Final, TypedDict, assert_never
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from typing_extensions import Required
from typing_extensions import ReadOnly, Required
import litellm
from litellm._logging import verbose_proxy_logger
@ -32,6 +33,15 @@ from litellm.proxy.a2a.agent_card import (
merge_agent_card,
normalize_protocol_version,
)
from litellm.proxy.agent_endpoints.agent_search import (
DEFAULT_AGENT_SEARCH_TOP_K,
AgentSearchEmbeddingFailed,
AgentSearchHits,
AgentSearchNotConfigured,
global_agent_search_index,
search_agents,
)
from litellm.proxy.agent_endpoints.auth.agent_permission_handler import accessible_agents
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user
from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity
@ -211,6 +221,41 @@ async def _check_agent_url_health(
}
class _AgentSearchErrorDetail(TypedDict):
error: ReadOnly[str]
message: ReadOnly[str]
def _agent_search_error(status_code: int, error: str, message: str) -> HTTPException:
detail: Final[_AgentSearchErrorDetail] = {"error": error, "message": message}
return HTTPException(status_code=status_code, detail=detail)
async def _rank_agents_by_query(
query: str, agents: Sequence[AgentResponse], top_k: int, user_api_key_dict: UserAPIKeyAuth
) -> tuple[AgentResponse, ...]:
from litellm.proxy.proxy_server import llm_router
outcome: Final = await search_agents(
query=query,
agents=agents,
top_k=top_k,
router=llm_router,
embedding_model=litellm.agent_search_embedding_model,
index=global_agent_search_index,
user_api_key_dict=user_api_key_dict,
)
match outcome:
case AgentSearchHits(hits):
return tuple(hit.agent.model_copy(update=MappingProxyType({"search_score": hit.score})) for hit in hits)
case AgentSearchNotConfigured(reason):
raise _agent_search_error(400, "agent_search_not_configured", reason)
case AgentSearchEmbeddingFailed(reason):
raise _agent_search_error(503, "agent_search_unavailable", reason)
case _:
assert_never(outcome)
@router.get(
"/v1/agents",
tags=["[beta] A2A Agents"],
@ -223,6 +268,17 @@ async def get_agents(
False,
description="When true, performs a GET request to each agent's URL. Agents with reachable URLs (HTTP status < 500) and agents without a URL are returned; unreachable agents are filtered out.",
),
query: Annotated[
str | None,
Query(
min_length=1,
description="Describe the task in natural language to rank the agents you can reach by semantic similarity over their name, description, and skills. Each result carries a search_score. Requires litellm_settings.agent_search_embedding_model.",
),
] = None,
top_k: Annotated[
int,
Query(ge=1, le=100, description="With query: the maximum number of ranked agents to return."),
] = DEFAULT_AGENT_SEARCH_TOP_K,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # Used for auth
):
"""
@ -240,37 +296,22 @@ async def get_agents(
-H "Authorization: Bearer your-key" \
```
Pass `?query=<task>` to get the best matching agents ranked by semantic similarity:
```
curl -X GET "http://localhost:4000/v1/agents?query=translate+a+PDF+document&top_k=5" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-key" \
```
Returns: List[AgentResponse]
"""
await check_feature_access_for_user(user_api_key_dict, "agents")
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
AgentRequestHandler,
RestrictedAgentAccess,
UnrestrictedAgentAccess,
)
try:
returned_agents: Sequence[AgentResponse] = ()
# Admin users get all agents
if (
user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
):
returned_agents = global_agent_registry.get_agent_list()
else:
# Get allowed agents from object_permission (key/team level)
agent_access: Final = await AgentRequestHandler.resolve_agent_access(user_api_key_auth=user_api_key_dict)
all_agents: Final = global_agent_registry.get_agent_list()
match agent_access:
case UnrestrictedAgentAccess():
returned_agents = all_agents
case RestrictedAgentAccess(allowed_agent_ids):
returned_agents = [agent for agent in all_agents if agent.agent_id in allowed_agent_ids]
returned_agents: Sequence[AgentResponse] = await accessible_agents(user_api_key_dict)
# Fetch current spend from DB for all returned agents
from litellm.proxy.proxy_server import prisma_client
@ -336,7 +377,9 @@ async def get_agents(
healthy_ids: Final = {result["agent_id"] for result in health_results if result["healthy"]}
returned_agents = [agent for agent in agents_with_url if agent.agent_id in healthy_ids] + agents_without_url
return returned_agents
if query is None:
return returned_agents
return await _rank_agents_by_query(query, returned_agents, top_k, user_api_key_dict)
except HTTPException:
raise
except Exception as e:

13
litellm/proxy/auth/auth_checks.py
View file

@ -2596,6 +2596,11 @@ async def _delete_cache_key_object(
dropped before the Redis round trip. Letting a cache-backend error raise here therefore reports
failure for work that succeeded without making the cache any less stale; the leftover Redis
entry expires at its TTL either way.
Also broadcasts the eviction to every other worker (LIT-3803): auth serves this object
cache-first with no freshness check, so a worker that never receives the broadcast keeps
admitting requests against the pre-mutation object (e.g. a just-reset spend) until its own
copy's TTL expires.
"""
key: Final = hashed_token
@ -2612,6 +2617,8 @@ async def _delete_cache_key_object(
e,
)
await publish_auth_cache_invalidation(cache_key=key)
async def delete_cache_key_objects(
hashed_tokens: Sequence[str],
@ -2623,8 +2630,9 @@ async def delete_cache_key_objects(
`/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left
cached after its row is gone keeps buying access until its TTL expires.
Evicting locally only reaches this worker, so each token is also broadcast: a deleted key left
in a peer worker's in-memory cache still authenticates there until its TTL expires.
Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each
token, so a deleted key left in a peer worker's in-memory cache still authenticates there until
its TTL expires.
Best-effort per key: the rows are already deleted by the time this runs, so an unreachable
cache backend must not abort the caller partway through its own cascade.
@ -2648,7 +2656,6 @@ async def delete_cache_key_objects(
hashed_token,
result,
)
await publish_auth_cache_invalidation(cache_key=hashed_token)
class _TeamNotFoundDetail(TypedDict):

37
litellm/proxy/common_request_processing.py
View file

@ -4,7 +4,7 @@ import json
import logging
import math
import traceback
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
@ -2372,6 +2372,21 @@ class ProxyBaseLLMRequestProcessing:
)
logging_obj._on_deferred_stream_complete = _on_deferred_stream_complete
elif (
_post_call_guardrails_active
and route_type == "anthropic_messages"
and self._is_streaming_response(response)
):
from litellm.litellm_core_utils.logging_worker import (
GLOBAL_LOGGING_WORKER,
)
async def _on_deferred_native_stream_complete(
logging_coroutine: Coroutine[object, object, object],
) -> None:
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
logging_obj._on_deferred_stream_complete = _on_deferred_native_stream_complete
if route_type == "allm_passthrough_route":
# Check if response is an async generator
@ -2551,16 +2566,6 @@ class ProxyBaseLLMRequestProcessing:
except Exception as e:
verbose_proxy_logger.exception("Error in orphaned streaming async logging: %s", e)
# Always return the client-requested model name (not provider-prefixed internal identifiers)
# for OpenAI-compatible responses.
if requested_model_from_client:
_override_openai_response_model(
response_obj=response,
requested_model=requested_model_from_client,
log_context=f"litellm_call_id={logging_obj.litellm_call_id}",
return_raw_model_name=_should_return_raw_model_name(self.data),
)
hidden_params = get_hidden_params_dict(response) # get any updated response headers
additional_headers = hidden_params.get("additional_headers", {}) or {}
@ -2586,6 +2591,16 @@ class ProxyBaseLLMRequestProcessing:
else llm_cost_for_headers
)
# Always return the client-requested model name (not provider-prefixed internal identifiers)
# for OpenAI-compatible responses.
if requested_model_from_client:
_override_openai_response_model(
response_obj=response,
requested_model=requested_model_from_client,
log_context=f"litellm_call_id={logging_obj.litellm_call_id}",
return_raw_model_name=_should_return_raw_model_name(self.data),
)
fastapi_response.headers.update(
ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,

4
litellm/proxy/common_utils/reset_budget_job.py
View file

@ -3,7 +3,7 @@ import json
import math
import time
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
from types import MappingProxyType
@ -224,7 +224,7 @@ class _BudgetCascade:
endusers: tuple[_EndUserRow, ...] = ()
counter_resets: tuple[tuple[str, float], ...] = ()
cache_keys: tuple[str, ...] = ()
rollover_caps: Mapping[str, float] = MappingProxyType({})
rollover_caps: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
@dataclass(frozen=True, slots=True)

1
litellm/proxy/db/daily_spend_bulk_upsert.py
View file

@ -62,6 +62,7 @@ _SPEND_COLUMNS: Final = (
"spend",
"compression_savings_spend",
"prompt_caching_savings_spend",
"gateway_injected_caching_savings_spend",
"autorouter_savings_spend",
)

4
litellm/proxy/db/db_spend_update_writer.py
View file

@ -62,6 +62,7 @@ from litellm.proxy.spend_tracking.savings import (
compute_savings_spend,
extract_cache_creation_tokens,
extract_cache_read_tokens,
marks_gateway_injection,
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.repositories.prisma_protocols import BatchTable
@ -315,6 +316,7 @@ class DBSpendUpdateWriter:
model=payload.get("model"),
custom_llm_provider=payload.get("custom_llm_provider"),
compression_saved_tokens=0,
gateway_injected_cache=marks_gateway_injection(metadata, payload.get("model_id")),
routing_decision=metadata.get("routing_decision"),
usage_object=usage_object_raw if isinstance(usage_object_raw, dict) else None,
model_id=payload.get("model_id"),
@ -1879,6 +1881,7 @@ class DBSpendUpdateWriter:
model=payload.get("model", None),
custom_llm_provider=payload.get("custom_llm_provider", None),
compression_saved_tokens=compression_saved_tokens,
gateway_injected_cache=marks_gateway_injection(_metadata, payload.get("model_id")),
routing_decision=_metadata.get("routing_decision"),
model_id=payload.get("model_id"),
llm_router=_get_llm_router,
@ -1911,6 +1914,7 @@ class DBSpendUpdateWriter:
compression_saved_tokens=compression_saved_tokens,
compression_savings_spend=savings_spend.compression,
prompt_caching_savings_spend=savings_spend.prompt_caching,
gateway_injected_caching_savings_spend=savings_spend.gateway_injected_caching,
autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter,
)
return daily_transaction

4
litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
View file

@ -134,6 +134,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
payload.get("prompt_caching_savings_spend", 0) or 0
) + daily_transaction.get("prompt_caching_savings_spend", 0)
daily_transaction["gateway_injected_caching_savings_spend"] = (
payload.get("gateway_injected_caching_savings_spend", 0) or 0
) + daily_transaction.get("gateway_injected_caching_savings_spend", 0)
daily_transaction["autorouter_savings_spend"] = (
payload.get("autorouter_savings_spend", 0) or 0
) + daily_transaction.get("autorouter_savings_spend", 0)

65
litellm/proxy/db/shadow_eval_funnel.py Normal file
View file

@ -0,0 +1,65 @@
"""Pod-local queue of shadow-eval funnel increments, drained by the spend-update job.
The shadow-eval success hook counts the sampled-traffic outcomes that never produce an
attempt row (a lost sampling dice roll, an unjudgeable request shape, a concurrency
shed), so a job's results can state what share of its eligible traffic the judged rows
represent. Counters are advisory coverage stats: a pod dying loses at most one flush
interval, and a failed flush drops its batch because a repeated increment is worse
than an undercount (same call as the auto-router session rollup flush).
"""
from typing import TYPE_CHECKING, Final, Literal
from litellm._logging import verbose_proxy_logger
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
ShadowEvalFunnelStage = Literal["not_sampled", "unjudgeable", "shed", "withheld"]
FUNNEL_STAGES: Final[tuple[ShadowEvalFunnelStage, ...]] = ("not_sampled", "unjudgeable", "shed", "withheld")
_pending: dict[str, dict[ShadowEvalFunnelStage, int]] = {} # mutable-ok: module-level queue, single event loop
_FUNNEL_PLACEHOLDERS: Final = ", ".join(f"${n + 2}" for n in range(len(FUNNEL_STAGES)))
_UPSERT_FUNNEL_SQL: Final = f"""
INSERT INTO "LiteLLM_ShadowEvalFunnel" (job_id, {", ".join(FUNNEL_STAGES)})
VALUES ($1, {_FUNNEL_PLACEHOLDERS})
ON CONFLICT (job_id) DO UPDATE SET
{", ".join(f'{stage} = "LiteLLM_ShadowEvalFunnel".{stage} + EXCLUDED.{stage}' for stage in FUNNEL_STAGES)}
"""
def pending_shadow_eval_funnel_events() -> int:
"""Queue census for the drain triggers: entries not yet flushed, so a funnel-only
batch still wakes the spend job that would otherwise skip an empty-queue run."""
return sum(sum(counters.values()) for counters in _pending.values())
def record_shadow_eval_funnel_event(job_id: str, stage: ShadowEvalFunnelStage) -> None:
"""Count one skipped request for one job leg; synchronous so the hook's read-modify-
write cannot interleave with the flush's snapshot on the shared event loop."""
counters: Final = _pending.setdefault(job_id, dict.fromkeys(FUNNEL_STAGES, 0)) # mutable-ok: queue entry
counters[stage] += 1
async def flush_shadow_eval_funnel(prisma_client: "PrismaClient") -> None:
if not _pending:
return
batch: Final = dict(_pending) # mutable-ok: snapshot drained from the queue
_pending.clear()
for job_id, counters in batch.items():
try:
await prisma_client.db.execute_raw(
_UPSERT_FUNNEL_SQL,
job_id,
*(counters[stage] for stage in FUNNEL_STAGES),
)
except Exception as flush_err: # noqa: BLE001 # drop this leg's batch: a repeated increment is worse than an undercount
verbose_proxy_logger.error(
"Spend tracking - shadow eval funnel flush failed for job %s, %s dropped: %s",
job_id,
counters,
flush_err,
)

24
litellm/proxy/guardrails/guardrail_endpoints.py
View file

@ -1218,6 +1218,30 @@ async def patch_guardrail(
verbose_proxy_logger.info(
"Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id
)
except (ValueError, TypeError) as update_error:
# The new config is invalid (e.g. an unsupported on_flagged combination):
# reinitialize_guardrail already restored the previous live instance, but
# update_guardrail_in_db above already persisted the rejected config to
# the DB. Roll that back too, so the DB and the live guardrail never
# disagree about what's actually enforcing, and surface the rejection to
# the caller instead of a misleading 200.
await GUARDRAIL_REGISTRY.update_guardrail_in_db(
guardrail_id=guardrail_id,
guardrail=Guardrail(
guardrail_id=guardrail_id,
guardrail_name=existing_guardrail.get("guardrail_name") or "",
litellm_params=LitellmParams(**existing_litellm_params),
guardrail_info=existing_guardrail.get(
"guardrail_info",
{}, # mutable-ok: Guardrail's own constructor takes a plain dict
),
),
prisma_client=prisma_client,
)
raise HTTPException(
status_code=422,
detail=f"Invalid guardrail configuration, update rejected: {update_error}",
) from update_error
except Exception as update_error:
verbose_proxy_logger.warning(
"Immediate sync: Failed to update '%s' (ID: %s) in memory: %s",

4
litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py
View file

@ -385,7 +385,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
return [_extract_text_from_message(msg) for msg in tail]
async def _call_or_fail_open(
self, payload: dict[str, Any], hook_name: str, request_data: dict
self, payload: dict[str, Any], hook_name: str, request_data: dict[str, object]
) -> _GuardChatCompletionsResult:
start_time: Final = time.time()
try:
@ -421,7 +421,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
structured_messages: list[AllMessageValues],
guard_output: _GuardInput,
sent_indices: tuple[int, ...],
request_data: dict,
request_data: dict[str, object],
) -> list[AllMessageValues] | None:
if effective_skip_system_message_for_guardrail(self) or effective_skip_tool_message_for_guardrail(self):
request_messages: Final = request_data.get("messages")

446
litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py
View file

@ -1,13 +1,25 @@
import copy
import os
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Final
from string import Formatter
from types import MappingProxyType
from typing import Final, Literal
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import (
DEFAULT_ADVISORY_MESSAGE,
CustomGuardrail,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
filter_messages_by_skip_flags,
merge_guardrailed_scoped_messages,
)
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -19,14 +31,190 @@ from litellm.proxy.guardrails._content_utils import (
has_non_string_content,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
from litellm.types.llms.openai import AllMessageValues
from litellm.types.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import (
LakeraAIBreakdownItem,
LakeraAIRequest,
LakeraAIResponse,
)
from litellm.types.utils import CallTypesLiteral, GuardrailStatus, ModelResponse
_DETECTOR_CATEGORY_PHRASES: Final[Mapping[str, str]] = MappingProxyType(
{
"prompt_injection": "a potential prompt injection attempt",
"prompt_attack": "a potential prompt injection attempt",
"pii": "personally identifiable information",
"moderated_content": "policy-violating content",
}
)
def humanize_lakera_block_reasons(breakdown: Sequence[LakeraAIBreakdownItem] | None) -> str:
"""
Turn a Lakera v2 ``breakdown`` list into a plain-language reason string
suitable for an advisory message shown to the LLM (e.g. "a potential
prompt injection attempt, personally identifiable information").
Falls back to a generic phrase when breakdown is empty or every detected
detector_type is unrecognized.
"""
if not breakdown:
return "a content safety concern"
categories: Final = (
(item.get("detector_type") or "").split("/")[0] for item in breakdown if item.get("detected", False)
)
phrases: Final = tuple(
dict.fromkeys(
_DETECTOR_CATEGORY_PHRASES.get(category) or category.replace("_", " ")
for category in categories
if category
)
)
return ", ".join(phrases) if phrases else "a content safety concern"
def _template_uses_reason_placeholder(template: str) -> bool:
"""True if ``template`` has a real ``{reason}`` format field, not just the
literal substring -- an escaped ``{{reason}}`` contains the substring but
formats to a literal "{reason}", never substituting the actual value."""
return any(field_name == "reason" for _, field_name, _, _ in Formatter().parse(template))
def _pre_masking_scope_indices(
guardrail: "LakeraAIGuardrail",
messages: Sequence[object],
) -> tuple[int, ...]:
"""Indices into ``messages`` that mask-in-place can safely target: has
non-empty string content, and survives the same skip_system_message_in_guardrail
/ skip_tool_message_in_guardrail scoping ``filter_messages_by_skip_flags``
applies. Content is guaranteed to already be a plain string here -- masking
is only attempted when ``has_non_string_content(data)`` is False.
Preserved in original order, so it lines up positionally with the
``messages_for_lakera`` list _build_lakera_inspection_messages/skip-filtering
produces from the same input: both apply the identical "has text" and
"not skipped by role" predicates over the same original sequence. Role
comparison is lowercased to match filter_messages_by_skip_flags's own
normalization (via its _message_role helper) -- an uppercase-cased
"System"/"TOOL" role must be excluded by both or the two lists disagree
on length and the caller's strict positional zip raises."""
skip_system: Final = effective_skip_system_message_for_guardrail(guardrail)
skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail)
return tuple(
idx
for idx, message in enumerate(messages)
if isinstance(message, dict)
and isinstance(message.get("content"), str)
and message["content"]
and not (skip_system and str(message.get("role") or "").lower() == "system")
and not (skip_tool and str(message.get("role") or "").lower() == "tool")
)
def _apply_redacted_messages_back_preserving_fields(
guardrail: "LakeraAIGuardrail",
data: dict[str, object], # mutable-ok: writes the redacted result back into the caller's request dict in place
redacted_messages: Sequence[AllMessageValues],
) -> None:
"""Write masked content back to ``data["messages"]`` without losing fields
the synthetic role/content-only ``redacted_messages`` never carried (e.g. a
tool message's tool_call_id, an assistant message's tool_calls, name,
cache_control). Falls back to the shared, wholesale-replacing
apply_redacted_messages_back when ``data["messages"]`` isn't a list (a pure
Responses-API ``input`` string, with no chat messages to merge into)."""
original_messages: Final = data.get("messages")
if not isinstance(original_messages, list):
redacted_list: Final = list(redacted_messages) # mutable-ok: apply_redacted_messages_back requires a list
apply_redacted_messages_back(data, redacted_list)
return
scope_indices: Final = _pre_masking_scope_indices(guardrail, original_messages)
guardrailed_scoped: Final = tuple(
{ # mutable-ok: fresh dict per iteration, not stored beyond this comprehension
**original_messages[original_idx],
"content": redacted["content"],
}
for original_idx, redacted in zip(scope_indices, redacted_messages, strict=True)
)
data["messages"] = merge_guardrailed_scoped_messages(
full_messages=original_messages,
scoped_indices=scope_indices,
guardrailed_scoped=guardrailed_scoped, # pyright: ignore[reportArgumentType] # plain dicts satisfy AllMessageValues's TypedDict shape at runtime
)
def _has_combined_messages_and_input(data: Mapping[str, object]) -> bool:
"""True if ``data`` carries both ``messages`` and ``input``.
build_inspection_messages flattens both into one synthetic list, so
mask-in-place would write input-derived content into data["messages"]
(and vice versa) even when a message dropped for having no text
coincidentally keeps the raw message count unchanged."""
return isinstance(data.get("messages"), list) and data.get("input") is not None
def _has_responses_instructions(guardrail: "LakeraAIGuardrail", data: Mapping[str, object]) -> bool:
"""True if ``data`` carries a Responses-API ``instructions`` field that
Lakera actually inspected. _build_lakera_inspection_messages includes
``instructions`` as a synthetic system message so Lakera can inspect it,
but apply_redacted_messages_back has no path to rewrite
``data["instructions"]`` -- masking here would either leave unredacted
content in the real instructions field the model reads, or write a
redacted duplicate into data["messages"] instead, which the Responses
API never consumes.
When skip_system_message_in_guardrail excludes that synthetic system
message before it ever reaches Lakera, none of this applies: Lakera never
saw ``instructions``, so it can't have flagged anything there, and
forcing a hard block anyway would defeat the whole point of the skip
flag for a response that only carries PII in the (maskable) non-system
content."""
instructions: Final = data.get("instructions")
return (
isinstance(instructions, str)
and bool(instructions)
and not effective_skip_system_message_for_guardrail(guardrail)
)
def _breakdown_has_pii_violation(lakera_response: LakeraAIResponse | None) -> bool:
"""True if any PII-category detector fired, regardless of whether other,
non-PII detectors (prompt injection, moderated content) also fired.
Unlike ``_is_only_pii_violation``, this doesn't require PII to be the
*only* thing detected -- it's used to decide whether masking/blocking is
even relevant at all before advisory mode's own logic runs."""
if not lakera_response:
return False
breakdown: Final = lakera_response.get("breakdown") or ()
return any(
item.get("detected", False) and (item.get("detector_type") or "").startswith("pii/") for item in breakdown
)
def _build_lakera_inspection_messages(data: Mapping[str, object]) -> Sequence[Mapping[str, str]]:
"""Like build_inspection_messages, but also covers the Responses-API
``instructions`` field, placed first since litellm later converts it
into the model's leading system message and a prompt-injection detector
should see the same conversation order the model actually receives.
Kept local to Lakera rather than folded into the shared
_content_utils.build_inspection_messages helper: doing that once made
``instructions`` visible to every guardrail sharing that helper (AIM,
presidio, bedrock, ...), but only Lakera has a masking-safety-guard
(_has_responses_instructions) accounting for apply_redacted_messages_back
having no write-back path for data["instructions"] -- other guardrails
would have silently mishandled a PII/redaction hit found there."""
instructions: Final = data.get("instructions")
leading: Final[Sequence[Mapping[str, str]]] = (
[{"role": "system", "content": instructions}] # mutable-ok: fresh list/dict, not stored
if isinstance(instructions, str) and instructions
else [] # mutable-ok: fresh empty list, not stored
)
return [ # mutable-ok: fresh list, not stored
*leading,
*build_inspection_messages(dict(data)), # mutable-ok: fresh shallow copy for the dict[str, Any] param
]
class LakeraAIGuardrail(CustomGuardrail):
@classmethod
@ -46,7 +234,10 @@ class LakeraAIGuardrail(CustomGuardrail):
breakdown: bool | None = True,
metadata: dict | None = None,
dev_info: bool | None = True,
on_flagged: str | None = "block",
on_flagged: Literal["block", "monitor", "inject_system_message"] | None = "block",
skip_system_message_in_guardrail: bool | None = None,
skip_tool_message_in_guardrail: bool | None = None,
advisory_system_message: str | None = None,
**kwargs,
):
"""
@ -65,7 +256,13 @@ class LakeraAIGuardrail(CustomGuardrail):
breakdown: Optional[bool] = True,
metadata: Optional[Dict] = None,
dev_info: Optional[bool] = True,
on_flagged: Optional[str] = "block", Action to take when content is flagged: "block" or "monitor"
on_flagged: Optional[str] = "block", Action to take when content is flagged:
"block", "monitor", or "inject_system_message"
skip_system_message_in_guardrail: Optional[bool] = None,
skip_tool_message_in_guardrail: Optional[bool] = None,
advisory_system_message: Optional[str] = None, custom advisory message template
(must contain a {reason} placeholder) used when on_flagged="inject_system_message".
Defaults to a generic message when unset.
"""
self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
self.lakera_api_key = api_key or os.environ.get("LAKERA_API_KEY") or ""
@ -75,13 +272,89 @@ class LakeraAIGuardrail(CustomGuardrail):
self.breakdown: bool | None = breakdown
self.metadata: dict | None = metadata
self.dev_info: bool | None = dev_info
self.skip_system_message_in_guardrail = skip_system_message_in_guardrail
self.skip_tool_message_in_guardrail = skip_tool_message_in_guardrail
self.on_flagged = on_flagged or "block"
self.advisory_system_message = advisory_system_message
kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
super().__init__(**kwargs)
self._validate_advisory_config(
on_flagged=self.on_flagged,
advisory_system_message=self.advisory_system_message,
payload=self.payload,
breakdown=self.breakdown,
)
def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None:
"""
The base implementation blindly ``setattr``s every field on ``litellm_params``
(including ``on_flagged``/``advisory_system_message``/``payload``/``breakdown``)
onto this live instance with no revalidation, so an in-place config update (via
the DB/UI, without a restart) could otherwise reintroduce the exact invalid
on_flagged combinations __init__ rejects. Validate the prospective post-update
state *before* mutating, so a rejected update leaves the live instance untouched
instead of raising after it's already been corrupted.
The base setattr also writes ``litellm_params.mode`` onto a new ``self.mode``
attribute rather than the ``self.event_hook`` dispatch actually reads
(LitellmParams has no field literally named ``event_hook``), so without the
explicit sync below a hot reload that changes mode would pass validation but
keep dispatching on the stale event_hook.
"""
new_event_hook: Final = getattr(litellm_params, "mode", None) or self.event_hook
prospective_payload: Final = getattr(litellm_params, "payload", None)
prospective_breakdown: Final = getattr(litellm_params, "breakdown", None)
self._validate_advisory_config(
on_flagged=getattr(litellm_params, "on_flagged", None) or self.on_flagged,
advisory_system_message=getattr(litellm_params, "advisory_system_message", None),
payload=self.payload if prospective_payload is None else prospective_payload,
breakdown=self.breakdown if prospective_breakdown is None else prospective_breakdown,
)
super().update_in_memory_litellm_params(litellm_params=litellm_params)
self.event_hook = new_event_hook
def _validate_advisory_config(
self,
on_flagged: str,
advisory_system_message: str | None,
payload: bool | None,
breakdown: bool | None,
) -> None:
if on_flagged == "inject_system_message" and advisory_system_message is not None:
if not _template_uses_reason_placeholder(advisory_system_message):
raise ValueError(
"Invalid advisory_system_message template: must include a real {reason} "
"placeholder (not an escaped {{reason}}) so the LLM sees why the request was flagged."
)
try:
advisory_system_message.format(reason="placeholder")
except (KeyError, IndexError, ValueError) as e:
raise ValueError(
f"Invalid advisory_system_message template: {e}. The template must be a valid "
"str.format() string using only the {reason} placeholder."
) from e
if on_flagged == "inject_system_message" and not (payload and breakdown):
raise ValueError(
"on_flagged='inject_system_message' requires payload=True and breakdown=True: advisory "
"mode masks any detected PII before appending the advisory note, and that masking can "
"only happen when Lakera's response carries both the violation breakdown and the "
"payload location data. Without them, PII would be forwarded to the model unredacted."
)
def _build_advisory_message(self, lakera_response: LakeraAIResponse | None) -> str:
"""Format the advisory message shown to the LLM when on_flagged='inject_system_message'."""
reason: Final = humanize_lakera_block_reasons(lakera_response.get("breakdown") if lakera_response else None)
template: Final = self.advisory_system_message or DEFAULT_ADVISORY_MESSAGE
return template.format(reason=reason)
def _filter_skipped_messages(
self, messages: Sequence[AllMessageValues]
) -> tuple[tuple[AllMessageValues, ...], bool]:
return filter_messages_by_skip_flags(self, messages)
async def call_v2_guard(
self,
messages: list[AllMessageValues],
messages: Sequence[AllMessageValues],
request_data: dict,
event_type: GuardrailEventHooks,
) -> tuple[LakeraAIResponse, dict]:
@ -143,10 +416,10 @@ class LakeraAIGuardrail(CustomGuardrail):
def _mask_pii_in_messages(
self,
messages: list[AllMessageValues],
messages: Sequence[AllMessageValues],
lakera_response: LakeraAIResponse | None,
masked_entity_count: dict,
) -> list[AllMessageValues]:
) -> Sequence[AllMessageValues]:
"""
Return a copy of messages with any detected PII replaced by
“[MASKED <TYPE>]” tokens.
@ -218,18 +491,38 @@ class LakeraAIGuardrail(CustomGuardrail):
verbose_proxy_logger.debug("Lakera AI: not running guardrail. Guardrail is disabled.")
return data
# Covers multimodal list content + Responses-API input.
new_messages: Final = build_inspection_messages(data)
if not new_messages:
# Covers multimodal list content + Responses-API input/instructions.
inspection_messages: Final = _build_lakera_inspection_messages(data)
if not inspection_messages:
verbose_proxy_logger.warning("Lakera AI: not running guardrail. No inspectable text in data")
return data
# Mask-in-place uses offsets returned by Lakera and can only
# preserve non-text parts (images, audio, …) when the original
# content is a plain string. For multimodal/Responses-API input
# we degrade to block-on-detect so we never silently strip image
# parts while attempting to redact text.
is_multimodal_input: Final = has_non_string_content(data)
new_messages, _ = self._filter_skipped_messages(
inspection_messages # pyright: ignore[reportArgumentType] # build_inspection_messages returns plain dicts, not typed message unions
)
if not new_messages:
verbose_proxy_logger.warning(
"Lakera AI: not running guardrail. All inspectable text was excluded by "
"skip_system_message_in_guardrail/skip_tool_message_in_guardrail"
)
return data
# Mask-in-place can only preserve non-text parts (images, audio) when
# the original content is a plain string, and can only merge a
# redacted result back into data["messages"] by position when
# messages and input aren't both present at once (build_inspection_messages
# flattens both into one list, so a position could mean either).
# Degrade to block-on-detect in either case. Skip-flag-excluded and
# no-text messages, and messages carrying fields beyond role/content
# (tool_call_id, name, tool_calls, cache_control), are otherwise
# handled safely by _apply_redacted_messages_back_preserving_fields's
# scope-index merge, which never touches a message outside the scope
# it actually redacted instead of reconstructing the list from scratch.
is_multimodal_input: Final = (
has_non_string_content(data)
or _has_combined_messages_and_input(data)
or _has_responses_instructions(self, data)
)
#########################################################
########## 1. Make the Lakera AI v2 guard API request ##########
@ -244,18 +537,52 @@ class LakeraAIGuardrail(CustomGuardrail):
########## 2. Handle flagged content ##########
#########################################################
if lakera_guardrail_response.get("flagged") is True:
# If only PII violations exist, mask the PII (string input only).
# PII-only violations get masked in place regardless of on_flagged: there's
# no reason to expose raw PII to satisfy an advisory note, and masking is
# strictly safer than either blocking or appending an advisory message next
# to unredacted PII.
if self._is_only_pii_violation(lakera_guardrail_response) and not is_multimodal_input:
redacted_messages: Final = self._mask_pii_in_messages(
messages=new_messages,
lakera_response=lakera_guardrail_response,
masked_entity_count=masked_entity_count,
)
# Write back to ``messages`` AND ``input``. The Responses-API
# backend reads ``input``; writing only to ``messages``
# would let unredacted PII reach the LLM for /v1/responses.
apply_redacted_messages_back(data, list(redacted_messages))
_apply_redacted_messages_back_preserving_fields(self, data, redacted_messages)
verbose_proxy_logger.debug("Lakera AI: Masked PII in messages instead of blocking request")
elif self.on_flagged == "inject_system_message":
if _breakdown_has_pii_violation(lakera_guardrail_response) and is_multimodal_input:
# There's PII in the mix and nothing here can be safely masked,
# so an advisory note next to this raw, unredacted PII would be
# no safer than a note next to nothing. Degrade to blocking
# instead, same as this on_flagged setting already does when
# the advisory itself has no field it can be delivered into.
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)
masked_pii_before_advisory: Final = _breakdown_has_pii_violation(lakera_guardrail_response)
if masked_pii_before_advisory:
# A mixed violation (PII plus something else, e.g. prompt
# injection): mask whatever Lakera returned location data for
# before advising about what remains, so the advisory is never
# shown next to raw PII that could have been redacted.
mixed_redacted_messages: Final = self._mask_pii_in_messages(
messages=new_messages,
lakera_response=lakera_guardrail_response,
masked_entity_count=masked_entity_count,
)
_apply_redacted_messages_back_preserving_fields(self, data, mixed_redacted_messages)
advisory_delivered: Final = self.inject_advisory_message(
data, self._build_advisory_message(lakera_guardrail_response)
)
if advisory_delivered:
verbose_proxy_logger.warning(
"Lakera Guardrail: Advisory mode - violation detected, %sappended advisory system message",
"masked PII and " if masked_pii_before_advisory else "",
)
else:
# Structured Responses-API input (a list, not a plain string)
# has no field this can safely append into -- degrade to
# blocking rather than silently letting the flagged request
# through with no advisory ever reaching the model.
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)
else:
# Check on_flagged setting
if self.on_flagged == "monitor":
@ -290,19 +617,26 @@ class LakeraAIGuardrail(CustomGuardrail):
if self.should_run_guardrail(data=data, event_type=event_type) is not True:
return
new_messages: Final = build_inspection_messages(data)
if not new_messages:
# Covers multimodal list content + Responses-API input/instructions.
inspection_messages: Final = _build_lakera_inspection_messages(data)
if not inspection_messages:
verbose_proxy_logger.warning("Lakera AI: not running guardrail. No inspectable text in data")
return
# See ``async_pre_call_hook`` — multimodal input degrades to
# block-on-detect because mask-in-place would drop image parts.
is_multimodal_input: Final = has_non_string_content(data)
new_messages, _ = self._filter_skipped_messages(
inspection_messages # pyright: ignore[reportArgumentType] # build_inspection_messages returns plain dicts, not typed message unions
)
if not new_messages:
verbose_proxy_logger.warning(
"Lakera AI: not running guardrail. All inspectable text was excluded by "
"skip_system_message_in_guardrail/skip_tool_message_in_guardrail"
)
return
#########################################################
########## 1. Make the Lakera AI v2 guard API request ##########
#########################################################
lakera_guardrail_response, masked_entity_count = await self.call_v2_guard(
lakera_guardrail_response, _ = await self.call_v2_guard(
messages=new_messages,
request_data=data,
event_type=GuardrailEventHooks.during_call,
@ -312,24 +646,29 @@ class LakeraAIGuardrail(CustomGuardrail):
########## 2. Handle flagged content ##########
#########################################################
if lakera_guardrail_response.get("flagged") is True:
if self._is_only_pii_violation(lakera_guardrail_response) and not is_multimodal_input:
redacted_messages: Final = self._mask_pii_in_messages(
messages=new_messages,
lakera_response=lakera_guardrail_response,
masked_entity_count=masked_entity_count,
)
# Write back to ``messages`` AND ``input``. The Responses-API
# backend reads ``input``; writing only to ``messages``
# would let unredacted PII reach the LLM for /v1/responses.
apply_redacted_messages_back(data, list(redacted_messages))
verbose_proxy_logger.debug("Lakera AI: Masked PII in messages instead of blocking request")
else:
if self.on_flagged == "monitor":
verbose_proxy_logger.warning(
"Lakera Guardrail: Monitoring mode - violation detected but allowing request"
)
elif self.on_flagged == "block":
# during_call runs concurrently with the LLM dispatch (see
# ProxyLogging.during_call_hook / common_request_processing.py), with
# no pre-call barrier: in the common path, the provider call already
# binds its messages kwarg before this coroutine gets a chance to run,
# let alone before the masking helper's own network round trip
# completes. Unlike async_pre_call_hook, mask-in-place here can never
# reliably reach the outgoing request, so PII is never masked in this
# hook -- only blocked (which still works, since raising here blocks
# the response from reaching the caller regardless of dispatch timing)
# or, for non-PII violations, logged and allowed same as monitor mode.
if self.on_flagged == "inject_system_message":
if _breakdown_has_pii_violation(lakera_guardrail_response):
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)
verbose_proxy_logger.warning(
"Lakera Guardrail: Advisory mode has no effect during during_call; "
"violation detected but allowing request"
)
elif self.on_flagged == "monitor":
verbose_proxy_logger.warning(
"Lakera Guardrail: Monitoring mode - violation detected but allowing request"
)
elif self.on_flagged == "block":
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)
#########################################################
########## 3. Add the guardrail to the applied guardrails header ##########
@ -355,9 +694,8 @@ class LakeraAIGuardrail(CustomGuardrail):
if self.should_run_guardrail(data=data, event_type=event_type) is not True:
return response
original_messages: list[AllMessageValues] | None = data.get("messages", [])
if original_messages is None:
original_messages = []
messages_or_none: Final[list[AllMessageValues] | None] = data.get("messages")
original_messages, _ = self._filter_skipped_messages(messages_or_none or [])
# Extract assistant messages from the response, keeping only role/content.
# Track choice indices so we write masked content back to the correct choice
@ -376,7 +714,7 @@ class LakeraAIGuardrail(CustomGuardrail):
choice_indices.append(i)
# Use a copy of original_messages so _mask_pii_in_messages does not mutate data["messages"]
post_call_messages: Final = copy.deepcopy(original_messages) + response_messages
post_call_messages: Final = list(copy.deepcopy(original_messages)) + response_messages # mutable-ok: needs list
# Call Lakera guardrail
lakera_guardrail_response, _ = await self.call_v2_guard(
@ -403,9 +741,13 @@ class LakeraAIGuardrail(CustomGuardrail):
add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name)
return ModelResponse(**response_dict)
if self.on_flagged == "monitor":
verbose_proxy_logger.warning("Lakera Guardrail: Post-call violation detected in monitor mode")
# Allow response to proceed
# inject_system_message has nothing left to inject into once a response
# already exists, so it is treated the same as monitor: log and allow.
if self.on_flagged in ("monitor", "inject_system_message"):
verbose_proxy_logger.warning(
"Lakera Guardrail: Post-call violation detected (on_flagged=%s) - allowing response",
self.on_flagged,
)
elif self.on_flagged == "block":
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)

29
litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py
View file

@ -22,7 +22,7 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
from litellm.types.llms.openai import AllMessageValues
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
from litellm.types.utils import GenericGuardrailAPIInputs
@ -87,6 +87,7 @@ class QualifireGuardrail(CustomGuardrail):
self.tool_selection_quality_check = tool_selection_quality_check
self.assertions = assertions
self.on_flagged = on_flagged or "block"
self._validate_on_flagged(self.on_flagged)
# If no checks are specified and no evaluation_id, default to prompt_injections
if not self._has_any_check_enabled() and not self.evaluation_id:
@ -98,6 +99,32 @@ class QualifireGuardrail(CustomGuardrail):
kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
super().__init__(**kwargs)
def _validate_on_flagged(self, on_flagged: str) -> None:
if on_flagged not in ("block", "monitor"):
# on_flagged is defined on LakeraV2GuardrailConfigModel but LitellmParams
# flattens every guardrail config mixin together, so a value Lakera
# supports (e.g. "inject_system_message") type-checks for any guardrail,
# including this one, which never implements it. Reject it explicitly
# instead of silently falling through to a block-on-anything-else branch.
raise ValueError(
f"Qualifire guardrail does not support on_flagged={on_flagged!r}; "
"only 'block' and 'monitor' are supported."
)
def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None:
"""
The base implementation blindly ``setattr``s every field on ``litellm_params``
(including ``on_flagged``) onto this live instance with no revalidation, so an
in-place config update (via the DB/UI, without a restart) could otherwise
reintroduce the exact invalid on_flagged value __init__ rejects. Validate the
prospective post-update value *before* mutating, so a rejected update leaves
the live instance untouched instead of raising after it's already been
corrupted. Mirrors LakeraAIGuardrail's own override of this same method.
"""
prospective_on_flagged: Final = getattr(litellm_params, "on_flagged", None) or self.on_flagged
self._validate_on_flagged(prospective_on_flagged)
super().update_in_memory_litellm_params(litellm_params=litellm_params)
def _has_any_check_enabled(self) -> bool:
"""Check if any evaluation check is explicitly enabled."""
return any(

3
litellm/proxy/guardrails/guardrail_initializers.py
View file

@ -73,6 +73,9 @@ def initialize_lakera_v2(litellm_params: LitellmParams, guardrail: Guardrail):
metadata=litellm_params.metadata,
dev_info=litellm_params.dev_info,
on_flagged=litellm_params.on_flagged,
skip_system_message_in_guardrail=litellm_params.skip_system_message_in_guardrail,
skip_tool_message_in_guardrail=litellm_params.skip_tool_message_in_guardrail,
advisory_system_message=litellm_params.advisory_system_message,
)
litellm.logging_callback_manager.add_litellm_callback(_lakera_v2_callback)
return _lakera_v2_callback

25
litellm/proxy/guardrails/guardrail_registry.py
View file

@ -413,6 +413,16 @@ class GuardrailRegistry:
raise Exception(f"Error getting guardrail from DB: {e}")
def _apply_configured_bool_override(instance: CustomGuardrail, litellm_params: LitellmParams, param_name: str) -> None:
"""Override ``instance.<param_name>`` only when ``litellm_params`` explicitly
sets it, preserving whatever default the guardrail's own constructor chose
otherwise (its constructor default may be True, so blindly copying an
absent/None config value would silently clobber it back to False)."""
configured: Final = getattr(litellm_params, param_name, None)
if configured is not None:
setattr(instance, param_name, bool(configured))
class InMemoryGuardrailHandler:
"""
Class that handles initializing guardrails and adding them to the CallbackManager
@ -534,9 +544,8 @@ class InMemoryGuardrailHandler:
"skip_tool_message_in_guardrail are enabled together, which excludes every message from "
"scanning, so no request content would ever be scanned. Remove one of the two."
)
configured_run_in_parallel: Final[bool | None] = getattr(litellm_params, "run_in_parallel", None)
if configured_run_in_parallel is not None:
custom_guardrail_callback.run_in_parallel = bool(configured_run_in_parallel)
for override_param in ("run_in_parallel", "scan_raw_request"):
_apply_configured_bool_override(custom_guardrail_callback, litellm_params, override_param)
parsed_guardrail: Final = Guardrail(
guardrail_id=guardrail.get("guardrail_id"),
@ -778,15 +787,23 @@ class InMemoryGuardrailHandler:
"""
Force re-initialization of a guardrail even if it exists in memory.
Removes old callback from litellm.callbacks and creates fresh instance.
If the new config fails to initialize (e.g. an invalid on_flagged
combination), the previous instance is restored rather than left
deleted: initialize_guardrail's own ValueError/TypeError propagate
uncaught, so a caller reaching this point after already deleting the
old instance would otherwise leave the guardrail providing no
protection at all, not merely "still enforcing the old config."
"""
guardrail_id: Final = guardrail.get("guardrail_id")
if not guardrail_id:
verbose_proxy_logger.error("Cannot reinitialize guardrail without guardrail_id")
return None
# Remove from memory if exists (also removes from callbacks)
previous_guardrail: Final = self.IN_MEMORY_GUARDRAILS.get(guardrail_id)
previous_source: Final = self._sources.get(guardrail_id, source)
# Remove from memory if exists (also removes from callbacks)
if guardrail_id in self.IN_MEMORY_GUARDRAILS:
self.delete_in_memory_guardrail(guardrail_id)

20
litellm/proxy/guardrails/init_guardrails.py
View file

@ -26,12 +26,20 @@ def init_guardrails_v2(
guardrail_list: Final[list[Guardrail]] = []
for guardrail in all_guardrails:
initialized_guardrail = IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(
guardrail=cast(Guardrail, guardrail),
config_file_path=config_file_path,
llm_router=llm_router,
source="config",
)
try:
initialized_guardrail = IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(
guardrail=cast(Guardrail, guardrail),
config_file_path=config_file_path,
llm_router=llm_router,
source="config",
)
except (ValueError, TypeError) as init_error:
verbose_proxy_logger.error(
"Skipping guardrail '%s': invalid configuration, proxy is starting WITHOUT this guardrail: %s",
guardrail.get("guardrail_name"),
init_error,
)
continue
if initialized_guardrail:
guardrail_list.append(initialized_guardrail)

1
litellm/proxy/list_api/__init__.py Normal file
View file

@ -0,0 +1 @@
"""Surface-neutral machinery for LiteLLM's own paginated list endpoints."""

104
litellm/proxy/list_api/common.py Normal file
View file

@ -0,0 +1,104 @@
"""Contract machinery shared by every LiteLLM-defined list route, on any surface."""
from typing import Final
from urllib.parse import urlencode
from fastapi import Request
from fastapi.dependencies.utils import get_flat_params
from fastapi.params import ParamTypes
from fastapi.responses import JSONResponse
from litellm.types.proxy.management_endpoints.management_v1 import (
ListLinks,
PageLinks,
ProblemDetail,
)
PROBLEM_CONTENT_TYPE: Final = "application/problem+json"
# A URN, not an https URL: RFC 9457 only asks that `type` identify the problem
# type, and an https URI promises documentation at that address. Switch to an
# https base only when pages actually exist to serve.
PROBLEM_TYPE_BASE: Final = "urn:litellm:error:"
class ManagementProblem(Exception):
"""Raised to return an RFC 9457 problem instead of the proxy's OpenAI error shape."""
def __init__(self, problem: ProblemDetail) -> None:
self.problem = problem
super().__init__(problem.detail)
def problem_response(problem: ProblemDetail) -> JSONResponse:
return JSONResponse(
status_code=problem.status,
content=problem.model_dump(exclude_none=True),
media_type=PROBLEM_CONTENT_TYPE,
)
def _declared_query_params(request: Request) -> frozenset[str]:
route: Final = request.scope.get("route")
dependant: Final = getattr(route, "dependant", None)
if dependant is None:
return frozenset()
# fastapi>=0.140.7 removed get_flat_dependant(); get_flat_params() returns the
# flattened (deduped) param list. Filter to query params to match the old behavior.
return frozenset(
field.alias
for field in get_flat_params(dependant)
if getattr(field.field_info, "in_", None) == ParamTypes.query
)
def escape_like(value: str) -> str:
"""Escape LIKE/ILIKE metacharacters. Ids routinely contain `_`, which is a wildcard unescaped."""
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
def unknown_query_param_problem(unknown: tuple[str, ...], allowed: tuple[str, ...]) -> ProblemDetail:
return ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}unknown-query-parameter",
title="Unknown query parameter",
status=400,
detail=f"Unrecognized query parameter(s): {', '.join(unknown)}.",
allowed=sorted(allowed),
)
async def reject_unknown_query_params(request: Request) -> None:
"""Reject any query param the route did not declare.
A silently ignored filter over-returns data, which is worse than a rejected
request; a fresh surface is the only chance to be strict about it.
"""
declared: Final = _declared_query_params(request)
unknown: Final[tuple[str, ...]] = tuple(sorted(name for name in request.query_params if name not in declared))
if not unknown:
return
raise ManagementProblem(unknown_query_param_problem(unknown=unknown, allowed=tuple(sorted(declared))))
def _page_url(request: Request, page: int) -> str:
others: Final = tuple((key, value) for key, value in request.query_params.multi_items() if key != "page")
return f"{request.url.path}?{urlencode((*others, ('page', page)))}"
def build_page_links(request: Request, page: int, has_more: bool) -> PageLinks:
return PageLinks(
self_link=_page_url(request, page),
prev=_page_url(request, page - 1) if page > 1 else None,
next=_page_url(request, page + 1) if has_more else None,
)
def build_list_links(request: Request, page: int, total_pages: int) -> ListLinks:
"""Page-mode links. `last` clamps to page 1 on an empty result set so every link still resolves."""
last: Final = max(total_pages, 1)
return ListLinks(
self_link=_page_url(request, page),
first=_page_url(request, 1),
prev=_page_url(request, page - 1) if page > 1 else None,
next=_page_url(request, page + 1) if page < last else None,
last=_page_url(request, last),
)

143
litellm/proxy/list_api/in_memory.py Normal file
View file

@ -0,0 +1,143 @@
"""An in-memory `ListExecutor`, for list resources whose rows are computed rather than queried.
Answers the same `QueryPlan` a SQL executor would render through `where_sql` / `order_by_sql`,
so a filter or a sort means the same thing on either. `enrich_page` runs on the page slice and
never on the whole match set.
"""
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from functools import reduce
from typing import Final, Generic, TypeAlias, TypeVar
from typing_extensions import assert_never
from litellm.proxy.list_api.list_framework import (
AnyOf,
Compare,
ComparisonOp,
FilterValue,
IsNull,
Predicate,
QueryPlan,
SortKey,
Within,
)
TRow: Final = TypeVar("TRow")
Cell: TypeAlias = str | int | float | datetime | None
# A tuple-valued cell is a row's repeated field (a model group's providers, say). A predicate
# holds against it when it holds against any one element, the way an SQL join would answer.
Cells: TypeAlias = Mapping[str, Cell | tuple[Cell, ...]]
def _sign(cell: Cell, value: FilterValue) -> int | None:
"""None when the two values are not orderable against each other."""
if isinstance(cell, str) and isinstance(value, str):
return (cell > value) - (cell < value)
if isinstance(cell, datetime) and isinstance(value, datetime):
return (cell > value) - (cell < value)
if isinstance(cell, (int, float)) and isinstance(value, (int, float)):
return (cell > value) - (cell < value)
return None
def _matches(cell: Cell, op: ComparisonOp, value: FilterValue) -> bool:
"""SQL's three-valued logic: a NULL cell satisfies no comparison, only `is_null`."""
if cell is None:
return False
sign: Final = _sign(cell, value)
match op:
case "eq":
return cell == value
case "not":
return cell != value
case "contains":
return str(value).casefold() in str(cell).casefold()
case "gt":
return sign is not None and sign > 0
case "gte":
return sign is not None and sign >= 0
case "lt":
return sign is not None and sign < 0
case "lte":
return sign is not None and sign <= 0
case _:
assert_never(op)
def _any_cell(cells: Cells, name: str, matches: Callable[[Cell], bool]) -> bool:
cell: Final = cells.get(name)
if isinstance(cell, tuple):
return any(matches(item) for item in cell)
return matches(cell)
def _leaf_holds(predicate: Compare | Within | IsNull, cells: Cells) -> bool:
match predicate:
case Compare(field=name, op=op, value=value):
return _any_cell(cells, name, lambda cell: _matches(cell, op, value))
case Within(field=name, values=values):
return _any_cell(cells, name, lambda cell: cell is not None and cell in values)
case IsNull(field=name, negated=negated):
return _any_cell(cells, name, lambda cell: (cell is None) != negated)
case _:
assert_never(predicate)
def _holds(predicate: Predicate, cells: Cells) -> bool:
if isinstance(predicate, AnyOf):
return any(_leaf_holds(clause, cells) for clause in predicate.clauses)
return _leaf_holds(predicate, cells)
def _sort_key(cells: Cells, key: SortKey) -> tuple[bool, Cell | tuple[Cell, ...]]:
"""NULLS LAST in both directions, matching `order_by_sql`.
The placeholder standing in for a null is only ever compared against another null's,
because the rank ahead of it already separates nulls from the rest.
"""
cell: Final = cells.get(key.field)
return (cell is None) != key.descending, 0 if cell is None else cell
def _ordered(
matched: Sequence[tuple[Cells, TRow]],
order: tuple[SortKey, ...],
) -> Sequence[tuple[Cells, TRow]]:
"""Least significant key first: Python's sort is stable, so the most significant pass wins."""
return reduce(
lambda rows, key: sorted(rows, key=lambda pair: _sort_key(pair[0], key), reverse=key.descending),
reversed(order),
matched,
)
async def _unchanged(rows: Sequence[TRow]) -> Sequence[TRow]:
return rows
@dataclass(frozen=True, slots=True)
class InMemoryListExecutor(Generic[TRow]):
"""`cells` projects a row down to the values the spec's filters, search and sort read, so a
plan can be applied without this module knowing the row type."""
rows: Sequence[TRow]
cells: Callable[[TRow], Cells]
enrich_page: Callable[[Sequence[TRow]], Awaitable[Sequence[TRow]]] = _unchanged
def _matching(self, where: tuple[Predicate, ...]) -> Sequence[tuple[Cells, TRow]]:
return tuple(
(cells, row)
for cells, row in ((self.cells(row), row) for row in self.rows)
if all(_holds(predicate, cells) for predicate in where)
)
async def count(self, where: tuple[Predicate, ...]) -> int:
return len(self._matching(where))
async def find_many(self, plan: QueryPlan) -> Sequence[TRow]:
page: Final = _ordered(self._matching(plan.where), plan.order)[plan.skip : plan.skip + plan.take]
return await self.enrich_page(tuple(row for _, row in page))

25
litellm/proxy/management_endpoints/management_v1/list_framework.py → litellm/proxy/list_api/list_framework.py
View file

@ -1,4 +1,4 @@
"""Generic list handling for `/management/v1` collection routes.
"""Generic list handling for LiteLLM-defined collection routes.
A resource declares a `ListSpec`; `build_query_plan` turns query parameters into a
`QueryPlan` or an RFC 9457 problem without touching a database, and `handle_list`
@ -24,7 +24,7 @@ from pydantic import TypeAdapter, ValidationError
from typing_extensions import assert_never
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.management_endpoints.management_v1.common import (
from litellm.proxy.list_api.common import (
PROBLEM_TYPE_BASE,
ManagementProblem,
build_list_links,
@ -85,9 +85,13 @@ class IsNull:
@dataclass(frozen=True, slots=True)
class AnyOf:
"""Disjunction of its clauses. `?q=` is the only producer today."""
"""Disjunction of its clauses. `?q=` is the only producer.
clauses: tuple["Predicate", ...]
Holding leaves rather than predicates keeps the disjunction one level deep by type, so
neither the SQL renderer nor an in-memory executor has to walk a tree to evaluate it.
"""
clauses: tuple[Compare, ...]
Predicate = Compare | Within | IsNull | AnyOf
@ -369,6 +373,19 @@ def _parse_sort(spec: ListSpec[TRow, TOut], params: Mapping[str, str]) -> tuple[
f"Cannot sort {spec.resource} by: {', '.join(repr(field) for field in rejected)}.",
tuple(spec.sortable),
)
# A repeated field cannot change the ordering, but an executor that sorts once per key
# does the work anyway. Rejecting repeats bounds that to the size of `sortable`, which
# matters because an unauthenticated caller can otherwise name one field a thousand times.
fields: Final = tuple(key.field for key in keys)
repeated: Final = tuple(sorted(frozenset(field for field in fields if fields.count(field) > 1)))
if repeated:
return _problem(
"duplicate-sort-field",
"Duplicate sort field",
400,
f"Sort field(s) named more than once: {', '.join(repeated)}. Each may appear once.",
tuple(spec.sortable),
)
return keys

22
litellm/proxy/litellm_pre_call_utils.py
View file

@ -51,6 +51,7 @@ from litellm.proxy.common_utils.callback_utils import (
strip_callback_config,
)
from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers
from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY
# Cache special headers as a frozenset for O(1) lookup performance
_SPECIAL_HEADERS_CACHE: Final = frozenset(v.value.lower() for v in SpecialHeaders._member_map_.values())
@ -221,6 +222,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
"policy_sources",
"guardrail_scan_ids",
"routing_decision",
GATEWAY_INJECTED_CACHE_METADATA_KEY,
"pillar_response_headers",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
@ -275,6 +277,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
"policy_sources",
"guardrail_scan_ids",
"routing_decision",
GATEWAY_INJECTED_CACHE_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
@ -310,6 +313,10 @@ _CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.mod
# into response_cost and spend; a client seeding it forges (even negative)
# guardrail cost.
_CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logging_guardrail_information"})
# ``attempted_fallbacks`` and ``original_model_group`` are written by the router
# and read by spend logs as fact; a client value has no legitimate meaning and no
# key or team setting keeps it, so the strip is never gated.
_ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset({"attempted_fallbacks", "original_model_group"})
_ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override"
# Request fields whose value, when URL-valued, becomes the outbound destination
@ -535,6 +542,20 @@ def _strip_client_pricing_overrides(data: dict[str, Any]) -> None:
)
def _strip_router_reserved_metadata(
data: dict[str, Any], # mutable-ok: strips in place on the request body the pre-call pipeline threads through
) -> None:
"""Drop the router-owned fallback stamps from any client-supplied metadata bucket."""
for metadata_key in ("metadata", "litellm_metadata"):
if not isinstance(metadata := data.get(metadata_key), dict):
continue
for field in _ROUTER_RESERVED_METADATA_FIELDS & metadata.keys():
metadata.pop(field)
verbose_proxy_logger.debug(
"Stripped router-reserved metadata field from request body: %s.%s", metadata_key, field
)
def _get_metadata_variable_name(request: Request) -> str:
"""
Helper to return what the "metadata" field should be called in the request data
@ -1879,6 +1900,7 @@ async def add_litellm_data_to_request(
# would silently skip the field.
if not _key_or_team_allows_client_pricing_override(user_api_key_dict):
_strip_client_pricing_overrides(data)
_strip_router_reserved_metadata(data)
# Same reason as the strips above: runs after the metadata string-to-dict parse
# so JSON-string metadata cannot smuggle callback credentials past the dict guard.

108
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -15,6 +15,7 @@ from uuid import uuid4
from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.exceptions import BudgetExceededError
from litellm.litellm_core_utils.llm_judge import judge_target
@ -119,6 +120,10 @@ class _ShadowEvalAttemptRow(Protocol):
def error(self) -> str | None: ...
class _ShadowEvalFunnelTable(Protocol):
async def create_many(self, data: Sequence[Mapping[str, object]], skip_duplicates: bool) -> int: ...
class _ShadowEvalAttemptTable(Protocol):
async def find_first(
self, *, where: Mapping[str, object], order: Mapping[str, str]
@ -137,6 +142,10 @@ def _shadow_eval_jobs(prisma_client: "PrismaClient") -> _ShadowEvalJobTable:
return prisma_client.db.litellm_shadowevaljob
def _shadow_eval_funnel(prisma_client: "PrismaClient") -> _ShadowEvalFunnelTable:
return prisma_client.db.litellm_shadowevalfunnel # pyright: ignore[reportAttributeAccessIssue] # generated client
def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTable:
return prisma_client.db.litellm_shadowevalattempt
@ -678,6 +687,20 @@ def _is_configured_pre_routing_strategy(llm_router: "Router", router_name: str)
)
def _sdk_model_is_missing_anthropic_credentials(model: str) -> bool:
_, provider, _, _ = litellm.get_llm_provider(model=model)
if provider != "anthropic" or litellm.anthropic_key or litellm.api_key:
return False
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.secret_managers.main import secret_manager_would_be_consulted
if AnthropicModelInfo.get_api_key() or AnthropicModelInfo.get_auth_token():
return False
return not any(
secret_manager_would_be_consulted(secret_name) for secret_name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")
)
def _validate_plain_model(
llm_router: "Router | None", model: str, field_name: str, team_ids: Sequence[str | None]
) -> None:
@ -694,14 +717,26 @@ def _validate_plain_model(
status_code=400,
detail=f"{field_name} '{model}' is an auto-router; it must be a plain model",
)
unreachable: Final = tuple(team for team in team_ids if judge_target(llm_router, model, team).via == "nothing")
if not unreachable:
targets: Final = tuple((team, judge_target(llm_router, model, team)) for team in team_ids)
unreachable: Final = tuple(team for team, target in targets if target.via == "nothing")
if unreachable:
raise HTTPException(
status_code=400,
detail=(
f"{field_name} '{model}' is neither a model configured on this proxy nor a "
"provider-qualified public model name (e.g. 'anthropic/claude-sonnet-5')" + _for_teams(unreachable)
),
)
sdk_teams: Final = tuple(team for team, target in targets if target.via == "sdk")
if not sdk_teams:
return
if not _sdk_model_is_missing_anthropic_credentials(model):
return
raise HTTPException(
status_code=400,
detail=(
f"{field_name} '{model}' is neither a model configured on this proxy nor a "
"provider-qualified public model name (e.g. 'anthropic/claude-sonnet-5')" + _for_teams(unreachable)
f"{field_name} '{model}' uses the LiteLLM SDK but required credentials are not configured: "
"ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN" + _for_teams(sdk_teams)
),
)
@ -820,6 +855,9 @@ class _AttemptAggRow(BaseModel):
shadow_wins: int
ties: int
avg_confidence: float | None
real_spend: float
shadow_spend: float
cache_hit_turns: int
_ATTEMPT_AGG_ROWS: Final = TypeAdapter(list[_AttemptAggRow])
@ -829,7 +867,10 @@ _ATTEMPT_AGG_SELECT: Final = """
COUNT(*) FILTER (WHERE outcome = 'real')::int AS real_wins,
COUNT(*) FILTER (WHERE outcome = 'shadow')::int AS shadow_wins,
COUNT(*) FILTER (WHERE outcome = 'tie')::int AS ties,
AVG(confidence)::float AS avg_confidence
AVG(confidence)::float AS avg_confidence,
COALESCE(SUM(real_cost + real_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS real_spend,
COALESCE(SUM(shadow_cost + shadow_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS shadow_spend,
COUNT(*) FILTER (WHERE real_cache_hit)::int AS cache_hit_turns
FROM "LiteLLM_ShadowEvalAttempt"
WHERE job_id = ANY($1::text[]) AND outcome != 'error'
GROUP BY 1
@ -850,7 +891,7 @@ WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL
OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns
OR (
j.max_budget IS NOT NULL
AND (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_budget
AND (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_budget
)
)
"""
@ -865,13 +906,24 @@ WHERE job_id = ANY($1::text[])
"""
_ATTEMPT_COUNTS_SQL: Final = """
SELECT a.job_id, COUNT(*)::int AS attempt_count, COALESCE(SUM(a.judge_cost + a.shadow_cost), 0)::float AS spend
SELECT a.job_id, COUNT(*)::int AS attempt_count, COALESCE(SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost), 0)::float AS spend
FROM "LiteLLM_ShadowEvalAttempt" a
JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id
WHERE a.job_id = ANY($1::text[]) AND (j.stopped_at IS NULL OR a.created_at <= j.stopped_at)
GROUP BY a.job_id
"""
_FUNNEL_TOTALS_SQL: Final = """
SELECT COUNT(*)::int AS legs_with_rows,
COALESCE(SUM(not_sampled), 0)::int AS not_sampled,
COALESCE(SUM(unjudgeable), 0)::int AS unjudgeable,
COALESCE(SUM(shed), 0)::int AS shed,
COALESCE(SUM(withheld), 0)::int AS withheld
FROM "LiteLLM_ShadowEvalFunnel"
WHERE job_id = ANY($1::text[])
"""
_STOP_JOB_SQL: Final = """
UPDATE "LiteLLM_ShadowEvalJob"
SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp)
@ -883,12 +935,20 @@ WHERE group_id = $1 AND stopped_by IS NULL
AND (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_turns
AND (
k.max_budget IS NULL
OR (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_budget
OR (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_budget
)
)
"""
class _FunnelTotalsRow(BaseModel):
legs_with_rows: int
not_sampled: int
unjudgeable: int
shed: int
withheld: int
class _AttemptCountRow(BaseModel):
job_id: str
attempt_count: int
@ -937,6 +997,9 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]:
shadow_win_rate_pct=_pct_of(row.shadow_wins, row.turn_count),
tie_rate_pct=_pct_of(row.ties, row.turn_count),
avg_judge_confidence=round(row.avg_confidence or 0.0, 3),
real_spend=row.real_spend,
shadow_spend=row.shadow_spend,
cache_hit_turns=row.cache_hit_turns,
)
for row in sorted(rows, key=lambda r: r.turn_count, reverse=True)
)
@ -1087,12 +1150,23 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le
for row in by_leg
)
total_turns: Final = sum(r.turn_count for r in by_tier)
funnel_rows: Final = await _query_raw(prisma_client, _FUNNEL_TOTALS_SQL, leg_ids)
counted: Final = _FunnelTotalsRow.model_validate(funnel_rows[0]) if funnel_rows else None
# Coverage only when EVERY leg has a funnel row: a partial seed (one leg's insert
# failed) must read as unknown, not as job-level counts missing a leg's traffic.
funnel: Final = counted if counted is not None and counted.legs_with_rows == len(leg_ids) else None
return ShadowEvalResult(
by_tier=_slices(by_tier),
by_current_model=_slices(by_model),
by_key=_slices(by_key),
overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns),
overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns),
sampled_real_spend=sum(r.real_spend for r in by_tier),
sampled_shadow_spend=sum(r.shadow_spend for r in by_tier),
not_sampled_count=funnel.not_sampled if funnel is not None else None,
unjudgeable_count=funnel.unjudgeable if funnel is not None else None,
shed_count=funnel.shed if funnel is not None else None,
withheld_count=funnel.withheld if funnel is not None else None,
)
@ -1191,8 +1265,14 @@ async def start_shadow_eval(
"ends_at": ends_at,
}
try:
# Leg ids are minted here rather than by the DB default so the funnel seed below
# writes from the same values with no read-back, which a lagging read replica
# (DATABASE_URL_READ_REPLICA) could otherwise return empty.
leg_ids: Final = tuple(str(uuid4()) for _ in data.api_key_ids)
await _shadow_eval_jobs(prisma_client).create_many(
data=[{**shared_config, "api_key_id": key} for key in data.api_key_ids] # mutable-ok: Prisma payload
data=[ # mutable-ok: Prisma payload
{**shared_config, "id": leg_id, "api_key_id": key} for leg_id, key in zip(leg_ids, data.api_key_ids)
]
)
except Exception as e:
if not _is_unique_violation(e):
@ -1203,6 +1283,16 @@ async def start_shadow_eval(
f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first."
),
) from e
# Seed a zero funnel row per leg NOW: a fully covered job never skips a request, so
# waiting for the first skip would leave it indistinguishable from a pre-funnel job
# (null coverage). A failed seed degrades this job to exactly that, nothing worse.
try:
await _shadow_eval_funnel(prisma_client).create_many(
data=[{"job_id": leg_id} for leg_id in leg_ids], # mutable-ok: Prisma payload
skip_duplicates=True,
)
except Exception as seed_err: # noqa: BLE001 # coverage is advisory; the job must still start
verbose_proxy_logger.error("shadow_eval: funnel seed failed for job %s: %s", group_id, seed_err)
labels: Final = MappingProxyType({row.token: row for row in token_rows})
return ShadowEvalJobResponse(
job_id=group_id,

14
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -94,6 +94,9 @@ class DailySpendRecord(Protocol):
@property
def prompt_caching_savings_spend(self) -> float: ...
@property
def gateway_injected_caching_savings_spend(self) -> float: ...
@property
def autorouter_savings_spend(self) -> float: ...
@ -137,6 +140,7 @@ class _GroupingSetsRow(SimpleNamespace):
compression_saved_tokens: int | None
compression_savings_spend: float | None
prompt_caching_savings_spend: float | None
gateway_injected_caching_savings_spend: float | None
autorouter_savings_spend: float | None
api_requests: int | None
successful_requests: int | None
@ -189,6 +193,9 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) ->
existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0
existing_metrics.compression_savings_spend += record.compression_savings_spend or 0
existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0
existing_metrics.gateway_injected_caching_savings_spend += ( # rebind-ok: this accumulator mutates its target in place for every metric on the row
record.gateway_injected_caching_savings_spend or 0
)
existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0
existing_metrics.api_requests += record.api_requests or 0
existing_metrics.successful_requests += record.successful_requests or 0
@ -721,6 +728,7 @@ def _build_aggregated_sql_query(
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(compression_savings_spend)::float AS compression_savings_spend,
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend,
SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
@ -799,6 +807,7 @@ def _build_entity_rollup_sql_query(
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(compression_savings_spend)::float AS compression_savings_spend,
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend,
SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
@ -934,6 +943,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
compression_saved_tokens=record.compression_saved_tokens or 0,
compression_savings_spend=record.compression_savings_spend or 0,
prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0,
gateway_injected_caching_savings_spend=record.gateway_injected_caching_savings_spend or 0,
autorouter_savings_spend=record.autorouter_savings_spend or 0,
api_requests=record.api_requests or 0,
successful_requests=record.successful_requests or 0,
@ -1200,6 +1210,7 @@ async def get_daily_activity(
total_compression_saved_tokens=metadata_metrics.compression_saved_tokens,
total_compression_savings_spend=metadata_metrics.compression_savings_spend,
total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend,
total_gateway_injected_caching_savings_spend=metadata_metrics.gateway_injected_caching_savings_spend,
total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend,
page=page,
total_pages=-(-total_count // page_size), # Ceiling division
@ -1372,6 +1383,9 @@ async def get_daily_activity_aggregated(
total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens,
total_compression_savings_spend=aggregated["totals"].compression_savings_spend,
total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend,
total_gateway_injected_caching_savings_spend=aggregated[
"totals"
].gateway_injected_caching_savings_spend,
total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend,
page=1,
total_pages=1,

158
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -62,6 +62,9 @@ from litellm.proxy.auth.auth_utils import (
enforce_output_token_estimates_are_admin_only,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
publish_auth_cache_invalidation,
)
from litellm.proxy.common_utils.callback_config_validation import logging_metadata_config_error
from litellm.proxy.common_utils.callback_utils import (
decrypt_callback_vars,
@ -5171,6 +5174,125 @@ def _validate_reset_spend_value(reset_to: object, key_in_db: LiteLLM_Verificatio
return reset_to
async def _set_spend_counter_with_floor_and_broadcast(counter_key: str, value: float) -> None:
"""
Set a Redis-backed spend counter to `value`, mirror it into the short-lived
spend_db_floor marker `_authoritative_floor_spend` reads, and broadcast both
to every worker (LIT-3803 pattern: setting, not deleting, means a worker's
own self-delivered broadcast still carries the reset value forward).
Without the floor marker, `_authoritative_floor_spend` can re-derive a
stale, pre-reset value from a marker another worker cached moments earlier
and raise the just-reset counter right back up via `_repair_stale_spend_counter`.
Without the broadcast, a worker that already cached the pre-reset key object
or floor marker keeps enforcing against it until its own TTL expires.
"""
from litellm.proxy.proxy_server import SPEND_DB_FLOOR_CACHE_TTL_SECONDS, spend_counter_cache
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=value, ttl=60)
if spend_counter_cache.redis_cache is not None:
try:
await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=value, ttl=60)
except Exception as redis_err:
verbose_proxy_logger.warning(
"Failed to update spend counter %s in Redis: %s. "
"Budget checks may use stale value until counter expires.",
counter_key,
redis_err,
)
floor_key: Final = f"spend_db_floor:{counter_key}"
spend_counter_cache.in_memory_cache.set_cache(key=floor_key, value=value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS)
await publish_auth_cache_invalidation(cache_key=counter_key, new_value=value, ttl=60)
await publish_auth_cache_invalidation(cache_key=floor_key, new_value=value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS)
def _budget_limit_windows(budget_limits: Sequence[object] | str | None) -> tuple[Mapping[str, object], ...]:
"""Coerce a key's stored `budget_limits` into a tuple of plain window dicts.
It is a DB Json column, so a caller reading it straight off `find_unique`
gets an already-parsed list; one reading it off `json.dumps`'d text (or a
raw SQL row) gets the string form. Either way each entry is a plain dict,
except wherever a caller already validated the field through a pydantic
model (e.g. `UserAPIKeyAuth.budget_limits`), which yields `BudgetLimitEntry`
objects instead -- coerced here via `model_dump()`, matching
`_set_budget_reset_at`'s identical coercion in team_endpoints.py.
"""
if not budget_limits:
return ()
raw_windows: Final = json.loads(budget_limits) if isinstance(budget_limits, str) else budget_limits
return tuple(raw_window if isinstance(raw_window, dict) else raw_window.model_dump() for raw_window in raw_windows)
def _advance_one_key_budget_window(window: Mapping[str, object]) -> Mapping[str, object]:
"""Restart one budget window from now, by advancing its `reset_at`.
`window_start` is derived elsewhere as `reset_at - budget_duration`
(`get_budget_window_start`), so `reset_at` must be set to `now +
budget_duration` -- a window floating from THIS moment -- to make
`window_start` land at `now` and exclude the historical spend that
triggered the block. Reusing `get_budget_reset_time`/
`ResetBudgetJob._reset_expired_window`'s calendar-standardized boundary
(e.g. "next midnight") would not do that: for a "1d" window `next
midnight - 1d` is simply the START of the calendar day already in
progress, which still covers that spend. That reuse is only safe for the
scheduled job, which runs right as `reset_at` naturally elapses, so the
elapsed boundary it computes is already close to "now". A manual reset
can happen at any point mid-window, so it needs the floating form
instead. A window with no `budget_duration` is returned unchanged.
"""
duration = window.get("budget_duration")
if not isinstance(duration, str) or not duration:
return window
new_reset_at: Final = datetime.now(timezone.utc) + timedelta(seconds=duration_in_seconds(duration))
return { # mutable-ok: this is the JSON payload persisted to budget_limits' Json column, which requires a plain dict
**window,
"reset_at": new_reset_at.isoformat(),
}
async def _reset_key_budget_windows(
prisma_client: PrismaClient,
hashed_api_key: str,
budget_limits: Sequence[object] | str | None,
) -> None:
"""Force-expire every one of a key's own `budget_limits` windows (extra
time-windowed caps layered on top of the lifetime max_budget, e.g. a daily
limit) so a manual spend reset also clears them, not just the lifetime
counter.
Persists the advanced `reset_at` boundaries BEFORE zeroing any window's
Redis counter, not after: a window counter reading zero is only durable
once every reader recomputing its floor from the DB sees the new
boundary too (`get_current_spend` re-derives a window counter from real
`LiteLLM_SpendLogs` rows inside `[window_start, now)` on every read below
max_budget, see its `is_window` branch). Zeroing first would let a
request racing the DB write compute `window_start` from the stale
pre-reset boundary, re-sum the unchanged historical spend, and put the
counter right back where it was before the write ever landed.
"""
windows: Final = _budget_limit_windows(budget_limits)
if not windows:
return
reset_windows: Final = tuple(_advance_one_key_budget_window(w) for w in windows)
# prisma-client-py's typed update() takes plain dict literals for `where`/`data`; there is no
# frozen-mapping equivalent to pass instead.
reset_payload: Final = {"budget_limits": json.dumps(reset_windows, default=str)} # mutable-ok: prisma data kwarg
await VerificationTokenRepository(prisma_client).table.update(
where={"token": hashed_api_key}, # mutable-ok: prisma where kwarg
data=reset_payload,
)
for window in reset_windows:
duration = window.get("budget_duration")
if isinstance(duration, str) and duration:
counter_key = f"spend:key:{hashed_api_key}:window:{duration}"
await _set_spend_counter_with_floor_and_broadcast(counter_key=counter_key, value=0.0)
@router.post(
"/key/{key:path}/reset_spend",
tags=["key management"],
@ -5236,30 +5358,30 @@ async def reset_key_spend_fn(
detail={"error": "Failed to update key spend"},
)
# Reset the lifetime spend counter to the new value (not 0.0, so partial
# resets are reflected correctly), and force-expire any of the key's own
# budget_limits windows, so get_current_spend() returns the correct
# amount for every enforcement check immediately instead of the stale
# pre-reset value.
_counter_key: Final = f"spend:key:{hashed_api_key}"
await _set_spend_counter_with_floor_and_broadcast(counter_key=_counter_key, value=reset_to)
await _reset_key_budget_windows(
prisma_client=prisma_client,
hashed_api_key=hashed_api_key,
budget_limits=_key_in_db.budget_limits,
)
# Evicting the cached key object LAST (after every DB write above has
# committed) matters: a request landing between an earlier eviction and
# a later write would re-fetch and re-cache the pre-write row, pinning
# that pod to the stale budget_limits/spend for the rest of its own
# cache TTL even though the DB is already correct.
await _delete_cache_key_object(
hashed_token=hashed_api_key,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
# Set Redis spend counter to the new value so get_current_spend()
# returns the correct amount immediately instead of the stale pre-reset value.
# We use reset_to (not 0.0) so partial resets are reflected correctly.
from litellm.proxy.proxy_server import spend_counter_cache
_counter_key: Final = f"spend:key:{hashed_api_key}"
spend_counter_cache.in_memory_cache.set_cache(key=_counter_key, value=reset_to, ttl=60)
if spend_counter_cache.redis_cache is not None:
try:
await spend_counter_cache.redis_cache.async_set_cache(key=_counter_key, value=reset_to, ttl=60)
except Exception as redis_err:
verbose_proxy_logger.warning(
"Failed to update spend counter %s in Redis: %s. "
"Budget checks may use stale value until counter expires.",
_counter_key,
redis_err,
)
max_budget: Final = updated_key.max_budget
budget_reset_at: Final = updated_key.budget_reset_at

6
litellm/proxy/management_endpoints/management_v1/budgets.py
View file

@ -16,12 +16,11 @@ from litellm.proxy._types import (
user_api_key_has_admin_view,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management_endpoints.management_v1.common import (
MANAGEMENT_V1_PREFIX,
from litellm.proxy.list_api.common import (
PROBLEM_TYPE_BASE,
ManagementProblem,
)
from litellm.proxy.management_endpoints.management_v1.list_framework import (
from litellm.proxy.list_api.list_framework import (
FilterSpec,
ListSpec,
Predicate,
@ -34,6 +33,7 @@ from litellm.proxy.management_endpoints.management_v1.list_framework import (
order_by_sql,
where_sql,
)
from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX
from litellm.proxy.utils import PrismaClient
from litellm.types.proxy.management_endpoints.management_v1 import (
ListResponse,

105
litellm/proxy/management_endpoints/management_v1/common.py
View file

@ -1,105 +1,8 @@
"""Contract machinery shared by every `/management/v1` route."""
"""Constants specific to the `/management/v1` control-plane surface.
The contract machinery every list route shares lives in `litellm.proxy.list_api`.
"""
from typing import Final
from urllib.parse import urlencode
from fastapi import Request
from fastapi.dependencies.utils import get_flat_params
from fastapi.params import ParamTypes
from fastapi.responses import JSONResponse
from litellm.types.proxy.management_endpoints.management_v1 import (
ListLinks,
PageLinks,
ProblemDetail,
)
MANAGEMENT_V1_PREFIX: Final = "/management/v1"
PROBLEM_CONTENT_TYPE: Final = "application/problem+json"
# A URN, not an https URL: RFC 9457 only asks that `type` identify the problem
# type, and an https URI promises documentation at that address. Switch to an
# https base only when pages actually exist to serve.
PROBLEM_TYPE_BASE: Final = "urn:litellm:error:"
class ManagementProblem(Exception):
"""Raised to return an RFC 9457 problem instead of the proxy's OpenAI error shape."""
def __init__(self, problem: ProblemDetail) -> None:
self.problem = problem
super().__init__(problem.detail)
def problem_response(problem: ProblemDetail) -> JSONResponse:
return JSONResponse(
status_code=problem.status,
content=problem.model_dump(exclude_none=True),
media_type=PROBLEM_CONTENT_TYPE,
)
def _declared_query_params(request: Request) -> frozenset[str]:
route: Final = request.scope.get("route")
dependant: Final = getattr(route, "dependant", None)
if dependant is None:
return frozenset()
# fastapi>=0.140.7 removed get_flat_dependant(); get_flat_params() returns the
# flattened (deduped) param list. Filter to query params to match the old behavior.
return frozenset(
field.alias
for field in get_flat_params(dependant)
if getattr(field.field_info, "in_", None) == ParamTypes.query
)
def escape_like(value: str) -> str:
"""Escape LIKE/ILIKE metacharacters. Ids routinely contain `_`, which is a wildcard unescaped."""
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
def unknown_query_param_problem(unknown: tuple[str, ...], allowed: tuple[str, ...]) -> ProblemDetail:
return ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}unknown-query-parameter",
title="Unknown query parameter",
status=400,
detail=f"Unrecognized query parameter(s): {', '.join(unknown)}.",
allowed=sorted(allowed),
)
async def reject_unknown_query_params(request: Request) -> None:
"""Reject any query param the route did not declare.
A silently ignored filter over-returns data, which is worse than a rejected
request; a fresh surface is the only chance to be strict about it.
"""
declared: Final = _declared_query_params(request)
unknown: Final[tuple[str, ...]] = tuple(sorted(name for name in request.query_params if name not in declared))
if not unknown:
return
raise ManagementProblem(unknown_query_param_problem(unknown=unknown, allowed=tuple(sorted(declared))))
def _page_url(request: Request, page: int) -> str:
others: Final = tuple((key, value) for key, value in request.query_params.multi_items() if key != "page")
return f"{request.url.path}?{urlencode((*others, ('page', page)))}"
def build_page_links(request: Request, page: int, has_more: bool) -> PageLinks:
return PageLinks(
self_link=_page_url(request, page),
prev=_page_url(request, page - 1) if page > 1 else None,
next=_page_url(request, page + 1) if has_more else None,
)
def build_list_links(request: Request, page: int, total_pages: int) -> ListLinks:
"""Page-mode links. `last` clamps to page 1 on an empty result set so every link still resolves."""
last: Final = max(total_pages, 1)
return ListLinks(
self_link=_page_url(request, page),
first=_page_url(request, 1),
prev=_page_url(request, page - 1) if page > 1 else None,
next=_page_url(request, page + 1) if page < last else None,
last=_page_url(request, last),
)

4
litellm/proxy/management_endpoints/management_v1/spend_logs.py
View file

@ -8,14 +8,14 @@ from fastapi import APIRouter, Depends, Query, Request
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management_endpoints.management_v1.common import (
MANAGEMENT_V1_PREFIX,
from litellm.proxy.list_api.common import (
PROBLEM_TYPE_BASE,
ManagementProblem,
build_page_links,
escape_like,
reject_unknown_query_params,
)
from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX
from litellm.proxy.utils import PrismaClient
from litellm.types.proxy.management_endpoints.management_v1 import (
FacetListResponse,

45
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -16,10 +16,10 @@ import json
from collections.abc import Awaitable, Mapping, Sequence
from json import JSONDecodeError
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, cast
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
@ -81,7 +81,10 @@ from litellm.router_strategy.complexity_router import (
ClassificationRubric,
ComplexityRouterConfig,
ComplexityTier,
TierDefinition,
classification_system_prompt,
custom_tier_classification_prompt,
normalize_classification_prompt,
)
from litellm.router_utils.auto_router_model_naming import (
STRATEGY_ROUTER_PARAM_FIELDS,
@ -2230,6 +2233,39 @@ def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[Complexity
) from e
class AutoRouterClassifierPromptPreviewRequest(BaseModel):
"""A POST rather than query params: classification_prompt is the operator's own text, which must
not reach access logs through a URL."""
tier_definitions: tuple[TierDefinition, ...]
context_window_size: Annotated[int, Field(ge=0)] = DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE
classification_prompt: str | None = None
_normalize_prompt = field_validator("classification_prompt")(normalize_classification_prompt)
@router.post(
"/auto_router/classifier/default_prompt",
description="Get the system prompt an auto-router's LLM classifier sends for an edited tier set",
tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list
dependencies=[Depends(user_api_key_auth)], # mutable-ok: fastapi's decorator signature types dependencies as a list
)
async def preview_auto_router_classifier_prompt(
request: AutoRouterClassifierPromptPreviewRequest,
) -> AutoRouterClassifierDefaultPromptResponse:
"""
Get the classifier system prompt an edited tier set sends, so the dashboard can show it.
Built by the same function the live classifier uses, so the preview cannot drift from what the
router sends. Payload validity beyond a renderable definition stays the dry-run's job.
"""
return AutoRouterClassifierDefaultPromptResponse(
system_prompt=custom_tier_classification_prompt(
request.tier_definitions, request.classification_prompt, request.context_window_size
)
)
@router.get(
"/auto_router/classifier/default_prompt",
description="Get the built-in system prompt used by an auto-router's LLM classifier",
@ -2242,13 +2278,16 @@ async def get_auto_router_classifier_default_prompt(
classification_rubric: ClassificationRubric | None = None,
) -> AutoRouterClassifierDefaultPromptResponse:
"""
Get the default classifier system prompt, so the dashboard's prompt editor can prefill it.
Get the classifier system prompt a router would send, so the dashboard can show it.
The prompt's closing line depends on whether prior conversation turns are quoted to the
classifier, its tier bullets are named by the router's tier_labels, and its calibration examples
come from the router's classification rubric, so the caller passes all three to get the text that router
would actually send rather than a rubric it does not use.
An edited tier set replaces the whole rubric; POST to this path for that prompt, which carries
the operator's own instructions and so must not ride in a query string.
Parameters:
- context_window_size: int - The router's classifier_context_window_size. Defaults to the
built-in default.

34
litellm/proxy/policy_engine/pipeline_executor.py
View file

@ -15,6 +15,7 @@ from litellm.integrations.custom_guardrail import (
ModifyResponseException,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import independent_snapshot
from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
UnifiedLLMGuardrails,
)
@ -41,6 +42,7 @@ class PipelineExecutor:
user_api_key_dict: Any,
call_type: str,
policy_name: str,
raw_request_snapshot: dict | None = None, # mutable-ok: same request-payload shape as data
) -> PipelineExecutionResult:
"""
Execute pipeline steps sequentially with conditional actions.
@ -52,6 +54,11 @@ class PipelineExecutor:
user_api_key_dict: User API key auth
call_type: Type of call (completion, etc.)
policy_name: Name of the owning policy (for logging)
raw_request_snapshot: pristine pre-pipeline, pre-guardrail request
(taken by the caller before any guardrail or pipeline ran), so a
step whose guardrail opted into ``scan_raw_request`` evaluates
the original request instead of whatever an earlier
``pass_data`` step in this same pipeline already rewrote.
Returns:
PipelineExecutionResult with terminal action and step results
@ -75,6 +82,7 @@ class PipelineExecutor:
data=working_data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
raw_request_snapshot=raw_request_snapshot,
)
duration = time.perf_counter() - start_time
@ -143,6 +151,7 @@ class PipelineExecutor:
data: dict,
user_api_key_dict: Any,
call_type: str,
raw_request_snapshot: dict | None = None, # mutable-ok: same request-payload shape as data
) -> tuple[
Literal["pass", "fail", "error"],
dict | None,
@ -172,20 +181,33 @@ class PipelineExecutor:
data["metadata"] = {}
data["metadata"]["guardrails"] = [step.guardrail]
# A scan_raw_request step evaluates the pristine pre-pipeline
# snapshot instead of `data` (which earlier pass_data steps in
# this same pipeline may have already rewritten), same reason
# the normal sequential/parallel guardrail loops do this.
scans_raw_request: Final = getattr(callback, "scan_raw_request", False)
hook_input: Final[dict] = ( # mutable-ok: same request-payload shape as data
independent_snapshot(raw_request_snapshot)
if scans_raw_request and raw_request_snapshot is not None
else data
)
if hook_input is not data:
hook_input.setdefault("metadata", {})["guardrails"] = [step.guardrail]
# Use unified_guardrail path if callback implements apply_guardrail
target: CustomLogger = callback
use_unified: Final = (
"apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks
)
if use_unified:
data["guardrail_to_apply"] = callback
hook_input["guardrail_to_apply"] = callback
target = UnifiedLLMGuardrails()
if mode == "pre_call":
response = await target.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=None,
data=data,
data=hook_input,
call_type=call_type,
)
if isinstance(callback, CustomGuardrail):
@ -201,9 +223,13 @@ class PipelineExecutor:
else:
return ("error", None, f"Unsupported pipeline mode: {mode}", None)
# Normal return means pass
# Normal return means pass. A scan_raw_request step is block-only,
# same contract as run_in_parallel/scan_raw_request elsewhere: any
# data it returned is discarded, since applying it on top of the
# raw snapshot would silently undo whatever an earlier step in
# this pipeline already did.
modified_data = None
if response is not None and isinstance(response, dict):
if response is not None and isinstance(response, dict) and not scans_raw_request:
modified_data = response
return ("pass", modified_data, None, None)

41
litellm/proxy/proxy_server.py
View file

@ -432,6 +432,11 @@ from litellm.proxy.hooks.prompt_injection_detection import (
)
from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger
from litellm.proxy.image_endpoints.endpoints import router as image_router
from litellm.proxy.list_api.common import (
PROBLEM_TYPE_BASE,
ManagementProblem,
problem_response,
)
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
rust_control_plane_router,
@ -488,12 +493,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
from litellm.proxy.management_endpoints.management_v1 import (
router as management_v1_router,
)
from litellm.proxy.management_endpoints.management_v1.common import (
MANAGEMENT_V1_PREFIX,
PROBLEM_TYPE_BASE,
ManagementProblem,
problem_response,
)
from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
router as model_access_group_management_router,
)
@ -600,6 +600,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
router as pass_through_router,
)
from litellm.proxy.public_endpoints import router as public_endpoints_router
from litellm.proxy.public_endpoints.public_v1 import router as public_v1_router
from litellm.proxy.rag_endpoints.endpoints import router as rag_router
from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router
from litellm.proxy.response_api_endpoints.endpoints import router as response_router
@ -672,7 +673,12 @@ from litellm.types.llms.anthropic import (
AnthropicResponseContentBlockText,
AnthropicResponseUsageBlock,
)
from litellm.types.llms.openai import HttpxBinaryResponseContent
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionSystemMessage,
ChatCompletionToolParam,
HttpxBinaryResponseContent,
)
from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
ModelGroupInfoProxy,
@ -12126,6 +12132,13 @@ async def _try_provider_token_count(
return result
def _system_message(system: object) -> ChatCompletionSystemMessage | None:
if not isinstance(system, (str, list)) or not system:
return None
message: Final[ChatCompletionSystemMessage] = {"role": "system", "content": system}
return message
@router.post(
"/utils/token_counter",
tags=["llm utils"],
@ -12224,10 +12237,21 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False)
_tokenizer_used: Final = litellm.utils._select_tokenizer(model=model_to_use, custom_tokenizer=custom_tokenizer)
tokenizer_used: Final = str(_tokenizer_used["type"])
system_message: Final = _system_message(system)
typed_messages: Final = cast( # cast-ok: request messages are raw chat-shaped dicts that token_counter normalizes
Sequence[AllMessageValues] | None, messages
)
counted_messages: Final = (
typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages)
)
counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats
list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None
)
total_tokens: Final = await asyncify(litellm.token_counter)(
model=model_to_use,
text=prompt,
messages=messages,
messages=counted_messages,
tools=counted_tools,
custom_tokenizer=_tokenizer_used,
)
return TokenCountResponse(
@ -17661,6 +17685,7 @@ async def get_routes():
app.include_router(router)
app.include_router(response_router)
app.include_router(public_endpoints_router)
app.include_router(public_v1_router)
app.include_router(rerank_router)
app.include_router(ocr_router)
app.include_router(rag_router)

14
litellm/proxy/public_endpoints/public_v1/__init__.py Normal file
View file

@ -0,0 +1,14 @@
"""The `/public/v1` unauthenticated public surface."""
from typing import Final
from fastapi import APIRouter
from litellm.proxy.public_endpoints.public_v1.model_hub import router as model_hub_router
PUBLIC_V1_PREFIX: Final = "/public/v1"
router: Final = APIRouter(prefix=PUBLIC_V1_PREFIX)
router.include_router(model_hub_router)
__all__ = ("PUBLIC_V1_PREFIX", "router")

242
litellm/proxy/public_endpoints/public_v1/model_hub.py Normal file
View file

@ -0,0 +1,242 @@
"""`GET /public/v1/model_hub`."""
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Annotated, Final, Protocol
from fastapi import APIRouter, Depends, Request
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem
from litellm.proxy.list_api.in_memory import Cells, InMemoryListExecutor
from litellm.proxy.list_api.list_framework import (
FilterSpec,
ListSpec,
Scope,
ScopeAll,
SortKey,
handle_list,
)
from litellm.proxy.utils import PrismaClient
from litellm.types.proxy.management_endpoints.management_v1 import (
ListResponse,
ProblemDetail,
)
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
ModelGroupInfoProxy,
)
router: Final = APIRouter()
@dataclass(frozen=True, slots=True)
class HealthSnapshot:
"""The health fields a model hub row carries, as the latest health check recorded them."""
status: str | None
response_time_ms: float | None
checked_at: str | None
class HealthSnapshotLookup(Protocol):
"""The health half of the list, injected so the page slice decides how much of it runs."""
async def latest_for(self, model_groups: Sequence[str]) -> Mapping[str, HealthSnapshot]: ...
@dataclass(frozen=True, slots=True)
class PrismaHealthSnapshotLookup:
prisma_client: PrismaClient
async def latest_for(self, model_groups: Sequence[str]) -> Mapping[str, HealthSnapshot]:
checks: Final = await self.prisma_client.get_latest_health_checks_for_models(model_groups)
return MappingProxyType(
{
check.model_name: HealthSnapshot(
status=check.status,
response_time_ms=check.response_time_ms,
checked_at=check.checked_at.isoformat() if check.checked_at else None,
)
for check in checks
}
)
class _HealthFields(TypedDict):
health_status: ReadOnly[str | None]
health_response_time: ReadOnly[float | None]
health_checked_at: ReadOnly[str | None]
def _with_health(row: ModelGroupInfoProxy, health: HealthSnapshot | None) -> ModelGroupInfoProxy:
if health is None:
return row
update: Final[_HealthFields] = {
"health_status": health.status,
"health_response_time": health.response_time_ms,
"health_checked_at": health.checked_at,
}
return row.model_copy(update=update)
@dataclass(frozen=True, slots=True)
class HealthEnricher:
"""Resolves health for exactly the rows handed to it, which is the page and never the match set."""
lookup: HealthSnapshotLookup
async def __call__(self, rows: Sequence[ModelGroupInfoProxy]) -> Sequence[ModelGroupInfoProxy]:
health: Final = await self.lookup.latest_for(tuple(row.model_group for row in rows))
return tuple(_with_health(row, health.get(row.model_group)) for row in rows)
def _cells(row: ModelGroupInfoProxy) -> Cells:
return MappingProxyType(
{
"model_group": row.model_group,
"mode": row.mode,
"providers": tuple(row.providers),
"max_input_tokens": row.max_input_tokens,
"max_output_tokens": row.max_output_tokens,
"input_cost_per_token": row.input_cost_per_token,
"output_cost_per_token": row.output_cost_per_token,
}
)
def _serialize(row: ModelGroupInfoProxy) -> ModelGroupInfoProxy:
"""The row shape is the wire shape: the rows served are the router's own model group records."""
return row
def _scope(_caller: UserAPIKeyAuth) -> Scope:
"""Unconditional, and `/public/v1` is the one surface where that is allowed.
Every row here is already a model group the operator published, so a public browse
caller seeing all of them is the answer, not a gap in the scoping.
"""
return ScopeAll()
MODEL_HUB_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
{
"mode": FilterSpec(type=str, ops=frozenset(("eq", "in"))),
"providers": FilterSpec(type=str, ops=frozenset(("contains",))),
}
)
MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ListSpec(
resource="model groups",
sortable=frozenset(
(
"model_group",
"mode",
"max_input_tokens",
"max_output_tokens",
"input_cost_per_token",
"output_cost_per_token",
)
),
searchable=frozenset(("model_group",)),
filters=MODEL_HUB_FILTERS,
default_sort=(SortKey(field="model_group", descending=False),),
default_page_size=50,
max_page_size=100,
scope=_scope,
serialize=_serialize,
tiebreaker="model_group",
)
def _executor(
rows: Sequence[ModelGroupInfoProxy],
prisma_client: PrismaClient | None,
) -> InMemoryListExecutor[ModelGroupInfoProxy]:
if prisma_client is None:
return InMemoryListExecutor(rows=rows, cells=_cells)
return InMemoryListExecutor(
rows=rows,
cells=_cells,
enrich_page=HealthEnricher(lookup=PrismaHealthSnapshotLookup(prisma_client=prisma_client)),
)
@router.get(
"/model_hub",
tags=["public", "model management"], # mutable-ok: fastapi types tags as list[str | Enum]
dependencies=(Depends(user_api_key_auth),),
response_model=ListResponse[ModelGroupInfoProxy],
)
async def public_model_hub_list(
request: Request,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> ListResponse[ModelGroupInfoProxy]:
"""
The public model groups this proxy publishes, paged, sortable, searchable and
filterable, for the public Model Hub page. No authentication.
A rejected request answers with the parameters, sort fields and filter operators
it would have accepted, so the accepted set stays discoverable from the endpoint
itself rather than from a copy of the spec kept here.
Example curl:
```
curl --location --globoff \
'http://0.0.0.0:4000/public/v1/model_hub?sort=-input_cost_per_token&filter[mode][in]=chat&page_size=25'
```
"""
try:
from litellm.proxy.proxy_server import (
_get_model_group_info, # pyright: ignore[reportPrivateUsage] # /public/model_hub imports it the same way
llm_router,
prisma_client,
)
if llm_router is None:
raise ManagementProblem(
ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}no-llm-router",
title="No models configured",
status=400,
detail=CommonProxyErrors.no_llm_router.value,
)
)
rows: Final[Sequence[ModelGroupInfoProxy]] = (
()
if litellm.public_model_groups is None
else tuple(
_get_model_group_info(
llm_router=llm_router,
all_models_str=litellm.public_model_groups,
model_group=None,
)
)
)
return await handle_list(
spec=MODEL_HUB_LIST_SPEC,
executor=_executor(rows, prisma_client),
request=request,
caller=user_api_key_dict,
)
except ManagementProblem:
raise
except Exception as e: # noqa: BLE001 # a router error answers as a problem document, not the OpenAI error shape
verbose_proxy_logger.exception(
"litellm.proxy.public_endpoints.public_v1.model_hub.public_model_hub_list(): Exception occured - %s", e
)
raise ManagementProblem(
ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}internal-server-error",
title="Internal server error",
status=500,
detail="Failed to list public model groups.",
)
)

21
litellm/proxy/schema.prisma
View file

@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -1527,12 +1533,27 @@ model LiteLLM_ShadowEvalAttempt {
confidence Float?
judge_cost Float @default(0)
shadow_cost Float @default(0)
real_cost Float? // NULL = row predates cost measurement; comparisons read only measured rows
real_classifier_cost Float @default(0)
shadow_classifier_cost Float @default(0)
real_cache_hit Boolean @default(false)
error String?
created_at DateTime @default(now())
@@index([job_id])
}
// Per-leg sampling funnel counters the attempt rows cannot derive: requests an
// admitting job saw but did not judge. attempted = the leg's attempt rows; the
// leg's eligible traffic = not_sampled + unjudgeable + shed + withheld + attempted.
model LiteLLM_ShadowEvalFunnel {
job_id String @id
not_sampled Int @default(0)
unjudgeable Int @default(0)
shed Int @default(0)
withheld Int @default(0)
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

55
litellm/proxy/spend_tracking/savings.py
View file

@ -15,6 +15,10 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token
from litellm.types.integrations.anthropic_cache_control_hook import (
GATEWAY_INJECTED_CACHE_METADATA_KEY,
GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
)
if TYPE_CHECKING:
from litellm.router import Router
@ -25,6 +29,7 @@ class SavingsSpend(NamedTuple):
compression: float
prompt_caching: float
autorouter: float = 0.0
gateway_injected_caching: float = 0.0
def _input_cache_read_and_write_cost(info: ModelInfo | None) -> tuple[float, float, float]:
@ -391,6 +396,28 @@ def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage |
return None
def marks_gateway_injection(metadata: Mapping[str, object] | None, model_id: str | None) -> bool:
"""Whether the gateway put cache breakpoints on the payload THIS row was billed for.
``AnthropicCacheControlHook.record_gateway_injection`` stamps the deployment it
injected for, and a row carries the deployment it was billed for, so the two agree
only on the leg that was actually injected. Every retry, failover and fallback of a
request shares one metadata bucket and one ``litellm_call_id``, so the deployment is
what tells those legs apart, and a marker left by a sibling reads here as no injection
without anyone having to strip it. An injection that ran before any deployment was
chosen is in the payload every leg sends, so it is marked for all of them and credits
each. Absent on requests the gateway never acted on
(client-supplied ``cache_control``, implicit provider caching) and on rows written
before the marker shipped; all of it is the fail-closed direction.
"""
if not metadata:
return False
injected_deployment: Final = metadata.get(GATEWAY_INJECTED_CACHE_METADATA_KEY)
if not isinstance(injected_deployment, str):
return False
return injected_deployment in (GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT, model_id)
def extract_cache_read_tokens(usage_object: Mapping[str, object] | None) -> int:
"""Cache-read tokens from a logged usage object, whatever shape recorded them.
@ -475,14 +502,11 @@ def autorouter_savings_for_request(
usage: Final = _usage_from_spend_log(usage_object)
if usage is None or not model:
return None
# The configured `autorouter_savings_baseline_model` wins; otherwise the baseline
# the deciding router recorded on its decision; neither means the driver is off.
decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {}
recorded: Final = decision.get("savings_baseline_model")
recorded_id: Final = decision.get("savings_baseline_deployment_id")
configured: Final = litellm.autorouter_savings_baseline_model
baseline_model: Final = configured or (recorded if isinstance(recorded, str) else None)
baseline_id: Final = recorded_id if configured is None and isinstance(recorded_id, str) else None
baseline_model: Final = recorded if isinstance(recorded, str) else None
baseline_id: Final = recorded_id if isinstance(recorded_id, str) else None
if not decision or not baseline_model:
return None
router_instance: Final = llm_router() if llm_router else None
@ -533,6 +557,7 @@ def compute_savings_spend(
model: str | None,
custom_llm_provider: str | None,
compression_saved_tokens: int,
gateway_injected_cache: bool,
routing_decision: Mapping[str, object] | None = None,
usage_object: Mapping[str, object] | None = None,
model_id: str | None = None,
@ -565,7 +590,23 @@ def compute_savings_spend(
A request that only writes cache and gets no hits therefore reports negative savings,
which is accurate: it really did cost more than the uncached call would have. The
daily rollup increments arithmetically, so those rows offset positive ones in the
same bucket. Auto-router savings compare the
same bucket.
Caching is reported twice. ``prompt_caching`` is every net dollar caching saved,
whoever caused it, which is what a customer means by "what did caching save me".
``gateway_injected_caching`` is the subset the gateway can claim credit for, carrying
a value only when ``gateway_injected_cache`` is set, i.e. litellm itself added the
``cache_control`` breakpoints (configured injection points or the auto prompt-caching
flag). A client that sent its own breakpoints, and a provider that
caches implicitly (OpenAI, Gemini), produce the same usage shape with no gateway
action, so they count toward the total and not toward the attributed figure.
Reporting both rather than gating the one column keeps the customer-facing number
stable across the change and leaves attribution a separate question. The attributed
figure is normally the smaller of the two, being a subset of the same requests, but
not always: a request that only writes cache and never reads it has negative net
savings, and dropping such a request from the attributed figure can lift it above
the total. Auto-router savings compare the
served ``model`` against the counterfactual baseline the router recorded on
its ``routing_decision``, and are zero unless the two differ. That record
also says whether the conversation was already underway, which is what tells
@ -602,6 +643,7 @@ def compute_savings_spend(
read_discount: Final = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0)
write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost)
prompt_caching: Final = read_discount - write_premium
gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0
# The figure the logging path recorded wins, before the usage gate on purpose: a row
# whose usage no longer parses still carries the number computed when it did.
@ -623,4 +665,5 @@ def compute_savings_spend(
compression=compression,
prompt_caching=prompt_caching,
autorouter=0.0 if autorouter is None else autorouter,
gateway_injected_caching=gateway_injected_caching,
)

1
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -137,6 +137,7 @@ def _get_spend_logs_metadata(
cost_breakdown=None,
compression_savings=None,
autorouter_savings=autorouter_savings,
litellm_gateway_injected_cache=None,
litellm_call_id=litellm_call_id,
)
verbose_proxy_logger.debug(

201
litellm/proxy/utils.py
View file

@ -91,7 +91,11 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus import PrometheusLogger
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert
from litellm.litellm_core_utils.core_helpers import coerce_token_limit, is_expected_client_error
from litellm.litellm_core_utils.core_helpers import (
coerce_token_limit,
independent_snapshot,
is_expected_client_error,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@ -1387,6 +1391,83 @@ class ProxyLogging:
return data
async def _run_sequential_guardrail_callback(
self,
callback: CustomGuardrail,
data: dict, # mutable-ok: matches _process_guardrail_callback's own request-payload typing
raw_request_snapshot: dict | None, # mutable-ok: same request-payload shape as data
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
) -> dict: # mutable-ok: callers reassign the loop's own data from this return value
"""
Run one guardrail from the sequential pre_call loop and return what the
rest of the loop should carry forward.
A guardrail opted into ``scan_raw_request`` always evaluates a fresh
copy of ``raw_request_snapshot`` (taken before any guardrail in this
hook ran) instead of ``data`` (the live, possibly already-mutated
payload), so its block/pass decision can never depend on where it's
declared relative to a guardrail that masks or rewrites content. It's
declared block-only, same contract as ``run_in_parallel``: any data it
returns is discarded, since applying its view on top of a stale
snapshot would silently undo whatever a later guardrail already did to
the live request. A guardrail that mutates content (e.g. PII masking)
should never set this flag -- if one does anyway, its returned
mutation is discarded and a warning is logged so the misconfiguration
is visible instead of silently forwarding unredacted content.
"""
scans_raw_request: Final = getattr(callback, "scan_raw_request", False)
should_use_raw_snapshot: Final = scans_raw_request and raw_request_snapshot is not None
input_data: Final = ( # mutable-ok: same request-payload shape as data
independent_snapshot(raw_request_snapshot) if should_use_raw_snapshot else data
)
# _process_guardrail_callback always calls mark_pre_call_hook_ran on a
# successful run, which unconditionally stamps bookkeeping metadata onto
# the dict regardless of whether the guardrail's own hook mutated
# anything -- so comparing `result` straight against `input_data` would
# warn on every single scan_raw_request call. Apply that same stamp to a
# throwaway, guaranteed-independent copy first (never the live request or
# raw_request_snapshot itself) so the comparison isolates the guardrail's
# own content mutation from this bookkeeping noise without risking a
# premature marker write into shared state.
expected_if_unmutated: Final[dict | None] = ( # mutable-ok: same request-payload shape as data
independent_snapshot(input_data) if scans_raw_request else None
)
if expected_if_unmutated is not None:
callback.mark_pre_call_hook_ran(expected_if_unmutated)
result: Final = await self._process_guardrail_callback(
callback=callback,
data=input_data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
event_type=GuardrailEventHooks.pre_call,
)
if (
scans_raw_request
and expected_if_unmutated is not None
and result is not None
and result != expected_if_unmutated
):
verbose_proxy_logger.warning(
"Guardrail '%s' has scan_raw_request=True but returned a modified payload; "
"scan_raw_request is for block-only guardrails and this mutation is being "
"discarded. Remove scan_raw_request from this guardrail's config if it needs "
"to mask/rewrite content.",
getattr(callback, "guardrail_name", None) or callback.__class__.__name__,
)
if scans_raw_request:
if result is not None:
# _process_guardrail_callback only stamped input_data (a throwaway
# snapshot copy), never the live data returned here -- without this,
# a deployment-level guardrail sharing this name would see no marker
# via _pre_call_hook_already_ran and re-run the same guardrail a
# second time on live kwargs.
callback.mark_pre_call_hook_ran(data)
return data
if result is None:
return data
return result
async def _process_prompt_template(
self,
data: dict,
@ -1442,6 +1523,7 @@ class ProxyLogging:
prompt_variables=data.pop("prompt_variables", None) or {},
prompt_label=data.pop("prompt_label", None) or {},
prompt_version=data.pop("prompt_version", None) or {},
request_kwargs=data,
)
data.update(optional_params)
@ -1495,6 +1577,7 @@ class ProxyLogging:
user_api_key_dict: UserAPIKeyAuth,
call_type: str,
event_hook: str,
raw_request_snapshot: dict | None = None, # mutable-ok: same request-payload shape as data
) -> dict:
"""
Execute guardrail pipelines if any are configured for this request.
@ -1502,6 +1585,11 @@ class ProxyLogging:
Checks metadata for pipelines resolved by the policy engine
and executes them. Handles the result (allow/block/modify_response).
``raw_request_snapshot`` (taken before any guardrail or pipeline ran)
is forwarded so a pipeline step whose guardrail opted into
``scan_raw_request`` evaluates the pristine request, not whatever an
earlier ``pass_data`` step in the same pipeline already rewrote.
Returns the (possibly modified) data dict.
"""
pipelines: Final = _policy_pipelines(data)
@ -1519,6 +1607,7 @@ class ProxyLogging:
user_api_key_dict=user_api_key_dict,
call_type=call_type,
policy_name=policy_name,
raw_request_snapshot=raw_request_snapshot,
)
data = self._handle_pipeline_result(
@ -1678,6 +1767,24 @@ class ProxyLogging:
call_type=call_type,
)
# Snapshotted here, before _maybe_execute_pipelines or any guardrail in
# this hook has run, so a scan_raw_request guardrail's block/pass
# decision never depends on its position in the guardrails list or on
# a pipeline that runs ahead of it: an earlier guardrail (pipelined or
# not) that masks/rewrites content can't hide a violation from a later
# one that opted into scanning the original request. Only computed
# when at least one registered guardrail actually opted in, and via
# independent_snapshot (not safe_deep_copy) since this isolation
# guarantee must hold even under litellm.safe_memory_mode, which
# otherwise makes deep copies return the original object.
needs_raw_request_snapshot: Final = any(
isinstance(cb, CustomGuardrail) and getattr(cb, "scan_raw_request", False)
for cb in ProxyLogging._callback_capabilities().resolved_callbacks
)
raw_request_snapshot: Final[dict | None] = ( # mutable-ok: same request-payload shape as data
independent_snapshot(data) if needs_raw_request_snapshot else None
)
try:
# Execute guardrail pipelines before the normal callback loop
data = await self._maybe_execute_pipelines(
@ -1685,6 +1792,7 @@ class ProxyLogging:
user_api_key_dict=user_api_key_dict,
call_type=call_type,
event_hook="pre_call",
raw_request_snapshot=raw_request_snapshot,
)
# Get pipeline-managed guardrails to skip in normal loop
@ -1725,16 +1833,13 @@ class ProxyLogging:
if getattr(_callback, "run_in_parallel", False):
continue
result = await self._process_guardrail_callback(
data = await self._run_sequential_guardrail_callback(
callback=_callback,
data=data,
raw_request_snapshot=raw_request_snapshot,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
event_type=GuardrailEventHooks.pre_call,
)
if result is None:
continue
data = result
elif (
_callback is not None
@ -1786,6 +1891,7 @@ class ProxyLogging:
await self._run_parallel_pre_call_guardrails(
guardrails=parallel_guardrails,
data=data,
raw_request_snapshot=raw_request_snapshot,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
)
@ -1806,6 +1912,7 @@ class ProxyLogging:
self,
guardrails: tuple[CustomGuardrail, ...],
data: dict,
raw_request_snapshot: dict | None, # mutable-ok: same request-payload shape as data
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
) -> None:
@ -1822,12 +1929,24 @@ class ProxyLogging:
the LLM, preserving the pre-call barrier that ``during_call`` guardrails
cannot provide. Per-guardrail latency is recorded by
``_process_guardrail_callback``'s own metrics.
A guardrail that also opted into ``scan_raw_request`` evaluates
``raw_request_snapshot`` (taken before the sequential loop ran) instead
of ``data`` (the sequential loop's output), for the same reason the
sequential branch does: its block decision must not depend on what a
sequential guardrail already masked or rewrote.
"""
def _input_for(callback: CustomGuardrail) -> dict: # mutable-ok: same request-payload shape as data
if not getattr(callback, "scan_raw_request", False) or raw_request_snapshot is None:
return data
return independent_snapshot(raw_request_snapshot)
results: Final = await asyncio.gather(
*(
self._process_guardrail_callback(
callback=callback,
data=data,
data=_input_for(callback),
user_api_key_dict=user_api_key_dict,
call_type=call_type,
event_type=GuardrailEventHooks.pre_call,
@ -1836,6 +1955,19 @@ class ProxyLogging:
),
return_exceptions=True,
)
for callback, result in zip(guardrails, results, strict=True):
# _process_guardrail_callback stamped mark_pre_call_hook_ran on
# _input_for's throwaway snapshot copy for a scan_raw_request
# guardrail, never on the live, shared `data` -- without this, a
# deployment-level guardrail sharing this name would see no marker
# via _pre_call_hook_already_ran and re-run it a second time on
# live kwargs.
if (
getattr(callback, "scan_raw_request", False)
and not isinstance(result, BaseException)
and result is not None
):
callback.mark_pre_call_hook_ran(data)
raised: Final = tuple(result for result in results if isinstance(result, BaseException))
blocking: Final = next((exc for exc in raised if not _exception_changes_request_flow(exc)), None)
if blocking is not None:
@ -3035,8 +3167,14 @@ class ProxyLogging:
# through each of them adds N pass-through trampolines per chunk for
# zero behavior change. Skip the chain entirely and stream through.
if not caps.iterator_overrides:
async for chunk in response:
yield chunk
try:
async for chunk in response:
yield chunk
except (GeneratorExit, asyncio.CancelledError):
raise
except Exception:
ProxyLogging._fire_deferred_stream_logging(request_data)
raise
ProxyLogging._fire_deferred_stream_logging(request_data)
return
@ -3089,9 +3227,14 @@ class ProxyLogging:
),
)
# Actually iterate through the chained async generator and yield chunks
async for chunk in current_response:
yield chunk
try:
async for chunk in current_response:
yield chunk
except (GeneratorExit, asyncio.CancelledError):
raise
except Exception:
ProxyLogging._fire_deferred_stream_logging(request_data)
raise
# Fire deferred logging AFTER all guardrail end-of-stream blocks
# completed. unified_guardrail writes guardrail_information during
@ -5832,6 +5975,29 @@ class PrismaClient:
verbose_proxy_logger.error("Error getting all latest health checks: %s", e)
return []
async def get_latest_health_checks_for_models(
self, model_names: "Sequence[str]"
) -> "Sequence[prisma_models.LiteLLM_HealthCheckTable]":
"""
Get the latest health check for each of the named models.
Same DISTINCT ON as ``get_all_latest_health_checks``, bounded to the models asked
about, so a paged caller reads health for its page instead of for the whole table.
"""
if not model_names:
return ()
latest_first: Final = (("model_id", "asc"), ("model_name", "asc"), ("checked_at", "desc"))
order: Final = [{field: direction} for field, direction in latest_first] # mutable-ok: prisma order is a list
try:
return await HealthCheckRepository(self).table.find_many(
where={"model_name": {"in": list(model_names)}}, # mutable-ok: prisma filters are dicts and lists
distinct=["model_id", "model_name"], # mutable-ok: prisma distinct takes a list
order=order,
)
except Exception as e: # noqa: BLE001 # health decorates a list; a driver error must not fail the page
verbose_proxy_logger.error("Error getting latest health checks for models: %s", e)
return ()
### HELPER FUNCTIONS ###
@ -6276,7 +6442,9 @@ async def _total_queued_spend_transactions(prisma_client: PrismaClient) -> int:
tool_queue_size: Final = len(prisma_client.tool_usage_transactions)
async with prisma_client._autorouter_turn_transactions_lock:
autorouter_queue_size: Final = len(prisma_client.autorouter_turn_transactions)
return spend_queue_size + tool_queue_size + autorouter_queue_size
from litellm.proxy.db.shadow_eval_funnel import pending_shadow_eval_funnel_events
return spend_queue_size + tool_queue_size + autorouter_queue_size + pending_shadow_eval_funnel_events()
async def update_daily_tag_spend(
@ -6418,6 +6586,13 @@ async def update_spend_logs_job(
autorouter_tracking_err,
)
try:
from litellm.proxy.db.shadow_eval_funnel import flush_shadow_eval_funnel
await flush_shadow_eval_funnel(prisma_client)
except Exception as funnel_err: # noqa: BLE001 # a drain bug must not abort the spend job
verbose_proxy_logger.error("Spend tracking - shadow eval funnel drain failed: %s", funnel_err)
MAX_SPEND_LOG_DRAIN_ITERATIONS: Final = 20

2
litellm/responses/main.py
View file

@ -537,6 +537,7 @@ async def aresponses(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
input = cast(
str | ResponseInputParam,
@ -692,6 +693,7 @@ def _apply_prompt_management_to_responses_call(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
input = cast(
str | ResponseInputParam,

163
litellm/router.py
View file

@ -64,6 +64,7 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.ptu_pricing import (
PTU_COST_ATTRIBUTION_ENV_VAR,
@ -229,6 +230,7 @@ from litellm.types.utils import (
StandardLoggingPayload,
StandardLoggingRoutingDecision,
Usage,
all_litellm_params,
shared_backend_model_info,
)
from litellm.types.utils import ModelInfo as ModelMapInfo
@ -243,6 +245,7 @@ from litellm.utils import (
get_secret,
get_utc_datetime,
is_region_allowed,
provider_rejectable_params,
set_live_deployment_replay,
)
@ -3998,6 +4001,7 @@ class Router:
prompt_id=prompt_id,
prompt_variables=prompt_variables,
prompt_label=prompt_label,
request_kwargs=kwargs,
)
# Filter out prompt management specific parameters from data before merging
@ -7047,13 +7051,11 @@ class Router:
_sibling_metadata_key: Final = (
"metadata" if _fallback_metadata_key == "litellm_metadata" else "litellm_metadata"
)
if isinstance(_sibling_metadata := kwargs.get(_sibling_metadata_key), dict) and (
"attempted_fallbacks" in _sibling_metadata or "original_model_group" in _sibling_metadata
):
_scrubbed_sibling_metadata: Final = _sibling_metadata.copy()
_scrubbed_sibling_metadata.pop("attempted_fallbacks", None)
_scrubbed_sibling_metadata.pop("original_model_group", None)
kwargs[_sibling_metadata_key] = _scrubbed_sibling_metadata
if isinstance(_sibling_metadata := kwargs.get(_sibling_metadata_key), dict):
# In place, like every other router bucket write: downstream resolves the bucket by
# key presence, so rebinding kwargs to a copy detaches the proxy's request_data write-backs
_sibling_metadata.pop("attempted_fallbacks", None)
_sibling_metadata.pop("original_model_group", None)
if isinstance(_fallback_metadata := kwargs.get(_fallback_metadata_key), dict):
_fallback_metadata["attempted_fallbacks"] = 0
if model_group is not None:
@ -10832,6 +10834,114 @@ class Router:
}
return {**deployment, "model_info": model_info} # mutable-ok: DeploymentTypedDict rows are plain dicts
TIER_PARAMS_NEVER_DROPPED: Final = frozenset(all_litellm_params) | frozenset(
{
"additional_drop_params",
"drop_params",
"messages",
"model",
"extra_headers",
"max_tokens",
"max_completion_tokens",
}
)
@staticmethod
def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]:
declared: Final = params.get("allowed_openai_params")
if not isinstance(declared, (list, tuple, set, frozenset)):
return frozenset()
return frozenset(entry for entry in declared if isinstance(entry, str))
@staticmethod
def _deployment_accepts_param(deployment: DeploymentTypedDict, group: str, param: str) -> bool:
deployment_params: Final = deployment.get("litellm_params")
if not deployment_params:
return True
if param in Router._declared_param_allowlist(deployment_params):
return True
if declared_authenticating_provider(
str(deployment_params.get("model") or ""), deployment_params.get("custom_llm_provider")
):
return True
deployment_model_info: Final = deployment.get("model_info")
base_model: Final = (
deployment_model_info.get("base_model") if deployment_model_info else None
) or deployment_params.get("base_model")
try:
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=deployment_params.get("model") or group,
custom_llm_provider=deployment_params.get("custom_llm_provider"),
)
supported: Final = litellm.get_supported_openai_params(
model=model,
custom_llm_provider=custom_llm_provider,
base_model=base_model if isinstance(base_model, str) else None,
)
except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
verbose_router_logger.debug(
"litellm.router.py::_deployment_accepts_param: keeping %s for model=%s. Got - %s", param, group, e
)
return True
return supported is None or param in supported
def _tier_params_the_target_accepts(
self, model: str, tier_params: Mapping[str, object], request_kwargs: Mapping[str, object]
) -> Mapping[str, object]:
"""Drop an OpenAI param that no deployment behind ``model`` declares.
A tier's litellm_params are an operator override applied to every request the tier routes,
so one the target cannot take turns that whole tier into a 400 raised before the request
leaves the proxy. The candidates are exactly what get_optional_params can reject, asked of
the module that raises, so credentials and endpoint controls are never at risk.
TIER_PARAMS_NEVER_DROPPED is excluded on top of that, for two reasons. No provider lists a
litellm control among its supported params, so "no deployment declares it" means litellm
consumes it rather than that the target refuses it, and dropping one changes litellm's own
behavior: dropping drop_params or additional_drop_params silently disables the sanitization
the operator configured. Providers do list extra_headers, but it carries auth, tenancy and
routing information, so sending fewer headers than configured is worse than today's error.
Token ceilings stay for the same reason: a tier's max_tokens or max_completion_tokens is a
cost bound, and dropping it would let a caller's own larger value through where today the
mismatch fails loudly.
The trade this filter makes is a param for a working request, which is right for one that
only shapes how the model answers and wrong for anything else.
A param survives if ANY deployment could take it, because routing has not chosen one yet,
and it survives both an unresolvable provider and a group with no deployments, because a
best-effort filter must never narrow what the request already did.
A github_copilot or chatgpt deployment counts as accepting everything, decided before any
lookup: resolving either provider runs its OAuth device flow, so a capability question
asked from the routing path can freeze the event loop for minutes waiting on a human.
allowed_openai_params is the documented escape hatch for an outdated or incomplete
supported-params list: request-time validation extends the supported list with it before
comparing. The filter asks the same question, so a param named by the allowlist on the tier
overlay, the request, or a deployment's own litellm_params is never a drop candidate.
"""
deployments: Final = self.get_model_list(model_name=model) or ()
if not deployments:
return tier_params
allowlisted: Final = self._declared_param_allowlist(tier_params) | self._declared_param_allowlist(
request_kwargs
)
candidates: Final = provider_rejectable_params(tier_params) - self.TIER_PARAMS_NEVER_DROPPED - allowlisted
unsupported: Final = frozenset(
param
for param in candidates
if not any(self._deployment_accepts_param(deployment, model, param) for deployment in deployments)
)
if not unsupported:
return tier_params
verbose_router_logger.warning(
"litellm.router.py: dropping tier params %s for model=%s, no deployment behind it declares them",
", ".join(sorted(unsupported)),
model,
)
return MappingProxyType({key: value for key, value in tier_params.items() if key not in unsupported})
def get_model_list(
self, model_name: str | None = None, team_id: str | None = None
) -> list[DeploymentTypedDict] | None:
@ -11842,6 +11952,33 @@ class Router:
return healthy_deployments
@staticmethod
def _pop_effort_from_nested_carrier(request_kwargs: dict[str, object], carrier: str) -> None:
nested: Final = request_kwargs.get(carrier)
if not isinstance(nested, dict):
return
nested.pop("effort", None)
if not nested:
request_kwargs.pop(carrier, None)
@staticmethod
def _drop_client_effort_carriers_a_tier_pin_supersedes(
request_kwargs: dict[str, object],
tier_litellm_params: Mapping[str, object],
) -> None:
"""Tier litellm_params are deliberate operator overrides, but provider
translations let a caller-supplied carrier of the same setting
(``thinking``, ``output_config.effort``, ``reasoning.effort``) outrank
the ``reasoning_effort`` alias, so a pinned effort only reaches the wire
if the client's other encodings are removed before the merge. Non-effort
fields a carrier also holds (``output_config.format``,
``reasoning.summary``) are kept."""
if "reasoning_effort" not in tier_litellm_params:
return
request_kwargs.pop("thinking", None)
Router._pop_effort_from_nested_carrier(request_kwargs, "output_config")
Router._pop_effort_from_nested_carrier(request_kwargs, "reasoning")
async def async_get_available_deployment(
self,
model: str,
@ -11887,7 +12024,11 @@ class Router:
model = pre_routing_hook_response.model
messages = pre_routing_hook_response.messages
if pre_routing_hook_response.litellm_params:
request_kwargs.update(pre_routing_hook_response.litellm_params)
accepted_tier_params: Final = self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs
)
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
request_kwargs.update(accepted_tier_params)
#########################################################
# Resolve the strategy and logger AFTER the pre-routing hook, since
@ -11998,7 +12139,11 @@ class Router:
model = pre_routing_hook_response.model
messages = pre_routing_hook_response.messages
if pre_routing_hook_response.litellm_params:
request_kwargs.update(pre_routing_hook_response.litellm_params)
accepted_tier_params: Final = self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs
)
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
request_kwargs.update(accepted_tier_params)
# 2. Get healthy deployments
healthy_deployments: Final = await self.async_get_healthy_deployments(

6
litellm/router_strategy/complexity_router/__init__.py
View file

@ -10,6 +10,7 @@ No external API calls - all scoring is local and <1ms.
from litellm.router_strategy.complexity_router.complexity_router import (
ComplexityRouter,
classification_system_prompt,
custom_tier_classification_prompt,
)
from litellm.router_strategy.complexity_router.config import (
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
@ -18,6 +19,8 @@ from litellm.router_strategy.complexity_router.config import (
ComplexityRouterConfig,
ComplexityTier,
ReminderMarkerPair,
TierDefinition,
normalize_classification_prompt,
)
__all__ = [
@ -28,5 +31,8 @@ __all__ = [
"ComplexityRouterConfig",
"ComplexityTier",
"ReminderMarkerPair",
"TierDefinition",
"classification_system_prompt",
"custom_tier_classification_prompt",
"normalize_classification_prompt",
]

139
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -48,6 +48,7 @@ from .config import (
DEFAULT_REASONING_KEYWORDS,
DEFAULT_SIMPLE_KEYWORDS,
DEFAULT_TECHNICAL_KEYWORDS,
HOUSEKEEPING_ASK_SENTINELS,
PLAN_MODE_SYSTEM_SENTINELS,
PLAN_MODE_TAIL_SENTINELS,
PLAN_MODE_TOOL_NAME,
@ -55,6 +56,7 @@ from .config import (
ClassificationRubric,
ComplexityRouterConfig,
ComplexityTier,
TierDefinition,
)
if TYPE_CHECKING:
@ -196,6 +198,26 @@ def _custom_tier_prompt(entries: Sequence[tuple[str, str]], preamble: str | None
)
def custom_tier_classification_prompt(
definitions: Sequence[TierDefinition],
classification_prompt: str | None,
context_window_size: int,
) -> str:
"""The classifier's system role for an operator-defined tier set.
The single owner of the built-in-criteria substitution, so the dashboard's preview resolves a
blank description exactly as the live classifier does.
"""
entries: Final = tuple(
(
definition.name,
definition.description or _CLASSIFICATION_TIER_CRITERIA[ComplexityTier[definition.name.upper()]],
)
for definition in definitions
)
return _custom_tier_prompt(entries, classification_prompt, _closing_line(context_window_size))
def classification_system_prompt(
context_window_size: int,
custom_prompt: str | None = None,
@ -679,8 +701,17 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo
on the floor's premium model after the user exits plan mode; leaving it unpinned means the
floor re-detects while plan mode lasts and the first ordinary turn classifies and pins as
if plan mode had never happened.
A housekeeping call is transient in the same way, and pinning it is the most expensive mistake
of the three: an agent names the conversation on its first turn, so the cheapest tier would be
the pin every session starts with, and the real work that follows would run there for the whole
TTL. It describes what that one call is, never what the session's traffic looks like.
"""
return decision is None or decision.get("cause") not in ("default_model_fallback", "plan_mode")
return decision is None or decision.get("cause") not in (
"default_model_fallback",
"plan_mode",
"housekeeping",
)
class DimensionScore:
@ -720,6 +751,7 @@ class ClassificationOutcome(NamedTuple):
"reasoning_override",
"llm_classifier",
"heuristic_first_short_circuit",
"housekeeping",
"classifier_plugin",
"classifier_fallback",
"default_model_fallback",
@ -881,17 +913,10 @@ class ComplexityRouter(CustomLogger):
raise ValueError("classifier_llm_config is not set")
definitions: Final = self.config.tier_definitions
if definitions is not None:
entries: Final = tuple(
(
definition.name,
definition.description or _CLASSIFICATION_TIER_CRITERIA[ComplexityTier[definition.name.upper()]],
)
for definition in definitions
)
return _custom_tier_prompt(
entries,
return custom_tier_classification_prompt(
definitions,
self.config.classification_prompt,
_closing_line(self.config.classifier_context_window_size),
self.config.classifier_context_window_size,
)
return classification_system_prompt(
self.config.classifier_context_window_size,
@ -922,17 +947,15 @@ class ComplexityRouter(CustomLogger):
def savings_baseline(self) -> Baseline | None:
"""The derived counterfactual this router's savings are measured against.
``None`` when `litellm_settings.autorouter_savings_baseline_model` is set (the
spend writer reads that setting directly and it wins) or when this router was
built with ``derive_savings_baseline=False``. Derived once on first use and
pinned for the instance's lifetime: creating or editing the router rebuilds
the instance, which re-derives. Deferred past ``__init__`` because during a
config load this router can be constructed before its tier deployments are.
``None`` when this router was built with ``derive_savings_baseline=False``.
Derived once on first use and pinned for the instance's lifetime: creating or
editing the router rebuilds the instance, which re-derives. Deferred past
``__init__`` because during a config load this router can be constructed
before its tier deployments are.
"""
import litellm
from litellm.router_strategy.savings_baseline import resolve_baseline
if not self._derive_savings_baseline or litellm.autorouter_savings_baseline_model is not None:
if not self._derive_savings_baseline:
return None
if not self._savings_baseline_derived:
self._savings_baseline = resolve_baseline(self.litellm_router_instance, self._hardest_tier_models())
@ -1738,12 +1761,20 @@ class ComplexityRouter(CustomLogger):
user_message: str,
request_kwargs: dict[str, Any] | None = None,
hard_floor: ComplexityTier | str | None = None,
hard_ceiling: ComplexityTier | str | None = None,
) -> str:
"""hard_floor excludes every candidate whose tiers all sit below it, turning this pick's
soft floors (a distance penalty a high-scoring cheap model can outweigh) into a hard
minimum for requests that carry one, e.g. the plan-mode floor. classified_tier arrives
already clamped to the floor, so the cold-start pool and the classified_tier eligibility
mode satisfy it by construction; only the "all" eligibility mode can reach below."""
mode satisfy it by construction; only the "all" eligibility mode can reach below.
hard_ceiling is the same bound in the other direction, for a request whose tier was decided
by what it IS rather than by how hard it is: a housekeeping call is placed at the cheapest
tier because that is all it is worth, so a bandit trading cost for quality has nothing to
win and must not reach above it. Without it the distance penalty is the only thing holding
the tier, and a deployment that lowers tier_distance_penalty silently gets the expensive
model back while the routing decision still reads as the cheapest tier."""
from litellm.router_strategy.adaptive_router.bandit import (
normalized_cost,
thompson_sample,
@ -1799,6 +1830,7 @@ class ComplexityRouter(CustomLogger):
penalty_weight: Final = self.config.tier_distance_penalty
floor_severity: Final = self._active_tier_severity(hard_floor) if hard_floor is not None else None
ceiling_severity: Final = self._active_tier_severity(hard_ceiling) if hard_ceiling is not None else None
best_model: str | None = None
best_score = float("-inf")
candidate_scores: Final[list[dict[str, Any]]] = []
@ -1808,6 +1840,11 @@ class ComplexityRouter(CustomLogger):
for model_tier in self._model_tiers.get(model, (classified_tier,))
):
continue
if ceiling_severity is not None and all(
self._active_tier_severity(model_tier) > ceiling_severity
for model_tier in self._model_tiers.get(model, (classified_tier,))
):
continue
cell = adaptive._cells[(request_type, model)]
quality_sample = thompson_sample(cell)
cost_score = normalized_cost(adaptive.model_to_cost.get(model, 0.0), all_costs)
@ -1881,6 +1918,44 @@ class ComplexityRouter(CustomLogger):
self._reminder_markers,
)
def _matched_housekeeping_sentinel(self, newest_ask: str | None) -> str | None:
"""The client housekeeping sentinel on this request's newest ask, or None.
Read from the newest ask alone, never the whole history, for the reason `_newest_turn_ask`
exists: a title request quoted into a later turn's context would otherwise keep matching and
route real work to the cheapest tier for the rest of the session.
Declines whenever an operator's classifier plugin owns the decision. The sentinels are
caller-controlled text, and displacing the built-in classifier with them only ever spends
less; displacing a plugin is different in kind, because a plugin is where an operator
encodes policy the tier ladder does not express, so a caller pasting a title prompt could
route a request past a sensitivity or identity rule to a pool that rule would have refused.
"""
if self.config.classifier_type == "custom" or not self.config.route_housekeeping_to_cheapest_tier:
return None
if not newest_ask:
return None
return next(
(
sentinel
for sentinel in (*HOUSEKEEPING_ASK_SENTINELS, *(self.config.housekeeping_patterns or ()))
if sentinel in newest_ask
),
None,
)
def _cheapest_configured_tier(self) -> ComplexityTier | str | None:
"""The least severe tier that has models, or None when none does.
Tiers can be declared without a pool, so this cannot assume the first name in the severity
order is routable; routing to an empty pool is what `default_fallback` exists to catch.
"""
pools: Final = self._tier_pools()
name: Final = next((name for name in self.config.tier_names() if pools.get(name)), None)
if name is None:
return None
return name if self.config.has_custom_tiers else ComplexityTier(name)
def _apply_plan_mode_floor(self, tier: ComplexityTier | str) -> ComplexityTier | str:
"""The higher of the decided tier and the plan-mode floor; identity when the floor is unset."""
floor: Final = self._resolve_plan_mode_floor()
@ -2476,8 +2551,14 @@ class ComplexityRouter(CustomLogger):
),
)
outcome: Final = await self.aclassify(
user_message, system_prompt, request_kwargs, resolved_messages, raw_messages=messages
housekeeping_sentinel: Final = self._matched_housekeeping_sentinel(newest_ask)
housekeeping_tier: Final = self._cheapest_configured_tier() if housekeeping_sentinel is not None else None
outcome: Final = (
ClassificationOutcome(tier=housekeeping_tier, score=None, signals=("housekeeping",), cause="housekeeping")
if housekeeping_tier is not None
else await self.aclassify(
user_message, system_prompt, request_kwargs, resolved_messages, raw_messages=messages
)
)
tier, score, signals = outcome.tier, outcome.score, outcome.signals
classified_tier: Final = tier
@ -2533,7 +2614,14 @@ class ComplexityRouter(CustomLogger):
# has plan_floored False, yet adaptive_eligible="all" scores every model and only
# penalizes tier distance, so without the floor the bandit could still route below
# it -- and a floor a bandit can slide under is not a floor.
routed_model = self._soft_floor_pick(tier, user_message, request_kwargs, hard_floor=plan_floor)
# The ceiling tracks the tier as raised, never the placement it started from: escalation
# and the plan-mode floor both move a housekeeping call up, and a ceiling still naming
# the cheapest tier would then contradict the floor and bound the pick below the tier
# the decision reports.
housekeeping_ceiling: Final = tier if outcome.cause == "housekeeping" else None
routed_model = self._soft_floor_pick(
tier, user_message, request_kwargs, hard_floor=plan_floor, hard_ceiling=housekeeping_ceiling
)
adaptive: Final = self._ensure_adaptive_router()
if adaptive is not None:
kwargs_metadata: Final = request_kwargs.setdefault("metadata", {})
@ -2582,6 +2670,9 @@ class ComplexityRouter(CustomLogger):
else signals
)
decision_cause: Final[RoutingDecisionCause] = "plan_mode" if plan_floored else outcome.cause
decision_keyword: Final = (
plan_mode_sentinel if plan_floored else (housekeeping_sentinel if outcome.cause == "housekeeping" else None)
)
return PreRoutingHookResponse(
model=routed_model,
messages=messages if has_original_messages else None,
@ -2593,7 +2684,7 @@ class ComplexityRouter(CustomLogger):
tier=classified_pool_tier,
score=score,
signals=decision_signals,
matched_keyword=plan_mode_sentinel if plan_floored else None,
matched_keyword=decision_keyword,
escalation_keyword=escalation_keyword,
escalated=escalated,
classifier_model=classifier_model,

69
litellm/router_strategy/complexity_router/config.py
View file

@ -99,6 +99,23 @@ MAX_TIER_DESCRIPTION_CHARS: Final[int] = 500
MAX_CLASSIFICATION_PROMPT_CHARS: Final[int] = 2000
def normalize_classification_prompt(value: str | None) -> str | None:
"""Strip, reject blank, and cap an operator-written classifier preamble.
The single owner of the rule, so the dashboard's prompt preview normalizes exactly what the
write gate stores: previewing the raw value would render leading whitespace the router strips,
or an over-long prompt the write then rejects.
"""
if value is None:
return None
stripped: Final = value.strip()
if not stripped:
raise ValueError("must be non-empty; omit the field instead")
if len(stripped) > MAX_CLASSIFICATION_PROMPT_CHARS:
raise ValueError(f"classification_prompt exceeds {MAX_CLASSIFICATION_PROMPT_CHARS} characters")
return stripped
class TierDefinition(BaseModel):
"""An operator-defined tier: the name the LLM classifier must return and its rubric description."""
@ -321,6 +338,18 @@ PLAN_MODE_TAIL_SENTINELS: Final[tuple[str, ...]] = (
"Plan mode still active",
)
PLAN_MODE_SYSTEM_SENTINELS: Final[tuple[str, ...]] = ('You are currently running in "Plan" mode.',)
# Taken verbatim from classifier payloads captured on a live gateway, 789 calls over one day: the
# first appears on 17 of them and the second on 2. A coding agent names the conversation by quoting
# the session and asking for a title, so the ask carries the session's engineering vocabulary while
# the task is the cheapest one the client performs. Only wording observed on the wire belongs here,
# never a paraphrase: a sentinel that matches nothing costs a substring scan per request and reads
# as coverage the router does not have. These are client-owned strings that drift with client
# releases, so operators extend coverage via housekeeping_patterns rather than editing these.
HOUSEKEEPING_ASK_SENTINELS: Final[tuple[str, ...]] = (
"Write the title in the predominant language of the session",
"You are coming up with a succinct title for a coding session",
)
PLAN_MODE_TOOL_NAME: Final[str] = "exit_plan_mode"
@ -770,6 +799,29 @@ class ComplexityRouterConfig(BaseModel):
"wording the built-ins don't cover, or after a client release changes its strings."
),
)
route_housekeeping_to_cheapest_tier: bool = Field(
default=True,
description=(
"Route a coding agent's own housekeeping calls to the cheapest configured tier "
"without classifying them. A client names the conversation by quoting the whole "
"session and asking for a title, so the ask reads as the session's engineering work "
"and lands on the most expensive tier, which is the reverse of what the call is "
"worth. Detection is a literal match against client-owned sentinels on the newest "
"ask only, so it cannot fire on an earlier turn, and it never lowers what anyone "
"else asked for: a keyword_tier_rule or a session pin still decides instead, and an "
"escalation keyword or the plan-mode floor still raises the tier from here. Only the "
"classifier is displaced, and its call is skipped, so a matched request costs "
"nothing to route. Set false to classify these calls like any other."
),
)
housekeeping_patterns: tuple[str, ...] | None = Field(
default=None,
description=(
"Additional case-sensitive literal sentinels that mark a request as client "
"housekeeping, on top of the built-in conversation-title ones. For clients whose "
"wording the built-ins don't cover, or after a client release changes its strings."
),
)
# Semantic (embedding) matching for keyword_tier_rules instead of literal text matching
semantic_keyword_matching: bool = Field(
@ -939,6 +991,15 @@ class ComplexityRouterConfig(BaseModel):
return None
return tuple(stripped for pattern in value if (stripped := pattern.strip()))
@field_validator("housekeeping_patterns")
@classmethod
def _normalize_housekeeping_patterns(cls, value: tuple[str, ...] | None) -> tuple[str, ...] | None:
"""Blank patterns are dropped: an empty string substring-matches every request, which would
silently route all traffic to the cheapest tier."""
if value is None:
return None
return tuple(stripped for pattern in value if (stripped := pattern.strip()))
@model_validator(mode="after")
def _validate_plan_mode_min_tier(self) -> "ComplexityRouterConfig":
if self.plan_mode_min_tier is None:
@ -1012,7 +1073,7 @@ class ComplexityRouterConfig(BaseModel):
)
return self
@field_validator("fallback_tier", "classification_prompt")
@field_validator("fallback_tier")
@classmethod
def _reject_blank_optional_text(cls, value: str | None) -> str | None:
if value is None:
@ -1024,10 +1085,8 @@ class ComplexityRouterConfig(BaseModel):
@field_validator("classification_prompt")
@classmethod
def _cap_classification_prompt(cls, value: str | None) -> str | None:
if value is not None and len(value) > MAX_CLASSIFICATION_PROMPT_CHARS:
raise ValueError(f"classification_prompt exceeds {MAX_CLASSIFICATION_PROMPT_CHARS} characters")
return value
def _normalize_classification_prompt_field(cls, value: str | None) -> str | None:
return normalize_classification_prompt(value)
@property
def has_custom_tiers(self) -> bool:

23
litellm/router_strategy/savings_baseline.py
View file

@ -1,16 +1,13 @@
"""The default counterfactual a complexity router's savings are measured against.
"""The counterfactual a complexity router's savings are measured against.
`litellm_settings.autorouter_savings_baseline_model` names the model the traffic would
have run on without a router. When the operator sets it, that answer wins and nothing
here runs. When they do not, the router's own tier ladder already names it: without a
router a deployment has to pick one model that can carry the hardest request it will
see, so the default baseline is the priciest model in the hardest configured tier. A
cheap tier is a choice the router made, not a ceiling it was bounded by.
The router's own tier ladder names the model the traffic would have run on without a
router: a deployment has to pick one model that can carry the hardest request it will
see, so the baseline is the priciest model in the hardest configured tier. A cheap
tier is a choice the router made, not a ceiling it was bounded by.
Candidates are ranked once against a fixed reference request, not against each request
that runs. Ranking per request means reading the request, and every input shape it can
take; a default must not carry that surface. An operator whose pool ordering genuinely
depends on request shape names the baseline in config, which skips this file entirely.
take; a per-router default must not carry that surface.
Baselines are always provider-qualified, because they travel to the spend writer as a
bare string with no provider beside them; an operator who writes ``deepseek-r1`` meaning
@ -54,9 +51,17 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str |
A deployment may name its vendor in the model prefix or in a separate
``custom_llm_provider``, and the bare name alone is not enough to price: it can
resolve to a different vendor's rates, or to nothing at all.
A github_copilot or chatgpt candidate is qualified by string alone: resolving either
provider runs its OAuth device flow, and for a declared pair the resolver's answer is
the declaration itself, so asking it buys nothing but the block.
"""
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
if declared is not None:
return f"{declared}/{model.removeprefix(f'{declared}/')}"
try:
resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline

31
litellm/router_utils/reasoning_effort_capability.py
View file

@ -87,6 +87,22 @@ def declared_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, .
return tuple(effort for effort in REASONING_EFFORT_ADVERTISEMENT_ORDER if effort in declared)
def declared_reasoning_efforts_for_model(model: str, custom_llm_provider: str) -> tuple[str, ...] | None:
"""The levels an entry declares, resolved from the model string a provider config holds rather
than from a router deployment's model_info.
None means the map has no opinion, either because the entry declares nothing or because it
describes no such model, so a caller keeps whatever it did before the entry was described. The
entry is read straight off the map rather than through get_model_info, which raises for a model
it does not know: a provider config runs on the request path for every model it serves, most of
which the map never named, and a lookup miss there must not fail the call.
"""
entry: Final = litellm.model_cost.get(f"{custom_llm_provider}/{model}") or litellm.model_cost.get(model)
if not isinstance(entry, dict):
return None
return declared_reasoning_efforts(entry)
def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: object) -> bool:
"""Opt-in only where a request path refuses the level. AzureOpenAIGPT5Config raises
UnsupportedParamsError on reasoning_effort='none' without an explicit true, and it is selected
@ -132,16 +148,25 @@ def resolve_supported_reasoning_efforts(
unset flag as () would let one custom deployment empty every level its mapped siblings agree
on. deployment_is_mapped is that provenance, and an operator who wants either answer for an
off-map deployment gets it by setting supports_reasoning explicitly.
If supports_reasoning is unset but at least one per-level flag (e.g.
supports_minimal_reasoning_effort) is present, treat it as implicitly True, since the
per-level flags are evidence the model supports reasoning. An explicit False always wins:
it is the operator's escape hatch and must not be overridden by inherited per-level flags.
"""
supports_reasoning: Final = model_info.get("supports_reasoning")
if supports_reasoning is not True:
return () if supports_reasoning is False or deployment_is_mapped else None
if supports_reasoning is False:
return ()
flags: Final = _declared_effort_flags(model_info)
has_per_level_flag: Final = any(value is not None for value in flags.values())
if supports_reasoning is not True and not has_per_level_flag:
return () if deployment_is_mapped else None
declared: Final = declared_reasoning_efforts(model_info)
if declared is not None:
return declared
flags: Final = _declared_effort_flags(model_info)
if all(value is None for value in flags.values()):
return None

1
litellm/types/agents.py
View file

@ -225,6 +225,7 @@ class AgentResponse(BaseModel):
static_headers: dict[str, str] | None = None
extra_headers: list[str] | None = None
keys: list[AgentKeySummary] | None = None
search_score: float | None = None
created_at: datetime | None = None
updated_at: datetime | None = None
created_by: str | None = None

23
litellm/types/guardrails.py
View file

@ -563,9 +563,15 @@ class LakeraV2GuardrailConfigModel(BaseModel):
default=True,
description="Whether to include developer information in the response",
)
on_flagged: Literal["block", "monitor"] | None = Field(
on_flagged: Literal["block", "monitor", "inject_system_message"] | None = Field(
default="block",
description="Action to take when content is flagged: 'block' (raise exception) or 'monitor' (log only)",
description="Action to take when content is flagged: 'block' (raise exception), 'monitor' (log only), "
"or 'inject_system_message' (append an advisory system message and let the LLM decide)",
)
advisory_system_message: str | None = Field(
default=None,
description="Custom advisory message template used when on_flagged='inject_system_message'. "
"Must contain a {reason} placeholder. Defaults to a generic advisory message if unset.",
)
@ -951,6 +957,17 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up
),
)
scan_raw_request: bool | None = Field(
default=None,
description=(
"When True, this pre_call guardrail always evaluates the request as it was before any "
"guardrail in this hook ran, regardless of its position in the guardrails list -- so the "
"YAML order of guardrails can never change whether this one blocks. Use only for "
"block-only guardrails: any data this guardrail returns is discarded, same contract as "
"run_in_parallel, since an earlier guardrail's masking must not be undone by this one."
),
)
@field_validator(
"mode",
"default_action",
@ -983,7 +1000,7 @@ class Mode(BaseModel):
default: str | list[str] | None = Field(default=None, description="Default mode when no tags match")
class LitellmParams(
class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # on_flagged literal diverges across mixins
CiscoAIDefenseGuardrailConfigModel,
PresidioConfigModel,
BedrockGuardrailConfigModel,

7
litellm/types/integrations/anthropic_cache_control_hook.py
View file

@ -1,9 +1,14 @@
from typing import Literal
from typing import Final, Literal
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.types.llms.openai import ChatCompletionCachedContent
GATEWAY_INJECTED_CACHE_METADATA_KEY: Final = "litellm_gateway_injected_cache"
# No deployment had been chosen when the injection happened, so it is in the payload
# every leg of the request sends. Never a real deployment id.
GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT: Final = ""
class CacheControlMessageInjectionPoint(TypedDict):
"""Type for message-level injection points."""

21
litellm/types/llms/anthropic.py
View file

@ -1,4 +1,4 @@
from collections.abc import Iterable
from collections.abc import Iterable, Sequence
from enum import Enum
from typing import Any, Final, Literal, TypeAlias
@ -254,6 +254,17 @@ class AnthropicContentParamSourceFileId(TypedDict):
file_id: str
class AnthropicContentParamSourceText(TypedDict):
type: ReadOnly[Literal["text"]]
media_type: ReadOnly[Literal["text/plain"]]
data: ReadOnly[str]
class AnthropicContentParamSourceContent(TypedDict):
type: ReadOnly[Literal["content"]]
content: ReadOnly[str | Sequence["AnthropicMessagesTextParam | AnthropicMessagesImageParam"]]
class AnthropicMessagesContainerUploadParam(TypedDict, total=False):
type: Required[Literal["container_upload"]]
file_id: str
@ -305,7 +316,13 @@ AnthropicCitation = AnthropicCitationPageLocation | AnthropicCitationCharLocatio
class AnthropicMessagesDocumentParam(TypedDict, total=False):
type: Required[Literal["document"]]
source: Required[AnthropicContentParamSource | AnthropicContentParamSourceFileId | AnthropicContentParamSourceUrl]
source: Required[
AnthropicContentParamSource
| AnthropicContentParamSourceFileId
| AnthropicContentParamSourceUrl
| AnthropicContentParamSourceText
| AnthropicContentParamSourceContent
]
cache_control: dict | ChatCompletionCachedContent | None
title: str
context: str

52
litellm/types/management_endpoints/auto_router_endpoints.py
View file

@ -362,6 +362,27 @@ class ShadowEvalSlice(BaseModel):
)
tie_rate_pct: float
avg_judge_confidence: float
real_spend: float = Field(
default=0.0,
description=(
"USD the real arm billed on this slice's judged turns, completion plus its own routing "
"classifier when it routed, excluding turns litellm's response cache served for free"
),
)
shadow_spend: float = Field(
default=0.0,
description=(
"USD the shadow arm billed on the same turns, completion plus its own routing classifier, "
"excluding the judge and the same cache-served turns, so the two spends compare like for like"
),
)
cache_hit_turns: int = Field(
default=0,
description=(
"Judged turns litellm's response cache served, excluded from both spends: an adopted router "
"would be served by the same cache, so those turns cost the same either way"
),
)
class ShadowEvalResult(BaseModel):
@ -382,6 +403,37 @@ class ShadowEvalResult(BaseModel):
)
overall_shadow_win_rate_pct: float
overall_tie_rate_pct: float
sampled_real_spend: float = Field(
default=0.0,
description="USD the real arm billed across all judged turns, cache-served turns excluded",
)
sampled_shadow_spend: float = Field(
default=0.0,
description="USD the shadow arm billed across the same turns, judge excluded, like for like",
)
not_sampled_count: int | None = Field(
default=None,
description=(
"Eligible requests the sampling dice skipped, summed over legs: the judged rows stand for "
"judged + this many requests. None for jobs from before the funnel existed"
),
)
unjudgeable_count: int | None = Field(
default=None,
description="Sampled requests whose shape could not be judged (tool-final turn, empty text)",
)
shed_count: int | None = Field(
default=None,
description="Sampled requests dropped by the per-pod concurrency cap, so quiet periods are overweighted",
)
withheld_count: int | None = Field(
default=None,
description=(
"Sampled requests the pipeline declined to spend on: no database to record into, an over-budget "
"key or team, or the eval budget unverifiable or already reached (the in-flight burst as a job "
"crosses max_budget lands here rather than vanishing from coverage)"
),
)
class ShadowEvalJobKeyResponse(BaseModel):

Some files were not shown because too many files have changed in this diff Show more