scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

chore: merge litellm_internal_staging

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-08-29 00:12:56 +00:00
commit 6de9ae107b
256 changed files with 25386 additions and 685 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/sync-together-ai-models.yml vendored
View file

@ -32,7 +32,7 @@ jobs:
echo "An open sync PR already exists on branch $open_pr; skipping this run."
fi
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
- name: Run the sync
if: steps.existing.outputs.open_pr == ''
run: |
@ -65,4 +65,4 @@ jobs:
--head "$branch" \
--base litellm_internal_staging
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}

2
.github/workflows/test-terraform-provider.yml vendored
View file

@ -114,4 +114,4 @@ jobs:
- name: Audit provider endpoints against the schema
working-directory: terraform/provider
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json" -coverage-allowlist ./tools/endpointaudit/coverage_allowlist.txt

10
basedpyright-code-budget.json
View file

@ -3,7 +3,7 @@
"limit": 18483
},
"reportArgumentType": {
"limit": 2564
"limit": 2557
},
"reportAssignmentType": {
"limit": 319
@ -57,7 +57,7 @@
"limit": 5659
},
"reportMissingTypeArgument": {
"limit": 15484
"limit": 15482
},
"reportMissingTypeStubs": {
"limit": 40
@ -105,13 +105,13 @@
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38782
"limit": 38779
},
"reportUnknownParameterType": {
"limit": 19829
"limit": 19827
},
"reportUnknownVariableType": {
"limit": 30349
"limit": 30348
},
"reportUnnecessaryCast": {
"limit": 117

9
ci_cd/generate_model_prices_schema.py
View file

@ -220,6 +220,15 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
"enum": ["low", "medium", "high", "max", "xhigh"],
},
"default_reasoning_effort": {
"type": "string",
"description": (
"Reasoning effort the provider applies when the request omits reasoning_effort. "
"Gates whether a non-default temperature or the top_p/logprobs sampling params are "
"accepted, which hold only when the effort resolves to 'none'."
),
"enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
},
"comment": STRING,
"audio_transcription_config": STRING,
}

22
litellm-proxy-extras/litellm_proxy_extras/migrations/20260828000000_shadow_eval_cost_comparison/migration.sql Normal file
View file

@ -0,0 +1,22 @@
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cost" DOUBLE PRECISION;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "shadow_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cache_hit" BOOLEAN NOT NULL DEFAULT false;
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalFunnel" (
"job_id" TEXT NOT NULL,
"not_sampled" INTEGER NOT NULL DEFAULT 0,
"unjudgeable" INTEGER NOT NULL DEFAULT 0,
"shed" INTEGER NOT NULL DEFAULT 0,
"withheld" INTEGER NOT NULL DEFAULT 0,
CONSTRAINT "LiteLLM_ShadowEvalFunnel_pkey" PRIMARY KEY ("job_id")
);

15
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1533,12 +1533,27 @@ model LiteLLM_ShadowEvalAttempt {
confidence Float?
judge_cost Float @default(0)
shadow_cost Float @default(0)
real_cost Float? // NULL = row predates cost measurement; comparisons read only measured rows
real_classifier_cost Float @default(0)
shadow_classifier_cost Float @default(0)
real_cache_hit Boolean @default(false)
error String?
created_at DateTime @default(now())
@@index([job_id])
}
// Per-leg sampling funnel counters the attempt rows cannot derive: requests an
// admitting job saw but did not judge. attempted = the leg's attempt rows; the
// leg's eligible traffic = not_sampled + unjudgeable + shed + withheld + attempted.
model LiteLLM_ShadowEvalFunnel {
job_id String @id
not_sampled Int @default(0)
unjudgeable Int @default(0)
shed Int @default(0)
withheld Int @default(0)
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

2
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -470,7 +470,7 @@ class ProxyExtrasDBManager:
ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
@staticmethod
def _mark_migrations_applied(migrations_dir: str):
def _mark_migrations_applied(migrations_dir: str) -> None:
migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
logger.info(f"Resolving {len(migration_names)} migrations")
for migration_name in migration_names:

1
litellm/__init__.py
View file

@ -274,7 +274,6 @@ databricks_key: Optional[str] = None
openai_like_key: Optional[str] = None
azure_key: Optional[str] = None
anthropic_key: Optional[str] = None
autorouter_savings_baseline_model: Optional[str] = None
replicate_key: Optional[str] = None
bytez_key: Optional[str] = None
gdc_key: Optional[str] = None

9
litellm/constants.py
View file

@ -629,6 +629,15 @@ LITELLM_CHAT_PROVIDERS: Final = [
"amazon_nova",
]
# Resolving these providers runs an OAuth device flow (their provider info IS the login), so any
# metadata or capability lookup against them can block for minutes waiting on a human.
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
{
"github_copilot",
"chatgpt",
}
)
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
"openai",
"azure",

24
litellm/cost_calculator.py
View file

@ -2,7 +2,7 @@
## File for 'response_cost' calculation in Logging
import logging
import time
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, cast
@ -739,6 +739,13 @@ def _get_provider_for_cost_calc(
return custom_llm_provider
def _get_hidden_str_for_cost_calc(hidden_params: object, key: str) -> str | None:
if not isinstance(hidden_params, Mapping):
return None
value: Final[object] = hidden_params.get(key)
return value if isinstance(value, str) and value else None
def _select_model_name_for_cost_calc(
model: str | None,
completion_response: object | None,
@ -755,7 +762,6 @@ def _select_model_name_for_cost_calc(
"""
return_model: str | None = None
region_name: str | None = None
custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
completion_response_model: str | None = None
@ -765,6 +771,14 @@ def _select_model_name_for_cost_calc(
elif isinstance(completion_response, dict):
completion_response_model = completion_response.get("model", None)
hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
explicit_pricing: Final = custom_pricing is True or base_model is not None
priced_from_response: Final = provider_response_model is not None or completion_response_model is not None
region_name: Final = (
_get_hidden_str_for_cost_calc(hidden_params, "region_name")
if not explicit_pricing and priced_from_response
else None
)
if custom_pricing is True:
if router_model_id is not None and router_model_id in litellm.model_cost:
@ -780,14 +794,12 @@ def _select_model_name_for_cost_calc(
else:
return_model = model
elif base_model is not None:
return_model = base_model
elif base_model is not None or provider_response_model is not None:
return_model = base_model if base_model is not None else provider_response_model
elif completion_response_model is None and hidden_params is not None:
if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
return_model = hidden_params.get("model", model)
elif hidden_params is not None and hidden_params.get("region_name", None) is not None:
region_name = hidden_params.get("region_name", None)
if return_model is None and completion_response_model is not None:
return_model = completion_response_model

2
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -1447,10 +1447,8 @@ Model Info:
from datetime import datetime
# Get the current timestamp
current_time: Final = datetime.now().strftime("%H:%M:%S")
_proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
# Use .name if it's an enum, otherwise use as is
alert_type_name: Final = getattr(alert_type, "name", alert_type)
alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
if alert_type == "daily_reports" or alert_type == "new_model_added":

91
litellm/integrations/custom_guardrail.py
View file

@ -60,6 +60,12 @@ _PRE_CALL_EXECUTED_TOKEN: Final = secrets.token_hex(16)
_GUARDRAIL_BLOCK_STATUS_CODES: Final = frozenset({400, 403, 422})
DEFAULT_ADVISORY_MESSAGE: Final = (
"The user's latest message was flagged for {reason} by a content safety "
"guardrail. This may be a false positive. Use your judgment: respond "
"helpfully if the request is legitimate, or decline if it is not."
)
_guardrail_self_recorded: Final[contextvars.ContextVar[bool]] = contextvars.ContextVar(
"litellm_guardrail_self_recorded", default=False
)
@ -158,6 +164,7 @@ class CustomGuardrail(CustomLogger):
sensitive_data_route_to_model: str | None = None,
sticky_session_routing: bool = True,
run_in_parallel: bool = False,
scan_raw_request: bool = False,
only_scan_new_messages: bool = False,
**kwargs,
):
@ -180,6 +187,13 @@ class CustomGuardrail(CustomLogger):
run_in_parallel: When True, this pre_call or post_call guardrail runs concurrently with
other opted-in guardrails of the same hook. Only safe for block-only guardrails that
do not mutate the request or response.
scan_raw_request: When True, this pre_call guardrail always evaluates the request as it
was before any guardrail in this hook ran, regardless of where it's declared in the
guardrails list -- so an earlier guardrail that masks/rewrites content (e.g. PII
redaction) can never hide a violation from this one. Only safe for block-only
guardrails: any data this guardrail returns is discarded, matching run_in_parallel's
contract, since applying its mutations on top of a stale snapshot would silently
undo whatever later guardrails already did to the live request.
"""
self.guardrail_name = guardrail_name
self.supported_event_hooks = supported_event_hooks
@ -195,6 +209,7 @@ class CustomGuardrail(CustomLogger):
self.sensitive_data_route_to_model: str | None = sensitive_data_route_to_model
self.sticky_session_routing: bool = sticky_session_routing
self.run_in_parallel: bool = run_in_parallel
self.scan_raw_request: bool = scan_raw_request
self.only_scan_new_messages: bool = only_scan_new_messages
if supported_event_hooks:
@ -281,6 +296,82 @@ class CustomGuardrail(CustomLogger):
original_response=original_response,
)
def inject_advisory_message(
self,
data: dict[str, Any], # mutable-ok: caller's dict is mutated in place, matching mark_pre_call_hook_ran
message: str,
) -> bool:
"""
Append an advisory system message to the request in place, so the LLM
itself can weigh a possible false-positive guardrail flag rather than
the request being hard-blocked or silently allowed.
Unlike raise_passthrough_exception, this does NOT short-circuit the LLM
call; the request proceeds normally with the extra message appended.
Guardrails should call this from on_flagged handling analogous to how
passthrough-supporting guardrails call raise_passthrough_exception.
Args:
data: The request data dictionary, mutated in place to append the
advisory message to its "messages" list and/or "input"/
"instructions" text.
message: The formatted advisory message to append as a system message.
Returns:
True if the advisory was actually written somewhere the model will
see it. False if ``data["input"]`` is a structured Responses-API
list (not a plain string) -- the Responses API reads only
``input``, so appending to ``messages`` would be inert regardless
of whether a ``messages`` list also happens to be present, and
there is no field this helper can safely append into. The caller
must treat this like any other case where the mitigation can't
land and degrade to blocking instead of silently letting the
flagged request through unmodified.
"""
advisory_message: Final = {"role": "system", "content": message} # mutable-ok: plain dict for live request
existing_messages: Final = data.get("messages")
existing_input: Final = data.get("input")
existing_instructions: Final = data.get("instructions")
if isinstance(existing_instructions, str):
# Responses API "instructions" is the privileged, developer-set
# system-level field the model treats as authoritative -- unlike
# "input", which the caller controls and could use to tell the
# model to disregard a trailing warning. Prefer it over "input"
# whenever present.
if isinstance(existing_messages, list):
messages_with_instructions_note: Final = [ # mutable-ok: fresh list
*existing_messages,
advisory_message,
]
data["messages"] = messages_with_instructions_note # rebind-ok: mutates caller's dict by design
data["instructions"] = f"{existing_instructions}\n\n{message}" # rebind-ok: mutates caller's dict by design
return True
if isinstance(existing_input, str):
# A plain-string "input" doesn't rule out "messages" also being a
# real, read field (e.g. a chat-completions call carrying a stray
# "input"), so write to both when both are present.
if isinstance(existing_messages, list):
messages_with_input_note: Final = [*existing_messages, advisory_message] # mutable-ok: fresh list
data["messages"] = messages_with_input_note # rebind-ok: mutates caller's dict by design
# The Responses API reads "input", not "messages" -- appending only to
# "messages" would leave the advisory unreachable for that endpoint.
data["input"] = f"{existing_input}\n\n{message}" # rebind-ok: mutates caller's dict by design
return True
if existing_input is not None:
# existing_input is a structured (non-string) Responses-API item
# list. That endpoint reads only "input", so appending to
# "messages" -- even if "messages" also happens to be present --
# would never reach the model. Leave data untouched and report
# non-delivery so the caller degrades to blocking.
return False
if isinstance(existing_messages, list):
messages_without_input_note: Final = [*existing_messages, advisory_message] # mutable-ok: fresh list
data["messages"] = messages_without_input_note # rebind-ok: mutates caller's dict by design
return True
sole_message: Final = [advisory_message] # mutable-ok: plain list for the live JSON request
data["messages"] = sole_message # rebind-ok: mutates caller's dict by design
return True
def raise_sensitive_data_route_exception(
self,
route_to_model: str,

161
litellm/integrations/shadow_eval_logger.py
View file

@ -38,6 +38,7 @@ from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalD
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
if TYPE_CHECKING:
from litellm.proxy.db.shadow_eval_funnel import ShadowEvalFunnelStage
from litellm.proxy.utils import PrismaClient
from litellm.router import Router
from litellm.types.utils import StandardLoggingPayload
@ -386,6 +387,13 @@ def _judge_user_prompt(conversation: str, response_a: str, response_b: str) -> s
)
def _leg_eval_spend(sums: Mapping[str, object]) -> float:
return sum(
float(raw) if isinstance(raw := sums.get(column), (int, float)) else 0.0
for column in ("judge_cost", "shadow_cost", "shadow_classifier_cost")
)
def _job_spend_counter_key(job_id: str) -> str:
return f"spend:shadow_eval:{job_id}"
@ -412,6 +420,15 @@ async def _add_job_spend_to_counter(counter_key: str, cost: float) -> None:
verbose_logger.warning("shadow_eval: spend counter increment failed for %s: %s", counter_key, e)
def _record_funnel_event(job_id: str, stage: "ShadowEvalFunnelStage") -> None:
try:
from litellm.proxy.db.shadow_eval_funnel import record_shadow_eval_funnel_event
record_shadow_eval_funnel_event(job_id, stage)
except Exception as e: # noqa: BLE001 # coverage stats are advisory; sampling must proceed
verbose_logger.debug("shadow_eval: funnel increment failed for %s: %s", job_id, e)
async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
"""Whether the shadowed key or its team is over budget, decided by the same owners
the request path uses, so counter keys and thresholds can never drift from auth's.
@ -474,6 +491,13 @@ def _routed_tier(metadata: Mapping[str, object]) -> str | None:
return str(raw) if raw is not None else None
def _decision_classifier_cost(metadata: Mapping[str, object]) -> float:
"""What the arm's own routing decision says its classifier call billed: the money a
completion cost alone omits, and 0 for a plain model that never classifies."""
raw: Final = _routing_decision(metadata).get("classifier_cost")
return float(raw) if isinstance(raw, (int, float)) else 0.0
def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool:
"""Whether the router under evaluation served this request, which is what decides
the direction it belongs to. A forward job skips its own router's traffic, since
@ -489,6 +513,7 @@ class _CallFailure:
error: str
cost: float = 0.0
classifier_cost: float = 0.0
@dataclass(frozen=True, slots=True)
@ -499,6 +524,7 @@ class _ShadowResponse:
model: str
tier: str | None
cost: float
classifier_cost: float
@dataclass(frozen=True, slots=True)
@ -575,6 +601,7 @@ class ShadowEvalLogger(CustomLogger):
jobs_cache: InMemoryCache | None = None,
job_spend_reader: Callable[[str, float, float], Awaitable[float]] | None = None,
job_spend_writer: Callable[[str, float], Awaitable[None]] | None = None,
funnel_recorder: Callable[[str, "ShadowEvalFunnelStage"], None] | None = None,
) -> None:
"""Providers are callables so the proxy's lazily-initialized globals are resolved
at call time, not at logger construction. The spend reader and writer wrap the
@ -584,6 +611,7 @@ class ShadowEvalLogger(CustomLogger):
self._jobs_cache = jobs_cache or _jobs_cache
self._read_job_spend = job_spend_reader or _job_spend_from_counter
self._write_job_spend = job_spend_writer or _add_job_spend_to_counter
self._record_funnel = funnel_recorder or _record_funnel_event
self._inflight_shadow_tasks: int = 0
# Starts per job since the last cache fill, never decremented within a
# generation; the refill absorbs written rows and resets.
@ -610,7 +638,8 @@ class ShadowEvalLogger(CustomLogger):
await prisma.db.litellm_shadowevalattempt.group_by(
by=["job_id"],
count=True,
sum={"judge_cost": True, "shadow_cost": True}, # mutable-ok: Prisma aggregate spec
# mutable-ok: Prisma aggregate spec
sum={"judge_cost": True, "shadow_cost": True, "shadow_classifier_cost": True},
where={"job_id": {"in": [str(record.id) for record in records]}}, # mutable-ok: Prisma filter
)
if records
@ -619,8 +648,7 @@ class ShadowEvalLogger(CustomLogger):
attempt_stats: Final = { # mutable-ok: frozen snapshot of the grouped read
str(row["job_id"]): (
int(row["_count"]["_all"]),
float((row["_sum"] or {}).get("judge_cost") or 0.0)
+ float((row["_sum"] or {}).get("shadow_cost") or 0.0),
_leg_eval_spend(row["_sum"] or _EMPTY_METADATA),
)
for row in grouped or []
}
@ -646,6 +674,32 @@ class ShadowEvalLogger(CustomLogger):
#### hook ####
def _sampled_jobs(
self,
active_jobs: Sequence[ActiveShadowEvalJob],
request_metadata: Mapping[str, object],
request_id: str,
) -> tuple[ActiveShadowEvalJob, ...]:
"""The jobs that sample this request. A key can hold one job per direction, and a
request routed by one job's router while bypassing the other's qualifies for both;
each is separately budgeted, so both fire. An admitting job that loses the sampling
dice is counted, so results can weigh judged rows against the traffic they stand for."""
eligible: list[ActiveShadowEvalJob] = [] # mutable-ok: bucketed per-job admission
now: Final = datetime.now(timezone.utc)
for job in active_jobs:
if (
now >= job.ends_at
or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns
or (job.max_budget is not None and job.spend >= job.max_budget)
or _request_was_routed_by(request_metadata, job.router_name) != (job.direction == "reverse")
):
continue
if not _sample_hits(request_id, job.id, job.shadow_percentage):
self._record_funnel(job.id, "not_sampled")
continue
eligible.append(job)
return tuple(eligible)
async def async_log_success_event(
self,
kwargs: Mapping[str, object],
@ -677,18 +731,8 @@ class ShadowEvalLogger(CustomLogger):
return # only surfaces this table can normalize are comparable; unknown types fail closed
if ops.wire_params and _request_mutating_guardrail_ran(request_metadata):
return # the wire-body snapshot predates the rewrite; replaying it would resurrect stripped content
# A key can hold one job per direction, and a request routed by one job's
# router while bypassing the other's qualifies for both. Each is separately
# budgeted, so both fire; the request is normalized once, and only when at
# least one job sampled it.
eligible: Final = tuple(
job
for job in (await self._active_jobs()).get(str(api_key_hash), ())
if datetime.now(timezone.utc) < job.ends_at
and job.attempts + self._job_starts.get(job.id, 0) < job.max_turns
and (job.max_budget is None or job.spend < job.max_budget)
and _sample_hits(request_id, job.id, job.shadow_percentage)
and _request_was_routed_by(request_metadata, job.router_name) == (job.direction == "reverse")
eligible: Final = self._sampled_jobs(
(await self._active_jobs()).get(str(api_key_hash), ()), request_metadata, request_id
)
if not eligible:
return
@ -699,12 +743,18 @@ class ShadowEvalLogger(CustomLogger):
response_obj,
)
if sample is None:
for job in eligible:
self._record_funnel(job.id, "unjudgeable")
return
messages, shadow_params, real_text = sample
control_tier: Final = _routed_tier(request_metadata)
real_cost: Final = float(payload.get("response_cost") or 0.0)
real_cache_hit: Final = payload.get("cache_hit") is True
real_classifier_cost: Final = _decision_classifier_cost(request_metadata)
for job in eligible:
if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS:
return
self._record_funnel(job.id, "shed")
continue
self._job_starts[job.id] = self._job_starts.get(job.id, 0) + 1
self._inflight_shadow_tasks += 1
asyncio.create_task(
@ -714,6 +764,9 @@ class ShadowEvalLogger(CustomLogger):
messages=messages,
real_text=real_text,
real_model=payload.get("model") or "",
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
control_tier=control_tier,
shadow_params=shadow_params,
parent_metadata=MappingProxyType(dict(request_metadata)), # mutable-ok: frozen snapshot
@ -734,37 +787,66 @@ class ShadowEvalLogger(CustomLogger):
messages: Sequence[Mapping[str, object]],
real_text: str,
real_model: str,
real_cost: float,
real_classifier_cost: float,
real_cache_hit: bool,
control_tier: str | None,
shadow_params: Mapping[str, object],
parent_metadata: Mapping[str, object],
) -> None:
"""Budget gate -> shadow call -> blind judge -> one attempt row. The prisma gate
sits above the dispatch so no provider spend happens without a place to record
the outcome, and the budget read lives here rather than in the success hook."""
"""Budget gate -> shadow call -> blind judge -> one attempt row, and every exit
in exactly one coverage bucket: the gates that decline to spend on an admitted
sample (no DB to record into, an over-budget key, an unverifiable or exhausted
eval budget) count it withheld, so eligible traffic still reconciles as
not_sampled + unjudgeable + shed + withheld + attempt rows. The prisma gate sits
above the dispatch so no provider spend happens without a place to record the
outcome, and the budget read lives here rather than in the success hook."""
prisma: Final = self._prisma_provider()
try:
if prisma is None:
self._record_funnel(job.id, "withheld")
return
if await _key_or_team_is_over_budget(parent_metadata):
self._record_funnel(job.id, "withheld")
return
if job.max_budget is not None:
try:
spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget)
except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it
verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e)
self._record_funnel(job.id, "withheld")
return
if spend >= job.max_budget:
self._record_funnel(job.id, "withheld")
return
shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata)
except Exception as e: # noqa: BLE001 # detached task: nothing billed yet, record and never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
await self._record_attempt(
prisma, job, request_id, control_tier, outcome="error", error=f"pipeline error: {e}"
prisma,
job,
request_id,
control_tier,
outcome="error",
error=f"pipeline error: {e}",
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
return
if isinstance(shadow, _CallFailure):
await self._record_attempt(
prisma, job, request_id, control_tier, outcome="error", error=shadow.error, shadow_cost=shadow.cost
prisma,
job,
request_id,
control_tier,
outcome="error",
error=shadow.error,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
return
# From here the shadow call has billed, so every exit records its cost.
@ -787,6 +869,10 @@ class ShadowEvalLogger(CustomLogger):
shadow=shadow,
judge_cost=verdict.cost,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
return
await self._record_attempt(
@ -800,6 +886,10 @@ class ShadowEvalLogger(CustomLogger):
confidence=verdict.confidence,
judge_cost=verdict.cost,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
except Exception as e: # noqa: BLE001 # detached task: the shadow call billed, record its cost, never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
@ -812,6 +902,10 @@ class ShadowEvalLogger(CustomLogger):
error=f"pipeline error: {e}",
shadow=shadow,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
)
async def _record_attempt(
@ -822,15 +916,20 @@ class ShadowEvalLogger(CustomLogger):
control_tier: str | None,
*,
outcome: str,
real_cost: float,
real_classifier_cost: float,
real_cache_hit: bool,
shadow: _ShadowResponse | None = None,
real_model: str = "",
confidence: float | None = None,
judge_cost: float = 0.0,
shadow_cost: float = 0.0,
shadow_classifier_cost: float = 0.0,
error: str | None = None,
) -> None:
if judge_cost + shadow_cost > 0:
await self._write_job_spend(_job_spend_counter_key(job.id), judge_cost + shadow_cost)
eval_spend: Final = judge_cost + shadow_cost + shadow_classifier_cost
if eval_spend > 0:
await self._write_job_spend(_job_spend_counter_key(job.id), eval_spend)
if prisma is None:
return
try:
@ -845,6 +944,10 @@ class ShadowEvalLogger(CustomLogger):
"confidence": confidence,
"judge_cost": judge_cost,
"shadow_cost": shadow_cost,
"shadow_classifier_cost": shadow_classifier_cost,
"real_cost": real_cost,
"real_classifier_cost": real_classifier_cost,
"real_cache_hit": real_cache_hit,
"error": error[:_MAX_ERROR_CHARS] if error else None,
}
)
@ -881,15 +984,23 @@ class ShadowEvalLogger(CustomLogger):
)
except Exception as e: # noqa: BLE001 # provider errors become error rows, not crashes
verbose_logger.debug("shadow_eval: router call failed: %s", e)
return _CallFailure(f"shadow router call failed: {_failure_detail(e)}")
return _CallFailure(
f"shadow router call failed: {_failure_detail(e)}",
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
text: Final = _chat_final_text(response)
if not text:
return _CallFailure("shadow router returned an empty response", cost=_call_cost(response))
return _CallFailure(
"shadow router returned an empty response",
cost=_call_cost(response),
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
return _ShadowResponse(
text=text,
model=str(getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or ""),
tier=_routed_tier(shadow_metadata),
cost=_call_cost(response),
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
async def _call_judge(

56
litellm/litellm_core_utils/core_helpers.py
View file

@ -454,6 +454,62 @@ def safe_deep_copy(data):
return new_data
def independent_snapshot(
data: dict, # mutable-ok: caller-defined request-payload shape
) -> dict: # mutable-ok: caller-defined request-payload shape
"""
A copy of ``data`` whose top-level keys are deep-copied independently
where possible -- always attempted, regardless of
``litellm.safe_memory_mode``. Unlike ``safe_deep_copy``, which can return
the *original* object outright under that mode (defeating any isolation
guarantee for every key, not just the ones that need it), this never
skips copying wholesale.
Real proxy requests carry ``data["litellm_logging_obj"]`` (a ``Logging``
instance nesting a live OTel span with a real lock) by the time
``pre_call_hook`` runs, which can never be deep-copied. Any individual
key that fails to deep-copy falls back to sharing its original
reference, same crash tolerance as ``safe_deep_copy``'s own per-key
fallback; callers needing true isolation (e.g. a guardrail's
``scan_raw_request`` snapshot) only depend on the keys that are plain,
cleanly-copyable structures (``messages``/``input``,
``metadata``/``litellm_metadata``).
"""
sanitized: Final = {
key: (
{ # mutable-ok: same request-payload shape as data
inner_key: ("placeholder" if inner_key == "litellm_parent_otel_span" else inner_value)
for inner_key, inner_value in value.items()
}
if key in ("metadata", "litellm_metadata") and isinstance(value, dict)
else value
)
for key, value in data.items()
}
def _copied_value(key: str, sanitized_value: object) -> object:
try:
copied_value: Final = copy.deepcopy(sanitized_value)
except Exception: # noqa: BLE001 # any unpicklable value falls back to the original reference for this key only
return data.get(key)
original_value: Final = data.get(key)
if (
key in ("metadata", "litellm_metadata")
and isinstance(copied_value, dict)
and isinstance(original_value, dict)
and "litellm_parent_otel_span" in original_value
):
return { # mutable-ok: same request-payload shape as data
**copied_value,
"litellm_parent_otel_span": original_value["litellm_parent_otel_span"],
}
return copied_value
return { # mutable-ok: same request-payload shape as data
key: _copied_value(key, value) for key, value in sanitized.items()
}
def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
"""
Recursively filter out Exception objects and callable objects from dicts/lists.

14
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -2,7 +2,7 @@ from typing import Final, cast
from urllib.parse import urlparse
import litellm
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.litellm_core_utils.fallback_generalizations import (
match_routing_generalization,
)
@ -127,6 +127,18 @@ def handle_anthropic_text_model_custom_llm_provider(
return model, custom_llm_provider
def declared_authenticating_provider(model: str, custom_llm_provider: str | None = None) -> str | None:
"""The authenticating provider this pair already names, or None.
get_llm_provider runs the OAuth device flow for github_copilot and chatgpt, because their
provider info includes the key it unlocks. For a metadata question that flow is pure hazard,
and for a declared pair the resolver's answer is the declaration itself, so metadata callers
adopt the declaration instead of resolving.
"""
declared: Final = custom_llm_provider or model.split("/", 1)[0]
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
def get_llm_provider(
model: str,
custom_llm_provider: str | None = None,

5
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -2,6 +2,7 @@ from typing import Final, Literal
import litellm
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.types.utils import LlmProviders, LlmProvidersSet
@ -30,6 +31,10 @@ def get_supported_openai_params(
- List if custom_llm_provider is mapped
- None if unmapped
"""
if not custom_llm_provider:
custom_llm_provider = declared_authenticating_provider(
model
) # rebind-ok: resolving would run the provider's OAuth flow
if not custom_llm_provider:
try:
custom_llm_provider = litellm.get_llm_provider(model=model)[1]

2
litellm/litellm_core_utils/litellm_logging.py
View file

@ -6058,7 +6058,7 @@ def get_standard_logging_object_payload(
prompt_tokens=usage_dict.get("prompt_tokens", 0),
completion_tokens=usage_dict.get("completion_tokens", 0),
request_tags=request_tags,
end_user=end_user_id or "",
end_user=end_user_id,
api_base=StandardLoggingPayloadSetup.strip_trailing_slash(litellm_params.get("api_base", "")) or "",
model_group=_model_group,
model_id=_model_id,

25
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -239,6 +239,22 @@ class ChunkProcessor:
model_response._hidden_params = chunk.get("_hidden_params", {})
return model_response
@staticmethod
def _get_provider_response_model(
chunks: Sequence["_BaseChunk"],
first_chunk_model: str,
) -> str | None:
models: Final = tuple(
model
for chunk in chunks
if isinstance((hidden_params := chunk.get("_hidden_params")), Mapping)
if isinstance((model := hidden_params.get("provider_response_model")), str) and model
)
return next(
(model for model in models if model != first_chunk_model),
models[0] if models else None,
)
@staticmethod
def apply_provider_assembled_streaming_metadata(
response: ModelResponse,
@ -360,6 +376,15 @@ class ChunkProcessor:
)
response = self.update_model_response_with_hidden_params(model_response=response, chunk=chunk)
provider_response_model: Final = self._get_provider_response_model(
chunks,
first_chunk_model,
)
if provider_response_model is not None:
response._hidden_params = dict( # pyright: ignore[reportPrivateUsage] # ModelResponse exposes no public hidden-params setter
response._hidden_params, # pyright: ignore[reportPrivateUsage] # ModelResponse exposes no public hidden-params getter
provider_response_model=provider_response_model,
)
return response
@staticmethod

50
litellm/litellm_core_utils/streaming_handler.py
View file

@ -187,17 +187,42 @@ class _ParsedChunkHiddenParams(BaseModel):
provider_specific_fields: Mapping[str, object] | None = None
def _provider_hidden_params(chunk: object) -> Mapping[str, object] | None:
hidden: Final[object] = getattr(chunk, "_hidden_params", None)
def _provider_response_model(chunk: object) -> str | None:
model: Final[object] = chunk.get("model") if isinstance(chunk, Mapping) else getattr(chunk, "model", None)
return model if isinstance(model, str) and model else None
def _parsed_provider_hidden_params(hidden: object) -> _ParsedChunkHiddenParams | None:
if not isinstance(hidden, dict):
return None
try:
parsed: Final = _ParsedChunkHiddenParams.model_validate(hidden)
return _ParsedChunkHiddenParams.model_validate(hidden)
except ValidationError:
return None
if not parsed.provider_specific_fields:
return None
return MappingProxyType({"provider_specific_fields": dict(parsed.provider_specific_fields)})
def _provider_hidden_params(
chunk: object,
provider_response_model: str | None,
) -> Mapping[str, object] | None:
hidden: Final[object] = getattr(chunk, "_hidden_params", None)
parsed: Final = _parsed_provider_hidden_params(hidden)
provider_specific_fields: Final[object | None] = (
dict(parsed.provider_specific_fields) # mutable-ok: stream assembly merges provider metadata into this dict
if parsed is not None and parsed.provider_specific_fields
else None
)
params: Final[Mapping[str, object]] = MappingProxyType(
{
key: value
for key, value in (
("provider_response_model", provider_response_model),
("provider_specific_fields", provider_specific_fields),
)
if value is not None
}
)
return params or None
class CustomStreamWrapper:
@ -229,6 +254,7 @@ class CustomStreamWrapper:
self.thinking_content = ""
self.system_fingerprint: str | None = None
self._provider_response_model: str | None = None
self.received_finish_reason: str | None = None
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
self.special_tokens = [
@ -819,7 +845,9 @@ class CustomStreamWrapper:
except Exception as e:
raise e
def model_response_creator(self, chunk: dict | None = None, hidden_params: Mapping[str, object] | None = None):
def model_response_creator(
self, chunk: dict | None = None, hidden_params: Mapping[str, object] | None = None
) -> ModelResponseStream:
_model: Final = self._cached_model_name
_logging_obj_llm_provider: Final = self._cached_logging_llm_provider
@ -1522,7 +1550,12 @@ class CustomStreamWrapper:
def chunk_creator(self, chunk: Any):
if hasattr(chunk, "id"):
self.response_id = chunk.id
model_response = self.model_response_creator(hidden_params=_provider_hidden_params(chunk))
provider_response_model: Final = _provider_response_model(chunk)
if provider_response_model is not None:
self._provider_response_model = provider_response_model
model_response = self.model_response_creator(
hidden_params=_provider_hidden_params(chunk, self._provider_response_model)
)
response_obj: dict[str, Any] = {}
try:
# return this for all models
@ -2336,6 +2369,7 @@ class CustomStreamWrapper:
partial_response: Final = litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
logging_obj=self.logging_obj,
)
if partial_response is None:
return

20
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -19,19 +19,19 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
GPT5_SERIES_ROUTE = "gpt5_series/"
@classmethod
def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool:
"""Override to handle gpt5_series/ prefix used for Azure routing.
def _model_map_lookup_name(cls, model: str) -> str:
"""Normalise an Azure routing name to its cost-map key.
The parent class calls ``_supports_factory(model, custom_llm_provider=None)``
which fails to resolve ``gpt5_series/gpt-5.1`` to the correct Azure model
entry. Strip the prefix and prepend ``azure/`` so the lookup finds
``azure/gpt-5.1`` in model_prices_and_context_window.json.
Neither ``gpt5_series/gpt-5.1`` nor a bare ``gpt-5.1`` is a key in
model_prices_and_context_window.json; ``azure/gpt-5.1`` is. Overriding the shared
resolver rather than one lookup means the supports, explicitly-disabled and
default-effort answers all read the same entry.
"""
if model.startswith(cls.GPT5_SERIES_ROUTE):
model = "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :]
elif not model.startswith("azure/"):
model = "azure/" + model
return super()._supports_reasoning_effort_level(model, level)
return "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :]
if model.startswith("azure/"):
return model
return "azure/" + model
@classmethod
def is_model_gpt_5_model(cls, model: str) -> bool:

16
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -158,6 +158,22 @@ def openai_messages_without_tool(
return tuple(m for m in messages if _message_role(m) != "tool")
def filter_messages_by_skip_flags(
guardrail_to_apply: object, messages: Sequence[AllMessageValues]
) -> tuple[tuple[AllMessageValues, ...], bool]:
system_filtered = (
openai_messages_without_system(messages)
if effective_skip_system_message_for_guardrail(guardrail_to_apply)
else tuple(messages)
)
fully_filtered = (
openai_messages_without_tool(system_filtered)
if effective_skip_tool_message_for_guardrail(guardrail_to_apply)
else system_filtered
)
return fully_filtered, len(fully_filtered) != len(messages)
def effective_scan_only_tool_results_for_guardrail(guardrail_to_apply: object) -> bool:
return getattr(guardrail_to_apply, "scan_only_tool_results", None) is True

80
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -6,11 +6,28 @@ import litellm
from litellm.utils import (
_is_explicitly_disabled_factory,
_supports_factory,
declared_value_factory,
)
from .gpt_transformation import OpenAIGPTConfig
def _catalogue_declares_default_effort() -> bool:
"""Whether the loaded cost map carries default_reasoning_effort for ANY entry.
The map is fetched from the published branch at import time, so it can be OLDER than the
code reading it. On such a map every model looks undeclared, and treating that as "reasoning
is active" would silently strip temperature from the gpt-5.1/5.2/5.4 deployments that accept
it - a regression caused purely by data lag rather than by anything about the model.
So the absence of the key is only meaningful once the catalogue is known to carry it at all.
A map that has never heard of the key predates the feature, and the honest answer there is
the one litellm gave before it existed. Scanning costs ~80us on the largest published map and
only on the fallback path, which is noise beside the request it precedes.
"""
return any(isinstance(entry, dict) and "default_reasoning_effort" in entry for entry in litellm.model_cost.values())
def _normalize_reasoning_effort_for_chat_completion(
value: str | dict | None,
) -> str | None:
@ -114,6 +131,17 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
except (ValueError, IndexError):
return False
@classmethod
def _model_map_lookup_name(cls, model: str) -> str:
"""The name this model is looked up by in the cost map.
Identity here, because an OpenAI model name is already its map key. Azure overrides
it: its routing prefixes are not map keys, so every capability lookup has to
normalise the name the same way, and doing that in ONE place is what keeps the
supports/disabled/default answers from disagreeing about which entry they read.
"""
return model
@classmethod
def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool:
"""Check if the model supports a specific reasoning_effort level.
@ -123,11 +151,40 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
Returns False for unknown models (safe fallback).
"""
return _supports_factory(
model=model,
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key=f"supports_{level}_reasoning_effort",
)
@classmethod
def effort_resolves_to_none(cls, model: str, effective_effort: str | None) -> bool:
"""Whether this request's reasoning effort ends up as "none", which is the single
condition under which the provider accepts a non-default temperature or the
top_p/logprobs sampling params.
An explicit reasoning_effort answers outright. When the request omits it the answer
is the model's DEFAULT effort, which only the map can state: supporting "none" is a
different fact from defaulting to it, and reading the former as the latter is what
forwarded temperature=0 to every gpt-5.5/5.6 deployment.
An undeclared default resolves to False. The map not saying is not the model
saying no, so the gate takes the conservative branch: a param the provider would
have rejected gets dropped or refused with an actionable error, and a model
released before its map entry declares a default needs no code change to be safe.
"""
if effective_effort is not None:
return effective_effort == "none"
declared: Final = declared_value_factory(
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key="default_reasoning_effort",
)
if declared is not None:
return declared == "none"
if not _catalogue_declares_default_effort():
return cls._supports_reasoning_effort_level(model, "none")
return False
@classmethod
def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool:
"""Return True only when the model map explicitly sets the capability to False.
@ -140,7 +197,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
Use this for opt-out checks where unknown models should be allowed through.
"""
return _is_explicitly_disabled_factory(
model=model,
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key=f"supports_{level}_reasoning_effort",
)
@ -260,15 +317,16 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
if supports_none:
sampling_params: Final = ["logprobs", "top_logprobs", "top_p"]
has_sampling: Final = any(p in non_default_params for p in sampling_params)
if has_sampling and effective_effort not in (None, "none"):
if has_sampling and not self.effort_resolves_to_none(model, effective_effort):
if litellm.drop_params or drop_params:
for p in sampling_params:
non_default_params.pop(p, None)
else:
raise litellm.utils.UnsupportedParamsError(
message=(
"gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when "
f"reasoning_effort='none'. Current reasoning_effort='{effective_effort}'. "
f"{model} only supports logprobs, top_p, top_logprobs when reasoning_effort "
"resolves to 'none', either set explicitly on the request or declared as the "
f"model's default_reasoning_effort. Current reasoning_effort={effective_effort!r}. "
"To drop unsupported params set `litellm.drop_params = True`"
),
status_code=400,
@ -277,17 +335,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
if "temperature" in non_default_params:
temperature_value: Final[float | None] = non_default_params.pop("temperature")
if temperature_value is not None:
# models supporting reasoning_effort="none" also support flexible temperature
if supports_none and (effective_effort == "none" or effective_effort is None) or temperature_value == 1:
# a non-default temperature rides on the effort resolving to "none", not on
# the model merely supporting it
if (supports_none and self.effort_resolves_to_none(model, effective_effort)) or temperature_value == 1:
optional_params["temperature"] = temperature_value
elif litellm.drop_params or drop_params:
pass
else:
raise litellm.utils.UnsupportedParamsError(
message=(
f"gpt-5 models (including gpt-5-codex) don't support temperature={temperature_value}. "
"Only temperature=1 is supported. "
"For gpt-5.1, temperature is supported when reasoning_effort='none' (or not specified, as it defaults to 'none'). "
f"{model} doesn't support temperature={temperature_value} while reasoning is "
"active. Only temperature=1 is supported unless reasoning_effort resolves to "
"'none', either set explicitly on the request or declared as the model's "
"default_reasoning_effort. "
"To drop unsupported params set `litellm.drop_params = True`"
),
status_code=400,

24
litellm/llms/openai/responses/transformation.py
View file

@ -61,6 +61,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
key="supports_none_reasoning_effort",
)
@staticmethod
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
"""Whether this request's reasoning effort ends up as "none", the one condition
under which a non-default temperature is accepted.
Delegates to the chat-completions gpt-5 config so both surfaces answer from one
rule: the Responses API reaches the same models over a different wire, and a second
copy of the rule here is what let this surface keep forwarding temperature after the
chat surface stopped.
"""
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
return OpenAIGPT5Config.effort_resolves_to_none(model, effort)
@staticmethod
def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None":
"""Raise sub-minimum max_output_tokens up to the OpenAI Responses API minimum.
@ -116,17 +130,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
reasoning: Final = params.get("reasoning") or {}
effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None
supports_none: Final = self._supports_reasoning_effort_none(model=model)
if supports_none and (effort == "none" or effort is None):
if supports_none and self._effort_resolves_to_none(model, effort):
pass # flexible temperature allowed
elif drop_params or litellm.drop_params:
params.pop("temperature", None)
else:
raise litellm.UnsupportedParamsError(
message=(
f"gpt-5 models don't support temperature={temperature}. "
"Only temperature=1 is supported. "
"For models like gpt-5.1/5.4, temperature is supported "
"when reasoning.effort='none' (or not specified). "
f"{model} doesn't support temperature={temperature} while reasoning is "
"active. Only temperature=1 is supported unless reasoning.effort resolves "
"to 'none', either set explicitly on the request or declared as the "
"model's default_reasoning_effort. "
"To drop unsupported params set `litellm.drop_params = True`"
),
status_code=400,

77
litellm/main.py
View file

@ -8590,6 +8590,47 @@ def stream_chunk_builder_text_completion(chunks: list, messages: list | None = N
return TextCompletionResponse(**response)
def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None:
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)):
return float(usage_cost)
if logging_obj is not None:
return None
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
"custom_llm_provider"
)
try:
return litellm.completion_cost(completion_response=response, custom_llm_provider=provider_hint)
except Exception:
return _stream_builder_model_map_cost(response)
def _joined_streamed_citations(streamed_citations: "tuple[object, ...]") -> "list[object]":
if all(isinstance(citation, list) for citation in streamed_citations):
return list(streamed_citations) # mutable-ok: JSON list field
return [list(streamed_citations)] # mutable-ok: JSON list field
def _stream_builder_model_map_cost(response: ModelResponse) -> float | None:
model_name: Final = getattr(response, "model", None)
usage: Final = getattr(response, "usage", None)
if not isinstance(model_name, str) or not model_name or not isinstance(usage, Usage):
return None
try:
prompt_cost, completion_tokens_cost = litellm.cost_per_token(model=model_name, usage_object=usage)
return prompt_cost + completion_tokens_cost
except Exception: # noqa: BLE001 # cost_per_token raises bare Exception for unpriceable models
return None
def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
response_cost: Final = _stream_builder_response_cost(response, logging_obj)
if response_cost is None:
return
hidden_params: Final = response._hidden_params # pyright: ignore[reportPrivateUsage] # no public accessor
hidden_params["response_cost"] = response_cost
def stream_chunk_builder(
chunks: list,
messages: list | None = None,
@ -8690,6 +8731,8 @@ def stream_chunk_builder(
"cost",
logging_obj._response_cost_calculator(result=response),
)
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
@ -8814,18 +8857,26 @@ def stream_chunk_builder(
]
if len(provider_specific_chunks) > 0:
combined_provider_fields: Final[dict[str, object]] = {}
for chunk in provider_specific_chunks:
fields = chunk["choices"][0]["delta"]["provider_specific_fields"]
if isinstance(fields, dict):
for key, value in fields.items():
if key not in combined_provider_fields:
combined_provider_fields[key] = value
elif isinstance(value, list) and isinstance(combined_provider_fields[key], list):
# For lists like web_search_results, take the last (most complete) one
combined_provider_fields[key] = value
else:
combined_provider_fields[key] = value
provider_field_dicts: Final = tuple(
fields
for chunk in provider_specific_chunks
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
if isinstance(fields, dict)
)
streamed_citations: Final = tuple(
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
)
citation_fields: Final = (
{"citations": _joined_streamed_citations(streamed_citations)} # mutable-ok: JSON dict field
if streamed_citations
else {} # mutable-ok: JSON dict field
)
combined_provider_fields: Final = { # mutable-ok: Message.provider_specific_fields is a plain dict field
key: value
for fields in (citation_fields, *provider_field_dicts)
for key, value in fields.items()
if key != "citation"
}
if combined_provider_fields:
_choice = cast(Choices, response.choices[0])
@ -8862,6 +8913,8 @@ def stream_chunk_builder(
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
setattr(usage, "cost", logging_obj._response_cost_calculator(result=response))
_set_stream_builder_response_cost(response, logging_obj)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
except Exception as e:

63
litellm/model_prices_and_context_window_backup.json
View file

@ -3409,6 +3409,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -3456,6 +3457,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -3589,6 +3591,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3630,6 +3633,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3671,6 +3675,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3712,6 +3717,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3937,7 +3943,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/eu/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.4e-07,
@ -3972,7 +3979,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/eu/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -4247,7 +4255,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/global/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.25e-07,
@ -4282,7 +4291,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/global/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -5367,6 +5377,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_minimal_reasoning_effort": true
},
"azure/gpt-5.1-chat-2025-11-13": {
@ -5404,7 +5415,8 @@
"supports_system_messages": true,
"supports_tool_choice": false,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/gpt-5.1-codex-2025-11-13": {
"cache_read_input_token_cost": 1.25e-07,
@ -5833,7 +5845,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.25e-07,
@ -5868,7 +5881,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -6315,6 +6329,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6354,6 +6369,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6393,6 +6409,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6438,6 +6455,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6477,6 +6495,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6516,6 +6535,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -7663,6 +7683,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-5.4-mini-2026-03-17": {
@ -7704,6 +7725,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-5.4-nano": {
@ -7745,6 +7767,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-5.4-nano-2026-03-17": {
@ -7786,6 +7809,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-image-1": {
@ -8856,7 +8880,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/us/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.4e-07,
@ -8891,7 +8916,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/us/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -26315,6 +26341,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true
},
@ -26359,6 +26386,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true
},
@ -26404,6 +26432,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true
},
@ -26449,6 +26478,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -26494,6 +26524,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -27318,6 +27349,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -27366,6 +27398,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -27515,6 +27548,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -27566,6 +27600,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -27614,6 +27649,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -27662,6 +27698,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -38682,7 +38719,7 @@
"together_ai/openai/gpt-oss-20b": {
"input_cost_per_token": 5e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 128000,
"max_input_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2e-07,
"source": "https://www.together.ai/models/gpt-oss-20b",
@ -38904,14 +38941,14 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/Qwen/Qwen3.8-2.4T-A95B": {
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1010000,
"max_output_tokens": 1010000,
"max_tokens": 1010000,
"mode": "chat",
"output_cost_per_token": 6.25e-06,
"output_cost_per_token": 6e-06,
"source": "https://docs.together.ai/docs/serverless-models",
"supports_prompt_caching": true
},

152
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1,5 +1,5 @@
import re
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
@ -67,6 +67,9 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
_EMPTY_TOOLSET_GRANTS: Final[Mapping[str, Sequence[str]]] = MappingProxyType({})
def _as_list(values: Sequence[str] | None) -> list[str] | None: # mutable-ok: resolver returns a list
"""Widen a read-only allowlist back to the mutable list the resolver's own contract returns,
preserving the ``None`` that means "no restriction"."""
@ -1497,7 +1500,11 @@ class MCPRequestHandler:
team_set: Final = set(allowed_mcp_servers_for_team)
grants_set: Final = set(key_access_group_grants)
has_lower_level_mcp_restrictions = bool(key_set or team_set or grants_set)
# A DECLARED toolset restricts even when it resolves to no servers: the org
# ceiling below may only cap it, never substitute the org's full server list.
has_lower_level_mcp_restrictions = bool(key_set or team_set or grants_set) or (
await MCPRequestHandler._key_or_team_declares_toolsets(user_api_key_auth)
)
# 1. Key/team ceiling. An empty set means "this level does not restrict".
if not team_set:
@ -1941,6 +1948,105 @@ class MCPRequestHandler:
return team_obj.object_permission
@staticmethod
async def _toolset_tool_permissions(
object_permission: LiteLLM_ObjectPermissionTable | None,
) -> Mapping[str, Sequence[str]]:
"""The ``server_id -> tool names`` grants of this permission row's toolsets, empty when it
declares none. The shared resolver for the team, org, and internal-user levels, so a toolset
behaves identically wherever it is attached.
RAISES ``UnloadableEntitlementError`` when the row DECLARES toolsets but resolution yields
nothing (deleted or unknown ids, a swallowed DB fault, or a toolset with no tools): that is a
KNOWN restriction with unknown contents, and every caller already turns this error into deny
rather than letting the level read as unrestricted."""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
if object_permission is None or not object_permission.mcp_toolsets:
return _EMPTY_TOOLSET_GRANTS
resolved: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions(
toolset_ids=object_permission.mcp_toolsets
)
if not resolved:
raise UnloadableEntitlementError(
f"declared mcp_toolsets {object_permission.mcp_toolsets!r} resolved to no grants"
)
return resolved
@staticmethod
async def _toolset_tools_for_server(
object_permission: LiteLLM_ObjectPermissionTable | None,
server_id: str,
) -> Sequence[str] | None:
"""Tool names this row's toolsets grant on ``server_id``, ``None`` when its toolsets place
no restriction on that server (it declares no toolsets, or none of them name it)."""
return (await MCPRequestHandler._toolset_tool_permissions(object_permission)).get(server_id)
@staticmethod
def _union_tool_grants(
direct: Sequence[str] | None,
via_toolsets: Sequence[str] | None,
) -> Sequence[str] | None:
"""Union of one level's direct tool grants and its toolset-granted tools on one server,
``None`` when neither source restricts (allow-all from this level)."""
if direct is None and via_toolsets is None:
return None
return tuple({*(direct or ()), *(via_toolsets or ())})
@staticmethod
async def _key_object_permission_hydrated(
user_api_key_auth: UserAPIKeyAuth,
) -> LiteLLM_ObjectPermissionTable | None:
"""The key's object_permission, loading it by ``object_permission_id`` when the main auth
flow cached the key with the relation unhydrated (its loader swallows a failed read and
caches the partial object)."""
loaded: Final = MCPRequestHandler._get_key_object_permission(user_api_key_auth)
if loaded is not None or not user_api_key_auth.object_permission_id:
return loaded
from litellm.proxy.auth.auth_checks import get_object_permission
from litellm.proxy.proxy_server import (
prisma_client,
proxy_logging_obj,
user_api_key_cache,
)
if prisma_client is None:
return None
return await get_object_permission(
object_permission_id=user_api_key_auth.object_permission_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=user_api_key_auth.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
@staticmethod
async def _key_or_team_declares_toolsets(user_api_key_auth: UserAPIKeyAuth | None) -> bool:
"""Whether the key or its team GRANTS any toolset, resolvable or not. A declared toolset is
a lower-level restriction even when it resolves to no servers (deleted or unknown ids), so the
org ceiling may only cap it; reading an empty resolution as "no restriction" would substitute
the org's entire server list for the narrowest grant an operator can write.
Falls back to the DB when the auth object carries ``object_permission_id`` unhydrated (the
main auth flow swallows a failed load and caches the partial object). An INDETERMINATE fault
answers False — no gate, org substitution as before the fault — mirroring how the org ceiling
keeps key auth open on a fault it cannot classify."""
if user_api_key_auth is None:
return False
try:
key_obj_perm: Final = await MCPRequestHandler._key_object_permission_hydrated(user_api_key_auth)
if key_obj_perm is not None and key_obj_perm.mcp_toolsets:
return True
if not user_api_key_auth.team_id:
return False
team_obj_perm: Final = await MCPRequestHandler._get_team_object_permission(user_api_key_auth)
return bool(team_obj_perm is not None and team_obj_perm.mcp_toolsets)
except Exception as e: # noqa: BLE001 # indeterminate fault: no gate, as before this level existed
verbose_logger.warning("Failed to check declared MCP toolsets, org ceiling unchanged: %s", e)
return False
@staticmethod
async def get_allowed_tools_for_server(
server_id: str,
@ -2004,12 +2110,17 @@ class MCPRequestHandler:
if key_direct_tools is not None or key_toolset_tools is not None
else None
)
team_tools: Final = (
team_direct_tools: Final = (
global_mcp_server_manager.expand_tool_permissions(team_obj_perm.mcp_tool_permissions).get(server_id)
if team_obj_perm
else None
)
# Tools granted through the team's toolsets restrict this server exactly
# as the team's direct tool permissions do, mirroring the key path above
team_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server(team_obj_perm, server_id)
team_tools: Final = MCPRequestHandler._union_tool_grants(team_direct_tools, team_toolset_tools)
# Apply same inheritance logic as get_allowed_mcp_servers
if team_tools:
if key_tools:
@ -2094,11 +2205,13 @@ class MCPRequestHandler:
e,
)
return allowed_tools
org_tools: Final = (
org_direct_tools: Final = (
global_mcp_server_manager.expand_tool_permissions(org_obj_perm.mcp_tool_permissions).get(server_id)
if org_obj_perm and org_obj_perm.mcp_tool_permissions
else None
)
org_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server(org_obj_perm, server_id)
org_tools: Final = MCPRequestHandler._union_tool_grants(org_direct_tools, org_toolset_tools)
if org_tools is not None:
allowed_tools = (
list(set(allowed_tools) & set(org_tools)) if allowed_tools is not None else list(org_tools)
@ -2340,7 +2453,8 @@ class MCPRequestHandler:
async def _team_granted_servers(team_obj: LiteLLM_TeamTable, team_access_group_servers: list[str]) -> set[str]:
"""The raw MCP-server set a team grants (before any org ceiling): its object_permission (direct
``mcp_servers``, the ``all_proxy_servers`` sentinel → the full registry, legacy access groups,
tool-perm-referenced servers) unioned with its unified ``access_group_ids`` servers."""
tool-perm-referenced servers, toolset-referenced servers) unioned with its unified
``access_group_ids`` servers."""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
@ -2357,6 +2471,7 @@ class MCPRequestHandler:
set(global_mcp_server_manager.expand_permission_list(object_permissions.mcp_servers or []))
| set(legacy_access_group_servers)
| set(global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).keys())
| (await MCPRequestHandler._toolset_tool_permissions(object_permissions)).keys()
| set(team_access_group_servers)
)
@ -2415,6 +2530,8 @@ class MCPRequestHandler:
servers: Final = await MCPRequestHandler._team_granted_servers(team_obj, team_access_group_servers)
return list(servers)
except Exception as e:
if isinstance(e, UnloadableEntitlementError):
raise
verbose_logger.warning("Failed to get allowed MCP servers for team: %s", e)
return []
@ -2546,7 +2663,13 @@ class MCPRequestHandler:
global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).keys()
)
all_servers: Final = direct_mcp_servers + access_group_servers + tool_perm_servers
# servers referenced by the org's toolset grants are part of the org ceiling,
# exactly as servers referenced by its inline tool permissions are
toolset_grants: Final = await MCPRequestHandler._toolset_tool_permissions(object_permissions)
all_servers: Final = tuple(
{*direct_mcp_servers, *access_group_servers, *tool_perm_servers, *toolset_grants}
)
return list(set(all_servers))
except Exception as e:
# None = ceiling UNRESOLVED, distinct from [] = org places no restriction. Collapsing them
@ -2740,8 +2863,8 @@ class MCPRequestHandler:
``[]`` means this human places no restriction (allow-all from this level); ``None`` means the
ceiling is UNRESOLVED, which the caller denies on. Servers named only under
``mcp_tool_permissions`` count as entitled, exactly as they do for a key or a team, so
granting one tool never requires naming its server twice.
``mcp_tool_permissions`` or reached through ``mcp_toolsets`` count as entitled, exactly as
they do for a key or a team, so granting one tool never requires naming its server twice.
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
@ -2759,7 +2882,8 @@ class MCPRequestHandler:
tool_perm_servers: Final = list(
global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).keys()
)
return list(set(direct_mcp_servers + access_group_servers + tool_perm_servers))
toolset_grants: Final = await MCPRequestHandler._toolset_tool_permissions(object_permissions)
return tuple({*direct_mcp_servers, *access_group_servers, *tool_perm_servers, *toolset_grants})
except Exception as e: # noqa: BLE001 # any resolution fault is an unresolved ceiling, never "no ceiling"
verbose_logger.warning("Failed to get allowed MCP servers for user: %s", e)
return None
@ -2860,12 +2984,14 @@ class MCPRequestHandler:
verbose_logger.warning("MCP user tool ceiling unresolvable, denying tools on %r: %s", server_id, e)
return []
if object_permissions is None or not object_permissions.mcp_tool_permissions:
if object_permissions is None:
return allowed_tools
user_tools = global_mcp_server_manager.expand_tool_permissions(object_permissions.mcp_tool_permissions).get(
server_id
)
user_direct_tools: Final = global_mcp_server_manager.expand_tool_permissions(
object_permissions.mcp_tool_permissions
).get(server_id)
user_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server(object_permissions, server_id)
user_tools: Final = MCPRequestHandler._union_tool_grants(user_direct_tools, user_toolset_tools)
if user_tools is None:
return allowed_tools
if allowed_tools is None:

41
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -9018,6 +9018,18 @@
"description": "When True, unified guardrails only evaluate tool results, the untrusted data an agent feeds back into the model, and skip system, user, and assistant content. Intended for agent harnesses whose own prompt scaffolding is trusted but often trips prompt-attack detectors.",
"title": "Scan Only Tool Results"
},
"scan_raw_request": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"description": "When True, this pre_call guardrail always evaluates the request as it was before any guardrail in this hook ran, regardless of its position in the guardrails list -- so the YAML order of guardrails can never change whether this one blocks. Use only for block-only guardrails: any data this guardrail returns is discarded, same contract as run_in_parallel, since an earlier guardrail's masking must not be undone by this one.",
"title": "Scan Raw Request"
},
"sensitive_data_route_to_model": {
"anyOf": [
{
@ -10068,6 +10080,18 @@
"description": "Additional provider-specific parameters for generic guardrail APIs",
"title": "Additional Provider Specific Params"
},
"advisory_system_message": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "Custom advisory message template used when on_flagged='inject_system_message'. Must contain a {reason} placeholder. Defaults to a generic advisory message if unset.",
"title": "Advisory System Message"
},
"akto_account_id": {
"anyOf": [
{
@ -11122,7 +11146,8 @@
{
"enum": [
"block",
"monitor"
"monitor",
"inject_system_message"
],
"type": "string"
},
@ -11131,7 +11156,7 @@
}
],
"default": "block",
"description": "Action to take when content is flagged: 'block' (raise exception) or 'monitor' (log only)",
"description": "Action to take when content is flagged: 'block' (raise exception), 'monitor' (log only), or 'inject_system_message' (append an advisory system message and let the LLM decide)",
"title": "On Flagged"
},
"on_flagged_action": {
@ -11641,6 +11666,18 @@
"description": "When True, unified guardrails only evaluate tool results, the untrusted data an agent feeds back into the model, and skip system, user, and assistant content. Intended for agent harnesses whose own prompt scaffolding is trusted but often trips prompt-attack detectors.",
"title": "Scan Only Tool Results"
},
"scan_raw_request": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"description": "When True, this pre_call guardrail always evaluates the request as it was before any guardrail in this hook ran, regardless of its position in the guardrails list -- so the YAML order of guardrails can never change whether this one blocks. Use only for block-only guardrails: any data this guardrail returns is discarded, same contract as run_in_parallel, since an earlier guardrail's masking must not be undone by this one.",
"title": "Scan Raw Request"
},
"send_user_api_key_alias": {
"anyOf": [
{

1
litellm/proxy/_lazy_openapi_snapshot.py
View file

@ -126,7 +126,6 @@ def _feature_fragment(app: "FastAPI", feat: "LazyFeature", used_operation_ids: s
full: Final = get_openapi(title=app.title, version=app.version, routes=feat_routes)
paths: Final = full.get("paths", {})
_normalize_operation_ids(paths)
# Group all of a feature's routes under one tag.
for path_ops in paths.values():
for method, op in path_ops.items():
if isinstance(op, dict):

2
litellm/proxy/_types.py
View file

@ -940,6 +940,8 @@ class LiteLLMRoutes(enum.Enum):
# Model cost map maintenance views (read-only status / source).
"/schedule/model_cost_map_reload/status",
"/model/cost_map/source",
# A pure read; POST only so the prompt does not ride in a URL.
"/auto_router/classifier/default_prompt",
]
# Spend tracking reads (/spend/logs, /spend/logs/ui, /spend/keys,
# /spend/users, /spend/tags, /spend/calculate, /cost/estimate). Admin

13
litellm/proxy/auth/auth_checks.py
View file

@ -2596,6 +2596,11 @@ async def _delete_cache_key_object(
dropped before the Redis round trip. Letting a cache-backend error raise here therefore reports
failure for work that succeeded without making the cache any less stale; the leftover Redis
entry expires at its TTL either way.
Also broadcasts the eviction to every other worker (LIT-3803): auth serves this object
cache-first with no freshness check, so a worker that never receives the broadcast keeps
admitting requests against the pre-mutation object (e.g. a just-reset spend) until its own
copy's TTL expires.
"""
key: Final = hashed_token
@ -2612,6 +2617,8 @@ async def _delete_cache_key_object(
e,
)
await publish_auth_cache_invalidation(cache_key=key)
async def delete_cache_key_objects(
hashed_tokens: Sequence[str],
@ -2623,8 +2630,9 @@ async def delete_cache_key_objects(
`/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left
cached after its row is gone keeps buying access until its TTL expires.
Evicting locally only reaches this worker, so each token is also broadcast: a deleted key left
in a peer worker's in-memory cache still authenticates there until its TTL expires.
Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each
token, so a deleted key left in a peer worker's in-memory cache still authenticates there until
its TTL expires.
Best-effort per key: the rows are already deleted by the time this runs, so an unreachable
cache backend must not abort the caller partway through its own cascade.
@ -2648,7 +2656,6 @@ async def delete_cache_key_objects(
hashed_token,
result,
)
await publish_auth_cache_invalidation(cache_key=hashed_token)
class _TeamNotFoundDetail(TypedDict):

20
litellm/proxy/common_request_processing.py
View file

@ -2551,16 +2551,6 @@ class ProxyBaseLLMRequestProcessing:
except Exception as e:
verbose_proxy_logger.exception("Error in orphaned streaming async logging: %s", e)
# Always return the client-requested model name (not provider-prefixed internal identifiers)
# for OpenAI-compatible responses.
if requested_model_from_client:
_override_openai_response_model(
response_obj=response,
requested_model=requested_model_from_client,
log_context=f"litellm_call_id={logging_obj.litellm_call_id}",
return_raw_model_name=_should_return_raw_model_name(self.data),
)
hidden_params = get_hidden_params_dict(response) # get any updated response headers
additional_headers = hidden_params.get("additional_headers", {}) or {}
@ -2586,6 +2576,16 @@ class ProxyBaseLLMRequestProcessing:
else llm_cost_for_headers
)
# Always return the client-requested model name (not provider-prefixed internal identifiers)
# for OpenAI-compatible responses.
if requested_model_from_client:
_override_openai_response_model(
response_obj=response,
requested_model=requested_model_from_client,
log_context=f"litellm_call_id={logging_obj.litellm_call_id}",
return_raw_model_name=_should_return_raw_model_name(self.data),
)
fastapi_response.headers.update(
ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,

4
litellm/proxy/common_utils/reset_budget_job.py
View file

@ -3,7 +3,7 @@ import json
import math
import time
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
from types import MappingProxyType
@ -224,7 +224,7 @@ class _BudgetCascade:
endusers: tuple[_EndUserRow, ...] = ()
counter_resets: tuple[tuple[str, float], ...] = ()
cache_keys: tuple[str, ...] = ()
rollover_caps: Mapping[str, float] = MappingProxyType({})
rollover_caps: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
@dataclass(frozen=True, slots=True)

65
litellm/proxy/db/shadow_eval_funnel.py Normal file
View file

@ -0,0 +1,65 @@
"""Pod-local queue of shadow-eval funnel increments, drained by the spend-update job.
The shadow-eval success hook counts the sampled-traffic outcomes that never produce an
attempt row (a lost sampling dice roll, an unjudgeable request shape, a concurrency
shed), so a job's results can state what share of its eligible traffic the judged rows
represent. Counters are advisory coverage stats: a pod dying loses at most one flush
interval, and a failed flush drops its batch because a repeated increment is worse
than an undercount (same call as the auto-router session rollup flush).
"""
from typing import TYPE_CHECKING, Final, Literal
from litellm._logging import verbose_proxy_logger
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
ShadowEvalFunnelStage = Literal["not_sampled", "unjudgeable", "shed", "withheld"]
FUNNEL_STAGES: Final[tuple[ShadowEvalFunnelStage, ...]] = ("not_sampled", "unjudgeable", "shed", "withheld")
_pending: dict[str, dict[ShadowEvalFunnelStage, int]] = {} # mutable-ok: module-level queue, single event loop
_FUNNEL_PLACEHOLDERS: Final = ", ".join(f"${n + 2}" for n in range(len(FUNNEL_STAGES)))
_UPSERT_FUNNEL_SQL: Final = f"""
INSERT INTO "LiteLLM_ShadowEvalFunnel" (job_id, {", ".join(FUNNEL_STAGES)})
VALUES ($1, {_FUNNEL_PLACEHOLDERS})
ON CONFLICT (job_id) DO UPDATE SET
{", ".join(f'{stage} = "LiteLLM_ShadowEvalFunnel".{stage} + EXCLUDED.{stage}' for stage in FUNNEL_STAGES)}
"""
def pending_shadow_eval_funnel_events() -> int:
"""Queue census for the drain triggers: entries not yet flushed, so a funnel-only
batch still wakes the spend job that would otherwise skip an empty-queue run."""
return sum(sum(counters.values()) for counters in _pending.values())
def record_shadow_eval_funnel_event(job_id: str, stage: ShadowEvalFunnelStage) -> None:
"""Count one skipped request for one job leg; synchronous so the hook's read-modify-
write cannot interleave with the flush's snapshot on the shared event loop."""
counters: Final = _pending.setdefault(job_id, dict.fromkeys(FUNNEL_STAGES, 0)) # mutable-ok: queue entry
counters[stage] += 1
async def flush_shadow_eval_funnel(prisma_client: "PrismaClient") -> None:
if not _pending:
return
batch: Final = dict(_pending) # mutable-ok: snapshot drained from the queue
_pending.clear()
for job_id, counters in batch.items():
try:
await prisma_client.db.execute_raw(
_UPSERT_FUNNEL_SQL,
job_id,
*(counters[stage] for stage in FUNNEL_STAGES),
)
except Exception as flush_err: # noqa: BLE001 # drop this leg's batch: a repeated increment is worse than an undercount
verbose_proxy_logger.error(
"Spend tracking - shadow eval funnel flush failed for job %s, %s dropped: %s",
job_id,
counters,
flush_err,
)

24
litellm/proxy/guardrails/guardrail_endpoints.py
View file

@ -1218,6 +1218,30 @@ async def patch_guardrail(
verbose_proxy_logger.info(
"Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id
)
except (ValueError, TypeError) as update_error:
# The new config is invalid (e.g. an unsupported on_flagged combination):
# reinitialize_guardrail already restored the previous live instance, but
# update_guardrail_in_db above already persisted the rejected config to
# the DB. Roll that back too, so the DB and the live guardrail never
# disagree about what's actually enforcing, and surface the rejection to
# the caller instead of a misleading 200.
await GUARDRAIL_REGISTRY.update_guardrail_in_db(
guardrail_id=guardrail_id,
guardrail=Guardrail(
guardrail_id=guardrail_id,
guardrail_name=existing_guardrail.get("guardrail_name") or "",
litellm_params=LitellmParams(**existing_litellm_params),
guardrail_info=existing_guardrail.get(
"guardrail_info",
{}, # mutable-ok: Guardrail's own constructor takes a plain dict
),
),
prisma_client=prisma_client,
)
raise HTTPException(
status_code=422,
detail=f"Invalid guardrail configuration, update rejected: {update_error}",
) from update_error
except Exception as update_error:
verbose_proxy_logger.warning(
"Immediate sync: Failed to update '%s' (ID: %s) in memory: %s",

4
litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py
View file

@ -385,7 +385,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
return [_extract_text_from_message(msg) for msg in tail]
async def _call_or_fail_open(
self, payload: dict[str, Any], hook_name: str, request_data: dict
self, payload: dict[str, Any], hook_name: str, request_data: dict[str, object]
) -> _GuardChatCompletionsResult:
start_time: Final = time.time()
try:
@ -421,7 +421,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
structured_messages: list[AllMessageValues],
guard_output: _GuardInput,
sent_indices: tuple[int, ...],
request_data: dict,
request_data: dict[str, object],
) -> list[AllMessageValues] | None:
if effective_skip_system_message_for_guardrail(self) or effective_skip_tool_message_for_guardrail(self):
request_messages: Final = request_data.get("messages")

446
litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py
View file

@ -1,13 +1,25 @@
import copy
import os
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Final
from string import Formatter
from types import MappingProxyType
from typing import Final, Literal
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import (
DEFAULT_ADVISORY_MESSAGE,
CustomGuardrail,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
filter_messages_by_skip_flags,
merge_guardrailed_scoped_messages,
)
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -19,14 +31,190 @@ from litellm.proxy.guardrails._content_utils import (
has_non_string_content,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
from litellm.types.llms.openai import AllMessageValues
from litellm.types.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import (
LakeraAIBreakdownItem,
LakeraAIRequest,
LakeraAIResponse,
)
from litellm.types.utils import CallTypesLiteral, GuardrailStatus, ModelResponse
_DETECTOR_CATEGORY_PHRASES: Final[Mapping[str, str]] = MappingProxyType(
{
"prompt_injection": "a potential prompt injection attempt",
"prompt_attack": "a potential prompt injection attempt",
"pii": "personally identifiable information",
"moderated_content": "policy-violating content",
}
)
def humanize_lakera_block_reasons(breakdown: Sequence[LakeraAIBreakdownItem] | None) -> str:
"""
Turn a Lakera v2 ``breakdown`` list into a plain-language reason string
suitable for an advisory message shown to the LLM (e.g. "a potential
prompt injection attempt, personally identifiable information").
Falls back to a generic phrase when breakdown is empty or every detected
detector_type is unrecognized.
"""
if not breakdown:
return "a content safety concern"
categories: Final = (
(item.get("detector_type") or "").split("/")[0] for item in breakdown if item.get("detected", False)
)
phrases: Final = tuple(
dict.fromkeys(
_DETECTOR_CATEGORY_PHRASES.get(category) or category.replace("_", " ")
for category in categories
if category
)
)
return ", ".join(phrases) if phrases else "a content safety concern"
def _template_uses_reason_placeholder(template: str) -> bool:
"""True if ``template`` has a real ``{reason}`` format field, not just the
literal substring -- an escaped ``{{reason}}`` contains the substring but
formats to a literal "{reason}", never substituting the actual value."""
return any(field_name == "reason" for _, field_name, _, _ in Formatter().parse(template))
def _pre_masking_scope_indices(
guardrail: "LakeraAIGuardrail",
messages: Sequence[object],
) -> tuple[int, ...]:
"""Indices into ``messages`` that mask-in-place can safely target: has
non-empty string content, and survives the same skip_system_message_in_guardrail
/ skip_tool_message_in_guardrail scoping ``filter_messages_by_skip_flags``
applies. Content is guaranteed to already be a plain string here -- masking
is only attempted when ``has_non_string_content(data)`` is False.
Preserved in original order, so it lines up positionally with the
``messages_for_lakera`` list _build_lakera_inspection_messages/skip-filtering
produces from the same input: both apply the identical "has text" and
"not skipped by role" predicates over the same original sequence. Role
comparison is lowercased to match filter_messages_by_skip_flags's own
normalization (via its _message_role helper) -- an uppercase-cased
"System"/"TOOL" role must be excluded by both or the two lists disagree
on length and the caller's strict positional zip raises."""
skip_system: Final = effective_skip_system_message_for_guardrail(guardrail)
skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail)
return tuple(
idx
for idx, message in enumerate(messages)
if isinstance(message, dict)
and isinstance(message.get("content"), str)
and message["content"]
and not (skip_system and str(message.get("role") or "").lower() == "system")
and not (skip_tool and str(message.get("role") or "").lower() == "tool")
)
def _apply_redacted_messages_back_preserving_fields(
guardrail: "LakeraAIGuardrail",
data: dict[str, object], # mutable-ok: writes the redacted result back into the caller's request dict in place
redacted_messages: Sequence[AllMessageValues],
) -> None:
"""Write masked content back to ``data["messages"]`` without losing fields
the synthetic role/content-only ``redacted_messages`` never carried (e.g. a
tool message's tool_call_id, an assistant message's tool_calls, name,
cache_control). Falls back to the shared, wholesale-replacing
apply_redacted_messages_back when ``data["messages"]`` isn't a list (a pure
Responses-API ``input`` string, with no chat messages to merge into)."""
original_messages: Final = data.get("messages")
if not isinstance(original_messages, list):
redacted_list: Final = list(redacted_messages) # mutable-ok: apply_redacted_messages_back requires a list
apply_redacted_messages_back(data, redacted_list)
return
scope_indices: Final = _pre_masking_scope_indices(guardrail, original_messages)
guardrailed_scoped: Final = tuple(
{ # mutable-ok: fresh dict per iteration, not stored beyond this comprehension
**original_messages[original_idx],
"content": redacted["content"],
}
for original_idx, redacted in zip(scope_indices, redacted_messages, strict=True)
)
data["messages"] = merge_guardrailed_scoped_messages(
full_messages=original_messages,
scoped_indices=scope_indices,
guardrailed_scoped=guardrailed_scoped, # pyright: ignore[reportArgumentType] # plain dicts satisfy AllMessageValues's TypedDict shape at runtime
)
def _has_combined_messages_and_input(data: Mapping[str, object]) -> bool:
"""True if ``data`` carries both ``messages`` and ``input``.
build_inspection_messages flattens both into one synthetic list, so
mask-in-place would write input-derived content into data["messages"]
(and vice versa) even when a message dropped for having no text
coincidentally keeps the raw message count unchanged."""
return isinstance(data.get("messages"), list) and data.get("input") is not None
def _has_responses_instructions(guardrail: "LakeraAIGuardrail", data: Mapping[str, object]) -> bool:
"""True if ``data`` carries a Responses-API ``instructions`` field that
Lakera actually inspected. _build_lakera_inspection_messages includes
``instructions`` as a synthetic system message so Lakera can inspect it,
but apply_redacted_messages_back has no path to rewrite
``data["instructions"]`` -- masking here would either leave unredacted
content in the real instructions field the model reads, or write a
redacted duplicate into data["messages"] instead, which the Responses
API never consumes.
When skip_system_message_in_guardrail excludes that synthetic system
message before it ever reaches Lakera, none of this applies: Lakera never
saw ``instructions``, so it can't have flagged anything there, and
forcing a hard block anyway would defeat the whole point of the skip
flag for a response that only carries PII in the (maskable) non-system
content."""
instructions: Final = data.get("instructions")
return (
isinstance(instructions, str)
and bool(instructions)
and not effective_skip_system_message_for_guardrail(guardrail)
)
def _breakdown_has_pii_violation(lakera_response: LakeraAIResponse | None) -> bool:
"""True if any PII-category detector fired, regardless of whether other,
non-PII detectors (prompt injection, moderated content) also fired.
Unlike ``_is_only_pii_violation``, this doesn't require PII to be the
*only* thing detected -- it's used to decide whether masking/blocking is
even relevant at all before advisory mode's own logic runs."""
if not lakera_response:
return False
breakdown: Final = lakera_response.get("breakdown") or ()
return any(
item.get("detected", False) and (item.get("detector_type") or "").startswith("pii/") for item in breakdown
)
def _build_lakera_inspection_messages(data: Mapping[str, object]) -> Sequence[Mapping[str, str]]:
"""Like build_inspection_messages, but also covers the Responses-API
``instructions`` field, placed first since litellm later converts it
into the model's leading system message and a prompt-injection detector
should see the same conversation order the model actually receives.
Kept local to Lakera rather than folded into the shared
_content_utils.build_inspection_messages helper: doing that once made
``instructions`` visible to every guardrail sharing that helper (AIM,
presidio, bedrock, ...), but only Lakera has a masking-safety-guard
(_has_responses_instructions) accounting for apply_redacted_messages_back
having no write-back path for data["instructions"] -- other guardrails
would have silently mishandled a PII/redaction hit found there."""
instructions: Final = data.get("instructions")
leading: Final[Sequence[Mapping[str, str]]] = (
[{"role": "system", "content": instructions}] # mutable-ok: fresh list/dict, not stored
if isinstance(instructions, str) and instructions
else [] # mutable-ok: fresh empty list, not stored
)
return [ # mutable-ok: fresh list, not stored
*leading,
*build_inspection_messages(dict(data)), # mutable-ok: fresh shallow copy for the dict[str, Any] param
]
class LakeraAIGuardrail(CustomGuardrail):
@classmethod
@ -46,7 +234,10 @@ class LakeraAIGuardrail(CustomGuardrail):
breakdown: bool | None = True,
metadata: dict | None = None,
dev_info: bool | None = True,
on_flagged: str | None = "block",
on_flagged: Literal["block", "monitor", "inject_system_message"] | None = "block",
skip_system_message_in_guardrail: bool | None = None,
skip_tool_message_in_guardrail: bool | None = None,
advisory_system_message: str | None = None,
**kwargs,
):
"""
@ -65,7 +256,13 @@ class LakeraAIGuardrail(CustomGuardrail):
breakdown: Optional[bool] = True,
metadata: Optional[Dict] = None,
dev_info: Optional[bool] = True,
on_flagged: Optional[str] = "block", Action to take when content is flagged: "block" or "monitor"
on_flagged: Optional[str] = "block", Action to take when content is flagged:
"block", "monitor", or "inject_system_message"
skip_system_message_in_guardrail: Optional[bool] = None,
skip_tool_message_in_guardrail: Optional[bool] = None,
advisory_system_message: Optional[str] = None, custom advisory message template
(must contain a {reason} placeholder) used when on_flagged="inject_system_message".
Defaults to a generic message when unset.
"""
self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
self.lakera_api_key = api_key or os.environ.get("LAKERA_API_KEY") or ""
@ -75,13 +272,89 @@ class LakeraAIGuardrail(CustomGuardrail):
self.breakdown: bool | None = breakdown
self.metadata: dict | None = metadata
self.dev_info: bool | None = dev_info
self.skip_system_message_in_guardrail = skip_system_message_in_guardrail
self.skip_tool_message_in_guardrail = skip_tool_message_in_guardrail
self.on_flagged = on_flagged or "block"
self.advisory_system_message = advisory_system_message
kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
super().__init__(**kwargs)
self._validate_advisory_config(
on_flagged=self.on_flagged,
advisory_system_message=self.advisory_system_message,
payload=self.payload,
breakdown=self.breakdown,
)
def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None:
"""
The base implementation blindly ``setattr``s every field on ``litellm_params``
(including ``on_flagged``/``advisory_system_message``/``payload``/``breakdown``)
onto this live instance with no revalidation, so an in-place config update (via
the DB/UI, without a restart) could otherwise reintroduce the exact invalid
on_flagged combinations __init__ rejects. Validate the prospective post-update
state *before* mutating, so a rejected update leaves the live instance untouched
instead of raising after it's already been corrupted.
The base setattr also writes ``litellm_params.mode`` onto a new ``self.mode``
attribute rather than the ``self.event_hook`` dispatch actually reads
(LitellmParams has no field literally named ``event_hook``), so without the
explicit sync below a hot reload that changes mode would pass validation but
keep dispatching on the stale event_hook.
"""
new_event_hook: Final = getattr(litellm_params, "mode", None) or self.event_hook
prospective_payload: Final = getattr(litellm_params, "payload", None)
prospective_breakdown: Final = getattr(litellm_params, "breakdown", None)
self._validate_advisory_config(
on_flagged=getattr(litellm_params, "on_flagged", None) or self.on_flagged,
advisory_system_message=getattr(litellm_params, "advisory_system_message", None),
payload=self.payload if prospective_payload is None else prospective_payload,
breakdown=self.breakdown if prospective_breakdown is None else prospective_breakdown,
)
super().update_in_memory_litellm_params(litellm_params=litellm_params)
self.event_hook = new_event_hook
def _validate_advisory_config(
self,
on_flagged: str,
advisory_system_message: str | None,
payload: bool | None,
breakdown: bool | None,
) -> None:
if on_flagged == "inject_system_message" and advisory_system_message is not None:
if not _template_uses_reason_placeholder(advisory_system_message):
raise ValueError(
"Invalid advisory_system_message template: must include a real {reason} "
"placeholder (not an escaped {{reason}}) so the LLM sees why the request was flagged."
)
try:
advisory_system_message.format(reason="placeholder")
except (KeyError, IndexError, ValueError) as e:
raise ValueError(
f"Invalid advisory_system_message template: {e}. The template must be a valid "
"str.format() string using only the {reason} placeholder."
) from e
if on_flagged == "inject_system_message" and not (payload and breakdown):
raise ValueError(
"on_flagged='inject_system_message' requires payload=True and breakdown=True: advisory "
"mode masks any detected PII before appending the advisory note, and that masking can "
"only happen when Lakera's response carries both the violation breakdown and the "
"payload location data. Without them, PII would be forwarded to the model unredacted."
)
def _build_advisory_message(self, lakera_response: LakeraAIResponse | None) -> str:
"""Format the advisory message shown to the LLM when on_flagged='inject_system_message'."""
reason: Final = humanize_lakera_block_reasons(lakera_response.get("breakdown") if lakera_response else None)
template: Final = self.advisory_system_message or DEFAULT_ADVISORY_MESSAGE
return template.format(reason=reason)
def _filter_skipped_messages(
self, messages: Sequence[AllMessageValues]
) -> tuple[tuple[AllMessageValues, ...], bool]:
return filter_messages_by_skip_flags(self, messages)
async def call_v2_guard(
self,
messages: list[AllMessageValues],
messages: Sequence[AllMessageValues],
request_data: dict,
event_type: GuardrailEventHooks,
) -> tuple[LakeraAIResponse, dict]:
@ -143,10 +416,10 @@ class LakeraAIGuardrail(CustomGuardrail):
def _mask_pii_in_messages(
self,
messages: list[AllMessageValues],
messages: Sequence[AllMessageValues],
lakera_response: LakeraAIResponse | None,
masked_entity_count: dict,
) -> list[AllMessageValues]:
) -> Sequence[AllMessageValues]:
"""
Return a copy of messages with any detected PII replaced by
“[MASKED <TYPE>]” tokens.
@ -218,18 +491,38 @@ class LakeraAIGuardrail(CustomGuardrail):
verbose_proxy_logger.debug("Lakera AI: not running guardrail. Guardrail is disabled.")
return data
# Covers multimodal list content + Responses-API input.
new_messages: Final = build_inspection_messages(data)
if not new_messages:
# Covers multimodal list content + Responses-API input/instructions.
inspection_messages: Final = _build_lakera_inspection_messages(data)
if not inspection_messages:
verbose_proxy_logger.warning("Lakera AI: not running guardrail. No inspectable text in data")
return data
# Mask-in-place uses offsets returned by Lakera and can only
# preserve non-text parts (images, audio, …) when the original
# content is a plain string. For multimodal/Responses-API input
# we degrade to block-on-detect so we never silently strip image
# parts while attempting to redact text.
is_multimodal_input: Final = has_non_string_content(data)
new_messages, _ = self._filter_skipped_messages(
inspection_messages # pyright: ignore[reportArgumentType] # build_inspection_messages returns plain dicts, not typed message unions
)
if not new_messages:
verbose_proxy_logger.warning(
"Lakera AI: not running guardrail. All inspectable text was excluded by "
"skip_system_message_in_guardrail/skip_tool_message_in_guardrail"
)
return data
# Mask-in-place can only preserve non-text parts (images, audio) when
# the original content is a plain string, and can only merge a
# redacted result back into data["messages"] by position when
# messages and input aren't both present at once (build_inspection_messages
# flattens both into one list, so a position could mean either).
# Degrade to block-on-detect in either case. Skip-flag-excluded and
# no-text messages, and messages carrying fields beyond role/content
# (tool_call_id, name, tool_calls, cache_control), are otherwise
# handled safely by _apply_redacted_messages_back_preserving_fields's
# scope-index merge, which never touches a message outside the scope
# it actually redacted instead of reconstructing the list from scratch.
is_multimodal_input: Final = (
has_non_string_content(data)
or _has_combined_messages_and_input(data)
or _has_responses_instructions(self, data)
)
#########################################################
########## 1. Make the Lakera AI v2 guard API request ##########
@ -244,18 +537,52 @@ class LakeraAIGuardrail(CustomGuardrail):
########## 2. Handle flagged content ##########
#########################################################
if lakera_guardrail_response.get("flagged") is True:
# If only PII violations exist, mask the PII (string input only).
# PII-only violations get masked in place regardless of on_flagged: there's
# no reason to expose raw PII to satisfy an advisory note, and masking is
# strictly safer than either blocking or appending an advisory message next
# to unredacted PII.
if self._is_only_pii_violation(lakera_guardrail_response) and not is_multimodal_input:
redacted_messages: Final = self._mask_pii_in_messages(
messages=new_messages,
lakera_response=lakera_guardrail_response,
masked_entity_count=masked_entity_count,
)
# Write back to ``messages`` AND ``input``. The Responses-API
# backend reads ``input``; writing only to ``messages``
# would let unredacted PII reach the LLM for /v1/responses.
apply_redacted_messages_back(data, list(redacted_messages))
_apply_redacted_messages_back_preserving_fields(self, data, redacted_messages)
verbose_proxy_logger.debug("Lakera AI: Masked PII in messages instead of blocking request")
elif self.on_flagged == "inject_system_message":
if _breakdown_has_pii_violation(lakera_guardrail_response) and is_multimodal_input:
# There's PII in the mix and nothing here can be safely masked,
# so an advisory note next to this raw, unredacted PII would be
# no safer than a note next to nothing. Degrade to blocking
# instead, same as this on_flagged setting already does when
# the advisory itself has no field it can be delivered into.
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)
masked_pii_before_advisory: Final = _breakdown_has_pii_violation(lakera_guardrail_response)
if masked_pii_before_advisory:
# A mixed violation (PII plus something else, e.g. prompt
# injection): mask whatever Lakera returned location data for
# before advising about what remains, so the advisory is never
# shown next to raw PII that could have been redacted.
mixed_redacted_messages: Final = self._mask_pii_in_messages(
messages=new_messages,
lakera_response=lakera_guardrail_response,
masked_entity_count=masked_entity_count,
)
_apply_redacted_messages_back_preserving_fields(self, data, mixed_redacted_messages)
advisory_delivered: Final = self.inject_advisory_message(
data, self._build_advisory_message(lakera_guardrail_response)
)
if advisory_delivered:
verbose_proxy_logger.warning(
"Lakera Guardrail: Advisory mode - violation detected, %sappended advisory system message",
"masked PII and " if masked_pii_before_advisory else "",
)
else:
# Structured Responses-API input (a list, not a plain string)
# has no field this can safely append into -- degrade to
# blocking rather than silently letting the flagged request
# through with no advisory ever reaching the model.
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)
else:
# Check on_flagged setting
if self.on_flagged == "monitor":
@ -290,19 +617,26 @@ class LakeraAIGuardrail(CustomGuardrail):
if self.should_run_guardrail(data=data, event_type=event_type) is not True:
return
new_messages: Final = build_inspection_messages(data)
if not new_messages:
# Covers multimodal list content + Responses-API input/instructions.
inspection_messages: Final = _build_lakera_inspection_messages(data)
if not inspection_messages:
verbose_proxy_logger.warning("Lakera AI: not running guardrail. No inspectable text in data")
return
# See ``async_pre_call_hook`` — multimodal input degrades to
# block-on-detect because mask-in-place would drop image parts.
is_multimodal_input: Final = has_non_string_content(data)
new_messages, _ = self._filter_skipped_messages(
inspection_messages # pyright: ignore[reportArgumentType] # build_inspection_messages returns plain dicts, not typed message unions
)
if not new_messages:
verbose_proxy_logger.warning(
"Lakera AI: not running guardrail. All inspectable text was excluded by "
"skip_system_message_in_guardrail/skip_tool_message_in_guardrail"
)
return
#########################################################
########## 1. Make the Lakera AI v2 guard API request ##########
#########################################################
lakera_guardrail_response, masked_entity_count = await self.call_v2_guard(
lakera_guardrail_response, _ = await self.call_v2_guard(
messages=new_messages,
request_data=data,
event_type=GuardrailEventHooks.during_call,
@ -312,24 +646,29 @@ class LakeraAIGuardrail(CustomGuardrail):
########## 2. Handle flagged content ##########
#########################################################
if lakera_guardrail_response.get("flagged") is True:
if self._is_only_pii_violation(lakera_guardrail_response) and not is_multimodal_input:
redacted_messages: Final = self._mask_pii_in_messages(
messages=new_messages,
lakera_response=lakera_guardrail_response,
masked_entity_count=masked_entity_count,
)
# Write back to ``messages`` AND ``input``. The Responses-API
# backend reads ``input``; writing only to ``messages``
# would let unredacted PII reach the LLM for /v1/responses.
apply_redacted_messages_back(data, list(redacted_messages))
verbose_proxy_logger.debug("Lakera AI: Masked PII in messages instead of blocking request")
else:
if self.on_flagged == "monitor":
verbose_proxy_logger.warning(
"Lakera Guardrail: Monitoring mode - violation detected but allowing request"
)
elif self.on_flagged == "block":
# during_call runs concurrently with the LLM dispatch (see
# ProxyLogging.during_call_hook / common_request_processing.py), with
# no pre-call barrier: in the common path, the provider call already
# binds its messages kwarg before this coroutine gets a chance to run,
# let alone before the masking helper's own network round trip
# completes. Unlike async_pre_call_hook, mask-in-place here can never
# reliably reach the outgoing request, so PII is never masked in this
# hook -- only blocked (which still works, since raising here blocks
# the response from reaching the caller regardless of dispatch timing)
# or, for non-PII violations, logged and allowed same as monitor mode.
if self.on_flagged == "inject_system_message":
if _breakdown_has_pii_violation(lakera_guardrail_response):
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)
verbose_proxy_logger.warning(
"Lakera Guardrail: Advisory mode has no effect during during_call; "
"violation detected but allowing request"
)
elif self.on_flagged == "monitor":
verbose_proxy_logger.warning(
"Lakera Guardrail: Monitoring mode - violation detected but allowing request"
)
elif self.on_flagged == "block":
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)
#########################################################
########## 3. Add the guardrail to the applied guardrails header ##########
@ -355,9 +694,8 @@ class LakeraAIGuardrail(CustomGuardrail):
if self.should_run_guardrail(data=data, event_type=event_type) is not True:
return response
original_messages: list[AllMessageValues] | None = data.get("messages", [])
if original_messages is None:
original_messages = []
messages_or_none: Final[list[AllMessageValues] | None] = data.get("messages")
original_messages, _ = self._filter_skipped_messages(messages_or_none or [])
# Extract assistant messages from the response, keeping only role/content.
# Track choice indices so we write masked content back to the correct choice
@ -376,7 +714,7 @@ class LakeraAIGuardrail(CustomGuardrail):
choice_indices.append(i)
# Use a copy of original_messages so _mask_pii_in_messages does not mutate data["messages"]
post_call_messages: Final = copy.deepcopy(original_messages) + response_messages
post_call_messages: Final = list(copy.deepcopy(original_messages)) + response_messages # mutable-ok: needs list
# Call Lakera guardrail
lakera_guardrail_response, _ = await self.call_v2_guard(
@ -403,9 +741,13 @@ class LakeraAIGuardrail(CustomGuardrail):
add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name)
return ModelResponse(**response_dict)
if self.on_flagged == "monitor":
verbose_proxy_logger.warning("Lakera Guardrail: Post-call violation detected in monitor mode")
# Allow response to proceed
# inject_system_message has nothing left to inject into once a response
# already exists, so it is treated the same as monitor: log and allow.
if self.on_flagged in ("monitor", "inject_system_message"):
verbose_proxy_logger.warning(
"Lakera Guardrail: Post-call violation detected (on_flagged=%s) - allowing response",
self.on_flagged,
)
elif self.on_flagged == "block":
raise self._get_http_exception_for_blocked_guardrail(lakera_guardrail_response)

29
litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py
View file

@ -22,7 +22,7 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
from litellm.types.llms.openai import AllMessageValues
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
from litellm.types.utils import GenericGuardrailAPIInputs
@ -87,6 +87,7 @@ class QualifireGuardrail(CustomGuardrail):
self.tool_selection_quality_check = tool_selection_quality_check
self.assertions = assertions
self.on_flagged = on_flagged or "block"
self._validate_on_flagged(self.on_flagged)
# If no checks are specified and no evaluation_id, default to prompt_injections
if not self._has_any_check_enabled() and not self.evaluation_id:
@ -98,6 +99,32 @@ class QualifireGuardrail(CustomGuardrail):
kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
super().__init__(**kwargs)
def _validate_on_flagged(self, on_flagged: str) -> None:
if on_flagged not in ("block", "monitor"):
# on_flagged is defined on LakeraV2GuardrailConfigModel but LitellmParams
# flattens every guardrail config mixin together, so a value Lakera
# supports (e.g. "inject_system_message") type-checks for any guardrail,
# including this one, which never implements it. Reject it explicitly
# instead of silently falling through to a block-on-anything-else branch.
raise ValueError(
f"Qualifire guardrail does not support on_flagged={on_flagged!r}; "
"only 'block' and 'monitor' are supported."
)
def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None:
"""
The base implementation blindly ``setattr``s every field on ``litellm_params``
(including ``on_flagged``) onto this live instance with no revalidation, so an
in-place config update (via the DB/UI, without a restart) could otherwise
reintroduce the exact invalid on_flagged value __init__ rejects. Validate the
prospective post-update value *before* mutating, so a rejected update leaves
the live instance untouched instead of raising after it's already been
corrupted. Mirrors LakeraAIGuardrail's own override of this same method.
"""
prospective_on_flagged: Final = getattr(litellm_params, "on_flagged", None) or self.on_flagged
self._validate_on_flagged(prospective_on_flagged)
super().update_in_memory_litellm_params(litellm_params=litellm_params)
def _has_any_check_enabled(self) -> bool:
"""Check if any evaluation check is explicitly enabled."""
return any(

3
litellm/proxy/guardrails/guardrail_initializers.py
View file

@ -73,6 +73,9 @@ def initialize_lakera_v2(litellm_params: LitellmParams, guardrail: Guardrail):
metadata=litellm_params.metadata,
dev_info=litellm_params.dev_info,
on_flagged=litellm_params.on_flagged,
skip_system_message_in_guardrail=litellm_params.skip_system_message_in_guardrail,
skip_tool_message_in_guardrail=litellm_params.skip_tool_message_in_guardrail,
advisory_system_message=litellm_params.advisory_system_message,
)
litellm.logging_callback_manager.add_litellm_callback(_lakera_v2_callback)
return _lakera_v2_callback

25
litellm/proxy/guardrails/guardrail_registry.py
View file

@ -413,6 +413,16 @@ class GuardrailRegistry:
raise Exception(f"Error getting guardrail from DB: {e}")
def _apply_configured_bool_override(instance: CustomGuardrail, litellm_params: LitellmParams, param_name: str) -> None:
"""Override ``instance.<param_name>`` only when ``litellm_params`` explicitly
sets it, preserving whatever default the guardrail's own constructor chose
otherwise (its constructor default may be True, so blindly copying an
absent/None config value would silently clobber it back to False)."""
configured: Final = getattr(litellm_params, param_name, None)
if configured is not None:
setattr(instance, param_name, bool(configured))
class InMemoryGuardrailHandler:
"""
Class that handles initializing guardrails and adding them to the CallbackManager
@ -534,9 +544,8 @@ class InMemoryGuardrailHandler:
"skip_tool_message_in_guardrail are enabled together, which excludes every message from "
"scanning, so no request content would ever be scanned. Remove one of the two."
)
configured_run_in_parallel: Final[bool | None] = getattr(litellm_params, "run_in_parallel", None)
if configured_run_in_parallel is not None:
custom_guardrail_callback.run_in_parallel = bool(configured_run_in_parallel)
for override_param in ("run_in_parallel", "scan_raw_request"):
_apply_configured_bool_override(custom_guardrail_callback, litellm_params, override_param)
parsed_guardrail: Final = Guardrail(
guardrail_id=guardrail.get("guardrail_id"),
@ -778,15 +787,23 @@ class InMemoryGuardrailHandler:
"""
Force re-initialization of a guardrail even if it exists in memory.
Removes old callback from litellm.callbacks and creates fresh instance.
If the new config fails to initialize (e.g. an invalid on_flagged
combination), the previous instance is restored rather than left
deleted: initialize_guardrail's own ValueError/TypeError propagate
uncaught, so a caller reaching this point after already deleting the
old instance would otherwise leave the guardrail providing no
protection at all, not merely "still enforcing the old config."
"""
guardrail_id: Final = guardrail.get("guardrail_id")
if not guardrail_id:
verbose_proxy_logger.error("Cannot reinitialize guardrail without guardrail_id")
return None
# Remove from memory if exists (also removes from callbacks)
previous_guardrail: Final = self.IN_MEMORY_GUARDRAILS.get(guardrail_id)
previous_source: Final = self._sources.get(guardrail_id, source)
# Remove from memory if exists (also removes from callbacks)
if guardrail_id in self.IN_MEMORY_GUARDRAILS:
self.delete_in_memory_guardrail(guardrail_id)

20
litellm/proxy/guardrails/init_guardrails.py
View file

@ -26,12 +26,20 @@ def init_guardrails_v2(
guardrail_list: Final[list[Guardrail]] = []
for guardrail in all_guardrails:
initialized_guardrail = IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(
guardrail=cast(Guardrail, guardrail),
config_file_path=config_file_path,
llm_router=llm_router,
source="config",
)
try:
initialized_guardrail = IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(
guardrail=cast(Guardrail, guardrail),
config_file_path=config_file_path,
llm_router=llm_router,
source="config",
)
except (ValueError, TypeError) as init_error:
verbose_proxy_logger.error(
"Skipping guardrail '%s': invalid configuration, proxy is starting WITHOUT this guardrail: %s",
guardrail.get("guardrail_name"),
init_error,
)
continue
if initialized_guardrail:
guardrail_list.append(initialized_guardrail)

19
litellm/proxy/litellm_pre_call_utils.py
View file

@ -313,6 +313,10 @@ _CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.mod
# into response_cost and spend; a client seeding it forges (even negative)
# guardrail cost.
_CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logging_guardrail_information"})
# ``attempted_fallbacks`` and ``original_model_group`` are written by the router
# and read by spend logs as fact; a client value has no legitimate meaning and no
# key or team setting keeps it, so the strip is never gated.
_ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset({"attempted_fallbacks", "original_model_group"})
_ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override"
# Request fields whose value, when URL-valued, becomes the outbound destination
@ -538,6 +542,20 @@ def _strip_client_pricing_overrides(data: dict[str, Any]) -> None:
)
def _strip_router_reserved_metadata(
data: dict[str, Any], # mutable-ok: strips in place on the request body the pre-call pipeline threads through
) -> None:
"""Drop the router-owned fallback stamps from any client-supplied metadata bucket."""
for metadata_key in ("metadata", "litellm_metadata"):
if not isinstance(metadata := data.get(metadata_key), dict):
continue
for field in _ROUTER_RESERVED_METADATA_FIELDS & metadata.keys():
metadata.pop(field)
verbose_proxy_logger.debug(
"Stripped router-reserved metadata field from request body: %s.%s", metadata_key, field
)
def _get_metadata_variable_name(request: Request) -> str:
"""
Helper to return what the "metadata" field should be called in the request data
@ -1882,6 +1900,7 @@ async def add_litellm_data_to_request(
# would silently skip the field.
if not _key_or_team_allows_client_pricing_override(user_api_key_dict):
_strip_client_pricing_overrides(data)
_strip_router_reserved_metadata(data)
# Same reason as the strips above: runs after the metadata string-to-dict parse
# so JSON-string metadata cannot smuggle callback credentials past the dict guard.

108
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -15,6 +15,7 @@ from uuid import uuid4
from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.exceptions import BudgetExceededError
from litellm.litellm_core_utils.llm_judge import judge_target
@ -119,6 +120,10 @@ class _ShadowEvalAttemptRow(Protocol):
def error(self) -> str | None: ...
class _ShadowEvalFunnelTable(Protocol):
async def create_many(self, data: Sequence[Mapping[str, object]], skip_duplicates: bool) -> int: ...
class _ShadowEvalAttemptTable(Protocol):
async def find_first(
self, *, where: Mapping[str, object], order: Mapping[str, str]
@ -137,6 +142,10 @@ def _shadow_eval_jobs(prisma_client: "PrismaClient") -> _ShadowEvalJobTable:
return prisma_client.db.litellm_shadowevaljob
def _shadow_eval_funnel(prisma_client: "PrismaClient") -> _ShadowEvalFunnelTable:
return prisma_client.db.litellm_shadowevalfunnel # pyright: ignore[reportAttributeAccessIssue] # generated client
def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTable:
return prisma_client.db.litellm_shadowevalattempt
@ -678,6 +687,20 @@ def _is_configured_pre_routing_strategy(llm_router: "Router", router_name: str)
)
def _sdk_model_is_missing_anthropic_credentials(model: str) -> bool:
_, provider, _, _ = litellm.get_llm_provider(model=model)
if provider != "anthropic" or litellm.anthropic_key or litellm.api_key:
return False
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.secret_managers.main import secret_manager_would_be_consulted
if AnthropicModelInfo.get_api_key() or AnthropicModelInfo.get_auth_token():
return False
return not any(
secret_manager_would_be_consulted(secret_name) for secret_name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN")
)
def _validate_plain_model(
llm_router: "Router | None", model: str, field_name: str, team_ids: Sequence[str | None]
) -> None:
@ -694,14 +717,26 @@ def _validate_plain_model(
status_code=400,
detail=f"{field_name} '{model}' is an auto-router; it must be a plain model",
)
unreachable: Final = tuple(team for team in team_ids if judge_target(llm_router, model, team).via == "nothing")
if not unreachable:
targets: Final = tuple((team, judge_target(llm_router, model, team)) for team in team_ids)
unreachable: Final = tuple(team for team, target in targets if target.via == "nothing")
if unreachable:
raise HTTPException(
status_code=400,
detail=(
f"{field_name} '{model}' is neither a model configured on this proxy nor a "
"provider-qualified public model name (e.g. 'anthropic/claude-sonnet-5')" + _for_teams(unreachable)
),
)
sdk_teams: Final = tuple(team for team, target in targets if target.via == "sdk")
if not sdk_teams:
return
if not _sdk_model_is_missing_anthropic_credentials(model):
return
raise HTTPException(
status_code=400,
detail=(
f"{field_name} '{model}' is neither a model configured on this proxy nor a "
"provider-qualified public model name (e.g. 'anthropic/claude-sonnet-5')" + _for_teams(unreachable)
f"{field_name} '{model}' uses the LiteLLM SDK but required credentials are not configured: "
"ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN" + _for_teams(sdk_teams)
),
)
@ -820,6 +855,9 @@ class _AttemptAggRow(BaseModel):
shadow_wins: int
ties: int
avg_confidence: float | None
real_spend: float
shadow_spend: float
cache_hit_turns: int
_ATTEMPT_AGG_ROWS: Final = TypeAdapter(list[_AttemptAggRow])
@ -829,7 +867,10 @@ _ATTEMPT_AGG_SELECT: Final = """
COUNT(*) FILTER (WHERE outcome = 'real')::int AS real_wins,
COUNT(*) FILTER (WHERE outcome = 'shadow')::int AS shadow_wins,
COUNT(*) FILTER (WHERE outcome = 'tie')::int AS ties,
AVG(confidence)::float AS avg_confidence
AVG(confidence)::float AS avg_confidence,
COALESCE(SUM(real_cost + real_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS real_spend,
COALESCE(SUM(shadow_cost + shadow_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS shadow_spend,
COUNT(*) FILTER (WHERE real_cache_hit)::int AS cache_hit_turns
FROM "LiteLLM_ShadowEvalAttempt"
WHERE job_id = ANY($1::text[]) AND outcome != 'error'
GROUP BY 1
@ -850,7 +891,7 @@ WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL
OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns
OR (
j.max_budget IS NOT NULL
AND (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_budget
AND (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_budget
)
)
"""
@ -865,13 +906,24 @@ WHERE job_id = ANY($1::text[])
"""
_ATTEMPT_COUNTS_SQL: Final = """
SELECT a.job_id, COUNT(*)::int AS attempt_count, COALESCE(SUM(a.judge_cost + a.shadow_cost), 0)::float AS spend
SELECT a.job_id, COUNT(*)::int AS attempt_count, COALESCE(SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost), 0)::float AS spend
FROM "LiteLLM_ShadowEvalAttempt" a
JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id
WHERE a.job_id = ANY($1::text[]) AND (j.stopped_at IS NULL OR a.created_at <= j.stopped_at)
GROUP BY a.job_id
"""
_FUNNEL_TOTALS_SQL: Final = """
SELECT COUNT(*)::int AS legs_with_rows,
COALESCE(SUM(not_sampled), 0)::int AS not_sampled,
COALESCE(SUM(unjudgeable), 0)::int AS unjudgeable,
COALESCE(SUM(shed), 0)::int AS shed,
COALESCE(SUM(withheld), 0)::int AS withheld
FROM "LiteLLM_ShadowEvalFunnel"
WHERE job_id = ANY($1::text[])
"""
_STOP_JOB_SQL: Final = """
UPDATE "LiteLLM_ShadowEvalJob"
SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp)
@ -883,12 +935,20 @@ WHERE group_id = $1 AND stopped_by IS NULL
AND (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_turns
AND (
k.max_budget IS NULL
OR (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_budget
OR (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_budget
)
)
"""
class _FunnelTotalsRow(BaseModel):
legs_with_rows: int
not_sampled: int
unjudgeable: int
shed: int
withheld: int
class _AttemptCountRow(BaseModel):
job_id: str
attempt_count: int
@ -937,6 +997,9 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]:
shadow_win_rate_pct=_pct_of(row.shadow_wins, row.turn_count),
tie_rate_pct=_pct_of(row.ties, row.turn_count),
avg_judge_confidence=round(row.avg_confidence or 0.0, 3),
real_spend=row.real_spend,
shadow_spend=row.shadow_spend,
cache_hit_turns=row.cache_hit_turns,
)
for row in sorted(rows, key=lambda r: r.turn_count, reverse=True)
)
@ -1087,12 +1150,23 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le
for row in by_leg
)
total_turns: Final = sum(r.turn_count for r in by_tier)
funnel_rows: Final = await _query_raw(prisma_client, _FUNNEL_TOTALS_SQL, leg_ids)
counted: Final = _FunnelTotalsRow.model_validate(funnel_rows[0]) if funnel_rows else None
# Coverage only when EVERY leg has a funnel row: a partial seed (one leg's insert
# failed) must read as unknown, not as job-level counts missing a leg's traffic.
funnel: Final = counted if counted is not None and counted.legs_with_rows == len(leg_ids) else None
return ShadowEvalResult(
by_tier=_slices(by_tier),
by_current_model=_slices(by_model),
by_key=_slices(by_key),
overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns),
overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns),
sampled_real_spend=sum(r.real_spend for r in by_tier),
sampled_shadow_spend=sum(r.shadow_spend for r in by_tier),
not_sampled_count=funnel.not_sampled if funnel is not None else None,
unjudgeable_count=funnel.unjudgeable if funnel is not None else None,
shed_count=funnel.shed if funnel is not None else None,
withheld_count=funnel.withheld if funnel is not None else None,
)
@ -1191,8 +1265,14 @@ async def start_shadow_eval(
"ends_at": ends_at,
}
try:
# Leg ids are minted here rather than by the DB default so the funnel seed below
# writes from the same values with no read-back, which a lagging read replica
# (DATABASE_URL_READ_REPLICA) could otherwise return empty.
leg_ids: Final = tuple(str(uuid4()) for _ in data.api_key_ids)
await _shadow_eval_jobs(prisma_client).create_many(
data=[{**shared_config, "api_key_id": key} for key in data.api_key_ids] # mutable-ok: Prisma payload
data=[ # mutable-ok: Prisma payload
{**shared_config, "id": leg_id, "api_key_id": key} for leg_id, key in zip(leg_ids, data.api_key_ids)
]
)
except Exception as e:
if not _is_unique_violation(e):
@ -1203,6 +1283,16 @@ async def start_shadow_eval(
f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first."
),
) from e
# Seed a zero funnel row per leg NOW: a fully covered job never skips a request, so
# waiting for the first skip would leave it indistinguishable from a pre-funnel job
# (null coverage). A failed seed degrades this job to exactly that, nothing worse.
try:
await _shadow_eval_funnel(prisma_client).create_many(
data=[{"job_id": leg_id} for leg_id in leg_ids], # mutable-ok: Prisma payload
skip_duplicates=True,
)
except Exception as seed_err: # noqa: BLE001 # coverage is advisory; the job must still start
verbose_proxy_logger.error("shadow_eval: funnel seed failed for job %s: %s", group_id, seed_err)
labels: Final = MappingProxyType({row.token: row for row in token_rows})
return ShadowEvalJobResponse(
job_id=group_id,

158
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -62,6 +62,9 @@ from litellm.proxy.auth.auth_utils import (
enforce_output_token_estimates_are_admin_only,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
publish_auth_cache_invalidation,
)
from litellm.proxy.common_utils.callback_config_validation import logging_metadata_config_error
from litellm.proxy.common_utils.callback_utils import (
decrypt_callback_vars,
@ -5171,6 +5174,125 @@ def _validate_reset_spend_value(reset_to: object, key_in_db: LiteLLM_Verificatio
return reset_to
async def _set_spend_counter_with_floor_and_broadcast(counter_key: str, value: float) -> None:
"""
Set a Redis-backed spend counter to `value`, mirror it into the short-lived
spend_db_floor marker `_authoritative_floor_spend` reads, and broadcast both
to every worker (LIT-3803 pattern: setting, not deleting, means a worker's
own self-delivered broadcast still carries the reset value forward).
Without the floor marker, `_authoritative_floor_spend` can re-derive a
stale, pre-reset value from a marker another worker cached moments earlier
and raise the just-reset counter right back up via `_repair_stale_spend_counter`.
Without the broadcast, a worker that already cached the pre-reset key object
or floor marker keeps enforcing against it until its own TTL expires.
"""
from litellm.proxy.proxy_server import SPEND_DB_FLOOR_CACHE_TTL_SECONDS, spend_counter_cache
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=value, ttl=60)
if spend_counter_cache.redis_cache is not None:
try:
await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=value, ttl=60)
except Exception as redis_err:
verbose_proxy_logger.warning(
"Failed to update spend counter %s in Redis: %s. "
"Budget checks may use stale value until counter expires.",
counter_key,
redis_err,
)
floor_key: Final = f"spend_db_floor:{counter_key}"
spend_counter_cache.in_memory_cache.set_cache(key=floor_key, value=value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS)
await publish_auth_cache_invalidation(cache_key=counter_key, new_value=value, ttl=60)
await publish_auth_cache_invalidation(cache_key=floor_key, new_value=value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS)
def _budget_limit_windows(budget_limits: Sequence[object] | str | None) -> tuple[Mapping[str, object], ...]:
"""Coerce a key's stored `budget_limits` into a tuple of plain window dicts.
It is a DB Json column, so a caller reading it straight off `find_unique`
gets an already-parsed list; one reading it off `json.dumps`'d text (or a
raw SQL row) gets the string form. Either way each entry is a plain dict,
except wherever a caller already validated the field through a pydantic
model (e.g. `UserAPIKeyAuth.budget_limits`), which yields `BudgetLimitEntry`
objects instead -- coerced here via `model_dump()`, matching
`_set_budget_reset_at`'s identical coercion in team_endpoints.py.
"""
if not budget_limits:
return ()
raw_windows: Final = json.loads(budget_limits) if isinstance(budget_limits, str) else budget_limits
return tuple(raw_window if isinstance(raw_window, dict) else raw_window.model_dump() for raw_window in raw_windows)
def _advance_one_key_budget_window(window: Mapping[str, object]) -> Mapping[str, object]:
"""Restart one budget window from now, by advancing its `reset_at`.
`window_start` is derived elsewhere as `reset_at - budget_duration`
(`get_budget_window_start`), so `reset_at` must be set to `now +
budget_duration` -- a window floating from THIS moment -- to make
`window_start` land at `now` and exclude the historical spend that
triggered the block. Reusing `get_budget_reset_time`/
`ResetBudgetJob._reset_expired_window`'s calendar-standardized boundary
(e.g. "next midnight") would not do that: for a "1d" window `next
midnight - 1d` is simply the START of the calendar day already in
progress, which still covers that spend. That reuse is only safe for the
scheduled job, which runs right as `reset_at` naturally elapses, so the
elapsed boundary it computes is already close to "now". A manual reset
can happen at any point mid-window, so it needs the floating form
instead. A window with no `budget_duration` is returned unchanged.
"""
duration = window.get("budget_duration")
if not isinstance(duration, str) or not duration:
return window
new_reset_at: Final = datetime.now(timezone.utc) + timedelta(seconds=duration_in_seconds(duration))
return { # mutable-ok: this is the JSON payload persisted to budget_limits' Json column, which requires a plain dict
**window,
"reset_at": new_reset_at.isoformat(),
}
async def _reset_key_budget_windows(
prisma_client: PrismaClient,
hashed_api_key: str,
budget_limits: Sequence[object] | str | None,
) -> None:
"""Force-expire every one of a key's own `budget_limits` windows (extra
time-windowed caps layered on top of the lifetime max_budget, e.g. a daily
limit) so a manual spend reset also clears them, not just the lifetime
counter.
Persists the advanced `reset_at` boundaries BEFORE zeroing any window's
Redis counter, not after: a window counter reading zero is only durable
once every reader recomputing its floor from the DB sees the new
boundary too (`get_current_spend` re-derives a window counter from real
`LiteLLM_SpendLogs` rows inside `[window_start, now)` on every read below
max_budget, see its `is_window` branch). Zeroing first would let a
request racing the DB write compute `window_start` from the stale
pre-reset boundary, re-sum the unchanged historical spend, and put the
counter right back where it was before the write ever landed.
"""
windows: Final = _budget_limit_windows(budget_limits)
if not windows:
return
reset_windows: Final = tuple(_advance_one_key_budget_window(w) for w in windows)
# prisma-client-py's typed update() takes plain dict literals for `where`/`data`; there is no
# frozen-mapping equivalent to pass instead.
reset_payload: Final = {"budget_limits": json.dumps(reset_windows, default=str)} # mutable-ok: prisma data kwarg
await VerificationTokenRepository(prisma_client).table.update(
where={"token": hashed_api_key}, # mutable-ok: prisma where kwarg
data=reset_payload,
)
for window in reset_windows:
duration = window.get("budget_duration")
if isinstance(duration, str) and duration:
counter_key = f"spend:key:{hashed_api_key}:window:{duration}"
await _set_spend_counter_with_floor_and_broadcast(counter_key=counter_key, value=0.0)
@router.post(
"/key/{key:path}/reset_spend",
tags=["key management"],
@ -5236,30 +5358,30 @@ async def reset_key_spend_fn(
detail={"error": "Failed to update key spend"},
)
# Reset the lifetime spend counter to the new value (not 0.0, so partial
# resets are reflected correctly), and force-expire any of the key's own
# budget_limits windows, so get_current_spend() returns the correct
# amount for every enforcement check immediately instead of the stale
# pre-reset value.
_counter_key: Final = f"spend:key:{hashed_api_key}"
await _set_spend_counter_with_floor_and_broadcast(counter_key=_counter_key, value=reset_to)
await _reset_key_budget_windows(
prisma_client=prisma_client,
hashed_api_key=hashed_api_key,
budget_limits=_key_in_db.budget_limits,
)
# Evicting the cached key object LAST (after every DB write above has
# committed) matters: a request landing between an earlier eviction and
# a later write would re-fetch and re-cache the pre-write row, pinning
# that pod to the stale budget_limits/spend for the rest of its own
# cache TTL even though the DB is already correct.
await _delete_cache_key_object(
hashed_token=hashed_api_key,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
# Set Redis spend counter to the new value so get_current_spend()
# returns the correct amount immediately instead of the stale pre-reset value.
# We use reset_to (not 0.0) so partial resets are reflected correctly.
from litellm.proxy.proxy_server import spend_counter_cache
_counter_key: Final = f"spend:key:{hashed_api_key}"
spend_counter_cache.in_memory_cache.set_cache(key=_counter_key, value=reset_to, ttl=60)
if spend_counter_cache.redis_cache is not None:
try:
await spend_counter_cache.redis_cache.async_set_cache(key=_counter_key, value=reset_to, ttl=60)
except Exception as redis_err:
verbose_proxy_logger.warning(
"Failed to update spend counter %s in Redis: %s. "
"Budget checks may use stale value until counter expires.",
_counter_key,
redis_err,
)
max_budget: Final = updated_key.max_budget
budget_reset_at: Final = updated_key.budget_reset_at

45
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -16,10 +16,10 @@ import json
from collections.abc import Awaitable, Mapping, Sequence
from json import JSONDecodeError
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, cast
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
@ -81,7 +81,10 @@ from litellm.router_strategy.complexity_router import (
ClassificationRubric,
ComplexityRouterConfig,
ComplexityTier,
TierDefinition,
classification_system_prompt,
custom_tier_classification_prompt,
normalize_classification_prompt,
)
from litellm.router_utils.auto_router_model_naming import (
STRATEGY_ROUTER_PARAM_FIELDS,
@ -2230,6 +2233,39 @@ def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[Complexity
) from e
class AutoRouterClassifierPromptPreviewRequest(BaseModel):
"""A POST rather than query params: classification_prompt is the operator's own text, which must
not reach access logs through a URL."""
tier_definitions: tuple[TierDefinition, ...]
context_window_size: Annotated[int, Field(ge=0)] = DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE
classification_prompt: str | None = None
_normalize_prompt = field_validator("classification_prompt")(normalize_classification_prompt)
@router.post(
"/auto_router/classifier/default_prompt",
description="Get the system prompt an auto-router's LLM classifier sends for an edited tier set",
tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list
dependencies=[Depends(user_api_key_auth)], # mutable-ok: fastapi's decorator signature types dependencies as a list
)
async def preview_auto_router_classifier_prompt(
request: AutoRouterClassifierPromptPreviewRequest,
) -> AutoRouterClassifierDefaultPromptResponse:
"""
Get the classifier system prompt an edited tier set sends, so the dashboard can show it.
Built by the same function the live classifier uses, so the preview cannot drift from what the
router sends. Payload validity beyond a renderable definition stays the dry-run's job.
"""
return AutoRouterClassifierDefaultPromptResponse(
system_prompt=custom_tier_classification_prompt(
request.tier_definitions, request.classification_prompt, request.context_window_size
)
)
@router.get(
"/auto_router/classifier/default_prompt",
description="Get the built-in system prompt used by an auto-router's LLM classifier",
@ -2242,13 +2278,16 @@ async def get_auto_router_classifier_default_prompt(
classification_rubric: ClassificationRubric | None = None,
) -> AutoRouterClassifierDefaultPromptResponse:
"""
Get the default classifier system prompt, so the dashboard's prompt editor can prefill it.
Get the classifier system prompt a router would send, so the dashboard can show it.
The prompt's closing line depends on whether prior conversation turns are quoted to the
classifier, its tier bullets are named by the router's tier_labels, and its calibration examples
come from the router's classification rubric, so the caller passes all three to get the text that router
would actually send rather than a rubric it does not use.
An edited tier set replaces the whole rubric; POST to this path for that prompt, which carries
the operator's own instructions and so must not ride in a query string.
Parameters:
- context_window_size: int - The router's classifier_context_window_size. Defaults to the
built-in default.

34
litellm/proxy/policy_engine/pipeline_executor.py
View file

@ -15,6 +15,7 @@ from litellm.integrations.custom_guardrail import (
ModifyResponseException,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import independent_snapshot
from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
UnifiedLLMGuardrails,
)
@ -41,6 +42,7 @@ class PipelineExecutor:
user_api_key_dict: Any,
call_type: str,
policy_name: str,
raw_request_snapshot: dict | None = None, # mutable-ok: same request-payload shape as data
) -> PipelineExecutionResult:
"""
Execute pipeline steps sequentially with conditional actions.
@ -52,6 +54,11 @@ class PipelineExecutor:
user_api_key_dict: User API key auth
call_type: Type of call (completion, etc.)
policy_name: Name of the owning policy (for logging)
raw_request_snapshot: pristine pre-pipeline, pre-guardrail request
(taken by the caller before any guardrail or pipeline ran), so a
step whose guardrail opted into ``scan_raw_request`` evaluates
the original request instead of whatever an earlier
``pass_data`` step in this same pipeline already rewrote.
Returns:
PipelineExecutionResult with terminal action and step results
@ -75,6 +82,7 @@ class PipelineExecutor:
data=working_data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
raw_request_snapshot=raw_request_snapshot,
)
duration = time.perf_counter() - start_time
@ -143,6 +151,7 @@ class PipelineExecutor:
data: dict,
user_api_key_dict: Any,
call_type: str,
raw_request_snapshot: dict | None = None, # mutable-ok: same request-payload shape as data
) -> tuple[
Literal["pass", "fail", "error"],
dict | None,
@ -172,20 +181,33 @@ class PipelineExecutor:
data["metadata"] = {}
data["metadata"]["guardrails"] = [step.guardrail]
# A scan_raw_request step evaluates the pristine pre-pipeline
# snapshot instead of `data` (which earlier pass_data steps in
# this same pipeline may have already rewritten), same reason
# the normal sequential/parallel guardrail loops do this.
scans_raw_request: Final = getattr(callback, "scan_raw_request", False)
hook_input: Final[dict] = ( # mutable-ok: same request-payload shape as data
independent_snapshot(raw_request_snapshot)
if scans_raw_request and raw_request_snapshot is not None
else data
)
if hook_input is not data:
hook_input.setdefault("metadata", {})["guardrails"] = [step.guardrail]
# Use unified_guardrail path if callback implements apply_guardrail
target: CustomLogger = callback
use_unified: Final = (
"apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks
)
if use_unified:
data["guardrail_to_apply"] = callback
hook_input["guardrail_to_apply"] = callback
target = UnifiedLLMGuardrails()
if mode == "pre_call":
response = await target.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=None,
data=data,
data=hook_input,
call_type=call_type,
)
if isinstance(callback, CustomGuardrail):
@ -201,9 +223,13 @@ class PipelineExecutor:
else:
return ("error", None, f"Unsupported pipeline mode: {mode}", None)
# Normal return means pass
# Normal return means pass. A scan_raw_request step is block-only,
# same contract as run_in_parallel/scan_raw_request elsewhere: any
# data it returned is discarded, since applying it on top of the
# raw snapshot would silently undo whatever an earlier step in
# this pipeline already did.
modified_data = None
if response is not None and isinstance(response, dict):
if response is not None and isinstance(response, dict) and not scans_raw_request:
modified_data = response
return ("pass", modified_data, None, None)

15
litellm/proxy/schema.prisma
View file

@ -1533,12 +1533,27 @@ model LiteLLM_ShadowEvalAttempt {
confidence Float?
judge_cost Float @default(0)
shadow_cost Float @default(0)
real_cost Float? // NULL = row predates cost measurement; comparisons read only measured rows
real_classifier_cost Float @default(0)
shadow_classifier_cost Float @default(0)
real_cache_hit Boolean @default(false)
error String?
created_at DateTime @default(now())
@@index([job_id])
}
// Per-leg sampling funnel counters the attempt rows cannot derive: requests an
// admitting job saw but did not judge. attempted = the leg's attempt rows; the
// leg's eligible traffic = not_sampled + unjudgeable + shed + withheld + attempted.
model LiteLLM_ShadowEvalFunnel {
job_id String @id
not_sampled Int @default(0)
unjudgeable Int @default(0)
shed Int @default(0)
withheld Int @default(0)
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

7
litellm/proxy/spend_tracking/savings.py
View file

@ -502,14 +502,11 @@ def autorouter_savings_for_request(
usage: Final = _usage_from_spend_log(usage_object)
if usage is None or not model:
return None
# The configured `autorouter_savings_baseline_model` wins; otherwise the baseline
# the deciding router recorded on its decision; neither means the driver is off.
decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {}
recorded: Final = decision.get("savings_baseline_model")
recorded_id: Final = decision.get("savings_baseline_deployment_id")
configured: Final = litellm.autorouter_savings_baseline_model
baseline_model: Final = configured or (recorded if isinstance(recorded, str) else None)
baseline_id: Final = recorded_id if configured is None and isinstance(recorded_id, str) else None
baseline_model: Final = recorded if isinstance(recorded, str) else None
baseline_id: Final = recorded_id if isinstance(recorded_id, str) else None
if not decision or not baseline_model:
return None
router_instance: Final = llm_router() if llm_router else None

156
litellm/proxy/utils.py
View file

@ -91,7 +91,11 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus import PrometheusLogger
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert
from litellm.litellm_core_utils.core_helpers import coerce_token_limit, is_expected_client_error
from litellm.litellm_core_utils.core_helpers import (
coerce_token_limit,
independent_snapshot,
is_expected_client_error,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@ -1387,6 +1391,83 @@ class ProxyLogging:
return data
async def _run_sequential_guardrail_callback(
self,
callback: CustomGuardrail,
data: dict, # mutable-ok: matches _process_guardrail_callback's own request-payload typing
raw_request_snapshot: dict | None, # mutable-ok: same request-payload shape as data
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
) -> dict: # mutable-ok: callers reassign the loop's own data from this return value
"""
Run one guardrail from the sequential pre_call loop and return what the
rest of the loop should carry forward.
A guardrail opted into ``scan_raw_request`` always evaluates a fresh
copy of ``raw_request_snapshot`` (taken before any guardrail in this
hook ran) instead of ``data`` (the live, possibly already-mutated
payload), so its block/pass decision can never depend on where it's
declared relative to a guardrail that masks or rewrites content. It's
declared block-only, same contract as ``run_in_parallel``: any data it
returns is discarded, since applying its view on top of a stale
snapshot would silently undo whatever a later guardrail already did to
the live request. A guardrail that mutates content (e.g. PII masking)
should never set this flag -- if one does anyway, its returned
mutation is discarded and a warning is logged so the misconfiguration
is visible instead of silently forwarding unredacted content.
"""
scans_raw_request: Final = getattr(callback, "scan_raw_request", False)
should_use_raw_snapshot: Final = scans_raw_request and raw_request_snapshot is not None
input_data: Final = ( # mutable-ok: same request-payload shape as data
independent_snapshot(raw_request_snapshot) if should_use_raw_snapshot else data
)
# _process_guardrail_callback always calls mark_pre_call_hook_ran on a
# successful run, which unconditionally stamps bookkeeping metadata onto
# the dict regardless of whether the guardrail's own hook mutated
# anything -- so comparing `result` straight against `input_data` would
# warn on every single scan_raw_request call. Apply that same stamp to a
# throwaway, guaranteed-independent copy first (never the live request or
# raw_request_snapshot itself) so the comparison isolates the guardrail's
# own content mutation from this bookkeeping noise without risking a
# premature marker write into shared state.
expected_if_unmutated: Final[dict | None] = ( # mutable-ok: same request-payload shape as data
independent_snapshot(input_data) if scans_raw_request else None
)
if expected_if_unmutated is not None:
callback.mark_pre_call_hook_ran(expected_if_unmutated)
result: Final = await self._process_guardrail_callback(
callback=callback,
data=input_data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
event_type=GuardrailEventHooks.pre_call,
)
if (
scans_raw_request
and expected_if_unmutated is not None
and result is not None
and result != expected_if_unmutated
):
verbose_proxy_logger.warning(
"Guardrail '%s' has scan_raw_request=True but returned a modified payload; "
"scan_raw_request is for block-only guardrails and this mutation is being "
"discarded. Remove scan_raw_request from this guardrail's config if it needs "
"to mask/rewrite content.",
getattr(callback, "guardrail_name", None) or callback.__class__.__name__,
)
if scans_raw_request:
if result is not None:
# _process_guardrail_callback only stamped input_data (a throwaway
# snapshot copy), never the live data returned here -- without this,
# a deployment-level guardrail sharing this name would see no marker
# via _pre_call_hook_already_ran and re-run the same guardrail a
# second time on live kwargs.
callback.mark_pre_call_hook_ran(data)
return data
if result is None:
return data
return result
async def _process_prompt_template(
self,
data: dict,
@ -1496,6 +1577,7 @@ class ProxyLogging:
user_api_key_dict: UserAPIKeyAuth,
call_type: str,
event_hook: str,
raw_request_snapshot: dict | None = None, # mutable-ok: same request-payload shape as data
) -> dict:
"""
Execute guardrail pipelines if any are configured for this request.
@ -1503,6 +1585,11 @@ class ProxyLogging:
Checks metadata for pipelines resolved by the policy engine
and executes them. Handles the result (allow/block/modify_response).
``raw_request_snapshot`` (taken before any guardrail or pipeline ran)
is forwarded so a pipeline step whose guardrail opted into
``scan_raw_request`` evaluates the pristine request, not whatever an
earlier ``pass_data`` step in the same pipeline already rewrote.
Returns the (possibly modified) data dict.
"""
pipelines: Final = _policy_pipelines(data)
@ -1520,6 +1607,7 @@ class ProxyLogging:
user_api_key_dict=user_api_key_dict,
call_type=call_type,
policy_name=policy_name,
raw_request_snapshot=raw_request_snapshot,
)
data = self._handle_pipeline_result(
@ -1679,6 +1767,24 @@ class ProxyLogging:
call_type=call_type,
)
# Snapshotted here, before _maybe_execute_pipelines or any guardrail in
# this hook has run, so a scan_raw_request guardrail's block/pass
# decision never depends on its position in the guardrails list or on
# a pipeline that runs ahead of it: an earlier guardrail (pipelined or
# not) that masks/rewrites content can't hide a violation from a later
# one that opted into scanning the original request. Only computed
# when at least one registered guardrail actually opted in, and via
# independent_snapshot (not safe_deep_copy) since this isolation
# guarantee must hold even under litellm.safe_memory_mode, which
# otherwise makes deep copies return the original object.
needs_raw_request_snapshot: Final = any(
isinstance(cb, CustomGuardrail) and getattr(cb, "scan_raw_request", False)
for cb in ProxyLogging._callback_capabilities().resolved_callbacks
)
raw_request_snapshot: Final[dict | None] = ( # mutable-ok: same request-payload shape as data
independent_snapshot(data) if needs_raw_request_snapshot else None
)
try:
# Execute guardrail pipelines before the normal callback loop
data = await self._maybe_execute_pipelines(
@ -1686,6 +1792,7 @@ class ProxyLogging:
user_api_key_dict=user_api_key_dict,
call_type=call_type,
event_hook="pre_call",
raw_request_snapshot=raw_request_snapshot,
)
# Get pipeline-managed guardrails to skip in normal loop
@ -1726,16 +1833,13 @@ class ProxyLogging:
if getattr(_callback, "run_in_parallel", False):
continue
result = await self._process_guardrail_callback(
data = await self._run_sequential_guardrail_callback(
callback=_callback,
data=data,
raw_request_snapshot=raw_request_snapshot,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
event_type=GuardrailEventHooks.pre_call,
)
if result is None:
continue
data = result
elif (
_callback is not None
@ -1787,6 +1891,7 @@ class ProxyLogging:
await self._run_parallel_pre_call_guardrails(
guardrails=parallel_guardrails,
data=data,
raw_request_snapshot=raw_request_snapshot,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
)
@ -1807,6 +1912,7 @@ class ProxyLogging:
self,
guardrails: tuple[CustomGuardrail, ...],
data: dict,
raw_request_snapshot: dict | None, # mutable-ok: same request-payload shape as data
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
) -> None:
@ -1823,12 +1929,24 @@ class ProxyLogging:
the LLM, preserving the pre-call barrier that ``during_call`` guardrails
cannot provide. Per-guardrail latency is recorded by
``_process_guardrail_callback``'s own metrics.
A guardrail that also opted into ``scan_raw_request`` evaluates
``raw_request_snapshot`` (taken before the sequential loop ran) instead
of ``data`` (the sequential loop's output), for the same reason the
sequential branch does: its block decision must not depend on what a
sequential guardrail already masked or rewrote.
"""
def _input_for(callback: CustomGuardrail) -> dict: # mutable-ok: same request-payload shape as data
if not getattr(callback, "scan_raw_request", False) or raw_request_snapshot is None:
return data
return independent_snapshot(raw_request_snapshot)
results: Final = await asyncio.gather(
*(
self._process_guardrail_callback(
callback=callback,
data=data,
data=_input_for(callback),
user_api_key_dict=user_api_key_dict,
call_type=call_type,
event_type=GuardrailEventHooks.pre_call,
@ -1837,6 +1955,19 @@ class ProxyLogging:
),
return_exceptions=True,
)
for callback, result in zip(guardrails, results, strict=True):
# _process_guardrail_callback stamped mark_pre_call_hook_ran on
# _input_for's throwaway snapshot copy for a scan_raw_request
# guardrail, never on the live, shared `data` -- without this, a
# deployment-level guardrail sharing this name would see no marker
# via _pre_call_hook_already_ran and re-run it a second time on
# live kwargs.
if (
getattr(callback, "scan_raw_request", False)
and not isinstance(result, BaseException)
and result is not None
):
callback.mark_pre_call_hook_ran(data)
raised: Final = tuple(result for result in results if isinstance(result, BaseException))
blocking: Final = next((exc for exc in raised if not _exception_changes_request_flow(exc)), None)
if blocking is not None:
@ -6300,7 +6431,9 @@ async def _total_queued_spend_transactions(prisma_client: PrismaClient) -> int:
tool_queue_size: Final = len(prisma_client.tool_usage_transactions)
async with prisma_client._autorouter_turn_transactions_lock:
autorouter_queue_size: Final = len(prisma_client.autorouter_turn_transactions)
return spend_queue_size + tool_queue_size + autorouter_queue_size
from litellm.proxy.db.shadow_eval_funnel import pending_shadow_eval_funnel_events
return spend_queue_size + tool_queue_size + autorouter_queue_size + pending_shadow_eval_funnel_events()
async def update_daily_tag_spend(
@ -6442,6 +6575,13 @@ async def update_spend_logs_job(
autorouter_tracking_err,
)
try:
from litellm.proxy.db.shadow_eval_funnel import flush_shadow_eval_funnel
await flush_shadow_eval_funnel(prisma_client)
except Exception as funnel_err: # noqa: BLE001 # a drain bug must not abort the spend job
verbose_proxy_logger.error("Spend tracking - shadow eval funnel drain failed: %s", funnel_err)
MAX_SPEND_LOG_DRAIN_ITERATIONS: Final = 20

162
litellm/router.py
View file

@ -64,6 +64,7 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.ptu_pricing import (
PTU_COST_ATTRIBUTION_ENV_VAR,
@ -229,6 +230,7 @@ from litellm.types.utils import (
StandardLoggingPayload,
StandardLoggingRoutingDecision,
Usage,
all_litellm_params,
shared_backend_model_info,
)
from litellm.types.utils import ModelInfo as ModelMapInfo
@ -243,6 +245,7 @@ from litellm.utils import (
get_secret,
get_utc_datetime,
is_region_allowed,
provider_rejectable_params,
set_live_deployment_replay,
)
@ -7048,13 +7051,11 @@ class Router:
_sibling_metadata_key: Final = (
"metadata" if _fallback_metadata_key == "litellm_metadata" else "litellm_metadata"
)
if isinstance(_sibling_metadata := kwargs.get(_sibling_metadata_key), dict) and (
"attempted_fallbacks" in _sibling_metadata or "original_model_group" in _sibling_metadata
):
_scrubbed_sibling_metadata: Final = _sibling_metadata.copy()
_scrubbed_sibling_metadata.pop("attempted_fallbacks", None)
_scrubbed_sibling_metadata.pop("original_model_group", None)
kwargs[_sibling_metadata_key] = _scrubbed_sibling_metadata
if isinstance(_sibling_metadata := kwargs.get(_sibling_metadata_key), dict):
# In place, like every other router bucket write: downstream resolves the bucket by
# key presence, so rebinding kwargs to a copy detaches the proxy's request_data write-backs
_sibling_metadata.pop("attempted_fallbacks", None)
_sibling_metadata.pop("original_model_group", None)
if isinstance(_fallback_metadata := kwargs.get(_fallback_metadata_key), dict):
_fallback_metadata["attempted_fallbacks"] = 0
if model_group is not None:
@ -10833,6 +10834,114 @@ class Router:
}
return {**deployment, "model_info": model_info} # mutable-ok: DeploymentTypedDict rows are plain dicts
TIER_PARAMS_NEVER_DROPPED: Final = frozenset(all_litellm_params) | frozenset(
{
"additional_drop_params",
"drop_params",
"messages",
"model",
"extra_headers",
"max_tokens",
"max_completion_tokens",
}
)
@staticmethod
def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]:
declared: Final = params.get("allowed_openai_params")
if not isinstance(declared, (list, tuple, set, frozenset)):
return frozenset()
return frozenset(entry for entry in declared if isinstance(entry, str))
@staticmethod
def _deployment_accepts_param(deployment: DeploymentTypedDict, group: str, param: str) -> bool:
deployment_params: Final = deployment.get("litellm_params")
if not deployment_params:
return True
if param in Router._declared_param_allowlist(deployment_params):
return True
if declared_authenticating_provider(
str(deployment_params.get("model") or ""), deployment_params.get("custom_llm_provider")
):
return True
deployment_model_info: Final = deployment.get("model_info")
base_model: Final = (
deployment_model_info.get("base_model") if deployment_model_info else None
) or deployment_params.get("base_model")
try:
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=deployment_params.get("model") or group,
custom_llm_provider=deployment_params.get("custom_llm_provider"),
)
supported: Final = litellm.get_supported_openai_params(
model=model,
custom_llm_provider=custom_llm_provider,
base_model=base_model if isinstance(base_model, str) else None,
)
except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
verbose_router_logger.debug(
"litellm.router.py::_deployment_accepts_param: keeping %s for model=%s. Got - %s", param, group, e
)
return True
return supported is None or param in supported
def _tier_params_the_target_accepts(
self, model: str, tier_params: Mapping[str, object], request_kwargs: Mapping[str, object]
) -> Mapping[str, object]:
"""Drop an OpenAI param that no deployment behind ``model`` declares.
A tier's litellm_params are an operator override applied to every request the tier routes,
so one the target cannot take turns that whole tier into a 400 raised before the request
leaves the proxy. The candidates are exactly what get_optional_params can reject, asked of
the module that raises, so credentials and endpoint controls are never at risk.
TIER_PARAMS_NEVER_DROPPED is excluded on top of that, for two reasons. No provider lists a
litellm control among its supported params, so "no deployment declares it" means litellm
consumes it rather than that the target refuses it, and dropping one changes litellm's own
behavior: dropping drop_params or additional_drop_params silently disables the sanitization
the operator configured. Providers do list extra_headers, but it carries auth, tenancy and
routing information, so sending fewer headers than configured is worse than today's error.
Token ceilings stay for the same reason: a tier's max_tokens or max_completion_tokens is a
cost bound, and dropping it would let a caller's own larger value through where today the
mismatch fails loudly.
The trade this filter makes is a param for a working request, which is right for one that
only shapes how the model answers and wrong for anything else.
A param survives if ANY deployment could take it, because routing has not chosen one yet,
and it survives both an unresolvable provider and a group with no deployments, because a
best-effort filter must never narrow what the request already did.
A github_copilot or chatgpt deployment counts as accepting everything, decided before any
lookup: resolving either provider runs its OAuth device flow, so a capability question
asked from the routing path can freeze the event loop for minutes waiting on a human.
allowed_openai_params is the documented escape hatch for an outdated or incomplete
supported-params list: request-time validation extends the supported list with it before
comparing. The filter asks the same question, so a param named by the allowlist on the tier
overlay, the request, or a deployment's own litellm_params is never a drop candidate.
"""
deployments: Final = self.get_model_list(model_name=model) or ()
if not deployments:
return tier_params
allowlisted: Final = self._declared_param_allowlist(tier_params) | self._declared_param_allowlist(
request_kwargs
)
candidates: Final = provider_rejectable_params(tier_params) - self.TIER_PARAMS_NEVER_DROPPED - allowlisted
unsupported: Final = frozenset(
param
for param in candidates
if not any(self._deployment_accepts_param(deployment, model, param) for deployment in deployments)
)
if not unsupported:
return tier_params
verbose_router_logger.warning(
"litellm.router.py: dropping tier params %s for model=%s, no deployment behind it declares them",
", ".join(sorted(unsupported)),
model,
)
return MappingProxyType({key: value for key, value in tier_params.items() if key not in unsupported})
def get_model_list(
self, model_name: str | None = None, team_id: str | None = None
) -> list[DeploymentTypedDict] | None:
@ -11843,6 +11952,33 @@ class Router:
return healthy_deployments
@staticmethod
def _pop_effort_from_nested_carrier(request_kwargs: dict[str, object], carrier: str) -> None:
nested: Final = request_kwargs.get(carrier)
if not isinstance(nested, dict):
return
nested.pop("effort", None)
if not nested:
request_kwargs.pop(carrier, None)
@staticmethod
def _drop_client_effort_carriers_a_tier_pin_supersedes(
request_kwargs: dict[str, object],
tier_litellm_params: Mapping[str, object],
) -> None:
"""Tier litellm_params are deliberate operator overrides, but provider
translations let a caller-supplied carrier of the same setting
(``thinking``, ``output_config.effort``, ``reasoning.effort``) outrank
the ``reasoning_effort`` alias, so a pinned effort only reaches the wire
if the client's other encodings are removed before the merge. Non-effort
fields a carrier also holds (``output_config.format``,
``reasoning.summary``) are kept."""
if "reasoning_effort" not in tier_litellm_params:
return
request_kwargs.pop("thinking", None)
Router._pop_effort_from_nested_carrier(request_kwargs, "output_config")
Router._pop_effort_from_nested_carrier(request_kwargs, "reasoning")
async def async_get_available_deployment(
self,
model: str,
@ -11888,7 +12024,11 @@ class Router:
model = pre_routing_hook_response.model
messages = pre_routing_hook_response.messages
if pre_routing_hook_response.litellm_params:
request_kwargs.update(pre_routing_hook_response.litellm_params)
accepted_tier_params: Final = self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs
)
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
request_kwargs.update(accepted_tier_params)
#########################################################
# Resolve the strategy and logger AFTER the pre-routing hook, since
@ -11999,7 +12139,11 @@ class Router:
model = pre_routing_hook_response.model
messages = pre_routing_hook_response.messages
if pre_routing_hook_response.litellm_params:
request_kwargs.update(pre_routing_hook_response.litellm_params)
accepted_tier_params: Final = self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs
)
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
request_kwargs.update(accepted_tier_params)
# 2. Get healthy deployments
healthy_deployments: Final = await self.async_get_healthy_deployments(

6
litellm/router_strategy/complexity_router/__init__.py
View file

@ -10,6 +10,7 @@ No external API calls - all scoring is local and <1ms.
from litellm.router_strategy.complexity_router.complexity_router import (
ComplexityRouter,
classification_system_prompt,
custom_tier_classification_prompt,
)
from litellm.router_strategy.complexity_router.config import (
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
@ -18,6 +19,8 @@ from litellm.router_strategy.complexity_router.config import (
ComplexityRouterConfig,
ComplexityTier,
ReminderMarkerPair,
TierDefinition,
normalize_classification_prompt,
)
__all__ = [
@ -28,5 +31,8 @@ __all__ = [
"ComplexityRouterConfig",
"ComplexityTier",
"ReminderMarkerPair",
"TierDefinition",
"classification_system_prompt",
"custom_tier_classification_prompt",
"normalize_classification_prompt",
]

48
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -56,6 +56,7 @@ from .config import (
ClassificationRubric,
ComplexityRouterConfig,
ComplexityTier,
TierDefinition,
)
if TYPE_CHECKING:
@ -197,6 +198,26 @@ def _custom_tier_prompt(entries: Sequence[tuple[str, str]], preamble: str | None
)
def custom_tier_classification_prompt(
definitions: Sequence[TierDefinition],
classification_prompt: str | None,
context_window_size: int,
) -> str:
"""The classifier's system role for an operator-defined tier set.
The single owner of the built-in-criteria substitution, so the dashboard's preview resolves a
blank description exactly as the live classifier does.
"""
entries: Final = tuple(
(
definition.name,
definition.description or _CLASSIFICATION_TIER_CRITERIA[ComplexityTier[definition.name.upper()]],
)
for definition in definitions
)
return _custom_tier_prompt(entries, classification_prompt, _closing_line(context_window_size))
def classification_system_prompt(
context_window_size: int,
custom_prompt: str | None = None,
@ -892,17 +913,10 @@ class ComplexityRouter(CustomLogger):
raise ValueError("classifier_llm_config is not set")
definitions: Final = self.config.tier_definitions
if definitions is not None:
entries: Final = tuple(
(
definition.name,
definition.description or _CLASSIFICATION_TIER_CRITERIA[ComplexityTier[definition.name.upper()]],
)
for definition in definitions
)
return _custom_tier_prompt(
entries,
return custom_tier_classification_prompt(
definitions,
self.config.classification_prompt,
_closing_line(self.config.classifier_context_window_size),
self.config.classifier_context_window_size,
)
return classification_system_prompt(
self.config.classifier_context_window_size,
@ -933,17 +947,15 @@ class ComplexityRouter(CustomLogger):
def savings_baseline(self) -> Baseline | None:
"""The derived counterfactual this router's savings are measured against.
``None`` when `litellm_settings.autorouter_savings_baseline_model` is set (the
spend writer reads that setting directly and it wins) or when this router was
built with ``derive_savings_baseline=False``. Derived once on first use and
pinned for the instance's lifetime: creating or editing the router rebuilds
the instance, which re-derives. Deferred past ``__init__`` because during a
config load this router can be constructed before its tier deployments are.
``None`` when this router was built with ``derive_savings_baseline=False``.
Derived once on first use and pinned for the instance's lifetime: creating or
editing the router rebuilds the instance, which re-derives. Deferred past
``__init__`` because during a config load this router can be constructed
before its tier deployments are.
"""
import litellm
from litellm.router_strategy.savings_baseline import resolve_baseline
if not self._derive_savings_baseline or litellm.autorouter_savings_baseline_model is not None:
if not self._derive_savings_baseline:
return None
if not self._savings_baseline_derived:
self._savings_baseline = resolve_baseline(self.litellm_router_instance, self._hardest_tier_models())

25
litellm/router_strategy/complexity_router/config.py
View file

@ -99,6 +99,23 @@ MAX_TIER_DESCRIPTION_CHARS: Final[int] = 500
MAX_CLASSIFICATION_PROMPT_CHARS: Final[int] = 2000
def normalize_classification_prompt(value: str | None) -> str | None:
"""Strip, reject blank, and cap an operator-written classifier preamble.
The single owner of the rule, so the dashboard's prompt preview normalizes exactly what the
write gate stores: previewing the raw value would render leading whitespace the router strips,
or an over-long prompt the write then rejects.
"""
if value is None:
return None
stripped: Final = value.strip()
if not stripped:
raise ValueError("must be non-empty; omit the field instead")
if len(stripped) > MAX_CLASSIFICATION_PROMPT_CHARS:
raise ValueError(f"classification_prompt exceeds {MAX_CLASSIFICATION_PROMPT_CHARS} characters")
return stripped
class TierDefinition(BaseModel):
"""An operator-defined tier: the name the LLM classifier must return and its rubric description."""
@ -1056,7 +1073,7 @@ class ComplexityRouterConfig(BaseModel):
)
return self
@field_validator("fallback_tier", "classification_prompt")
@field_validator("fallback_tier")
@classmethod
def _reject_blank_optional_text(cls, value: str | None) -> str | None:
if value is None:
@ -1068,10 +1085,8 @@ class ComplexityRouterConfig(BaseModel):
@field_validator("classification_prompt")
@classmethod
def _cap_classification_prompt(cls, value: str | None) -> str | None:
if value is not None and len(value) > MAX_CLASSIFICATION_PROMPT_CHARS:
raise ValueError(f"classification_prompt exceeds {MAX_CLASSIFICATION_PROMPT_CHARS} characters")
return value
def _normalize_classification_prompt_field(cls, value: str | None) -> str | None:
return normalize_classification_prompt(value)
@property
def has_custom_tiers(self) -> bool:

23
litellm/router_strategy/savings_baseline.py
View file

@ -1,16 +1,13 @@
"""The default counterfactual a complexity router's savings are measured against.
"""The counterfactual a complexity router's savings are measured against.
`litellm_settings.autorouter_savings_baseline_model` names the model the traffic would
have run on without a router. When the operator sets it, that answer wins and nothing
here runs. When they do not, the router's own tier ladder already names it: without a
router a deployment has to pick one model that can carry the hardest request it will
see, so the default baseline is the priciest model in the hardest configured tier. A
cheap tier is a choice the router made, not a ceiling it was bounded by.
The router's own tier ladder names the model the traffic would have run on without a
router: a deployment has to pick one model that can carry the hardest request it will
see, so the baseline is the priciest model in the hardest configured tier. A cheap
tier is a choice the router made, not a ceiling it was bounded by.
Candidates are ranked once against a fixed reference request, not against each request
that runs. Ranking per request means reading the request, and every input shape it can
take; a default must not carry that surface. An operator whose pool ordering genuinely
depends on request shape names the baseline in config, which skips this file entirely.
take; a per-router default must not carry that surface.
Baselines are always provider-qualified, because they travel to the spend writer as a
bare string with no provider beside them; an operator who writes ``deepseek-r1`` meaning
@ -54,9 +51,17 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str |
A deployment may name its vendor in the model prefix or in a separate
``custom_llm_provider``, and the bare name alone is not enough to price: it can
resolve to a different vendor's rates, or to nothing at all.
A github_copilot or chatgpt candidate is qualified by string alone: resolving either
provider runs its OAuth device flow, and for a declared pair the resolver's answer is
the declaration itself, so asking it buys nothing but the block.
"""
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
if declared is not None:
return f"{declared}/{model.removeprefix(f'{declared}/')}"
try:
resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline

23
litellm/types/guardrails.py
View file

@ -563,9 +563,15 @@ class LakeraV2GuardrailConfigModel(BaseModel):
default=True,
description="Whether to include developer information in the response",
)
on_flagged: Literal["block", "monitor"] | None = Field(
on_flagged: Literal["block", "monitor", "inject_system_message"] | None = Field(
default="block",
description="Action to take when content is flagged: 'block' (raise exception) or 'monitor' (log only)",
description="Action to take when content is flagged: 'block' (raise exception), 'monitor' (log only), "
"or 'inject_system_message' (append an advisory system message and let the LLM decide)",
)
advisory_system_message: str | None = Field(
default=None,
description="Custom advisory message template used when on_flagged='inject_system_message'. "
"Must contain a {reason} placeholder. Defaults to a generic advisory message if unset.",
)
@ -951,6 +957,17 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up
),
)
scan_raw_request: bool | None = Field(
default=None,
description=(
"When True, this pre_call guardrail always evaluates the request as it was before any "
"guardrail in this hook ran, regardless of its position in the guardrails list -- so the "
"YAML order of guardrails can never change whether this one blocks. Use only for "
"block-only guardrails: any data this guardrail returns is discarded, same contract as "
"run_in_parallel, since an earlier guardrail's masking must not be undone by this one."
),
)
@field_validator(
"mode",
"default_action",
@ -983,7 +1000,7 @@ class Mode(BaseModel):
default: str | list[str] | None = Field(default=None, description="Default mode when no tags match")
class LitellmParams(
class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # on_flagged literal diverges across mixins
CiscoAIDefenseGuardrailConfigModel,
PresidioConfigModel,
BedrockGuardrailConfigModel,

52
litellm/types/management_endpoints/auto_router_endpoints.py
View file

@ -362,6 +362,27 @@ class ShadowEvalSlice(BaseModel):
)
tie_rate_pct: float
avg_judge_confidence: float
real_spend: float = Field(
default=0.0,
description=(
"USD the real arm billed on this slice's judged turns, completion plus its own routing "
"classifier when it routed, excluding turns litellm's response cache served for free"
),
)
shadow_spend: float = Field(
default=0.0,
description=(
"USD the shadow arm billed on the same turns, completion plus its own routing classifier, "
"excluding the judge and the same cache-served turns, so the two spends compare like for like"
),
)
cache_hit_turns: int = Field(
default=0,
description=(
"Judged turns litellm's response cache served, excluded from both spends: an adopted router "
"would be served by the same cache, so those turns cost the same either way"
),
)
class ShadowEvalResult(BaseModel):
@ -382,6 +403,37 @@ class ShadowEvalResult(BaseModel):
)
overall_shadow_win_rate_pct: float
overall_tie_rate_pct: float
sampled_real_spend: float = Field(
default=0.0,
description="USD the real arm billed across all judged turns, cache-served turns excluded",
)
sampled_shadow_spend: float = Field(
default=0.0,
description="USD the shadow arm billed across the same turns, judge excluded, like for like",
)
not_sampled_count: int | None = Field(
default=None,
description=(
"Eligible requests the sampling dice skipped, summed over legs: the judged rows stand for "
"judged + this many requests. None for jobs from before the funnel existed"
),
)
unjudgeable_count: int | None = Field(
default=None,
description="Sampled requests whose shape could not be judged (tool-final turn, empty text)",
)
shed_count: int | None = Field(
default=None,
description="Sampled requests dropped by the per-pod concurrency cap, so quiet periods are overweighted",
)
withheld_count: int | None = Field(
default=None,
description=(
"Sampled requests the pipeline declined to spend on: no database to record into, an over-budget "
"key or team, or the eval budget unverifiable or already reached (the in-flight burst as a job "
"crosses max_budget lands here rather than vanishing from coverage)"
),
)
class ShadowEvalJobKeyResponse(BaseModel):

1
litellm/types/utils.py
View file

@ -165,6 +165,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_xhigh_reasoning_effort: bool | None
supports_max_reasoning_effort: bool | None
reasoning_effort_levels: ReadOnly[Sequence[str] | None]
default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None]
supports_output_config: bool | None
supports_image_size: bool | None
bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None

97
litellm/utils.py
View file

@ -76,6 +76,7 @@ from litellm.constants import (
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
NON_INFERENCE_CALL_TYPES,
OPENAI_EMBEDDING_PARAMS,
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
)
from litellm.litellm_core_utils.fallback_generalizations import (
@ -2555,10 +2556,19 @@ def _supports_factory(model: str, custom_llm_provider: str | None, key: str) ->
Raises:
Exception: If the given model is not found or there's an error in retrieval.
"""
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
try:
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider
)
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
if declared is not None:
model = model.removeprefix(
f"{declared}/"
) # rebind-ok: mirrors get_llm_provider's split without its OAuth flow
custom_llm_provider = declared # rebind-ok: same
else:
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider
)
model_info: Final = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
@ -2596,6 +2606,46 @@ def _supports_factory(model: str, custom_llm_provider: str | None, key: str) ->
return False
def declared_value_factory(model: str, custom_llm_provider: str | None, key: str) -> str | None:
"""Return a string value the model map declares for *key*, or ``None`` when it says nothing.
The string-valued sibling of :func:`_supports_factory` and
:func:`_is_explicitly_disabled_factory`, public where those two are not because it is read
from the provider configs rather than from this module, sharing their
``get_llm_provider`` -> ``_get_model_info_helper`` chain and their unprefixed-twin
fallback (#20885), so a provider-prefixed entry that omits the key still answers
from the bare entry that carries it.
``None`` means "the map does not say", never "the map says no" - callers decide what
an unknown declaration implies, and for a capability gate that decision must be the
conservative one.
"""
try:
resolved: Final = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
resolved_model: Final = resolved[0]
resolved_provider: Final = resolved[1]
model_info: Final = _get_model_info_helper(model=resolved_model, custom_llm_provider=resolved_provider)
declared: Final = model_info.get(key)
if isinstance(declared, str):
return declared
bare_model_key: Final = _get_model_cost_key(resolved_model)
bare_entry: Final = litellm.model_cost.get(bare_model_key) if bare_model_key is not None else None
if isinstance(bare_entry, dict):
bare_declared: Final = bare_entry.get(key)
if isinstance(bare_declared, str):
return bare_declared
return None
except Exception as e: # noqa: BLE001 # an unreadable map entry means "not declared", never a failed call
verbose_logger.debug(
"Model not found or error in reading %s. You passed model=%s, custom_llm_provider=%s. Error: %s",
key,
model,
custom_llm_provider,
e,
)
return None
def _is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, key: str) -> bool:
"""Return True only when the model map explicitly sets *key* to ``False``.
@ -2991,12 +3041,7 @@ def register_model(
for _registered_key, _registered_value in _registrations.items():
_runtime_registered_model_cost[_registered_key] = dict(_registered_value) # mutable-ok: caller-owned
# Providers that trigger side effects (e.g., OAuth flows) when get_model_info is called
# Skip get_model_info for these providers during model registration
_skip_get_model_info_providers: Final = {
LlmProviders.GITHUB_COPILOT.value,
LlmProviders.CHATGPT.value,
}
_skip_get_model_info_providers: Final = PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
for key, value in loaded_model_cost.items():
## get model info ##
@ -4150,17 +4195,11 @@ def get_optional_params(
unsupported_params: Final = {}
for k in non_default_params:
if k not in supported_params:
if k == "user" or k == "stream_options" or k == "stream":
if k in PROVIDER_UNVALIDATED_PARAMS:
continue
if k == "n" and n == 1: # langchain sends n=1 as a default value
continue # skip this param
if (
k == "max_retries"
): # TODO: This is a patch. We support max retries for OpenAI, Azure. For non OpenAI LLMs we need to add support for max retries
continue # skip this param
# Always keeps this in elif code blocks
else:
unsupported_params[k] = non_default_params[k]
unsupported_params[k] = non_default_params[k]
if unsupported_params:
if litellm.drop_params is True or (drop_params is not None and drop_params is True):
@ -4729,6 +4768,22 @@ def _apply_openai_param_overrides(optional_params: dict, non_default_params: dic
return optional_params
PROVIDER_UNVALIDATED_PARAMS: Final = frozenset({"user", "stream_options", "stream", "max_retries"})
def provider_rejectable_params(passed_params: Mapping[str, object]) -> frozenset[str]:
"""The params a provider can actually be rejected for, i.e. the ones _check_valid_arg compares
against its supported list.
Anything outside this set never reaches that comparison. Endpoint and transport controls such as
base_url, timeout, default_headers, organization and deployment_id are not chat completion
params at all, so a caller filtering on "is this an OpenAI param" would discard configuration the
request needs while never touching what the provider would have rejected.
"""
params: Final = dict(passed_params) # mutable-ok: get_non_default_params takes a dict
return frozenset(get_non_default_params(params)) - PROVIDER_UNVALIDATED_PARAMS
def get_non_default_params(passed_params: dict) -> dict:
# filter out those parameters that were passed with non-default values
non_default_params: Final = {
@ -5544,6 +5599,8 @@ def _get_model_info_helper(
"""
Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's
"""
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
try:
azure_llms: Final = {**litellm.azure_llms, **litellm.azure_embedding_models}
if model in azure_llms:
@ -5558,7 +5615,9 @@ def _get_model_info_helper(
):
model = model + "@latest"
##########################
potential_model_names: Final = _get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider)
potential_model_names: Final = _get_potential_model_names(
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
)
verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names)
@ -5866,6 +5925,7 @@ def _get_model_info_helper(
supports_response_schema=_model_info.get("supports_response_schema", None),
supports_vision=_model_info.get("supports_vision", None),
supports_function_calling=_model_info.get("supports_function_calling", None),
supports_parallel_function_calling=_model_info.get("supports_parallel_function_calling", None),
supports_tool_choice=_model_info.get("supports_tool_choice", None),
supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None),
supports_prompt_caching=_model_info.get("supports_prompt_caching", None),
@ -5890,6 +5950,7 @@ def _get_model_info_helper(
supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None),
supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None),
reasoning_effort_levels=_model_info.get("reasoning_effort_levels", None),
default_reasoning_effort=_model_info.get("default_reasoning_effort", None),
bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None),
bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None),
supports_computer_use=_model_info.get("supports_computer_use", None),

63
model_prices_and_context_window.json
View file

@ -3409,6 +3409,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -3456,6 +3457,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -3589,6 +3591,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3630,6 +3633,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3671,6 +3675,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3712,6 +3717,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -3937,7 +3943,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/eu/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.4e-07,
@ -3972,7 +3979,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/eu/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -4247,7 +4255,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/global/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.25e-07,
@ -4282,7 +4291,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/global/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -5367,6 +5377,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_minimal_reasoning_effort": true
},
"azure/gpt-5.1-chat-2025-11-13": {
@ -5404,7 +5415,8 @@
"supports_system_messages": true,
"supports_tool_choice": false,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/gpt-5.1-codex-2025-11-13": {
"cache_read_input_token_cost": 1.25e-07,
@ -5833,7 +5845,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.25e-07,
@ -5868,7 +5881,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -6315,6 +6329,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6354,6 +6369,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6393,6 +6409,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6438,6 +6455,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6477,6 +6495,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -6516,6 +6535,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -7663,6 +7683,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-5.4-mini-2026-03-17": {
@ -7704,6 +7725,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-5.4-nano": {
@ -7745,6 +7767,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-5.4-nano-2026-03-17": {
@ -7786,6 +7809,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-image-1": {
@ -8856,7 +8880,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/us/gpt-5.1-chat": {
"cache_read_input_token_cost": 1.4e-07,
@ -8891,7 +8916,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none"
},
"azure/us/gpt-5.1-codex": {
"deprecation_date": "2027-05-15",
@ -26315,6 +26341,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true
},
@ -26359,6 +26386,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true
},
@ -26404,6 +26432,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true
},
@ -26449,6 +26478,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -26494,6 +26524,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -27318,6 +27349,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -27366,6 +27398,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
@ -27515,6 +27548,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -27566,6 +27600,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -27614,6 +27649,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -27662,6 +27698,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"default_reasoning_effort": "none",
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
@ -38682,7 +38719,7 @@
"together_ai/openai/gpt-oss-20b": {
"input_cost_per_token": 5e-08,
"litellm_provider": "together_ai",
"max_input_tokens": 128000,
"max_input_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2e-07,
"source": "https://www.together.ai/models/gpt-oss-20b",
@ -38904,14 +38941,14 @@
"source": "https://docs.together.ai/docs/serverless-models"
},
"together_ai/Qwen/Qwen3.8-2.4T-A95B": {
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1010000,
"max_output_tokens": 1010000,
"max_tokens": 1010000,
"mode": "chat",
"output_cost_per_token": 6.25e-06,
"output_cost_per_token": 6e-06,
"source": "https://docs.together.ai/docs/serverless-models",
"supports_prompt_caching": true
},

12
model_prices_and_context_window.schema.json
View file

@ -179,6 +179,18 @@
"comment": {
"type": "string"
},
"default_reasoning_effort": {
"type": "string",
"description": "Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.",
"enum": [
"none",
"minimal",
"low",
"medium",
"high",
"xhigh"
]
},
"deprecation_date": {
"type": "string",
"description": "Date the provider deprecates the model, YYYY-MM-DD.",

2
ruff-strict-budget.json
View file

@ -9,7 +9,7 @@
"limit": 827
},
"ANN201": {
"limit": 2012
"limit": 2011
},
"ANN202": {
"limit": 847

15
schema.prisma
View file

@ -1533,12 +1533,27 @@ model LiteLLM_ShadowEvalAttempt {
confidence Float?
judge_cost Float @default(0)
shadow_cost Float @default(0)
real_cost Float? // NULL = row predates cost measurement; comparisons read only measured rows
real_classifier_cost Float @default(0)
shadow_classifier_cost Float @default(0)
real_cache_hit Boolean @default(false)
error String?
created_at DateTime @default(now())
@@index([job_id])
}
// Per-leg sampling funnel counters the attempt rows cannot derive: requests an
// admitting job saw but did not judge. attempted = the leg's attempt rows; the
// leg's eligible traffic = not_sampled + unjudgeable + shed + withheld + attempted.
model LiteLLM_ShadowEvalFunnel {
job_id String @id
not_sampled Int @default(0)
unjudgeable Int @default(0)
shed Int @default(0)
withheld Int @default(0)
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

20
terraform/provider/CHANGELOG.md
View file

@ -17,11 +17,31 @@ longer signal it.
### Added
- **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them
- **jwt_key_mapping**: New `litellm_jwt_key_mapping` resource for the proxy's JWT to virtual key mappings, so JWT clients identified by a claim (`client_id`, `azp`, `sub`) map to virtual keys and inherit their models, budgets and rate limits. Supports `description` and `is_active`, rotating the mapped key in place, and forces replacement when the claim name or value changes
- **team**: `soft_budget`, `tags`, and `soft_budget_alerting_emails` attributes on `litellm_team`, matching what `/team/new` and `/team/update` already accept; `soft_budget_alerting_emails` is sent under `metadata`, where the proxy reads it
- **user**: New `litellm_user` resource and `litellm_user` / `litellm_users` data sources for managing internal users
- **budget**: New `litellm_budget` resource and `litellm_budget` / `litellm_budgets` data sources for reusable budget objects
- **tag**: New `litellm_tag` resource and `litellm_tag` / `litellm_tags` data sources for spend and routing tags
- **project**: New `litellm_project` resource and `litellm_project` / `litellm_projects` data sources
- **guardrail**: New `litellm_guardrail` resource and `litellm_guardrail` / `litellm_guardrails` data sources; `litellm_params` is sensitive and never read back into state
- **prompt**: New `litellm_prompt` resource and `litellm_prompt` / `litellm_prompts` data sources for prompt templates
- **agent**: New `litellm_agent` resource and `litellm_agent` / `litellm_agents` data sources for A2A agents
- **search_tool**: New `litellm_search_tool` resource and `litellm_search_tool` / `litellm_search_tools` data sources
- **access groups**: New `litellm_access_group` and `litellm_unified_access_group` resources with matching singular and plural data sources
- **fallback**: New `litellm_fallback` resource and data source for per-model fallbacks (general, context window and content policy)
- **block resources**: New `litellm_key_block` and `litellm_team_block` resources to manage the blocked state of existing keys and teams
- **data sources for existing resources**: New `litellm_key` / `litellm_keys`, `litellm_team` / `litellm_teams`, `litellm_model` / `litellm_models`, `litellm_organization` / `litellm_organizations` and `litellm_mcp_server` / `litellm_mcp_servers` data sources
- **key**: New arguments `budget_id`, `enforced_params`, `allowed_routes`, `allowed_passthrough_routes`, `rpm_limit_type`, `tpm_limit_type`, `prompts`, `organization_id` and `project_id`
- **team**: New arguments `model_aliases`, `guardrails`, `prompts`, `team_member_budget`, `team_member_budget_duration`, `team_member_rpm_limit`, `team_member_tpm_limit`, `team_member_key_duration`, `model_rpm_limit`, `model_tpm_limit`, `allowed_passthrough_routes`, `rpm_limit_type` and `tpm_limit_type`
- **import**: `terraform import` support for `litellm_team`, `litellm_model`, `litellm_organization`, `litellm_mcp_server`, `litellm_vector_store` and every new resource
### Fixed
- **team**: Read now decodes the `team_info` envelope `/team/info` actually returns, so team attributes refresh from the proxy instead of always falling back to the prior state
- **key**: Read now unwraps the `info` envelope `/key/info` actually returns; previously reads mapped nothing back into state, so drift on a key was never detected
- **key**: Updates no longer send an empty `budget_duration`, which the proxy rejects with a 400; any update to a key without a configured `budget_duration` previously failed outright
- **key**: A config-supplied `key` value (write-only) is now forwarded to `/key/generate`; previously it was silently dropped and the proxy generated a random key instead
- **security**: The `litellm_key` data source and `litellm_key_block` resource normalize raw `sk-` keys to their SHA-256 token hash before building request URLs and resource IDs, so plaintext keys no longer land in reverse-proxy access logs, Terraform plan output, or state IDs
### Changed

5
terraform/provider/README.md
View file

@ -1,10 +1,10 @@
# LiteLLM Terraform Provider
This Terraform provider allows you to manage LiteLLM resources through Infrastructure as Code. It provides support for managing models, teams, team members, and API keys via the LiteLLM REST API.
This Terraform provider allows you to manage LiteLLM resources through Infrastructure as Code. It provides support for managing models, teams, team members, API keys, users, organizations, budgets, tags, projects, guardrails, prompts, agents, search tools, access groups, fallbacks, MCP servers, credentials and vector stores via the LiteLLM REST API, along with read-only data sources for each of them.
## Source of truth
This directory (`terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm)) is the source of truth for the provider. [BerriAI/terraform-provider-litellm](https://github.com/BerriAI/terraform-provider-litellm) is a thin release mirror that the public Terraform Registry ingests from; do not open PRs there. Changes land here, where CI builds the provider, runs its tests, and statically audits every endpoint the provider calls against the proxy's generated OpenAPI schema (`tools/endpointaudit/`), so the provider cannot drift from the LiteLLM API silently. Releases are published by mirroring this directory into the split repo and tagging it, which triggers the goreleaser workflow there (see `RELEASING.md`)
This directory (`terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm)) is the source of truth for the provider. [BerriAI/terraform-provider-litellm](https://github.com/BerriAI/terraform-provider-litellm) is a thin release mirror that the public Terraform Registry ingests from; do not open PRs there. Changes land here, where CI builds the provider, runs its tests, and statically audits every endpoint the provider calls against the proxy's generated OpenAPI schema (`tools/endpointaudit/`), so the provider cannot drift from the LiteLLM API silently. The same audit runs in reverse as a coverage gate: every management endpoint in the schema must be covered by a resource or data source, or carry a documented entry in `tools/endpointaudit/coverage_allowlist.txt`, and stale allowlist entries fail CI. Releases are published by mirroring this directory into the split repo and tagging it, which triggers the goreleaser workflow there (see `RELEASING.md`)
## Versioning
@ -151,6 +151,7 @@ For full details on the <code>litellm_key</code> resource, see the [key resource
- <code>litellm_mcp_server</code>: Manage MCP (Model Context Protocol) servers. [Documentation](docs/resources/mcp_server.md)
- <code>litellm_credential</code>: Manage credentials for secure authentication. [Documentation](docs/resources/credential.md)
- <code>litellm_vector_store</code>: Manage vector stores for embeddings and RAG. [Documentation](docs/resources/vector_store.md)
- <code>litellm_jwt_key_mapping</code>: Map JWT claim values to virtual keys for per-client budgets and limits. [Documentation](docs/resources/jwt_key_mapping.md)
### Available Data Sources

34
terraform/provider/docs/data-sources/access_group.md Normal file
View file

@ -0,0 +1,34 @@
---
page_title: "litellm_access_group Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Retrieves information about an existing LiteLLM model access group.
---
# litellm_access_group (Data Source)
Retrieves information about an existing LiteLLM model access group by name.
## Example Usage
```terraform
data "litellm_access_group" "production" {
access_group = "production-models"
}
output "production_models" {
value = data.litellm_access_group.production.model_names
}
```
## Argument Reference
* `access_group` - (Required) Name of the access group to look up.
## Attribute Reference
* `id` - The access group name.
* `model_names` - List of model names in the access group.
* `deployment_count` - Number of deployments tagged with this access group.

33
terraform/provider/docs/data-sources/access_groups.md Normal file
View file

@ -0,0 +1,33 @@
---
page_title: "litellm_access_groups Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Retrieves all LiteLLM model access groups.
---
# litellm_access_groups (Data Source)
Retrieves all LiteLLM model access groups configured on the proxy.
## Example Usage
```terraform
data "litellm_access_groups" "all" {}
output "access_group_names" {
value = data.litellm_access_groups.all.ids
}
```
## Argument Reference
This data source takes no arguments.
## Attribute Reference
* `access_groups` - List of access groups. Each entry exports:
* `access_group` - The access group name.
* `model_names` - List of model names in the access group.
* `deployment_count` - Number of deployments tagged with this access group.
* `ids` - List of all access group names.

43
terraform/provider/docs/data-sources/agent.md Normal file
View file

@ -0,0 +1,43 @@
# litellm_agent Data Source
Retrieves information about an existing A2A agent on the LiteLLM proxy.
## Example Usage
```hcl
data "litellm_agent" "existing" {
agent_id = "123e4567-e89b-12d3-a456-426614174000"
}
output "agent_card" {
value = jsondecode(data.litellm_agent.existing.agent_card_params)
}
```
## Argument Reference
The following arguments are supported:
* `agent_id` - (Required) Unique identifier of the agent to retrieve.
## Attribute Reference
In addition to all arguments above, the following attributes are exported:
* `agent_name` - Name of the agent.
* `agent_card_params` - The A2A agent card as a JSON object string (decode with `jsondecode`).
* `object_permission` - Access control permissions as a JSON object string.
* `extra_headers` - List of incoming request header names forwarded to the agent.
* `tpm_limit` - Tokens per minute limit.
* `rpm_limit` - Requests per minute limit.
* `session_tpm_limit` - Per-session tokens per minute limit.
* `session_rpm_limit` - Per-session requests per minute limit.
* `spend` - Total spend recorded for this agent.
* `created_at` - Timestamp when the agent was created.
* `updated_at` - Timestamp when the agent was last updated.
* `created_by` - User who created the agent.
* `updated_by` - User who last updated the agent.
## Security Note
`litellm_params` and `static_headers` are not exposed through this data source because they may hold API keys or tokens.

42
terraform/provider/docs/data-sources/agents.md Normal file
View file

@ -0,0 +1,42 @@
# litellm_agents Data Source
Retrieves the list of A2A agents registered on the LiteLLM proxy.
## Example Usage
```hcl
data "litellm_agents" "all" {}
output "agent_ids" {
value = data.litellm_agents.all.ids
}
# Only agents whose URL is currently reachable (or that have no URL)
data "litellm_agents" "healthy" {
health_check = true
}
```
## Argument Reference
The following arguments are supported:
* `health_check` - (Optional, default `false`) When true, the proxy probes each agent's URL and only returns agents that are reachable or have no URL.
## Attribute Reference
The following attributes are exported:
* `ids` - List of agent IDs.
* `agents` - List of agents. Each entry exports:
* `agent_id` - The unique agent ID.
* `agent_name` - Name of the agent.
* `tpm_limit` - Tokens per minute limit.
* `rpm_limit` - Requests per minute limit.
* `session_tpm_limit` - Per-session tokens per minute limit.
* `session_rpm_limit` - Per-session requests per minute limit.
* `spend` - Total spend recorded for the agent.
* `created_at` - Timestamp when the agent was created.
* `updated_at` - Timestamp when the agent was last updated.
* `created_by` - User who created the agent.
* `updated_by` - User who last updated the agent.

31
terraform/provider/docs/data-sources/budget.md Normal file
View file

@ -0,0 +1,31 @@
# litellm_budget Data Source
Retrieves information about an existing LiteLLM budget by ID
## Example Usage
```hcl
data "litellm_budget" "engineering" {
budget_id = "engineering-monthly"
}
output "engineering_max_budget" {
value = data.litellm_budget.engineering.max_budget
}
```
## Argument Reference
- `budget_id` (Required) - ID of the budget to retrieve
## Attribute Reference
- `id` - The budget ID
- `max_budget` - Hard budget limit in USD
- `soft_budget` - Soft budget limit in USD that triggers alerts
- `max_parallel_requests` - Maximum concurrent requests allowed for this budget
- `tpm_limit` - Maximum tokens per minute allowed for this budget
- `rpm_limit` - Maximum requests per minute allowed for this budget
- `budget_duration` - Budget reset period
- `model_max_budget` - JSON string of per-model budget config
- `budget_reset_at` - Datetime when the budget is reset

31
terraform/provider/docs/data-sources/budgets.md Normal file
View file

@ -0,0 +1,31 @@
# litellm_budgets Data Source
Retrieves all budgets configured on the LiteLLM proxy
## Example Usage
```hcl
data "litellm_budgets" "all" {}
output "budget_ids" {
value = data.litellm_budgets.all.ids
}
```
## Argument Reference
This data source takes no arguments
## Attribute Reference
- `budgets` - All budgets configured on the proxy. Each entry has:
- `budget_id` - The budget ID
- `max_budget` - Hard budget limit in USD
- `soft_budget` - Soft budget limit in USD that triggers alerts
- `max_parallel_requests` - Maximum concurrent requests allowed for this budget
- `tpm_limit` - Maximum tokens per minute allowed for this budget
- `rpm_limit` - Maximum requests per minute allowed for this budget
- `budget_duration` - Budget reset period
- `model_max_budget` - JSON string of per-model budget config
- `budget_reset_at` - Datetime when the budget is reset
- `ids` - IDs of all budgets configured on the proxy

38
terraform/provider/docs/data-sources/fallback.md Normal file
View file

@ -0,0 +1,38 @@
# litellm_fallback (Data Source)
Retrieves the fallback configuration for a LiteLLM model. Use this to reference fallbacks that were configured outside of Terraform.
## Example Usage
```hcl
data "litellm_fallback" "gpt4" {
model = "gpt-4"
}
output "gpt4_fallback_models" {
value = data.litellm_fallback.gpt4.fallback_models
}
```
### Specific Fallback Type
```hcl
data "litellm_fallback" "gpt4_context_window" {
model = "gpt-4"
fallback_type = "context_window"
}
```
## Argument Reference
The following arguments are supported:
* `model` - (Required) The model name to get fallbacks for.
* `fallback_type` - (Optional) Type of fallback to retrieve. One of `general` (default), `context_window`, or `content_policy`.
## Attribute Reference
In addition to the arguments above, the following attributes are exported:
* `id` - The primary model name.
* `fallback_models` - List of fallback model names in order of priority.

27
terraform/provider/docs/data-sources/guardrail.md Normal file
View file

@ -0,0 +1,27 @@
# litellm_guardrail Data Source
Retrieves information about an existing LiteLLM guardrail by ID. Sensitive `litellm_params` are not exposed.
## Example Usage
```hcl
data "litellm_guardrail" "existing" {
guardrail_id = "123e4567-e89b-12d3-a456-426614174000"
}
output "guardrail_name" {
value = data.litellm_guardrail.existing.guardrail_name
}
```
## Argument Reference
* `guardrail_id` - (Required) Unique identifier of the guardrail to retrieve.
## Attribute Reference
* `guardrail_name` - Human-readable name of the guardrail.
* `guardrail_info` - Map of additional metadata for the guardrail.
* `guardrail_definition_location` - Where the guardrail is defined: `config` or `db`.
* `created_at` - Timestamp when the guardrail was created.
* `updated_at` - Timestamp when the guardrail was last updated.

32
terraform/provider/docs/data-sources/guardrails.md Normal file
View file

@ -0,0 +1,32 @@
# litellm_guardrails Data Source
Retrieves the list of all guardrails configured on the LiteLLM proxy (from both config and DB). Sensitive `litellm_params` are not exposed.
## Example Usage
```hcl
data "litellm_guardrails" "all" {}
output "guardrail_ids" {
value = data.litellm_guardrails.all.ids
}
output "guardrail_names" {
value = [for g in data.litellm_guardrails.all.guardrails : g.guardrail_name]
}
```
## Argument Reference
This data source takes no arguments.
## Attribute Reference
* `guardrails` - List of guardrails. Each entry contains:
* `guardrail_id` - Unique identifier of the guardrail.
* `guardrail_name` - Human-readable name of the guardrail.
* `guardrail_info` - Map of additional metadata for the guardrail.
* `guardrail_definition_location` - Where the guardrail is defined: `config` or `db`.
* `created_at` - Timestamp when the guardrail was created.
* `updated_at` - Timestamp when the guardrail was last updated.
* `ids` - List of all guardrail IDs.

57
terraform/provider/docs/data-sources/key.md Normal file
View file

@ -0,0 +1,57 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_key Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Retrieves information about an existing LiteLLM API key.
---
# litellm_key (Data Source)
Retrieves information about an existing LiteLLM API key via `/key/info`. Pass either the raw key or its hashed token. The raw key value is never written to state beyond the input you provide; the data source ID is the hashed token.
## Example Usage
```terraform
data "litellm_key" "ci" {
key = var.ci_key_hash
}
output "ci_key_team" {
value = data.litellm_key.ci.team_id
}
```
## Argument Reference
The following arguments are supported:
* `key` - (Required, Sensitive) The API key (or its hash) to look up.
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `token_id` - Hashed token identifier of the key (safe to store in state).
* `key_name` - Redacted display name of the key.
* `key_alias` - User-friendly alias for the key.
* `models` - List of models this key can access.
* `spend` - Amount spent by this key.
* `max_budget` - Maximum budget for this key.
* `user_id` - User ID associated with this key.
* `team_id` - Team ID associated with this key.
* `organization_id` - Organization ID associated with this key.
* `tpm_limit` - Tokens per minute limit.
* `rpm_limit` - Requests per minute limit.
* `max_parallel_requests` - Maximum parallel requests allowed.
* `budget_duration` - Budget reset duration.
* `metadata` - Map of string metadata values for the key.
* `tags` - Tags attached to the key.
* `blocked` - Whether the key is blocked.
* `expires` - Expiry timestamp, if set.
* `created_at` - Timestamp when the key was created.
* `updated_at` - Timestamp when the key was last updated.
## Security Note
The raw key value is only used to perform the lookup; it is never exported as an attribute or used as the data source ID.

62
terraform/provider/docs/data-sources/keys.md Normal file
View file

@ -0,0 +1,62 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_keys Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Lists LiteLLM API keys with optional server-side filters.
---
# litellm_keys (Data Source)
Lists LiteLLM API keys via `/key/list`. Supports server-side filtering and pagination. Raw key values are never returned; each entry is identified by its hashed token.
## Example Usage
```terraform
data "litellm_keys" "team_keys" {
team_id = litellm_team.ml.id
size = 50
}
output "team_key_aliases" {
value = [for k in data.litellm_keys.team_keys.keys : k.key_alias]
}
```
## Argument Reference
The following arguments are supported:
* `page` - (Optional) Page number for pagination. Defaults to `1`.
* `size` - (Optional) Number of keys per page. Defaults to `100`.
* `user_id` - (Optional) Filter keys by user ID.
* `team_id` - (Optional) Filter keys by team ID.
* `organization_id` - (Optional) Filter keys by organization ID.
* `key_alias` - (Optional) Filter keys by key alias.
* `include_team_keys` - (Optional) Include all keys for teams the caller is an admin of.
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `total_count` - Total number of keys matching the filters.
* `total_pages` - Total number of pages.
* `current_page` - The page returned.
* `ids` - Hashed token identifiers of the returned keys.
* `keys` - List of key objects. Each entry exports:
* `token_id` - Hashed token identifier.
* `key_name` - Redacted display name.
* `key_alias` - User-friendly alias.
* `spend` - Amount spent by the key.
* `max_budget` - Maximum budget.
* `models` - Models the key can access.
* `user_id` - Associated user ID.
* `team_id` - Associated team ID.
* `organization_id` - Associated organization ID.
* `tpm_limit` - Tokens per minute limit.
* `rpm_limit` - Requests per minute limit.
* `budget_duration` - Budget reset duration.
* `blocked` - Whether the key is blocked.
* `expires` - Expiry timestamp, if set.
* `created_at` - Creation timestamp.
* `updated_at` - Last update timestamp.

58
terraform/provider/docs/data-sources/mcp_server.md Normal file
View file

@ -0,0 +1,58 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_mcp_server Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Retrieves information about an existing LiteLLM MCP server.
---
# litellm_mcp_server (Data Source)
Retrieves information about an existing MCP server via `/v1/mcp/server/{server_id}`. Secret material (environment variables, credentials, and static header values) is never exposed.
## Example Usage
```terraform
data "litellm_mcp_server" "github" {
server_id = "srv-1234"
}
output "github_mcp_url" {
value = data.litellm_mcp_server.github.url
}
```
## Argument Reference
The following arguments are supported:
* `server_id` - (Required) Unique identifier of the MCP server to retrieve.
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `server_name` - Name of the MCP server.
* `alias` - Alias for the MCP server.
* `description` - Description of the MCP server.
* `url` - URL of the MCP server.
* `transport` - Transport type (`http`, `sse`, `stdio`).
* `spec_version` - MCP specification version.
* `auth_type` - Authentication type (`none`, `bearer`, `basic`, ...).
* `mcp_access_groups` - Access groups for the MCP server.
* `allowed_tools` - Tools allowed on this server.
* `extra_headers` - Names of request headers forwarded to the MCP server.
* `command` - Command for stdio transport.
* `args` - Arguments for the command (stdio transport).
* `allow_all_keys` - Whether all keys can access the server.
* `status` - Health status (`healthy`, `unhealthy`, `unknown`).
* `last_health_check` - Timestamp of the last health check.
* `health_check_error` - Error message from the last health check, if any.
* `created_at` - Timestamp when the server was created.
* `created_by` - User who created the server.
* `updated_at` - Timestamp when the server was last updated.
* `updated_by` - User who last updated the server.
## Security Note
For security reasons, `env`, `credentials`, and `static_headers` are not exposed through this data source since they may hold secrets.

50
terraform/provider/docs/data-sources/mcp_servers.md Normal file
View file

@ -0,0 +1,50 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_mcp_servers Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Lists LiteLLM MCP servers.
---
# litellm_mcp_servers (Data Source)
Lists MCP servers via `/v1/mcp/server`. Secret material is never exposed.
## Example Usage
```terraform
data "litellm_mcp_servers" "all" {}
data "litellm_mcp_servers" "team_scoped" {
team_id = litellm_team.ml.id
}
output "mcp_server_urls" {
value = [for s in data.litellm_mcp_servers.all.mcp_servers : s.url]
}
```
## Argument Reference
The following arguments are supported:
* `team_id` - (Optional) Filter to servers this team can access plus globally available (`allow_all_keys`) servers.
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `ids` - IDs of the returned MCP servers.
* `mcp_servers` - List of MCP server objects. Each entry exports:
* `server_id` - Unique identifier of the MCP server.
* `server_name` - Name of the MCP server.
* `alias` - Alias for the MCP server.
* `description` - Description of the MCP server.
* `url` - URL of the MCP server.
* `transport` - Transport type (`http`, `sse`, `stdio`).
* `spec_version` - MCP specification version.
* `auth_type` - Authentication type.
* `allow_all_keys` - Whether all keys can access the server.
* `status` - Health status (`healthy`, `unhealthy`, `unknown`).
* `created_at` - Creation timestamp.
* `updated_at` - Last update timestamp.

50
terraform/provider/docs/data-sources/model.md Normal file
View file

@ -0,0 +1,50 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_model Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Retrieves information about a model deployment on the LiteLLM proxy.
---
# litellm_model (Data Source)
Retrieves information about a single model deployment via `/v1/model/info`. Sensitive `litellm_params` fields (API keys and other credentials) are never exposed; only safe routing metadata is exported.
## Example Usage
```terraform
data "litellm_model" "gpt4o" {
model_id = "0e5x74fab24a7a5245d2ced3536dd8f5"
}
output "gpt4o_provider" {
value = data.litellm_model.gpt4o.custom_llm_provider
}
```
## Argument Reference
The following arguments are supported:
* `model_id` - (Required) LiteLLM model ID (the `x-litellm-model-id` response header value).
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `model_name` - Public model name used for routing.
* `model` - The underlying `litellm_params` model, e.g. `openai/gpt-4o`.
* `custom_llm_provider` - Provider for the model.
* `model_api_base` - API base URL, if configured.
* `api_version` - API version, if configured.
* `tpm` - Tokens per minute limit for the deployment.
* `rpm` - Requests per minute limit for the deployment.
* `base_model` - Base model used for pricing and capabilities.
* `tier` - Model tier (`free` or `paid`).
* `mode` - Model mode, e.g. `chat` or `embedding`.
* `team_id` - Team the deployment is scoped to, if any.
* `db_model` - Whether the deployment is stored in the database (as opposed to config).
## Security Note
Credential material inside `litellm_params` (such as `api_key`, `aws_secret_access_key`, and `vertex_credentials`) is never exported by this data source.

44
terraform/provider/docs/data-sources/models.md Normal file
View file

@ -0,0 +1,44 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_models Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Lists model deployments on the LiteLLM proxy.
---
# litellm_models (Data Source)
Lists all model deployments via `/v1/model/info`. Sensitive `litellm_params` fields (API keys and other credentials) are never exposed.
## Example Usage
```terraform
data "litellm_models" "all" {}
output "model_names" {
value = [for m in data.litellm_models.all.models : m.model_name]
}
```
## Argument Reference
The following arguments are supported:
* `team_id` - (Optional) Filter models to those accessible by this team.
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `ids` - LiteLLM model IDs of the returned models.
* `models` - List of model objects. Each entry exports:
* `id` - LiteLLM model ID.
* `model_name` - Public model name used for routing.
* `model` - The underlying `litellm_params` model.
* `custom_llm_provider` - Provider for the model.
* `model_api_base` - API base URL, if configured.
* `base_model` - Base model used for pricing and capabilities.
* `tier` - Model tier (`free` or `paid`).
* `mode` - Model mode, e.g. `chat` or `embedding`.
* `team_id` - Team the deployment is scoped to, if any.
* `db_model` - Whether the deployment is stored in the database.

48
terraform/provider/docs/data-sources/organization.md Normal file
View file

@ -0,0 +1,48 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_organization Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Retrieves information about an existing LiteLLM organization.
---
# litellm_organization (Data Source)
Retrieves information about an existing LiteLLM organization via `/organization/info`, including its attached budget settings.
## Example Usage
```terraform
data "litellm_organization" "main" {
organization_id = "org-1234"
}
resource "litellm_team" "ml" {
team_alias = "ml-team"
organization_id = data.litellm_organization.main.organization_id
}
```
## Argument Reference
The following arguments are supported:
* `organization_id` - (Required) Unique identifier of the organization to retrieve.
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `organization_alias` - User-friendly name of the organization.
* `budget_id` - ID of the attached budget.
* `models` - Models the organization can access.
* `spend` - Amount spent by the organization.
* `metadata` - Map of string metadata values for the organization.
* `max_budget` - Maximum budget from the attached budget.
* `soft_budget` - Soft budget alert threshold from the attached budget.
* `tpm_limit` - Tokens per minute limit from the attached budget.
* `rpm_limit` - Requests per minute limit from the attached budget.
* `max_parallel_requests` - Maximum parallel requests from the attached budget.
* `budget_duration` - Budget reset duration from the attached budget.
* `created_at` - Timestamp when the organization was created.
* `updated_at` - Timestamp when the organization was last updated.

45
terraform/provider/docs/data-sources/organizations.md Normal file
View file

@ -0,0 +1,45 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_organizations Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Lists LiteLLM organizations.
---
# litellm_organizations (Data Source)
Lists LiteLLM organizations via `/organization/list`.
## Example Usage
```terraform
data "litellm_organizations" "all" {}
output "organization_ids" {
value = data.litellm_organizations.all.ids
}
```
## Argument Reference
The following arguments are supported:
* `org_alias` - (Optional) Filter organizations by alias.
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `ids` - IDs of the returned organizations.
* `organizations` - List of organization objects. Each entry exports:
* `organization_id` - Unique identifier of the organization.
* `organization_alias` - User-friendly name of the organization.
* `budget_id` - ID of the attached budget.
* `models` - Models the organization can access.
* `spend` - Amount spent by the organization.
* `max_budget` - Maximum budget from the attached budget.
* `tpm_limit` - Tokens per minute limit from the attached budget.
* `rpm_limit` - Requests per minute limit from the attached budget.
* `budget_duration` - Budget reset duration from the attached budget.
* `created_at` - Creation timestamp.
* `updated_at` - Last update timestamp.

43
terraform/provider/docs/data-sources/project.md Normal file
View file

@ -0,0 +1,43 @@
# litellm_project (Data Source)
Retrieves information about an existing LiteLLM project, including its budget settings
## Example Usage
```hcl
data "litellm_project" "ml_experiments" {
project_id = "4a422a4c-e246-4d02-a1eb-13e835cd0725"
}
output "project_spend" {
value = data.litellm_project.ml_experiments.spend
}
```
## Argument Reference
The following arguments are supported:
* `project_id` - (Required) Unique identifier of the project to retrieve
## Attribute Reference
In addition to all arguments above, the following attributes are exported:
* `project_alias` - Human-friendly name for the project
* `description` - Description of the project
* `team_id` - The team ID this project belongs to
* `budget_id` - Budget ID associated with this project
* `models` - List of models the project can access
* `max_budget` - Maximum budget for this project
* `soft_budget` - Soft budget limit for warnings
* `budget_duration` - Budget reset duration
* `tpm_limit` - Tokens per minute limit
* `rpm_limit` - Requests per minute limit
* `max_parallel_requests` - Maximum parallel requests allowed
* `blocked` - Whether the project is blocked from making requests
* `spend` - Current spend for the project
* `created_at` - Timestamp when the project was created
* `updated_at` - Timestamp when the project was last updated
* `created_by` - User that created the project
* `updated_by` - User that last updated the project

40
terraform/provider/docs/data-sources/projects.md Normal file
View file

@ -0,0 +1,40 @@
# litellm_projects (Data Source)
Retrieves the list of all LiteLLM projects visible to the caller
## Example Usage
```hcl
data "litellm_projects" "all" {}
output "project_ids" {
value = data.litellm_projects.all.ids
}
output "project_aliases" {
value = [for p in data.litellm_projects.all.projects : p.project_alias]
}
```
## Argument Reference
This data source takes no arguments
## Attribute Reference
The following attributes are exported:
* `ids` - IDs of all projects
* `projects` - List of projects. Each entry exports:
* `project_id` - The project ID
* `project_alias` - Human-friendly name for the project
* `description` - Description of the project
* `team_id` - The team ID this project belongs to
* `budget_id` - Budget ID associated with this project
* `models` - List of models the project can access
* `blocked` - Whether the project is blocked from making requests
* `spend` - Current spend for the project
* `created_at` - Timestamp when the project was created
* `updated_at` - Timestamp when the project was last updated
* `created_by` - User that created the project
* `updated_by` - User that last updated the project

43
terraform/provider/docs/data-sources/prompt.md Normal file
View file

@ -0,0 +1,43 @@
# litellm_prompt Data Source
Retrieves information about an existing LiteLLM prompt by ID. The provider API key is not exposed.
## Example Usage
```hcl
data "litellm_prompt" "existing" {
prompt_id = "my-langfuse-prompt"
}
output "prompt_integration" {
value = data.litellm_prompt.existing.prompt_integration
}
```
### With Environment
```hcl
data "litellm_prompt" "prod" {
prompt_id = "my-langfuse-prompt"
environment = "production"
}
```
## Argument Reference
* `prompt_id` - (Required) Unique identifier of the prompt to retrieve.
* `environment` - (Optional) Environment to fetch the prompt from (e.g. `development`, `production`).
## Attribute Reference
* `prompt_integration` - The prompt integration provider.
* `api_base` - Base URL for the prompt provider API.
* `provider_specific_query_params` - JSON string of provider-specific query parameters.
* `ignore_prompt_manager_model` - Whether the model specified in the prompt manager is ignored.
* `ignore_prompt_manager_optional_params` - Whether optional params from the prompt manager are ignored.
* `dotprompt_content` - Content for the dotprompt integration.
* `prompt_type` - Type of prompt: `config` or `db`.
* `version` - Version number of the prompt.
* `environments` - List of environments this prompt exists in.
* `created_at` - Timestamp when the prompt was created.
* `updated_at` - Timestamp when the prompt was last updated.

37
terraform/provider/docs/data-sources/prompts.md Normal file
View file

@ -0,0 +1,37 @@
# litellm_prompts Data Source
Retrieves the list of all prompts configured on the LiteLLM proxy.
## Example Usage
```hcl
data "litellm_prompts" "all" {}
output "prompt_ids" {
value = data.litellm_prompts.all.ids
}
```
### Filter by Environment
```hcl
data "litellm_prompts" "production" {
environment = "production"
}
```
## Argument Reference
* `environment` - (Optional) Filter prompts by environment (e.g. `development`, `production`).
## Attribute Reference
* `prompts` - List of prompts. Each entry contains:
* `prompt_id` - Unique identifier of the prompt.
* `prompt_integration` - The prompt integration provider.
* `prompt_type` - Type of prompt: `config` or `db`.
* `version` - Version number of the prompt.
* `environment` - Environment the prompt belongs to.
* `created_at` - Timestamp when the prompt was created.
* `updated_at` - Timestamp when the prompt was last updated.
* `ids` - List of all prompt IDs.

34
terraform/provider/docs/data-sources/search_tool.md Normal file
View file

@ -0,0 +1,34 @@
# litellm_search_tool Data Source
Retrieves information about an existing search tool on the LiteLLM proxy.
## Example Usage
```hcl
data "litellm_search_tool" "existing" {
search_tool_id = "123e4567-e89b-12d3-a456-426614174000"
}
output "search_tool_name" {
value = data.litellm_search_tool.existing.search_tool_name
}
```
## Argument Reference
The following arguments are supported:
* `search_tool_id` - (Required) Unique identifier of the search tool to retrieve.
## Attribute Reference
In addition to all arguments above, the following attributes are exported:
* `search_tool_name` - Name of the search tool.
* `search_tool_info` - Additional metadata as a JSON object string (decode with `jsondecode`).
* `created_at` - Timestamp when the search tool was created.
* `updated_at` - Timestamp when the search tool was last updated.
## Security Note
`litellm_params` is not exposed through this data source because it may hold provider API keys.

34
terraform/provider/docs/data-sources/search_tools.md Normal file
View file

@ -0,0 +1,34 @@
# litellm_search_tools Data Source
Retrieves the list of search tools configured on the LiteLLM proxy, from both the database and the proxy config.
## Example Usage
```hcl
data "litellm_search_tools" "all" {}
output "search_tool_ids" {
value = data.litellm_search_tools.all.ids
}
```
## Argument Reference
This data source takes no arguments.
## Attribute Reference
The following attributes are exported:
* `ids` - List of search tool IDs.
* `search_tools` - List of search tools. Each entry exports:
* `search_tool_id` - The unique search tool ID.
* `search_tool_name` - Name of the search tool.
* `search_tool_info` - Additional metadata as a JSON object string.
* `is_from_config` - Whether the search tool comes from the proxy config file rather than the database.
* `created_at` - Timestamp when the search tool was created.
* `updated_at` - Timestamp when the search tool was last updated.
## Security Note
`litellm_params` is not exposed through this data source because it may hold provider API keys.

38
terraform/provider/docs/data-sources/tag.md Normal file
View file

@ -0,0 +1,38 @@
# litellm_tag (Data Source)
Retrieves information about an existing LiteLLM tag, including its budget settings
## Example Usage
```hcl
data "litellm_tag" "production" {
name = "production"
}
output "production_tag_budget" {
value = data.litellm_tag.production.max_budget
}
```
## Argument Reference
The following arguments are supported:
* `name` - (Required) Name of the tag to retrieve
## Attribute Reference
In addition to all arguments above, the following attributes are exported:
* `description` - Description of the tag
* `models` - Model IDs this tag applies to
* `budget_id` - Budget ID associated with this tag
* `max_budget` - Max budget in USD for this tag
* `soft_budget` - Soft budget in USD for this tag
* `max_parallel_requests` - Max concurrent requests allowed for this tag
* `tpm_limit` - Max tokens per minute for this tag
* `rpm_limit` - Max requests per minute for this tag
* `budget_duration` - Duration for budget reset
* `created_at` - Timestamp when the tag was created
* `updated_at` - Timestamp when the tag was last updated
* `created_by` - User that created the tag

50
terraform/provider/docs/data-sources/tags.md Normal file
View file

@ -0,0 +1,50 @@
# litellm_tags (Data Source)
Retrieves the list of all LiteLLM tags. This includes stored tags created via `litellm_tag` or the API, and dynamic tags that were passed on requests
## Example Usage
```hcl
data "litellm_tags" "all" {}
output "tag_names" {
value = data.litellm_tags.all.ids
}
```
## Example Usage with Date Filter
```hcl
# Limit dynamic tags to those active in a window; stored tags are always returned
data "litellm_tags" "january" {
start_date = "2026-01-01"
end_date = "2026-01-31"
}
```
## Argument Reference
The following arguments are supported:
* `start_date` - (Optional) Start date (YYYY-MM-DD) limiting dynamic tags to those active in the window. Must be given with `end_date`
* `end_date` - (Optional) End date (YYYY-MM-DD). Must be given with `start_date`
## Attribute Reference
The following attributes are exported:
* `ids` - Names of all tags (tag names are their IDs)
* `tags` - List of tags. Each entry exports:
* `name` - The tag name
* `description` - Description of the tag
* `models` - Model IDs this tag applies to
* `budget_id` - Budget ID associated with this tag
* `max_budget` - Max budget in USD
* `soft_budget` - Soft budget in USD
* `max_parallel_requests` - Max concurrent requests allowed
* `tpm_limit` - Max tokens per minute
* `rpm_limit` - Max requests per minute
* `budget_duration` - Duration for budget reset
* `created_at` - Timestamp when the tag was created
* `updated_at` - Timestamp when the tag was last updated
* `created_by` - User that created the tag

52
terraform/provider/docs/data-sources/team.md Normal file
View file

@ -0,0 +1,52 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_team Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Retrieves information about an existing LiteLLM team.
---
# litellm_team (Data Source)
Retrieves information about an existing LiteLLM team via `/team/info`. Use it to reference teams created outside of Terraform or in other configurations.
## Example Usage
```terraform
data "litellm_team" "ml" {
team_id = "team-1234"
}
resource "litellm_key" "ml_key" {
team_id = data.litellm_team.ml.team_id
models = data.litellm_team.ml.models
}
```
## Argument Reference
The following arguments are supported:
* `team_id` - (Required) Unique identifier of the team to retrieve.
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `team_alias` - User-friendly name of the team.
* `organization_id` - Organization the team belongs to.
* `models` - Models the team can access.
* `metadata` - Map of string metadata values for the team.
* `tags` - Tags for spend tracking and tag-based routing.
* `soft_budget_alerting_emails` - Email addresses alerted when the team crosses `soft_budget`.
* `tpm_limit` - Tokens per minute limit.
* `rpm_limit` - Requests per minute limit.
* `max_parallel_requests` - Maximum parallel requests allowed.
* `max_budget` - Maximum budget for the team.
* `soft_budget` - Soft budget alert threshold.
* `spend` - Amount spent by the team.
* `budget_duration` - Budget reset duration.
* `blocked` - Whether the team is blocked.
* `team_member_permissions` - Permissions granted to team members.
* `created_at` - Timestamp when the team was created.
* `updated_at` - Timestamp when the team was last updated.

49
terraform/provider/docs/data-sources/teams.md Normal file
View file

@ -0,0 +1,49 @@
---
# generated by https://github.com/hashicorp/terraform-plugin-docs
page_title: "litellm_teams Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Lists LiteLLM teams with optional server-side filters.
---
# litellm_teams (Data Source)
Lists LiteLLM teams via `/team/list`. Supports filtering by user and organization.
## Example Usage
```terraform
data "litellm_teams" "org_teams" {
organization_id = litellm_organization.main.id
}
output "team_ids" {
value = data.litellm_teams.org_teams.ids
}
```
## Argument Reference
The following arguments are supported:
* `user_id` - (Optional) Only return teams this user belongs to.
* `organization_id` - (Optional) Only return teams in this organization.
## Attributes Reference
In addition to all arguments above, the following attributes are exported:
* `ids` - IDs of the returned teams.
* `teams` - List of team objects. Each entry exports:
* `team_id` - Unique identifier of the team.
* `team_alias` - User-friendly name of the team.
* `organization_id` - Organization the team belongs to.
* `models` - Models the team can access.
* `spend` - Amount spent by the team.
* `max_budget` - Maximum budget for the team.
* `tpm_limit` - Tokens per minute limit.
* `rpm_limit` - Requests per minute limit.
* `budget_duration` - Budget reset duration.
* `blocked` - Whether the team is blocked.
* `created_at` - Creation timestamp.
* `updated_at` - Last update timestamp.

52
terraform/provider/docs/data-sources/unified_access_group.md Normal file
View file

@ -0,0 +1,52 @@
---
page_title: "litellm_unified_access_group Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Retrieves information about an existing LiteLLM unified access group.
---
# litellm_unified_access_group (Data Source)
Retrieves information about an existing LiteLLM unified access group by ID.
## Example Usage
```terraform
data "litellm_unified_access_group" "engineering" {
access_group_id = "b6e5f9d0-..."
}
output "engineering_models" {
value = data.litellm_unified_access_group.engineering.access_model_names
}
```
## Argument Reference
* `access_group_id` - (Required) ID of the unified access group to look up.
## Attribute Reference
* `id` - The unified access group ID.
* `access_group_name` - Display name of the unified access group.
* `description` - Description of the unified access group.
* `access_model_names` - Model names the access group grants access to.
* `access_mcp_server_ids` - MCP server IDs the access group grants access to.
* `access_agent_ids` - Agent IDs the access group grants access to.
* `assigned_team_ids` - Team IDs the access group is assigned to.
* `assigned_key_ids` - Key IDs the access group is assigned to.
* `created_at` - Timestamp when the access group was created.
* `created_by` - User who created the access group.
* `updated_at` - Timestamp when the access group was last updated.
* `updated_by` - User who last updated the access group.

30
terraform/provider/docs/data-sources/unified_access_groups.md Normal file
View file

@ -0,0 +1,30 @@
---
page_title: "litellm_unified_access_groups Data Source - terraform-provider-litellm"
subcategory: ""
description: |-
Retrieves all LiteLLM unified access groups.
---
# litellm_unified_access_groups (Data Source)
Retrieves all LiteLLM unified access groups configured on the proxy.
## Example Usage
```terraform
data "litellm_unified_access_groups" "all" {}
output "unified_access_group_ids" {
value = data.litellm_unified_access_groups.all.ids
}
```
## Argument Reference
This data source takes no arguments.
## Attribute Reference
* `access_groups` - List of unified access groups. Each entry exports the same attributes as the `litellm_unified_access_group` data source: `access_group_id`, `access_group_name`, `description`, `access_model_names`, `access_mcp_server_ids`, `access_agent_ids`, `assigned_team_ids`, `assigned_key_ids`, `created_at`, `created_by`, `updated_at`, and `updated_by`.
* `ids` - List of all unified access group IDs.

36
terraform/provider/docs/data-sources/user.md Normal file
View file

@ -0,0 +1,36 @@
# litellm_user Data Source
Retrieves information about an existing LiteLLM user by ID
## Example Usage
```hcl
data "litellm_user" "alice" {
user_id = "alice-user-id"
}
output "alice_email" {
value = data.litellm_user.alice.user_email
}
```
## Argument Reference
- `user_id` (Required) - ID of the user to retrieve
## Attribute Reference
- `id` - The user ID
- `user_email` - Email address of the user
- `user_alias` - Descriptive name for the user
- `user_role` - Role of the user on the proxy
- `teams` - List of team IDs the user belongs to
- `models` - Models the user is allowed to call
- `max_budget` - Maximum budget in USD for the user
- `spend` - Current spend in USD for the user
- `budget_duration` - Budget reset period for the user
- `tpm_limit` - Tokens per minute limit
- `rpm_limit` - Requests per minute limit
- `max_parallel_requests` - Maximum number of parallel requests
- `metadata` - Map of metadata for the user
- `model_max_budget` - JSON string of per-model budget config

47
terraform/provider/docs/data-sources/users.md Normal file
View file

@ -0,0 +1,47 @@
# litellm_users Data Source
Retrieves a page of LiteLLM users, with optional server-side filters
## Example Usage
```hcl
data "litellm_users" "internal" {
role = "internal_user"
page = 1
page_size = 100
}
output "internal_user_ids" {
value = data.litellm_users.internal.ids
}
```
## Argument Reference
- `role` (Optional) - Filter users by role
- `user_ids` (Optional) - Comma-separated list of user IDs to filter by
- `user_email` (Optional) - Filter users by partial email match
- `team` (Optional) - Filter users by team ID
- `page` (Optional, Default `1`) - Page number to fetch
- `page_size` (Optional, Default `25`) - Number of users per page, max 100
- `sort_by` (Optional) - Column to sort by, e.g. `user_id`, `user_email`, `created_at`
- `sort_order` (Optional) - Sort order, `asc` or `desc`
## Attribute Reference
- `users` - Users returned for the requested page. Each entry has:
- `user_id` - The user ID
- `user_email` - Email address of the user
- `user_alias` - Descriptive name for the user
- `user_role` - Role of the user on the proxy
- `teams` - List of team IDs the user belongs to
- `models` - Models the user is allowed to call
- `max_budget` - Maximum budget in USD
- `spend` - Current spend in USD
- `tpm_limit` - Tokens per minute limit
- `rpm_limit` - Requests per minute limit
- `key_count` - Number of API keys owned by the user
- `created_at` - Timestamp when the user was created
- `ids` - IDs of the users returned for the requested page
- `total` - Total number of users matching the filters
- `total_pages` - Total number of pages available

1
terraform/provider/docs/index.md
View file

@ -51,6 +51,7 @@ The LiteLLM provider supports the following resources:
* [`litellm_mcp_server`](./resources/mcp_server) - Manage MCP (Model Context Protocol) servers
* [`litellm_credential`](./resources/credential) - Manage credentials for various providers
* [`litellm_vector_store`](./resources/vector_store) - Manage vector stores
* [`litellm_jwt_key_mapping`](./resources/jwt_key_mapping) - Map JWT claim values to virtual keys
## Available Data Sources

49
terraform/provider/docs/resources/access_group.md Normal file
View file

@ -0,0 +1,49 @@
---
page_title: "litellm_access_group Resource - terraform-provider-litellm"
subcategory: ""
description: |-
Manages a LiteLLM model access group.
---
# litellm_access_group (Resource)
Manages a LiteLLM model access group. Access groups bundle model deployments under one name so keys and teams can be granted access to the whole group at once.
## Example Usage
```terraform
resource "litellm_access_group" "production" {
access_group = "production-models"
model_names = ["gpt-4", "claude-3-sonnet"]
}
# Target specific deployments by model ID instead of model name
resource "litellm_access_group" "pinned" {
access_group = "pinned-deployments"
model_ids = ["4dbd9f43-...", "9a1e2c77-..."]
}
```
## Argument Reference
* `access_group` - (Required, Forces new resource) Name of the access group.
* `model_names` - (Optional) List of model names (the `model_name` of each deployment) to include in the group. At least one of `model_names` or `model_ids` must be set.
* `model_ids` - (Optional) List of specific deployment model IDs to include in the group. Takes precedence over `model_names` when both are set.
## Attribute Reference
In addition to the arguments above, the following attributes are exported:
* `id` - The access group name.
* `deployment_count` - Number of deployments currently tagged with this access group.
## Import
Access groups can be imported using the access group name:
```shell
terraform import litellm_access_group.production production-models
```

88
terraform/provider/docs/resources/agent.md Normal file
View file

@ -0,0 +1,88 @@
# litellm_agent Resource
Manages an A2A (Agent-to-Agent) agent on the LiteLLM proxy. Agents are AI-powered entities that can be discovered, invoked, and composed using the A2A protocol.
## Example Usage
```hcl
resource "litellm_agent" "hello_world" {
agent_name = "hello-world-agent"
agent_card_params = jsonencode({
protocolVersion = "1.0"
name = "Hello World Agent"
description = "Just a hello world agent"
url = "http://localhost:9999/"
version = "1.0.0"
defaultInputModes = ["text"]
defaultOutputModes = ["text"]
capabilities = {
streaming = true
}
skills = [
{
id = "hello_world"
name = "Returns hello world"
description = "just returns hello world"
tags = ["hello world"]
examples = ["hi", "hello world"]
}
]
})
litellm_params = jsonencode({
make_public = false
})
object_permission = jsonencode({
models = ["gpt-4-proxy"]
mcp_servers = ["my-mcp-server-id"]
})
static_headers = {
"x-api-key" = var.agent_api_key
}
extra_headers = ["x-request-id"]
tpm_limit = 100000
rpm_limit = 1000
session_tpm_limit = 10000
session_rpm_limit = 100
}
```
## Argument Reference
The following arguments are supported:
* `agent_name` - (Required) Name of the agent. Must be unique on the proxy.
* `agent_card_params` - (Required) The A2A agent card as a JSON object string (use `jsonencode`). Supports the standard A2A card fields: `name`, `description`, `url`, `version`, `protocolVersion`, `capabilities`, `skills`, `defaultInputModes`, `defaultOutputModes`, `preferredTransport`, `iconUrl`, `provider`, `documentationUrl`, and more. The proxy merges LiteLLM-fronting fields (such as `supportedInterfaces`) into the stored card, so the value you configure stays authoritative in state.
* `litellm_params` - (Optional, Sensitive) LiteLLM-specific parameters as a JSON object string. May include secrets such as `api_key`, so the value is never read back from the API; the configured value is authoritative.
* `object_permission` - (Optional) Access control permissions as a JSON object string with keys `mcp_servers`, `mcp_access_groups`, `mcp_tool_permissions`, `models`, and `agents`.
* `static_headers` - (Optional, Sensitive) Map of static headers sent with agent requests. May hold tokens, so it is never read back from the API.
* `extra_headers` - (Optional) List of incoming request header names to forward to the agent.
* `tpm_limit` - (Optional) Tokens per minute limit for the agent.
* `rpm_limit` - (Optional) Requests per minute limit for the agent.
* `session_tpm_limit` - (Optional) Per-session tokens per minute limit.
* `session_rpm_limit` - (Optional) Per-session requests per minute limit.
## Attribute Reference
In addition to all arguments above, the following attributes are exported:
* `id` - The agent ID assigned by LiteLLM.
* `created_at` - Timestamp when the agent was created.
* `updated_at` - Timestamp when the agent was last updated.
* `created_by` - User who created the agent.
* `updated_by` - User who last updated the agent.
## Import
Agents can be imported using the agent ID:
```shell
terraform import litellm_agent.example <agent_id>
```
Note: `litellm_params` and `static_headers` cannot be recovered on import because the API never returns their unmasked values; re-apply after import to set them.

48
terraform/provider/docs/resources/budget.md Normal file
View file

@ -0,0 +1,48 @@
# litellm_budget Resource
Manages a budget object on the LiteLLM proxy. Budgets can be attached to keys, teams, organizations, and end users to enforce spend limits
## Example Usage
```hcl
resource "litellm_budget" "engineering" {
budget_id = "engineering-monthly"
max_budget = 500.0
soft_budget = 400.0
budget_duration = "30d"
tpm_limit = 500000
rpm_limit = 5000
max_parallel_requests = 100
model_max_budget = jsonencode({
"gpt-4o" = {
max_budget = 100.0
budget_duration = "1d"
}
})
}
```
## Argument Reference
- `budget_id` (Optional, Forces new resource) - Unique ID for the budget. Generated by the server if not provided
- `max_budget` (Optional) - Requests fail if this budget in USD is exceeded
- `soft_budget` (Optional) - Requests do not fail if this is exceeded, but alerts fire
- `max_parallel_requests` (Optional) - Maximum concurrent requests allowed for this budget
- `tpm_limit` (Optional) - Maximum tokens per minute allowed for this budget
- `rpm_limit` (Optional) - Maximum requests per minute allowed for this budget
- `budget_duration` (Optional) - Budget reset period, e.g. `1hr`, `1d`, `28d`
- `model_max_budget` (Optional) - JSON string of per-model budget config, e.g. `jsonencode({"gpt-4o" = {max_budget = 10.0}})`
## Attribute Reference
- `id` - The budget ID
- `budget_reset_at` - Datetime when the budget is reset
## Import
Budgets can be imported using the budget ID:
```shell
terraform import litellm_budget.engineering <budget-id>
```

48
terraform/provider/docs/resources/fallback.md Normal file
View file

@ -0,0 +1,48 @@
# litellm_fallback Resource
Manages a fallback configuration for a model in LiteLLM. Fallbacks are triggered when a call to the primary model fails after retries.
## Example Usage
### Basic Fallback Configuration
```hcl
resource "litellm_fallback" "gpt4_fallbacks" {
model = "gpt-4"
fallback_models = ["claude-3-sonnet", "gpt-3.5-turbo"]
}
```
### Context Window Fallback
```hcl
resource "litellm_fallback" "gpt4_context_window" {
model = "gpt-4"
fallback_models = ["claude-3-sonnet"]
fallback_type = "context_window"
}
```
## Argument Reference
The following arguments are supported:
* `model` - (Required, Forces new resource) The model name to configure fallbacks for. The model must already exist on the proxy.
* `fallback_models` - (Required) List of fallback model names in order of priority. Each model must exist on the proxy, and the primary model cannot be its own fallback.
* `fallback_type` - (Optional, Forces new resource) Type of fallback. One of `general` (default), `context_window`, or `content_policy`.
## Attribute Reference
In addition to the arguments above, the following attribute is exported:
* `id` - The primary model name.
## Import
Fallback configurations can be imported using the primary model name:
```shell
terraform import litellm_fallback.example gpt-4
```
Note: import always reads the `general` fallback type. Fallbacks of type `context_window` or `content_policy` cannot be imported.

57
terraform/provider/docs/resources/guardrail.md Normal file
View file

@ -0,0 +1,57 @@
# litellm_guardrail Resource
Manages a guardrail in LiteLLM. Guardrails provide content filtering, PII detection, prompt injection protection, and more.
## Example Usage
```hcl
resource "litellm_guardrail" "bedrock_guard" {
guardrail_name = "my-bedrock-guard"
guardrail = "bedrock"
mode = "pre_call"
default_on = true
litellm_params = jsonencode({
guardrailIdentifier = "ff6ujrregl1q"
guardrailVersion = "DRAFT"
})
guardrail_info = {
description = "Bedrock content moderation guardrail"
}
}
```
### Multiple Modes
```hcl
resource "litellm_guardrail" "pii_guard" {
guardrail_name = "presidio-pii"
guardrail = "presidio"
mode = jsonencode(["pre_call", "post_call"])
}
```
## Argument Reference
* `guardrail_name` - (Required) Human-readable name for the guardrail.
* `guardrail` - (Required) The guardrail integration type (e.g. `bedrock`, `lakera`, `presidio`, `openai_moderation`, `hide_secrets`).
* `mode` - (Required) When to apply the guardrail. A single value (`pre_call`, `post_call`, `during_call`, `logging_only`) or a JSON array of values.
* `default_on` - (Optional) Whether the guardrail is enabled by default for all requests.
* `litellm_params` - (Optional, Sensitive) JSON string with additional provider-specific parameters merged into `litellm_params` (may contain API keys). The API masks these values, so the configured value stays authoritative in state.
* `guardrail_info` - (Optional) Map of additional metadata for the guardrail.
## Attribute Reference
* `id` - The guardrail ID assigned by LiteLLM.
* `created_at` - Timestamp when the guardrail was created.
## Import
Guardrails can be imported using the guardrail ID:
```shell
terraform import litellm_guardrail.example 123e4567-e89b-12d3-a456-426614174000
```
Note: `guardrail`, `mode`, `default_on` and `litellm_params` are not returned unmasked by the API, so after import you must set them in configuration to match the server.

Some files were not shown because too many files have changed in this diff Show more