scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_govcloud_profiles_lit6421

Browse source
This commit is contained in:
mateo-berri 2026-09-04 20:08:16 -07:00
commit 036d104533
229 changed files with 25724 additions and 2074 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
.circleci/config.yml
View file

@ -575,7 +575,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
-v \
--junitxml=test-results/junit.xml \
--durations=5 \
-k \"langfuse\""
@ -630,7 +630,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
-v \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -737,7 +737,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
-v \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -782,7 +782,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
-v \
--junitxml=test-results/junit.xml \
--durations=5 \
-k \"assistants\""
@ -909,7 +909,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
-vv -s \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
@ -999,7 +999,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
-vv -s \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1054,7 +1054,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
-v \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 8 \
@ -1090,7 +1090,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
-vv \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1134,7 +1134,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
-vv \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1178,7 +1178,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
-vv -s \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1222,7 +1222,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
-vv -s \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1267,7 +1267,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
-vv \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1312,7 +1312,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
-v \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1391,7 +1391,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
-vv -s \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
@ -1444,7 +1444,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
-vv -s \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
@ -1705,7 +1705,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
-v \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
@ -1794,7 +1794,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-s -v -x \
-s -v \
--junitxml=test-results/junit.xml \
-n 4 \
--durations=5"
@ -2012,7 +2012,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
-v \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
@ -2092,7 +2092,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
-vv \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -2195,7 +2195,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
-vv \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -2266,7 +2266,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
-vv \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -2350,7 +2350,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
-vv \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
@ -2446,7 +2446,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
-v \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -2516,7 +2516,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
-vv -s \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m

2
.github/workflows/_test-unit-base.yml vendored
View file

@ -116,7 +116,7 @@ jobs:
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra mongodb
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
- name: Cache Prisma binaries

10
basedpyright-code-budget.json
View file

@ -57,7 +57,7 @@
"limit": 5601
},
"reportMissingTypeArgument": {
"limit": 15285
"limit": 15284
},
"reportMissingTypeStubs": {
"limit": 40
@ -93,13 +93,13 @@
"limit": 181
},
"reportTypedDictNotRequiredAccess": {
"limit": 24
"limit": 22
},
"reportUndefinedVariable": {
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44360
"limit": 44358
},
"reportUnknownLambdaType": {
"limit": 109
@ -108,10 +108,10 @@
"limit": 38309
},
"reportUnknownParameterType": {
"limit": 19622
"limit": 19621
},
"reportUnknownVariableType": {
"limit": 29846
"limit": 29844
},
"reportUnnecessaryCast": {
"limit": 111

1
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -161,6 +161,7 @@ class CheckBatchCost:
metadata: dict[str, object] = {
"user_api_key_user_id": job.created_by,
"user_api_key": api_key,
"user_api_key_hash": api_key,
"user_api_key_team_id": team_id,
**(await self._get_user_info(batch_id, job.created_by)),
}

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000001_add_guardrail_usage_units_cost/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyGuardrailUsageUnits" ADD COLUMN IF NOT EXISTS "cost" DOUBLE PRECISION;
ALTER TABLE "LiteLLM_DailyGuardrailUsageUnits" ADD COLUMN IF NOT EXISTS "untracked_units" BIGINT NOT NULL DEFAULT 0;

2
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1124,6 +1124,8 @@ model LiteLLM_DailyGuardrailUsageUnits {
api_key String // hashed virtual key; empty string when unknown
usage_unit String // provider counter name, e.g. Bedrock's contentPolicyUnits
units BigInt @default(0)
cost Float? // USD for the priced share of units; null only on rows written before this column existed
untracked_units BigInt @default(0) // units recorded with no known price, the share cost leaves out
created_at DateTime @default(now())
updated_at DateTime @updatedAt

16
litellm/integrations/cloudzero/database.py
View file

@ -94,8 +94,18 @@ class LiteLLMDatabase:
try:
db_response: Final = await client.db.query_raw(query, *params)
# Convert the response to polars DataFrame with full schema inference
# This prevents schema mismatch errors when data types vary across rows
return pl.DataFrame(db_response, infer_schema_length=None)
from litellm.proxy.spend_tracking.key_metadata_recovery import (
fill_missing_api_key_aliases,
)
usage_rows: Final = (
db_response.to_dicts()
if isinstance(db_response, pl.DataFrame)
else db_response
if isinstance(db_response, list)
else []
)
recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows)
return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None)
except Exception as e:
raise Exception(f"Error retrieving usage data: {e}")

64
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -19,7 +19,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_BY_LITELM_STRING
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.integrations.datadog.datadog_handler import (
get_datadog_base_url_from_env,
@ -46,9 +46,10 @@ from litellm.llms.custom_httpx.http_handler import (
from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens
from litellm.types.integrations.datadog_llm_obs import *
from litellm.types.utils import (
AUDIT_GUARDRAIL_FIELDS,
PROMPT_CARRYING_GUARDRAIL_FIELDS,
PROMPT_QUOTING_ROUTING_DECISION_FIELDS,
CallTypes,
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
@ -60,6 +61,8 @@ _SAFE_REDACTED_MESSAGE_ROLES: Final = frozenset(
{"agent", "assistant", "developer", "function", "model", "system", "tool", "user"}
)
_CLASSIFIED_GUARDRAIL_FIELDS: Final = AUDIT_GUARDRAIL_FIELDS | PROMPT_CARRYING_GUARDRAIL_FIELDS
_PROMPT_CARRYING_METADATA_FIELDS: Final = frozenset(
{
"routing_decision",
@ -108,6 +111,49 @@ def _router_span_fields(
)
def _guardrail_entries(guardrail_information: object) -> tuple[Mapping[str, object], ...]:
"""The guardrail records as a sequence, whatever shape the payload carries.
`guardrail_information` is typed as a list, but a guardrail that writes the metadata key itself
can leave a single record there; Prometheus normalizes the same shape at
`_guardrail_overhead_seconds`.
"""
if isinstance(guardrail_information, Mapping):
return (guardrail_information,)
if isinstance(guardrail_information, (list, tuple)):
return tuple(entry for entry in guardrail_information if isinstance(entry, Mapping))
return ()
def _guardrail_entry_without_prompt_carriers(entry: Mapping[str, object]) -> Mapping[str, object]:
"""One guardrail record kept as its audit fields, with the prompt-quoting ones marked redacted.
Built as an allow-list rather than a deny-list: a key neither set classifies is dropped, so a
guardrail that records its own extra detail cannot put the caller's prompt on a redacted span.
"""
return { # mutable-ok: a fresh record built per entry, handed straight to the span serializer
field: REDACTED_BY_LITELM_STRING if field in PROMPT_CARRYING_GUARDRAIL_FIELDS else value
for field, value in entry.items()
if field in _CLASSIFIED_GUARDRAIL_FIELDS
}
def _guardrail_information_without_prompt_carriers(
guardrail_information: object,
) -> tuple[Mapping[str, object], ...] | None:
"""The guardrail records reduced to what a redacted span may carry.
Redaction removes the prompt, not the record that a guardrail ran: the name, mode, status,
timings and masked-entity counts are what an operator reads to answer whether a guardrail
caught anything on a request, and none of them reproduce the prompt. Field-level rather than
dropping the list, which is what `_sanitize_guardrail_information_for_spend_logs` already does
for spend logs.
"""
if guardrail_information is None:
return None
return tuple(_guardrail_entry_without_prompt_carriers(entry) for entry in _guardrail_entries(guardrail_information))
def _metadata_without_prompt_carriers(standard_logging_metadata: Mapping[str, Any]) -> Mapping[str, Any]:
"""The metadata minus the records that quote prompts, tool arguments, tool results, or retrieved text."""
return MappingProxyType(
@ -872,7 +918,9 @@ class DataDogLLMObsLogger(CustomBatchLogger):
"cache_key": standard_logging_payload.get("cache_key", "unknown"),
"saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0),
"guardrail_information": (
None if redact_prompt_text else standard_logging_payload.get("guardrail_information", None)
_guardrail_information_without_prompt_carriers(standard_logging_payload.get("guardrail_information"))
if redact_prompt_text
else standard_logging_payload.get("guardrail_information", None)
),
"is_streamed_request": self._get_stream_value_from_payload(standard_logging_payload),
"latency_metrics": dict(self._get_latency_metrics(standard_logging_payload)),
@ -904,14 +952,12 @@ class DataDogLLMObsLogger(CustomBatchLogger):
latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms
# Guardrail overhead latency
guardrail_info: Final[list[StandardLoggingGuardrailInformation] | None] = standard_logging_payload.get(
"guardrail_information"
)
if guardrail_info is not None:
guardrail_info: Final = _guardrail_entries(standard_logging_payload.get("guardrail_information"))
if guardrail_info:
total_duration = 0.0
for info in guardrail_info:
_guardrail_duration_seconds: float | None = info.get("duration")
if _guardrail_duration_seconds is not None:
_guardrail_duration_seconds = info.get("duration")
if isinstance(_guardrail_duration_seconds, (int, float, str)):
total_duration += float(_guardrail_duration_seconds)
if total_duration > 0:

14
litellm/integrations/focus/database.py
View file

@ -96,7 +96,19 @@ class FocusLiteLLMDatabase:
try:
db_response: Final = await client.db.query_raw(query, *query_params)
return pl.DataFrame(db_response, infer_schema_length=None)
from litellm.proxy.spend_tracking.key_metadata_recovery import (
fill_missing_api_key_aliases,
)
usage_rows: Final = (
db_response.to_dicts()
if isinstance(db_response, pl.DataFrame)
else db_response
if isinstance(db_response, list)
else []
)
recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows)
return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None)
except Exception as exc:
raise RuntimeError(f"Error retrieving usage data: {exc}") from exc

163
litellm/integrations/shadow_eval_logger.py
View file

@ -165,28 +165,91 @@ def _chat_request_from_responses(
)
def _chat_final_text(response_obj: object) -> str:
"""The assistant's text, or empty when the turn carries tool calls: only text-final
turns produce a judgeable A/B comparison."""
def _chat_choice(response_obj: object) -> object | None:
"""The response's first choice, from a payload mapping or a duck-typed ModelResponse."""
try:
message: Final = (
response_obj["choices"][0]["message"]
if isinstance(response_obj, Mapping)
else response_obj.choices[0].message # pyright: ignore[reportAttributeAccessIssue] # duck-typed ModelResponse
)
if isinstance(response_obj, Mapping):
return response_obj["choices"][0]
return response_obj.choices[0] # pyright: ignore[reportAttributeAccessIssue] # duck-typed ModelResponse
except (AttributeError, KeyError, IndexError, TypeError):
return None
def _field_reader(obj: object) -> Callable[[str], object]:
return obj.get if isinstance(obj, Mapping) else lambda key: getattr(obj, key, None)
def _chat_message_reader(response_obj: object) -> Callable[[str], object] | None:
"""Field access over the assistant message of a chat response, or None for a payload
with no readable message."""
choice: Final = _chat_choice(response_obj)
if choice is None:
return None
message: Final = _field_reader(choice)("message")
return _field_reader(message) if message is not None else None
def _chat_final_text(response_obj: object) -> str:
"""The turn's judgeable text: prose, or every tool call serialized alongside it as
`[tool call] name(arguments)` when the assistant chose to act instead of, or as well
as, answering directly. A tool call is a real turn, not a gap, so this is what both
the real arm's sampling decision and the shadow arm's reply compare against."""
read: Final = _chat_message_reader(response_obj)
if read is None:
return ""
read: Final = message.get if isinstance(message, Mapping) else lambda key: getattr(message, key, None)
if read("tool_calls") or read("function_call"):
return ""
return extract_text_from_content(read("content"))
prose: Final = extract_text_from_content(read("content"))
if not (read("tool_calls") or read("function_call")):
return prose
serialized: Final = _serialize_tool_calls(read)
return f"{prose} {serialized}".strip() if prose else serialized
def _chat_finish_reason(response_obj: object) -> str:
choice: Final = _chat_choice(response_obj)
raw: Final = _field_reader(choice)("finish_reason") if choice is not None else None
return str(raw) if raw else "unknown"
_RESPONSES_TOOL_CALL_TYPES: Final = frozenset(("function_call", "custom_tool_call"))
def _tool_calls_list(read: Callable[[str], object]) -> tuple[object, ...]:
calls: Final = read("tool_calls")
listed: Final = tuple(calls) if isinstance(calls, Sequence) and not isinstance(calls, str) else ()
single: Final = read("function_call")
return listed if listed else ((single,) if single is not None else ())
def _tool_call_invocation(call: object) -> str:
"""One tool call as `name(arguments)`. Custom tool calls name themselves and carry their
arguments under `custom` rather than `function`."""
read_call: Final = _field_reader(call)
payload: Final = read_call("function") or read_call("custom") or call
read_payload: Final = _field_reader(payload)
name: Final = read_payload("name")
arguments: Final = read_payload("arguments") or read_payload("input") or ""
return f"{name or 'unnamed'}({arguments})"
def _serialize_tool_calls(read: Callable[[str], object]) -> str:
"""Every tool call in a reply as text a judge built for prose can still read."""
return ", ".join(f"[tool call] {_tool_call_invocation(call)}" for call in _tool_calls_list(read))
def _shadow_empty_reply_error(response_obj: object, routed_model: str) -> str:
"""Why a shadow reply yielded no judgeable text at all: no prose, and no tool call to
serialize either. The stable sentence comes first and every varying part after the
semicolon, so grouping rows by error still yields one row per cause."""
detail: Final = f"finish_reason={_chat_finish_reason(response_obj)}, model={routed_model or 'unknown'}"
return f"shadow router returned an empty response; {detail}"
def _responses_final_text(response_obj: object) -> str:
"""The turn's aggregated output text, or empty when the turn carries tool calls. A
dict-shaped payload is validated into the owner type first, because ``output_text``
is a derived property rather than a serialized field, so it never exists on a dict;
a dict the owner type rejects is unjudgeable and skipped."""
"""The turn's judgeable text: the aggregated output plus any tool call serialized
alongside it, the same way the chat surface renders one. A dict-shaped payload is
validated into the owner type first, because ``output_text`` is a derived property
rather than a serialized field, so it never exists on a dict; a dict the owner type
rejects is unjudgeable and skipped."""
from litellm.types.llms.openai import ResponsesAPIResponse
try:
@ -199,11 +262,16 @@ def _responses_final_text(response_obj: object) -> str:
if not isinstance(output, Sequence):
return ""
items: Final = tuple(item.model_dump() if isinstance(item, BaseModel) else item for item in output)
if any(
not isinstance(item, Mapping) or item.get("type") in ("function_call", "custom_tool_call") for item in items
):
if any(not isinstance(item, Mapping) for item in items):
return ""
return str(getattr(response, "output_text", "") or "")
calls: Final = tuple(
item for item in items if isinstance(item, Mapping) and item.get("type") in _RESPONSES_TOOL_CALL_TYPES
)
prose: Final = str(getattr(response, "output_text", "") or "")
if not calls:
return prose
serialized: Final = ", ".join(f"[tool call] {_tool_call_invocation(call)}" for call in calls)
return f"{prose} {serialized}".strip() if prose else serialized
class _SurfaceOps:
@ -273,8 +341,8 @@ def _judgeable_sample(
response_obj: object,
) -> tuple[tuple[Mapping[str, object], ...], Mapping[str, object], str] | None:
"""The normalized chat conversation, the forwardable generation params, and the
judgeable final text; None when this request's shapes cannot be sampled (tool-final
turn, empty text, or a shape the owner transformations reject)."""
judgeable final text; None when this request's shapes cannot be sampled (no text and no
tool call to serialize, or a shape the owner transformations reject)."""
try:
request: Final = ops.chat_request(kwargs, model_parameters)
items: Final = _MESSAGE_ITEMS_ADAPTER.validate_python(request.get("messages"))
@ -307,6 +375,11 @@ PAIRWISE_JUDGE_SYSTEM_PROMPT: Final = """You are an impartial quality judge comp
The responses are labeled A and B in random order. You do not know which system produced which.
A response may be prose, or a tool call shown as `[tool call] name(arguments)` if the
assistant chose to act instead of answering directly. A tool call is not a defect: judge
whether calling that tool was the right response to the conversation, the same as you
would judge prose.
Criteria: correctness, completeness, clarity, conciseness.
Return ONLY valid JSON in this exact format, no other text:
@ -376,14 +449,37 @@ def _unmask_preference(raw_preference: str, real_is_a: bool) -> str:
return "tie"
def _judge_user_prompt(conversation: str, response_a: str, response_b: str) -> str:
_MAX_JUDGE_TOOL_DEFS_CHARS: Final = 2_000
def _tool_definitions_text(tools: object) -> str:
"""The tools available to both arms, name and description only: enough for the judge
to tell whether the chosen tool, and not some other one, was the right call, without
forwarding parameter schemas it does not need to score that."""
if not isinstance(tools, Sequence) or isinstance(tools, str):
return ""
entries: Final = tuple(
_field_reader(t)("function") or _field_reader(t)("custom") or t for t in tools if not isinstance(t, str)
)
lines: Final = tuple(
f"- {_field_reader(e)('name') or 'unnamed'}: {_field_reader(e)('description') or 'no description'}"
for e in entries
)
if not lines:
return ""
return ("Tools available to both responses:\n" + "\n".join(lines))[:_MAX_JUDGE_TOOL_DEFS_CHARS]
def _judge_user_prompt(conversation: str, response_a: str, response_b: str, tool_definitions: str = "") -> str:
"""The judge prompt under one total character budget: each response is capped, and
the conversation tail gets whatever budget the responses left over."""
the conversation tail gets whatever budget the responses and tool definitions left
over."""
a: Final = response_a[:_MAX_JUDGE_RESPONSE_CHARS]
b: Final = response_b[:_MAX_JUDGE_RESPONSE_CHARS]
conversation_budget: Final = _MAX_JUDGE_PROMPT_CHARS - len(a) - len(b)
prefix: Final = f"{tool_definitions}\n\n" if tool_definitions else ""
conversation_budget: Final = _MAX_JUDGE_PROMPT_CHARS - len(a) - len(b) - len(prefix)
return (
f"Conversation:\n{conversation[-conversation_budget:]}\n\n"
f"{prefix}Conversation:\n{conversation[-conversation_budget:]}\n\n"
f"Response A:\n{a}\n\n"
f"Response B:\n{b}\n\n"
"Which response is better?"
@ -942,6 +1038,7 @@ class ShadowEvalLogger(CustomLogger):
messages=messages,
real_text=real_text,
shadow_text=shadow.text,
tools=shadow_params.get("tools"),
parent_metadata=parent_metadata,
)
if isinstance(verdict, _CallFailure):
@ -1080,15 +1177,18 @@ class ShadowEvalLogger(CustomLogger):
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
text: Final = _chat_final_text(response)
routed_model: Final = str(
getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or ""
)
if not text:
return _CallFailure(
"shadow router returned an empty response",
_shadow_empty_reply_error(response, routed_model),
cost=_call_cost(response),
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
return _ShadowResponse(
text=text,
model=str(getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or ""),
model=routed_model,
tier=_routed_tier(shadow_metadata),
cost=_call_cost(response),
classifier_cost=_decision_classifier_cost(shadow_metadata),
@ -1100,9 +1200,12 @@ class ShadowEvalLogger(CustomLogger):
messages: Sequence[Mapping[str, object]],
real_text: str,
shadow_text: str,
tools: object,
parent_metadata: Mapping[str, object],
) -> "_JudgeVerdict | _CallFailure":
"""Blind pairwise judge with A/B labels randomized to cancel position bias."""
"""Blind pairwise judge with A/B labels randomized to cancel position bias. Both
arms were offered the same tools, so the judge is shown their definitions too: a
tool call is only assessable against what else was available to call instead."""
real_is_a: Final = random.random() < 0.5
response_a: Final = real_text if real_is_a else shadow_text
response_b: Final = shadow_text if real_is_a else real_text
@ -1117,7 +1220,7 @@ class ShadowEvalLogger(CustomLogger):
{"role": "system", "content": PAIRWISE_JUDGE_SYSTEM_PROMPT}, # mutable-ok: SDK message
{
"role": "user",
"content": _judge_user_prompt(conversation, response_a, response_b),
"content": _judge_user_prompt(conversation, response_a, response_b, _tool_definitions_text(tools)),
}, # mutable-ok: SDK message
]
try:

8
litellm/litellm_core_utils/litellm_logging.py
View file

@ -10,7 +10,7 @@ import subprocess
import sys
import time
import traceback
from collections.abc import Callable, Iterator, Mapping, Sequence
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from datetime import datetime as dt_object
from functools import lru_cache
from types import MappingProxyType, TracebackType
@ -576,6 +576,7 @@ class Logging(LiteLLMLoggingBaseClass):
# enqueue closure here instead of firing it immediately.
self._defer_async_logging: bool = False
self._enqueue_deferred_logging: Callable[[], None] | None = None
self._on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
def set_response_timing_metrics(self, timing_metrics: Mapping[str, float]) -> None:
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
@ -1894,6 +1895,11 @@ class Logging(LiteLLMLoggingBaseClass):
**kwargs,
)
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
"""Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
async def dispatch_failure_handlers(
self,
exception: Exception,

56
litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py
View file

@ -1,8 +1,8 @@
import math
from collections.abc import Mapping
from typing import Final
from typing import Annotated, Final
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
@ -30,6 +30,31 @@ class GuardrailCostEntry(BaseModel):
_GUARDRAIL_COST_ENTRY_ADAPTER: Final[TypeAdapter[GuardrailCostEntry]] = TypeAdapter(GuardrailCostEntry)
class GuardrailCostByUnitEntry(BaseModel):
"""The rollup-side view of a ``guardrail_information`` entry, validated apart from
``GuardrailCostEntry`` so a forged per-counter map can never zero the spend path."""
model_config = ConfigDict(extra="ignore", frozen=True)
guardrail_cost_by_unit: Mapping[str, Annotated[float, Field(ge=0, allow_inf_nan=False)] | None] | None = None
guardrail_cost_in_spend: bool | None = True
_GUARDRAIL_COST_BY_UNIT_ADAPTER: Final[TypeAdapter[GuardrailCostByUnitEntry]] = TypeAdapter(GuardrailCostByUnitEntry)
def billed_guardrail_cost_by_unit(raw: object) -> Mapping[str, float | None] | None:
"""Per-counter USD the daily rollup may record for one raw ``guardrail_information``
entry; None when the entry is unpriced, report-only, or malformed, and None per
counter the hook had no price for."""
try:
entry: Final = _GUARDRAIL_COST_BY_UNIT_ADAPTER.validate_python(raw)
except ValidationError as e:
verbose_logger.warning("Ignoring malformed guardrail_information entry for guardrail cost rollup: %s", e)
return None
return None if entry.guardrail_cost_in_spend is False else entry.guardrail_cost_by_unit
def _bedrock_guardrail_pricing(aws_region_name: str | None) -> GuardrailPricing | None:
regional_key: Final = f"bedrock/{aws_region_name}/guardrails" if aws_region_name else None
for key in (regional_key, BEDROCK_GUARDRAIL_PRICING_KEY):
@ -42,11 +67,32 @@ def _bedrock_guardrail_pricing(aws_region_name: str | None) -> GuardrailPricing
return None
def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str | None) -> float:
def _priced_units(units: int, price_per_unit: float | None) -> float | None:
return None if price_per_unit is None else units * price_per_unit
def bedrock_guardrail_cost_by_unit(
usage_units: Mapping[str, int], aws_region_name: str | None
) -> Mapping[str, float | None] | None:
"""USD per counter, keyed like ``usage_units``; None when no pricing entry exists,
and None for a counter the entry has no price for, since only an explicit 0.0 means free."""
pricing: Final = _bedrock_guardrail_pricing(aws_region_name)
if pricing is None:
return 0.0
return sum(units * pricing.guardrail_cost_per_unit.get(counter, 0.0) for counter, units in usage_units.items())
return None
return { # mutable-ok: stamped into guardrail_information, which safe_dumps only serializes as a plain dict
counter: _priced_units(units, pricing.guardrail_cost_per_unit.get(counter))
for counter, units in usage_units.items()
}
def guardrail_cost_total(cost_by_unit: Mapping[str, float | None] | None) -> float:
"""The scalar the spend path bills: unknown-priced counters count as 0 here, the
rollup keeps them unknown."""
return sum(cost for cost in cost_by_unit.values() if cost is not None) if cost_by_unit is not None else 0.0
def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str | None) -> float:
return guardrail_cost_total(bedrock_guardrail_cost_by_unit(usage_units, aws_region_name))
AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT: Final = "text_records"

39
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -229,6 +229,45 @@ def _content_parts_contain_image(parts: Sequence[object]) -> bool:
return False
def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> str | None:
"""Data or remote URL for an Anthropic ``source`` block, in the form chat completions expects."""
source_type: Final = image_source.get("type")
if source_type == "base64":
media_type: Final = image_source.get("media_type") or "image/jpeg"
image_data: Final = image_source.get("data") or ""
return f"data:{media_type};base64,{image_data}" if image_data else None
if source_type == "url":
url: Final = image_source.get("url")
return url if isinstance(url, str) else ""
return None
def _image_part_url(part: Mapping[str, object]) -> str | None:
"""The image URL carried by one content part, whichever of the three dialects wrote it."""
part_type: Final = part.get("type")
if part_type == "image_url":
image_url: Final = part.get("image_url")
if isinstance(image_url, str):
return image_url
return image_url.get("url") if isinstance(image_url, Mapping) else None
if part_type == "input_image":
responses_url: Final = part.get("image_url")
return responses_url if isinstance(responses_url, str) else None
if part_type == "image":
source: Final = part.get("source")
return anthropic_image_source_to_openai_url(source) if isinstance(source, Mapping) else None
return None
def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObject | None:
"""One image content part rewritten into chat-completions dialect, or None when it is not one.
Rebuilt rather than forwarded so no caller-controlled key beyond the URL rides along.
"""
url: Final = _image_part_url(part)
return {"type": "image_url", "image_url": {"url": url}} if url else None
def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool:
"""Whether any message carries an image content part, across the dialects that reach
pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``,

48
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -1,4 +1,5 @@
from collections.abc import Mapping, Sequence
from collections.abc import Set as AbstractSet
from typing import Any, Final
from pydantic import BaseModel
@ -6,38 +7,45 @@ from pydantic import BaseModel
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH, DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
from litellm.litellm_core_utils.secret_redaction import REDACTED
_DEFAULT_SENSITIVE_PATTERNS: Final = frozenset(
(
"password",
"secret",
"key",
"token",
"auth",
"authorization",
"credential",
# Plural form: Vertex uses ``vertex_credentials``; segment-exact
# matching otherwise misses it because "credential" != "credentials".
"credentials",
"access",
"private",
"certificate",
"fingerprint",
"tenancy",
)
)
class SensitiveDataMasker:
def __init__(
self,
sensitive_patterns: set[str] | None = None,
non_sensitive_overrides: set[str] | None = None,
sensitive_patterns: AbstractSet[str] | None = None,
non_sensitive_overrides: AbstractSet[str] | None = None,
visible_prefix: int = 4,
visible_suffix: int = 4,
mask_char: str = "*",
mask_short_values: bool = True,
extra_sensitive_patterns: AbstractSet[str] | None = None,
):
self.sensitive_patterns = sensitive_patterns or {
"password",
"secret",
"key",
"token",
"auth",
"authorization",
"credential",
# Plural form: Vertex uses ``vertex_credentials``; segment-exact
# matching otherwise misses it because "credential" != "credentials".
"credentials",
"access",
"private",
"certificate",
"fingerprint",
"tenancy",
}
self.sensitive_patterns = (sensitive_patterns or _DEFAULT_SENSITIVE_PATTERNS) | (
extra_sensitive_patterns or frozenset()
)
# If any key segment matches one of these, the key is not considered sensitive
# even if it also matches a sensitive pattern. For example, "input_cost_per_token"
# contains "token" but "cost" overrides that — it's a pricing field, not a secret.
self.non_sensitive_overrides = non_sensitive_overrides or {"cost"}
self.non_sensitive_overrides = non_sensitive_overrides or frozenset(("cost",))
self.visible_prefix = visible_prefix
self.visible_suffix = visible_suffix

22
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -99,6 +99,7 @@ def create_tool_name_mapping(
from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice
from litellm.litellm_core_utils.prompt_templates.common_utils import (
anthropic_image_source_to_openai_url,
parse_tool_call_arguments,
reasoning_content_from_thinking_blocks,
with_prompt_cache_breakpoint,
@ -524,18 +525,20 @@ class LiteLLMAnthropicMessagesAdapter:
self._add_cache_control_if_applicable(content, tool_call, model)
tool_calls.append(tool_call)
elif content.get("type") == "thinking":
# Anthropic's schema has no cache_control on thinking or
# redacted_thinking blocks, and anthropic_messages_pt replays
# these verbatim at content[0], so carrying one here (or
# inventing an empty one) is a guaranteed 400 on the way back.
thinking_block = ChatCompletionThinkingBlock(
type="thinking",
thinking=content.get("thinking") or "",
signature=content.get("signature") or "",
cache_control=content.get("cache_control", {}),
)
thinking_blocks.append(thinking_block)
elif content.get("type") == "redacted_thinking":
redacted_thinking_block = ChatCompletionRedactedThinkingBlock(
type="redacted_thinking",
data=content.get("data") or "",
cache_control=content.get("cache_control", {}),
)
thinking_blocks.append(redacted_thinking_block)
@ -1223,20 +1226,7 @@ class LiteLLMAnthropicMessagesAdapter:
"""
if not isinstance(image_source, dict):
return None
source_type: Final = image_source.get("type")
if source_type == "base64":
# Base64 image format
media_type: Final = image_source.get("media_type", "image/jpeg")
image_data: Final = image_source.get("data", "")
if image_data:
return f"data:{media_type};base64,{image_data}"
elif source_type == "url":
# URL-referenced image format
return image_source.get("url", "")
return None
return anthropic_image_source_to_openai_url(image_source)
def _tool_result_content(self, raw_content: object) -> ToolResultContent:
if isinstance(raw_content, str):

54
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -1,6 +1,6 @@
import asyncio
import json
from collections.abc import AsyncIterator, Mapping
from collections.abc import AsyncIterator, Mapping, Sequence
from datetime import datetime
from typing import Any, Final, Protocol, runtime_checkable
@ -177,12 +177,6 @@ def _try_claim_detached_drain_slot() -> bool:
def _exception_left_unconsumed(queue: "asyncio.Queue[bytes | None | BaseException]", exc: BaseException) -> bool:
"""After client detach the relay never reads the queue again, so drain it here.
The forwarded exception still sitting in the queue means the relay tore
down before re-raising it, so the proxy's failure handling never ran and
the caller must salvage spend itself.
"""
remaining: Final = tuple(queue.get_nowait() for _ in range(queue.qsize()))
return any(item is exc for item in remaining)
@ -671,27 +665,41 @@ class BaseAnthropicMessagesStreamingIterator:
self,
queue: "asyncio.Queue[bytes | None | BaseException]",
client_detached: "asyncio.Event",
collected_chunks: list[bytes], # mutable-ok: SSE buffer forwarded to list-typed _bill_collected_chunks
exc: BaseException,
collected_chunks: Sequence[bytes],
exc: Exception,
) -> None:
"""Forward a provider error to a still-connected client, else salvage partial spend.
"""Log the request as failed with its partial usage, then make sure the proxy's failure hook runs once.
Handing the original exception to the client-facing generator lets it
re-raise so the proxy's failure handling keeps the provider status and
owns logging (no success-bill). If the client already went away, or
disconnects before ever consuming the queued exception, no failure hook
runs, so bill the partial instead of dropping the request.
A still-connected client gets the original exception through the queue,
the relay re-raises it, and the proxy's own failure handling records the
failed spend. When the client already left, or leaves before consuming
the queued exception, that handling never runs, so the detached-failure
hook the proxy armed on the logging object fires here instead.
"""
from litellm._logging import verbose_proxy_logger
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
PassThroughStreamingHandler.schedule_stream_failure_logging(
litellm_logging_obj=self.litellm_logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
request_body=self.request_body,
raw_bytes=collected_chunks,
exception=exc,
)
if not client_detached.is_set() and await self._enqueue_for_client(queue, client_detached, exc):
await client_detached.wait()
if not _exception_left_unconsumed(queue, exc):
return
verbose_proxy_logger.warning(
"async_sse_wrapper upstream pump failed after client disconnect (%d chunks): %s(%s)",
len(collected_chunks),
type(exc).__name__,
exc,
)
await self._bill_collected_chunks(collected_chunks, stream_teardown=True)
await self._fire_detached_failure_hook(exc)
async def _fire_detached_failure_hook(self, exc: Exception) -> None:
from litellm._logging import verbose_proxy_logger
on_detached_failure: Final = getattr(self.litellm_logging_obj, "_on_detached_stream_failure", None)
if on_detached_failure is None:
return
try:
await on_detached_failure(exc)
except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not crash the detached pump
verbose_proxy_logger.warning(
"async_sse_wrapper detached failure hook raised: %s(%s)", type(hook_failure).__name__, hook_failure
)

9
litellm/llms/azure/responses/transformation.py
View file

@ -6,6 +6,7 @@ from openai.types.responses import ResponseReasoningItem
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import *
@ -29,6 +30,14 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.AZURE
@staticmethod
def _supports_reasoning_effort_none(model: str) -> bool:
return AzureOpenAIGPT5Config._supports_reasoning_effort_level(model, "none")
@staticmethod
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)
def get_supported_openai_params(self, model: str) -> list:
"""
Azure Responses API does not support context_management (compaction).

27
litellm/llms/bedrock/passthrough/guardrail_translation/handler.py
View file

@ -89,11 +89,24 @@ def _extract_converse_texts(
top-level ``text`` blocks this scans the arbitrary-JSON fields a caller can
hide prompt content in -- ``toolUse.input`` and
``toolResult.content[].json`` (alongside ``toolResult.content[].text``) --
as well as the request-level fields still forwarded to Bedrock that a caller
can route blocked content through: ``toolConfig.tools`` (tool names,
descriptions and input schemas) and ``additionalModelRequestFields``. Tool
message blocks are skipped when tool messages are excluded, but tool
definitions are always scanned to match the chat-completions guardrail path.
as well as ``additionalModelRequestFields``, a free-form model-parameter bag
with no schema that a caller can route blocked content through.
``toolConfig.tools`` is deliberately NOT scanned. Tool definitions are
app-authored config, so their names, descriptions and JSON-schema strings
("object", property names, titles, type names, enum values) would each reach
the guardrail as a separate INPUT item, producing false positives and
inflating guardrail usage for a request whose only prompt is one user
message. No other guardrail translation handler puts tool definitions in
``texts``; the chat and messages handlers carry them in the structured
``tools`` input instead, which this handler does not populate because a
Bedrock ``toolSpec`` is not the OpenAI tool shape those consumers expect.
``additionalModelRequestFields`` is treated differently on purpose. Bedrock
gives ``toolConfig.tools`` a fixed schema whose contents are tool metadata by
contract, while ``additionalModelRequestFields`` is free-form and defined by
the target model, so what it carries cannot be classified without knowing
that model. Scanning it stays the fail-closed default.
"""
holders: Final[list[_StringHolder]] = []
@ -121,10 +134,6 @@ def _extract_converse_texts(
_collect_block_text(inner, holders)
_collect_strings(inner.get("json"), holders)
tool_config: Final = body.get("toolConfig")
if isinstance(tool_config, dict):
_collect_strings(tool_config.get("tools"), holders)
_collect_strings(body.get("additionalModelRequestFields"), holders)
texts: Final = [container[key] for container, key in holders]

2
litellm/llms/bedrock_mantle/common_utils.py
View file

@ -29,7 +29,7 @@ from litellm.secret_managers.main import get_secret_str
BEDROCK_MANTLE_DEFAULT_REGION: Final = "us-east-1"
# Standard Mantle host: https://bedrock-mantle.<region>.api.aws (group 1 = region).
MANTLE_HOST_RE: Final = re.compile(r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE)
MANTLE_HOST_RE: Final = re.compile(r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws(?=/|$)", re.IGNORECASE)
def resolve_mantle_bearer_token(api_key: str | None) -> str | None:

8
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -272,11 +272,15 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
)
# Only add tool_choice for models that explicitly support it
if supports_tool_choice(model=model, custom_llm_provider="fireworks_ai"):
if self._get_model_cost_capability_exact(
model=model, capability="supports_tool_choice"
) or supports_tool_choice(model=model, custom_llm_provider="fireworks_ai"):
supported_params.append("tool_choice")
# Only add reasoning params for models that support it
if supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
if self._get_model_cost_capability_exact(model=model, capability="supports_reasoning") or supports_reasoning(
model=model, custom_llm_provider="fireworks_ai"
):
supported_params.append("reasoning_effort")
supported_params.append("reasoning_history")
supported_params.append("thinking")

57
litellm/llms/fireworks_ai/cost_calculator.py
View file

@ -2,6 +2,8 @@
For calculating cost of fireworks ai serverless inference models.
"""
import math
from datetime import datetime
from typing import Final
from litellm.constants import (
@ -10,9 +12,12 @@ from litellm.constants import (
FIREWORKS_AI_56_B_MOE,
FIREWORKS_AI_176_B_MOE,
)
from litellm.types.utils import Usage
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
from litellm.types.utils import ModelInfo, Usage
from litellm.utils import get_model_info
NO_CACHE_READ_RATE: Final = float("nan")
# Extract the number of billion parameters from the model name
# only used for together_computer LLMs
@ -54,44 +59,50 @@ def get_base_model_for_pricing(model_name: str) -> str:
return "fireworks-ai-default"
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
def _resolve_model_info(model: str) -> ModelInfo:
try:
return get_model_info(model=model, custom_llm_provider="fireworks_ai")
except Exception:
base_model: Final = get_base_model_for_pricing(model_name=model)
return get_model_info(model=base_model, custom_llm_provider="fireworks_ai")
def cost_per_token(model: str, usage: Usage, current_time: datetime | None = None) -> tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
Calculates the cost per token for a given model, prompt tokens, and completion tokens,
swapping in the model's off_peak_pricing rates while one of its windows is open.
Input:
- model: str, the model name without provider prefix
- usage: LiteLLM Usage block, containing anthropic caching information
- current_time: the moment the request is billed at; defaults to now, UTC
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
## check if model mapped, else use default pricing
try:
model_info = get_model_info(model=model, custom_llm_provider="fireworks_ai")
except Exception:
base_model: Final = get_base_model_for_pricing(model_name=model)
model_info: Final = _resolve_model_info(model)
standard_cache_read_rate: Final = model_info.get("cache_read_input_token_cost")
rates: Final = apply_off_peak_pricing(
model_info,
current_time,
TokenRates(
input_rate=model_info["input_cost_per_token"] or 0.0,
output_rate=model_info["output_cost_per_token"] or 0.0,
cache_read_rate=standard_cache_read_rate if standard_cache_read_rate is not None else NO_CACHE_READ_RATE,
cache_creation_rate=0.0,
reasoning_rate=None,
),
)
cache_read_rate: Final[float] = rates.input_rate if math.isnan(rates.cache_read_rate) else rates.cache_read_rate
## GET MODEL INFO
model_info = get_model_info(model=base_model, custom_llm_provider="fireworks_ai")
## CALCULATE INPUT COST
prompt_tokens_details: Final = usage.prompt_tokens_details
cached_tokens: Final[int] = (
prompt_tokens_details.cached_tokens
if prompt_tokens_details is not None and prompt_tokens_details.cached_tokens is not None
else 0
)
input_cost_per_token: Final[float] = model_info["input_cost_per_token"] or 0.0
cache_read_input_token_cost: Final = model_info.get("cache_read_input_token_cost")
cache_read_cost_per_token: Final[float] = (
cache_read_input_token_cost if cache_read_input_token_cost is not None else input_cost_per_token
)
non_cached_prompt_tokens: Final[int] = max(usage.prompt_tokens - cached_tokens, 0)
prompt_cost: float = non_cached_prompt_tokens * input_cost_per_token + cached_tokens * cache_read_cost_per_token
## CALCULATE OUTPUT COST
output_cost_per_token: Final[float] = model_info["output_cost_per_token"] or 0.0
completion_cost: Final[float] = usage.completion_tokens * output_cost_per_token
prompt_cost: Final[float] = non_cached_prompt_tokens * rates.input_rate + cached_tokens * cache_read_rate
completion_cost: Final[float] = usage.completion_tokens * rates.output_rate
return prompt_cost, completion_cost

0
litellm/llms/mongodb/__init__.py Normal file
View file

303
litellm/llms/mongodb/common_utils.py Normal file
View file

@ -0,0 +1,303 @@
"""Shared helpers for the MongoDB integrations. pymongo lives in the optional ``mongodb`` extra,
so every import of it is deferred to call time."""
import asyncio
import threading
import weakref
from asyncio import AbstractEventLoop
from collections import OrderedDict
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, TypeAlias, TypeVar
from litellm.exceptions import BadRequestError, ServiceUnavailableError, Timeout
if TYPE_CHECKING:
from pymongo import AsyncMongoClient, MongoClient
PYMONGO_INSTALL_HINT: Final = (
"The MongoDB vector store requires the 'pymongo' package. "
"Run 'pip install litellm[mongodb]' (or 'pip install pymongo') to install it."
)
MONGODB_PROVIDER: Final = "mongodb"
def config_error(message: str) -> BadRequestError:
"""400 rather than the 500 a bare ValueError becomes once litellm.exception_type wraps it."""
return BadRequestError(message=message, model=None, llm_provider=MONGODB_PROVIDER)
def timeout_error(message: str) -> Timeout:
return Timeout(message=message, model=None, llm_provider=MONGODB_PROVIDER)
def unavailable_error(message: str) -> ServiceUnavailableError:
"""litellm only retries 408, 409, 429 and 5xx, so a 400 here would make a failover permanent."""
return ServiceUnavailableError(message=message, model=None, llm_provider=MONGODB_PROVIDER)
DEFAULT_CONNECT_TIMEOUT_MS: Final = 10_000
DEFAULT_SOCKET_TIMEOUT_MS: Final = 30_000
DEFAULT_SERVER_SELECTION_TIMEOUT_MS: Final = 10_000
_MAX_CACHED_CLIENTS: Final = 32
_APP_NAME: Final = "litellm"
@dataclass(frozen=True, slots=True)
class MongoClientKey:
connection_string: str
connect_timeout_ms: int
socket_timeout_ms: int
server_selection_timeout_ms: int
SyncClientFactory: TypeAlias = Callable[..., "MongoClient"]
AsyncClientFactory: TypeAlias = Callable[..., "AsyncMongoClient"]
_K = TypeVar("_K")
_V = TypeVar("_V")
_AsyncClientCacheKey: TypeAlias = tuple[MongoClientKey, int]
# CPython recycles id() aggressively, so the id alone would hand a new loop a closed loop's client
_AsyncClientEntry: TypeAlias = tuple["weakref.ref[AbstractEventLoop]", "AsyncMongoClient"]
_SyncClientCache: TypeAlias = "OrderedDict[MongoClientKey, MongoClient]"
_AsyncClientCache: TypeAlias = "OrderedDict[_AsyncClientCacheKey, _AsyncClientEntry]"
_sync_clients: Final[_SyncClientCache] = OrderedDict() # mutable-ok: process-level client cache
_async_clients: Final[_AsyncClientCache] = OrderedDict() # mutable-ok: same cache, per loop
# async searches reach the sync client through executor threads, so both caches are shared state
_cache_lock: Final = threading.Lock()
def _store_bounded(cache: "OrderedDict[_K, _V]", cache_key: "_K", value: "_V") -> None:
"""Eviction only drops this cache's reference; an in-flight search keeps its client alive."""
with _cache_lock:
cache[cache_key] = value # mutable-ok: an LRU cache is mutable state by definition
cache.move_to_end(cache_key)
while len(cache) > _MAX_CACHED_CLIENTS:
cache.popitem(last=False)
def _mark_used(cache: "OrderedDict[_K, _V]", cache_key: "_K") -> None:
with _cache_lock:
if cache_key in cache:
cache.move_to_end(cache_key)
def import_sync_mongo_client() -> "type[MongoClient]":
try:
from pymongo import MongoClient as SyncMongoClient
except ImportError as e:
raise config_error(PYMONGO_INSTALL_HINT) from e
return SyncMongoClient
def import_async_mongo_client() -> "type[AsyncMongoClient]":
try:
from pymongo import AsyncMongoClient as AsyncMongoClientClass
except ImportError as e:
raise config_error(PYMONGO_INSTALL_HINT) from e
return AsyncMongoClientClass
def _client_kwargs(key: MongoClientKey) -> Mapping[str, object]:
return MappingProxyType(
{
"connectTimeoutMS": key.connect_timeout_ms,
"socketTimeoutMS": key.socket_timeout_ms,
"serverSelectionTimeoutMS": key.server_selection_timeout_ms,
"appname": _APP_NAME,
}
)
def get_sync_client(key: MongoClientKey, client_class: SyncClientFactory | None = None) -> "MongoClient":
cached: Final = _sync_clients.get(key)
if cached is not None:
_mark_used(_sync_clients, key)
return cached
build: Final = client_class if client_class is not None else import_sync_mongo_client()
client: Final = build(key.connection_string, **_client_kwargs(key))
_store_bounded(_sync_clients, key, client)
return client
def _purge_dead_loops() -> None:
"""A cached client holds its loop alive, so a closed loop's entry would pin that client and its
sockets for the life of the process."""
with _cache_lock:
for stale in tuple(
cache_key
for cache_key, (loop_ref, _) in _async_clients.items()
if (cached_loop := loop_ref()) is None or cached_loop.is_closed()
):
del _async_clients[stale]
def get_async_client(key: MongoClientKey, client_class: AsyncClientFactory | None = None) -> "AsyncMongoClient":
"""Async clients bind to the loop that created them, so the cache is keyed per loop."""
loop: Final = asyncio.get_running_loop()
loop_key: Final = (key, id(loop))
cached: Final = _async_clients.get(loop_key)
if cached is not None and cached[0]() is loop:
_mark_used(_async_clients, loop_key)
return cached[1]
_purge_dead_loops()
build: Final = client_class if client_class is not None else import_async_mongo_client()
client: Final = build(key.connection_string, **_client_kwargs(key))
_store_bounded(_async_clients, loop_key, (weakref.ref(loop), client))
return client
def reset_client_cache() -> None:
with _cache_lock:
_sync_clients.clear()
_async_clients.clear()
_AUTHENTICATION_FAILED_CODE: Final = 18
_UNAUTHORIZED_CODE: Final = 13
# Atlas reports a rejected user as code 8000 "AtlasError" where a self-managed mongod reports 18
_AUTHENTICATION_MESSAGE_MARKERS: Final = ("bad auth", "authentication failed", "not authorized")
_RESOLUTION_TIMEOUT_MARKERS: Final = ("resolution lifetime expired", "dns operation timed out")
_UNKNOWN_HOSTNAME_MARKERS: Final = ("dns query name does not exist", "name or service not known")
_CREDENTIAL_ESCAPING_MARKERS: Final = ("must be escaped according to rfc 3986", "bad database name")
def _index_hint(index_name: str, database: str, collection: str) -> str:
return (
f"No queryable MongoDB Vector Search index named '{index_name}' was found on "
f"'{database}.{collection}'. Confirm the index exists on that exact collection, that its "
"status is READY rather than still building, and that the vector store id matches the index name."
)
def missing_index_error(index_name: str, database: str, collection: str) -> BadRequestError:
"""$vectorSearch against a missing index, database or collection returns zero documents rather
than failing, so an empty result set is checked against the catalogue and reported as this."""
return config_error(
f"{_index_hint(index_name, database, collection)} A vector search against a database, "
"collection or index that does not exist returns no results rather than an error, so this "
"was reported as an empty result set by MongoDB."
)
def index_not_ready_error(index_name: str, database: str, collection: str, status: str) -> BadRequestError:
return config_error(
f"The MongoDB Vector Search index '{index_name}' on '{database}.{collection}' is not queryable "
f"yet; its status is {status}. Searches against it return no results until the build finishes."
)
def translate_mongo_error(error: Exception, index_name: str, database: str, collection: str) -> Exception:
"""Returns the exception to raise, so callers keep the driver error as ``__cause__``."""
try:
from pymongo.errors import (
ConfigurationError,
ConnectionFailure,
ExecutionTimeout,
InvalidOperation,
NetworkTimeout,
OperationFailure,
ServerSelectionTimeoutError,
)
except ImportError:
return error
if isinstance(error, ServerSelectionTimeoutError):
return timeout_error(
"Could not reach the MongoDB deployment before the timeout. On Atlas this is usually the "
"project's IP access list not containing this host, or a paused cluster. On a self-managed "
"deployment it is usually the host or port in the URI, or a firewall between this process "
f"and mongod. Either way it can also be an unresolvable hostname. Driver detail: {error}"
)
# ExecutionTimeout subclasses OperationFailure, so it has to be matched before it
if isinstance(error, (NetworkTimeout, ExecutionTimeout)):
return timeout_error(
f"The MongoDB vector search against '{database}.{collection}' timed out before returning. "
f"Driver detail: {error}"
)
# ServerSelectionTimeoutError and NetworkTimeout also subclass ConnectionFailure, so this only
# sees what those branches left
if isinstance(error, ConnectionFailure):
return unavailable_error(
f"The connection to '{database}.{collection}' was dropped or refused. That is usually a "
"replica set failover or a restarted node, so the search is worth retrying. If it keeps "
"happening: on Atlas the usual cause is a connection string with no username and password, "
"or a TLS failure, so confirm the URI is the one Atlas shows under Connect, Drivers; on a "
"self-managed deployment, check that mongod is listening on the host and port in the URI. "
f"Driver detail: {error}"
)
if isinstance(error, OperationFailure):
code: Final = error.code
detail: Final = str(error).lower()
if code in (_AUTHENTICATION_FAILED_CODE, _UNAUTHORIZED_CODE) or any(
marker in detail for marker in _AUTHENTICATION_MESSAGE_MARKERS
):
return config_error(
"MongoDB rejected the credentials in mongodb_connection_string, or the database user "
f"lacks read access to '{database}.{collection}'. Driver detail: {error.details}"
)
if "dimension" in detail:
return config_error(
"The query embedding does not match the vector dimensions the index was built for. "
"litellm_embedding_model must be the same model that produced the stored vectors. "
f"Driver detail: {error}"
)
if "is not indexed as vector" in detail:
return config_error(
"mongodb_embedding_field names a field the MongoDB Vector Search index does not cover. "
f"It must match the 'path' the index '{index_name}' was created on. Driver detail: {error}"
)
if "index" in detail and ("not found" in detail or "does not exist" in detail or "unknown" in detail):
return config_error(f"{_index_hint(index_name, database, collection)} Driver detail: {error}")
return config_error(
f"MongoDB rejected the vector search against '{database}.{collection}' using index "
f"'{index_name}'. Driver detail: {error}"
)
if isinstance(error, ConfigurationError):
configuration_detail: Final = str(error).lower()
if any(marker in configuration_detail for marker in _RESOLUTION_TIMEOUT_MARKERS):
return timeout_error(
"The DNS lookup for the cluster in mongodb_connection_string did not finish in time. "
"A mongodb+srv:// URI needs an SRV lookup before any connection is attempted, so this "
f"is DNS or the configured timeout, not MongoDB. Driver detail: {error}"
)
if any(marker in configuration_detail for marker in _UNKNOWN_HOSTNAME_MARKERS):
return config_error(
"The hostname in mongodb_connection_string does not exist in DNS. On Atlas, check the "
"cluster name against the URI shown under Connect, Drivers. On a self-managed deployment, "
f"check that the hostname resolves from this process. Driver detail: {error}"
)
if any(marker in configuration_detail for marker in _CREDENTIAL_ESCAPING_MARKERS):
return config_error(
"mongodb_connection_string could not be parsed. A username or password containing "
"'@', '/', ':' or '%' has to be percent-encoded per RFC 3986, so 'p@ss/word' becomes "
"'p%40ss%2Fword'. If the credentials are already encoded, check the database name in "
f"the URI path instead. Driver detail: {error}"
)
return config_error(
f"mongodb_connection_string is not a usable MongoDB connection string. Driver detail: {error}"
)
if isinstance(error, InvalidOperation):
return config_error(f"The MongoDB client was already closed or is unusable. Driver detail: {error}")
# An unreadable tlsCAFile or tlsCertificateKeyFile raises OSError, not a PyMongoError
if isinstance(error, OSError) and error.filename:
return config_error(
f"'{error.filename}', named by a TLS option in mongodb_connection_string, could not be read. "
"Check that tlsCAFile and tlsCertificateKeyFile point at files this process can open; inside "
f"a container that is the path in the container, not on the host. Driver detail: {error}"
)
# pymongo raises a plain ValueError, not a PyMongoError, for an unusable port
if isinstance(error, ValueError):
return config_error(
"The host and port in mongodb_connection_string could not be parsed. If the port is a "
"number between 0 and 65535, the cause is usually an unescaped ':' in the password, which "
f"has to be percent-encoded per RFC 3986 as '%3A'. Driver detail: {error}"
)
return error

0
litellm/llms/mongodb/vector_stores/__init__.py Normal file
View file

431
litellm/llms/mongodb/vector_stores/transformation.py Normal file
View file

@ -0,0 +1,431 @@
"""MongoDB Vector Search has no HTTP query API, so this is a direct provider that runs the
``$vectorSearch`` aggregation through pymongo. ``vector_store_id`` is the search index name."""
from collections.abc import Callable, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, NoReturn
import httpx
from pydantic import BaseModel, ConfigDict
from litellm.llms.base_llm.vector_store.transformation import (
BaseDirectVectorStoreConfig,
LiteLLMVectorStoreEmbeddingExecutor,
VectorStoreEmbeddingExecutor,
)
from litellm.llms.mongodb.common_utils import (
DEFAULT_CONNECT_TIMEOUT_MS,
DEFAULT_SERVER_SELECTION_TIMEOUT_MS,
DEFAULT_SOCKET_TIMEOUT_MS,
MongoClientKey,
config_error,
get_async_client,
get_sync_client,
index_not_ready_error,
missing_index_error,
translate_mongo_error,
)
from litellm.types.utils import EmbeddingResponse
from litellm.types.vector_stores import (
VectorStoreCreateOptionalRequestParams,
VectorStoreResultContent,
VectorStoreSearchOptionalRequestParams,
VectorStoreSearchResponse,
VectorStoreSearchResult,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
DEFAULT_EMBEDDING_FIELD_NAME: Final = "embedding"
DEFAULT_TEXT_FIELD_NAME: Final = "text"
SCORE_FIELD_NAME: Final = "score"
DEFAULT_MAX_NUM_RESULTS: Final = 10
MIN_MAX_NUM_RESULTS: Final = 1
MAX_MAX_NUM_RESULTS: Final = 50
NUM_CANDIDATES_MULTIPLIER: Final = 10
MIN_NUM_CANDIDATES: Final = 100
MAX_NUM_CANDIDATES: Final = 10_000
MAX_QUERY_CHARACTERS: Final = 32_000
_EMPTY_EMBEDDING_CONFIG: Final = MappingProxyType({})
_SEARCH_ONLY_MESSAGE: Final = (
"MongoDB vector store is search-only. Create the collection and its MongoDB Vector Search "
"index in MongoDB directly, then register it here by index name."
)
class _MongoDBSearchParams(BaseModel):
"""Typed view over the vector store's litellm_params; unrelated keys are ignored."""
model_config = ConfigDict(frozen=True, extra="ignore")
litellm_embedding_model: str | None = None
litellm_embedding_config: Mapping[str, object] | None = None
mongodb_connection_string: str | None = None
mongodb_database: str | None = None
mongodb_collection: str | None = None
mongodb_text_field: str | None = None
mongodb_embedding_field: str | None = None
mongodb_num_candidates: int | None = None
@property
def text_field(self) -> str:
return self.mongodb_text_field or DEFAULT_TEXT_FIELD_NAME
@property
def embedding_field(self) -> str:
return self.mongodb_embedding_field or DEFAULT_EMBEDDING_FIELD_NAME
def require_embedding_model(self) -> str:
if not self.litellm_embedding_model:
raise config_error(
"litellm_embedding_model is required in litellm_params for the MongoDB vector store. "
"It must be the same model that produced the vectors stored in "
f"'{self.mongodb_collection or '<collection>'}.{self.embedding_field}', or search results "
"will be meaningless. Example: litellm_embedding_model: openai/text-embedding-3-small"
)
return self.litellm_embedding_model
def require_connection_string(self) -> str:
if not self.mongodb_connection_string:
raise config_error(
"mongodb_connection_string is required in litellm_params for the MongoDB vector store. "
"Example: mongodb+srv://<user>:<password>@<cluster>.mongodb.net for Atlas, or "
"mongodb://<user>:<password>@<host>:27017 for a self-managed deployment"
)
scheme: Final = self.mongodb_connection_string.split("://", 1)[0].lower()
if scheme not in ("mongodb", "mongodb+srv"):
raise config_error(
"mongodb_connection_string must start with 'mongodb://' or 'mongodb+srv://', "
f"got '{self.mongodb_connection_string.split('://', 1)[0]}://'"
)
return self.mongodb_connection_string
def require_database(self) -> str:
if not self.mongodb_database:
raise config_error(
"mongodb_database is required in litellm_params for the MongoDB vector store. "
"Example: mongodb_database: sample_mflix"
)
return self.mongodb_database
def require_collection(self) -> str:
if not self.mongodb_collection:
raise config_error(
"mongodb_collection is required in litellm_params for the MongoDB vector store. "
"Example: mongodb_collection: embedded_movies"
)
return self.mongodb_collection
_MONGODB_PARAM_PREFIX: Final = "mongodb_"
_KNOWN_MONGODB_PARAMS: Final = frozenset(
name for name in _MongoDBSearchParams.model_fields if name.startswith(_MONGODB_PARAM_PREFIX)
)
class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig):
def __init__(
self,
embedding_executor: VectorStoreEmbeddingExecutor | None = None,
sync_client_factory: Callable[[MongoClientKey], object] | None = None,
async_client_factory: Callable[[MongoClientKey], object] | None = None,
) -> None:
super().__init__()
self.embedding_executor: Final[VectorStoreEmbeddingExecutor] = (
embedding_executor if embedding_executor is not None else LiteLLMVectorStoreEmbeddingExecutor()
)
self.sync_client_factory: Final[Callable[[MongoClientKey], object]] = (
sync_client_factory if sync_client_factory is not None else get_sync_client
)
self.async_client_factory: Final[Callable[[MongoClientKey], object]] = (
async_client_factory if async_client_factory is not None else get_async_client
)
@staticmethod
def _reject_unknown_params(litellm_params: Mapping[str, object]) -> None:
"""Without this a mistyped mongodb_collection reads as 'mongodb_collection is required',
naming a key the reader can see they have set."""
unknown: Final = sorted(
key for key in litellm_params if key.startswith(_MONGODB_PARAM_PREFIX) and key not in _KNOWN_MONGODB_PARAMS
)
if unknown:
raise config_error(
f"Unrecognised MongoDB vector store parameter(s): {', '.join(unknown)}. "
f"Supported: {', '.join(sorted(_KNOWN_MONGODB_PARAMS))}."
)
@staticmethod
def _query_text(query: str | Sequence[str]) -> str:
text: Final = query if isinstance(query, str) else " ".join(query)
if not text.strip():
raise config_error("query must not be empty")
if len(text) > MAX_QUERY_CHARACTERS:
raise config_error(f"query must be at most {MAX_QUERY_CHARACTERS} characters, got {len(text)}")
return text
@staticmethod
def _limit(vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams) -> int:
requested: Final = vector_store_search_optional_params.get("max_num_results")
if requested is None:
return DEFAULT_MAX_NUM_RESULTS
if not MIN_MAX_NUM_RESULTS <= requested <= MAX_MAX_NUM_RESULTS:
raise config_error(
f"max_num_results must be between {MIN_MAX_NUM_RESULTS} and {MAX_MAX_NUM_RESULTS}, got {requested}"
)
return requested
@staticmethod
def _num_candidates(limit: int, configured: int | None) -> int:
if configured is not None:
if not limit <= configured <= MAX_NUM_CANDIDATES:
raise config_error(
f"mongodb_num_candidates must be between max_num_results ({limit}) and "
f"{MAX_NUM_CANDIDATES}, got {configured}"
)
return configured
return min(max(limit * NUM_CANDIDATES_MULTIPLIER, MIN_NUM_CANDIDATES), MAX_NUM_CANDIDATES)
@staticmethod
def _timeout_ms(timeout: float | httpx.Timeout | None) -> tuple[int, int]:
"""The connect and socket budgets pymongo is built with, in that order."""
if isinstance(timeout, httpx.Timeout):
return (
int((timeout.connect or DEFAULT_CONNECT_TIMEOUT_MS / 1000) * 1000),
int((timeout.read or DEFAULT_SOCKET_TIMEOUT_MS / 1000) * 1000),
)
if timeout is None:
return DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_SOCKET_TIMEOUT_MS
return min(int(float(timeout) * 1000), DEFAULT_CONNECT_TIMEOUT_MS), int(float(timeout) * 1000)
@classmethod
def _client_key(cls, params: _MongoDBSearchParams, timeout: float | httpx.Timeout | None) -> MongoClientKey:
connect_ms, socket_ms = cls._timeout_ms(timeout)
return MongoClientKey(
connection_string=params.require_connection_string(),
connect_timeout_ms=connect_ms,
socket_timeout_ms=socket_ms,
server_selection_timeout_ms=min(connect_ms, DEFAULT_SERVER_SELECTION_TIMEOUT_MS),
)
@classmethod
def _pipeline(
cls,
vector_store_id: str,
query_vector: Sequence[float],
params: _MongoDBSearchParams,
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
) -> Sequence[Mapping[str, object]]:
if vector_store_search_optional_params.get("filters") is not None:
raise config_error(
"MongoDB vector store does not support the filters parameter yet. "
"Restrict the collection or the MongoDB Vector Search index definition instead."
)
if vector_store_search_optional_params.get("ranking_options") is not None:
raise config_error(
"MongoDB vector store does not support the ranking_options parameter yet. "
"Every result already carries the vectorSearchScore, so filter or re-rank "
"on that rather than having the threshold silently ignored."
)
if vector_store_search_optional_params.get("rewrite_query") is not None:
raise config_error(
"MongoDB vector store does not support the rewrite_query parameter. The query is "
"embedded exactly as sent; rewrite it before calling if you need that."
)
limit: Final = cls._limit(vector_store_search_optional_params)
search: Final = MappingProxyType(
{
"index": vector_store_id,
"path": params.embedding_field,
"queryVector": tuple(query_vector),
"numCandidates": cls._num_candidates(limit, params.mongodb_num_candidates),
"limit": limit,
}
)
projection: Final = MappingProxyType(
{params.text_field: 1, SCORE_FIELD_NAME: MappingProxyType({"$meta": "vectorSearchScore"})}
)
return [ # mutable-ok: pymongo rejects any non-list pipeline in common.validate_list
MappingProxyType({"$vectorSearch": search}),
MappingProxyType({"$project": projection}),
]
@classmethod
def _field_value(cls, document: Mapping[str, object], dotted_path: str) -> str | None:
"""None means absent, which is what separates a mistyped field from genuinely empty text."""
head, _, rest = dotted_path.partition(".")
if head not in document:
return None
value: Final = document[head]
if not rest:
return None if value is None else str(value)
return cls._field_value(value, rest) if isinstance(value, Mapping) else None
@classmethod
def _to_result(cls, document: Mapping[str, object], text_field: str) -> VectorStoreSearchResult:
document_id: Final = document.get("_id")
identifier: Final = None if document_id is None else str(document_id)
content: Final = [ # mutable-ok: VectorStoreSearchResult declares a list of content parts
VectorStoreResultContent(text=cls._field_value(document, text_field) or "", type="text")
]
raw_score: Final = document.get(SCORE_FIELD_NAME)
return VectorStoreSearchResult(
score=float(raw_score) if isinstance(raw_score, (int, float)) else None,
content=content,
file_id=identifier,
filename=identifier,
)
@classmethod
def _raise_for_missing_text_field(
cls, documents: Sequence[Mapping[str, object]], text_field: str, database: str, collection: str
) -> None:
"""$vectorSearch matches documents carrying no text, so a mistyped mongodb_text_field
returns well-scored results with empty content instead of failing."""
if documents and all(cls._field_value(document, text_field) is None for document in documents):
raise config_error(
f"None of the {len(documents)} matched documents in '{database}.{collection}' has a "
f"'{text_field}' field, so every result would carry empty text. Set mongodb_text_field "
"to the field holding the readable text; it accepts a dotted path such as metadata.body."
)
@classmethod
def _to_response(
cls, documents: Sequence[Mapping[str, object]], query_text: str, text_field: str
) -> VectorStoreSearchResponse:
return VectorStoreSearchResponse(
object="vector_store.search_results.page",
search_query=query_text,
data=[ # mutable-ok: VectorStoreSearchResponse declares data as a list
cls._to_result(document, text_field) for document in documents
],
)
@staticmethod
def _raise_for_unusable_index(
catalogue: Sequence[Mapping[str, object]], index_name: str, database: str, collection: str
) -> None:
"""mongod returns zero documents both for a query that matched nothing and for a missing
database, collection or index, so the catalogue decides which one happened."""
if not catalogue:
raise missing_index_error(index_name, database, collection)
entry: Final = catalogue[0]
if not entry.get("queryable"):
raise index_not_ready_error(index_name, database, collection, str(entry.get("status") or "unknown"))
@staticmethod
def _embedding_vector(embedding_response: EmbeddingResponse) -> Sequence[float]:
data: Final = embedding_response.data
if not data:
raise config_error(
"The embedding model returned no embedding for the search query, so there is nothing "
"to search MongoDB with. Check the embedding deployment named by litellm_embedding_model."
)
return data[0]["embedding"]
def execute_search_vector_store_request(
self,
vector_store_id: str,
query: str | Sequence[str],
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
litellm_logging_obj: "LiteLLMLoggingObj",
litellm_params: Mapping[str, object],
embedding_executor: VectorStoreEmbeddingExecutor | None = None,
timeout: float | httpx.Timeout | None = None,
) -> VectorStoreSearchResponse:
self._reject_unknown_params(litellm_params)
params: Final = _MongoDBSearchParams.model_validate(litellm_params)
query_text: Final = self._query_text(query)
key: Final = self._client_key(params, timeout)
database: Final = params.require_database()
collection: Final = params.require_collection()
embedding_response: Final = (embedding_executor or self.embedding_executor).embed(
params.require_embedding_model(),
query_text,
params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG,
)
pipeline: Final = self._pipeline(
vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params
)
try:
client: Final = self.sync_client_factory(key)
target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted
documents: Final = tuple(target.aggregate(pipeline))
except Exception as e:
raise translate_mongo_error(e, index_name=vector_store_id, database=database, collection=collection) from e
if not documents:
try:
catalogue: Final = tuple(target.list_search_indexes(vector_store_id))
except Exception as e:
raise translate_mongo_error(
e, index_name=vector_store_id, database=database, collection=collection
) from e
self._raise_for_unusable_index(catalogue, vector_store_id, database, collection)
self._raise_for_missing_text_field(documents, params.text_field, database, collection)
return self._to_response(documents, query_text, params.text_field)
async def aexecute_search_vector_store_request(
self,
vector_store_id: str,
query: str | Sequence[str],
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
litellm_logging_obj: "LiteLLMLoggingObj",
litellm_params: Mapping[str, object],
embedding_executor: VectorStoreEmbeddingExecutor | None = None,
timeout: float | httpx.Timeout | None = None,
) -> VectorStoreSearchResponse:
self._reject_unknown_params(litellm_params)
params: Final = _MongoDBSearchParams.model_validate(litellm_params)
query_text: Final = self._query_text(query)
key: Final = self._client_key(params, timeout)
database: Final = params.require_database()
collection: Final = params.require_collection()
embedding_response: Final = await (embedding_executor or self.embedding_executor).aembed(
params.require_embedding_model(),
query_text,
params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG,
)
pipeline: Final = self._pipeline(
vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params
)
try:
client: Final = self.async_client_factory(key)
target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted
cursor: Final = await target.aggregate(pipeline)
documents: Final = [ # mutable-ok: an async comprehension cannot build a tuple directly
document async for document in cursor
]
except Exception as e:
raise translate_mongo_error(e, index_name=vector_store_id, database=database, collection=collection) from e
if not documents:
try:
index_cursor: Final = await target.list_search_indexes(vector_store_id)
catalogue: Final = [ # mutable-ok: an async comprehension cannot build a tuple directly
entry async for entry in index_cursor
]
except Exception as e:
raise translate_mongo_error(
e, index_name=vector_store_id, database=database, collection=collection
) from e
self._raise_for_unusable_index(catalogue, vector_store_id, database, collection)
self._raise_for_missing_text_field(documents, params.text_field, database, collection)
return self._to_response(documents, query_text, params.text_field)
def transform_create_vector_store_request(
self,
vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams,
api_base: str,
) -> NoReturn:
raise config_error(_SEARCH_ONLY_MESSAGE)
def transform_create_vector_store_response(self, response: httpx.Response) -> NoReturn:
raise config_error(_SEARCH_ONLY_MESSAGE)

23
litellm/llms/perplexity/cost_calculator.py
View file

@ -3,19 +3,23 @@ Helper util for handling perplexity-specific cost calculation
- e.g.: citation tokens, search queries
"""
from datetime import datetime
from typing import Final
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
from litellm.types.utils import Usage
from litellm.utils import get_model_info
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
def cost_per_token(model: str, usage: Usage, current_time: datetime | None = None) -> tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
The manual fallback swaps in the model's off_peak_pricing rates while one of its windows is open.
Input:
- model: str, the model name without provider prefix
- usage: LiteLLM Usage block, containing perplexity-specific usage information
- current_time: the moment the request is billed at; defaults to now, UTC
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
@ -48,8 +52,21 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
except (ValueError, TypeError):
return default
rates: Final = apply_off_peak_pricing(
model_info,
current_time,
TokenRates(
input_rate=_safe_float_cast(model_info.get("input_cost_per_token")),
output_rate=_safe_float_cast(model_info.get("output_cost_per_token")),
cache_read_rate=0.0,
cache_creation_rate=0.0,
reasoning_rate=None,
),
)
input_cost_per_token: Final = rates.input_rate
output_cost_per_token: Final = rates.output_rate
## CALCULATE INPUT COST
input_cost_per_token: Final = _safe_float_cast(model_info.get("input_cost_per_token"))
prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token
## ADD CITATION TOKENS COST (if present)
@ -60,8 +77,6 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
prompt_cost += citation_tokens * citation_cost_per_token
## CALCULATE OUTPUT COST
output_cost_per_token: Final = _safe_float_cast(model_info.get("output_cost_per_token"))
reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0
if reasoning_tokens == 0 and hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:
reasoning_tokens = getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0

5008
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

3
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -928,6 +928,7 @@ def _resolve_openapi_tool_auth(
mcp_server_auth_headers,
alias=mcp_server.alias,
server_name=mcp_server.server_name,
access_groups=mcp_server.access_groups,
)
if mcp_server_auth_headers
else None
@ -3296,6 +3297,7 @@ class MCPServerManager:
mcp_server_auth_headers,
alias=server.alias,
server_name=server.server_name,
access_groups=server.access_groups,
)
# Fall back to deprecated mcp_auth_header if no server-specific header found
@ -5373,6 +5375,7 @@ class MCPServerManager:
mcp_server_auth_headers,
alias=mcp_server.alias,
server_name=mcp_server.server_name,
access_groups=mcp_server.access_groups,
)
# Fall back to deprecated mcp_auth_header if no server-specific header found

7
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -257,7 +257,7 @@ if MCP_AVAILABLE:
)
def _get_server_auth_header(
server,
server: MCPServer,
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
mcp_auth_header: str | None,
) -> dict[str, str] | str | None:
@ -269,8 +269,9 @@ if MCP_AVAILABLE:
if mcp_server_auth_headers:
server_auth: Final = lookup_mcp_server_auth_in_headers(
mcp_server_auth_headers,
alias=getattr(server, "alias", None),
server_name=getattr(server, "server_name", None),
alias=server.alias,
server_name=server.server_name,
access_groups=server.access_groups,
)
if server_auth is not None:
return server_auth

6
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -1612,7 +1612,10 @@ if MCP_AVAILABLE:
)
server_headers: Final = lookup_mcp_server_auth_in_headers(
mcp_server_auth_headers, alias=server.alias, server_name=server.server_name
mcp_server_auth_headers,
alias=server.alias,
server_name=server.server_name,
access_groups=server.access_groups,
)
if isinstance(server_headers, str):
return bool(server_headers.strip())
@ -1712,6 +1715,7 @@ if MCP_AVAILABLE:
mcp_server_auth_headers,
alias=server.alias,
server_name=server.server_name,
access_groups=server.access_groups,
)
extra_headers: dict[str, str] | None = None

49
litellm/proxy/_experimental/mcp_server/utils.py
View file

@ -8,11 +8,12 @@ import json
import os
import re
import typing
from collections.abc import Iterable, Iterator, Mapping, MutableMapping, MutableSequence
from collections.abc import Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Sequence
from collections.abc import Set as AbstractSet
from typing import Any, Final, Protocol
from urllib.parse import quote
from litellm._logging import verbose_logger
from litellm.types.mcp_server.mcp_server_manager import MCPServer
if typing.TYPE_CHECKING:
@ -169,34 +170,58 @@ def sanitize_mcp_alias_for_header(alias: str) -> str:
return sanitized.strip("_")
def _header_keys_for_identifier(identifier: str) -> tuple[str, ...]:
lowered: Final = identifier.lower()
sanitized: Final = sanitize_mcp_alias_for_header(identifier)
return (lowered,) if not sanitized or sanitized == lowered else (lowered, sanitized)
def _matching_header_key(normalized_headers: Mapping[str, object], identifier: str) -> str | None:
return next((key for key in _header_keys_for_identifier(identifier) if key in normalized_headers), None)
def lookup_mcp_server_auth_in_headers(
mcp_server_auth_headers: Mapping[str, str | dict[str, str]],
*,
alias: str | None = None,
server_name: str | None = None,
access_groups: Sequence[str] | None = None,
) -> str | dict[str, str] | None:
"""
Resolve server-specific auth headers with case-insensitive matching.
Tries the raw alias/server_name (lowercased) and the header-safe sanitized
alias so dashboard clients using sanitize_mcp_alias_for_header() still match.
When no server-level header matches, an ``x-mcp-{access_group}-*`` header is
used as the default for every server in that group. If the server belongs to
several groups that each carry a different credential, nothing is returned so
a token is never forwarded to a server it may not have been meant for.
"""
if not mcp_server_auth_headers:
return None
normalized_headers: Final = {k.lower(): v for k, v in mcp_server_auth_headers.items()}
for identifier in (alias, server_name):
if not identifier:
continue
keys_to_try = [identifier.lower()]
sanitized = sanitize_mcp_alias_for_header(identifier)
if sanitized and sanitized not in keys_to_try:
keys_to_try.append(sanitized)
for key in keys_to_try:
if key in normalized_headers:
return normalized_headers[key]
return None
server_keys: Final = (
_matching_header_key(normalized_headers, identifier) for identifier in (alias, server_name) if identifier
)
server_key: Final = next((key for key in server_keys if key is not None), None)
if server_key is not None:
return normalized_headers[server_key]
group_keys: Final = (_matching_header_key(normalized_headers, group) for group in access_groups or ())
group_matches: Final = tuple(normalized_headers[key] for key in group_keys if key is not None)
if not group_matches:
return None
if any(match != group_matches[0] for match in group_matches[1:]):
verbose_logger.debug(
"Ambiguous MCP group auth headers for server alias=%s (groups=%s); not forwarding any group credential",
alias,
access_groups,
)
return None
return group_matches[0]
MCP_TOOL_ALLOWLIST_ENFORCED_KEY: Final = "tool_allowlist_enforced"

242
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -741,6 +741,32 @@
"title": "AccessGroupInfo",
"type": "object"
},
"AccessGroupResource": {
"description": "A resource referenced by an access group. `name` is null when the id no longer resolves or has no alias.",
"properties": {
"id": {
"title": "Id",
"type": "string"
},
"name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Name"
}
},
"required": [
"id",
"name"
],
"title": "AccessGroupResource",
"type": "object"
},
"AccessGroupResponse": {
"properties": {
"access_agent_ids": {
@ -750,6 +776,13 @@
"title": "Access Agent Ids",
"type": "array"
},
"access_agents": {
"items": {
"$ref": "#/components/schemas/AccessGroupResource"
},
"title": "Access Agents",
"type": "array"
},
"access_group_id": {
"title": "Access Group Id",
"type": "string"
@ -765,6 +798,13 @@
"title": "Access Mcp Server Ids",
"type": "array"
},
"access_mcp_servers": {
"items": {
"$ref": "#/components/schemas/AccessGroupResource"
},
"title": "Access Mcp Servers",
"type": "array"
},
"access_model_names": {
"items": {
"type": "string"
@ -779,6 +819,13 @@
"title": "Assigned Key Ids",
"type": "array"
},
"assigned_keys": {
"items": {
"$ref": "#/components/schemas/AccessGroupResource"
},
"title": "Assigned Keys",
"type": "array"
},
"assigned_team_ids": {
"items": {
"type": "string"
@ -786,6 +833,13 @@
"title": "Assigned Team Ids",
"type": "array"
},
"assigned_teams": {
"items": {
"$ref": "#/components/schemas/AccessGroupResource"
},
"title": "Assigned Teams",
"type": "array"
},
"created_at": {
"format": "date-time",
"title": "Created At",
@ -838,6 +892,10 @@
"access_agent_ids",
"assigned_team_ids",
"assigned_key_ids",
"access_mcp_servers",
"access_agents",
"assigned_teams",
"assigned_keys",
"created_at",
"updated_at"
],
@ -3151,6 +3209,17 @@
}
],
"title": "Team Id"
},
"user_email": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "User Email"
}
},
"title": "KeyMetadata",
@ -13039,6 +13108,59 @@
],
"title": "Avgscore"
},
"cost": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Cost"
},
"cost_by_key": {
"additionalProperties": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
]
},
"title": "Cost By Key",
"type": "object"
},
"cost_by_team": {
"additionalProperties": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
]
},
"title": "Cost By Team",
"type": "object"
},
"cost_by_unit": {
"additionalProperties": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
]
},
"title": "Cost By Unit",
"type": "object"
},
"description": {
"anyOf": [
{
@ -13089,6 +13211,13 @@
"title": "Type",
"type": "string"
},
"untracked_usage_units": {
"additionalProperties": {
"type": "integer"
},
"title": "Untracked Usage Units",
"type": "object"
},
"usage_units": {
"additionalProperties": {
"type": "integer"
@ -13140,7 +13269,12 @@
"usage_units",
"usage_units_daily",
"usage_units_by_team",
"usage_units_by_key"
"usage_units_by_key",
"cost",
"cost_by_unit",
"cost_by_team",
"cost_by_key",
"untracked_usage_units"
],
"title": "UsageDetailResponse",
"type": "object"
@ -13295,10 +13429,28 @@
"title": "Totalblocked",
"type": "integer"
},
"totalCost": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Totalcost"
},
"totalRequests": {
"title": "Totalrequests",
"type": "integer"
},
"totalUntrackedUsageUnits": {
"additionalProperties": {
"type": "integer"
},
"title": "Totaluntrackedusageunits",
"type": "object"
},
"totalUsageUnits": {
"additionalProperties": {
"type": "integer"
@ -13313,7 +13465,9 @@
"totalRequests",
"totalBlocked",
"passRate",
"totalUsageUnits"
"totalUsageUnits",
"totalCost",
"totalUntrackedUsageUnits"
],
"title": "UsageOverviewResponse",
"type": "object"
@ -13342,6 +13496,18 @@
],
"title": "Avgscore"
},
"cost": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"description": "USD for the priced share of usageUnits over the window; null when no unit was priced",
"title": "Cost"
},
"failRate": {
"title": "Failrate",
"type": "number"
@ -13374,6 +13540,14 @@
"title": "Type",
"type": "string"
},
"untrackedUsageUnits": {
"additionalProperties": {
"type": "integer"
},
"description": "The share of usageUnits that cost leaves out: units recorded with no known price, per counter",
"title": "Untrackedusageunits",
"type": "object"
},
"usageUnits": {
"additionalProperties": {
"type": "integer"
@ -13393,13 +13567,26 @@
"avgLatency",
"status",
"trend",
"usageUnits"
"usageUnits",
"cost",
"untrackedUsageUnits"
],
"title": "UsageOverviewRow",
"type": "object"
},
"UsageUnitsDailyPoint": {
"properties": {
"cost": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Cost"
},
"date": {
"title": "Date",
"type": "string"
@ -13414,7 +13601,8 @@
},
"required": [
"date",
"units"
"units",
"cost"
],
"title": "UsageUnitsDailyPoint",
"type": "object"
@ -28773,10 +28961,28 @@
"title": "Totalblocked",
"type": "integer"
},
"totalCost": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Totalcost"
},
"totalRequests": {
"title": "Totalrequests",
"type": "integer"
},
"totalUntrackedUsageUnits": {
"additionalProperties": {
"type": "integer"
},
"title": "Totaluntrackedusageunits",
"type": "object"
},
"totalUsageUnits": {
"additionalProperties": {
"type": "integer"
@ -28791,7 +28997,9 @@
"totalRequests",
"totalBlocked",
"passRate",
"totalUsageUnits"
"totalUsageUnits",
"totalCost",
"totalUntrackedUsageUnits"
],
"title": "UsageOverviewResponse",
"type": "object"
@ -28820,6 +29028,18 @@
],
"title": "Avgscore"
},
"cost": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"description": "USD for the priced share of usageUnits over the window; null when no unit was priced",
"title": "Cost"
},
"failRate": {
"title": "Failrate",
"type": "number"
@ -28852,6 +29072,14 @@
"title": "Type",
"type": "string"
},
"untrackedUsageUnits": {
"additionalProperties": {
"type": "integer"
},
"description": "The share of usageUnits that cost leaves out: units recorded with no known price, per counter",
"title": "Untrackedusageunits",
"type": "object"
},
"usageUnits": {
"additionalProperties": {
"type": "integer"
@ -28871,7 +29099,9 @@
"avgLatency",
"status",
"trend",
"usageUnits"
"usageUnits",
"cost",
"untrackedUsageUnits"
],
"title": "UsageOverviewRow",
"type": "object"

2
litellm/proxy/_types.py
View file

@ -670,6 +670,7 @@ class LiteLLMRoutes(enum.Enum):
"/team/permissions_bulk_update",
"/team/daily/activity",
"/team/daily/activity/aggregated",
"/team/spend/by_user",
# gateway request counts (SGR); deployment-wide, admin-only
"/gateway/daily/activity",
# model
@ -832,6 +833,7 @@ class LiteLLMRoutes(enum.Enum):
"/team/permissions_update",
"/team/daily/activity",
"/team/daily/activity/aggregated",
"/team/spend/by_user",
"/team/{team_id}/members/me",
"/model/new",
"/model/update",

19
litellm/proxy/auth/auth_checks.py
View file

@ -142,6 +142,8 @@ class _PrismaDictableRow(Protocol):
class _PrismaJWTKeyMappingRow(Protocol):
token: str
jwt_claim_name: str
jwt_claim_value: str
class _PrismaModelDumpRow(Protocol):
@ -3466,6 +3468,23 @@ async def _fetch_key_object_from_db_with_reconnect(
raise
def jwt_key_mapping_cache_key(jwt_claim_name: str, jwt_claim_value: str) -> str:
"""Cache key under which ``_resolve_jwt_to_virtual_key`` stores a JWT-claim-to-key mapping."""
return f"jwt_key_mapping:{jwt_claim_name}:{jwt_claim_value}"
@log_db_metrics
async def get_jwt_key_mapping_cache_keys_for_token(
hashed_token: str,
prisma_client: PrismaClient,
) -> tuple[str, ...]:
"""Cache keys of every JWT claim mapped to the given virtual key."""
mappings: Final = await _jwt_key_mapping_table(JWTKeyMappingRepository(prisma_client)).find_many(
where={"token": hashed_token}
)
return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value) for m in mappings)
@log_db_metrics
async def get_jwt_key_mapping_object(
jwt_claim_name: str,

3
litellm/proxy/auth/user_api_key_auth.py
View file

@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_checks import (
get_team_object,
get_user_object,
is_valid_fallback_model,
jwt_key_mapping_cache_key,
resolve_and_validate_end_user_id,
)
from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler
@ -970,7 +971,7 @@ async def _resolve_jwt_to_virtual_key(
)
return None
cache_key: Final = f"jwt_key_mapping:{virtual_key_claim_field}:{claim_value}"
cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value))
cached_mapping: Final = await user_api_key_cache.async_get_cache(cache_key)
if cached_mapping == _JWT_PROXY_ADMIN_SENTINEL:

2
litellm/proxy/client/cli/README.md
View file

@ -489,7 +489,7 @@ lite codex exec "summarize the repo"
Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, and older versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol).
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, and older versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway.
Options (these belong to the wrapper, so put them before the agent's own flags):

176
litellm/proxy/client/cli/commands/agents.py
View file

@ -3,10 +3,13 @@ import shutil
import subprocess
import sys
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
import click
import requests
from pydantic import BaseModel, TypeAdapter, ValidationError
from .auth import context_secret_vault, get_stored_api_key, login
from .cmd_quoting import quote_for_cmd
@ -20,6 +23,12 @@ ENABLE_GATEWAY_MODEL_DISCOVERY_ENV: Final = "CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DI
ENABLE_GATEWAY_MODEL_DISCOVERY_VALUE: Final = "1"
OPENAI_BASE_URL_ENV: Final = "OPENAI_BASE_URL"
OPENAI_API_KEY_ENV: Final = "OPENAI_API_KEY"
OPENCODE_CONFIG_CONTENT_ENV: Final = "OPENCODE_CONFIG_CONTENT"
OPENCODE_PROVIDER_ID: Final = "litellm"
OPENCODE_PROVIDER_NAME: Final = "LiteLLM"
OPENCODE_PROVIDER_NPM: Final = "@ai-sdk/openai-compatible"
_SKIP_VERIFY_FLAG: Final = "--skip-verify"
PROFILE_ANTHROPIC: Final = "anthropic"
PROFILE_OPENAI: Final = "openai"
@ -131,6 +140,139 @@ def agent_launch_args(command: str, base_url: str) -> list[str]:
return builder(base_url) if builder else []
class ListedModel(BaseModel):
"""The fields of a /v1/models entry that an OpenCode model entry is built from."""
id: str
mode: str | None = None
max_input_tokens: int | None = None
max_output_tokens: int | None = None
class _ModelListing(BaseModel):
data: tuple[ListedModel, ...]
_MODEL_LISTING: Final = TypeAdapter(_ModelListing)
_OPENCODE_CHAT_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"})
_NO_EXTRA_ENV: Final[Mapping[str, str]] = MappingProxyType({})
@dataclass(frozen=True, slots=True)
class ModelSyncSkipped:
reason: str
class _OpenCodeLimit(BaseModel):
context: int
output: int
class _OpenCodeModel(BaseModel):
name: str
limit: _OpenCodeLimit | None = None
class _OpenCodeProviderOptions(BaseModel):
baseURL: str
apiKey: str
class _OpenCodeProvider(BaseModel):
npm: str
name: str
options: _OpenCodeProviderOptions
models: Mapping[str, _OpenCodeModel]
class _OpenCodeConfig(BaseModel):
provider: Mapping[str, _OpenCodeProvider]
def _opencode_model_entry(model: ListedModel) -> _OpenCodeModel:
if model.max_input_tokens is None or model.max_output_tokens is None:
return _OpenCodeModel(name=model.id)
return _OpenCodeModel(
name=model.id, limit=_OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens)
)
def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> str:
"""OPENCODE_CONFIG_CONTENT declaring the proxy as OpenCode provider `litellm`.
One model entry per chat-capable /v1/models row (mode chat, responses, or
unknown), so OpenCode's model picker mirrors what the key can call. The key
is read back through {env:OPENAI_API_KEY}, which build_agent_env exports, so
it never lands in the config text. OpenCode merges this inline config over
the user's own files, leaving unrelated keys and providers untouched.
"""
chat_models: Final = tuple(m for m in models if m.mode is None or m.mode in _OPENCODE_CHAT_MODES)
provider: Final = _OpenCodeProvider(
npm=OPENCODE_PROVIDER_NPM,
name=OPENCODE_PROVIDER_NAME,
options=_OpenCodeProviderOptions(
baseURL=base_url.rstrip("/") + "/v1",
apiKey=f"{{env:{OPENAI_API_KEY_ENV}}}",
),
models=MappingProxyType({m.id: _opencode_model_entry(m) for m in chat_models}),
)
config: Final = _OpenCodeConfig(provider=MappingProxyType({OPENCODE_PROVIDER_ID: provider}))
return config.model_dump_json(exclude_none=True)
def opencode_model_sync_env(
base_env: Mapping[str, str],
base_url: str,
api_key: str,
*,
get: Callable[..., requests.Response] = requests.get,
) -> Mapping[str, str] | ModelSyncSkipped:
"""Env addition that hands OpenCode the proxy's model list, or why it was skipped.
Fetches /v1/models with the key and packs it into OPENCODE_CONFIG_CONTENT.
An OPENCODE_CONFIG_CONTENT already in the environment is left alone, and a
failed fetch is reported rather than raised: OpenCode still launches on the
plain OPENAI_* env, just without a synced model list.
"""
if OPENCODE_CONFIG_CONTENT_ENV in base_env:
return ModelSyncSkipped(f"{OPENCODE_CONFIG_CONTENT_ENV} is already set")
url: Final = base_url.rstrip("/") + "/v1/models"
try:
resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
except requests.RequestException as e:
return ModelSyncSkipped(f"could not reach {url}: {e}")
if resp.status_code != 200:
return ModelSyncSkipped(f"{url} returned HTTP {resp.status_code}")
try:
listing: Final = _MODEL_LISTING.validate_json(resp.content)
except ValidationError:
return ModelSyncSkipped(f"{url} returned an unexpected body")
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing.data)})
def agent_model_sync_env(
command: str,
base_env: Mapping[str, str],
base_url: str,
api_key: str,
skip_verify: bool,
*,
get: Callable[..., requests.Response] = requests.get,
) -> Mapping[str, str] | ModelSyncSkipped:
"""Extra env an agent needs to see the proxy's model list.
Only OpenCode needs one: Claude Code discovers models through
CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY and Codex takes the model by name.
skip_verify means the caller wants no pre-launch proxy call at all, so the
listing is skipped too rather than hanging on an offline proxy.
"""
if os.path.basename(command) != "opencode":
return _NO_EXTRA_ENV
if skip_verify:
return ModelSyncSkipped(f"{_SKIP_VERIFY_FLAG} was passed")
return opencode_model_sync_env(base_env, base_url, api_key, get=get)
def verify_proxy_key(
base_url: str,
api_key: str,
@ -246,6 +388,10 @@ def _restore_controlling_terminal() -> None:
os.close(fd)
def _warn(message: str) -> None:
click.echo(message, err=True)
def run_agent(
base_url: str,
api_key: str,
@ -255,6 +401,10 @@ def run_agent(
base_env: Mapping[str, str] | None = None,
which: Callable[[str], str | None] = shutil.which,
verify: Callable[[str, str], None] = verify_proxy_key,
sync_models: Callable[[str, Mapping[str, str], str, str, bool], Mapping[str, str] | ModelSyncSkipped] = (
agent_model_sync_env
),
warn: Callable[[str], None] = _warn,
launcher: Callable[[str, Sequence[str], Mapping[str, str]], None] = _hand_off,
reattach_terminal: Callable[[], None] | None = None,
) -> None:
@ -262,13 +412,15 @@ def run_agent(
On success this never returns: POSIX replaces the current process, Windows
waits on the agent and exits with its status. Raises AgentRunError for
missing binaries, an unreachable proxy, or a rejected key.
missing binaries, an unreachable proxy, or a rejected key. The model list is
synced only once the key check passed, so an unreachable proxy costs one
timeout rather than two, and --skip-verify keeps the launch fully offline.
reattach_terminal, when given, runs just before handoff to restore stdin.
"""
if not command:
raise AgentRunError("Nothing to run.")
_, profiles = agent_profile(command[0])
display_name, profiles = agent_profile(command[0])
binary: Final = which(command[0])
if binary is None:
docs: Final = _INSTALL_DOCS.get(os.path.basename(command[0]))
@ -278,11 +430,16 @@ def run_agent(
if not skip_verify:
verify(base_url, api_key)
env: Final = build_agent_env(
base_env if base_env is not None else os.environ,
base_url,
api_key,
profiles,
env_before_sync: Final = base_env if base_env is not None else os.environ
synced: Final = sync_models(command[0], env_before_sync, base_url, api_key, skip_verify)
if isinstance(synced, ModelSyncSkipped):
warn(f"litellm: not syncing {display_name} models from the proxy: {synced.reason}")
env: Final = MappingProxyType(
{
**build_agent_env(env_before_sync, base_url, api_key, profiles),
**(_NO_EXTRA_ENV if isinstance(synced, ModelSyncSkipped) else synced),
}
)
extra_args: Final = agent_launch_args(command[0], base_url)
if reattach_terminal is not None:
@ -365,10 +522,15 @@ def agent_commands() -> tuple[click.Command, ...]:
__all__ = [
"AgentRunError",
"ListedModel",
"ModelSyncSkipped",
"agent_commands",
"agent_launch_args",
"agent_model_sync_env",
"agent_profile",
"build_agent_env",
"opencode_model_sync_env",
"opencode_provider_config",
"resolve_api_key",
"run_agent",
"verify_proxy_key",

33
litellm/proxy/common_request_processing.py
View file

@ -2520,6 +2520,11 @@ class ProxyBaseLLMRequestProcessing:
# This handles cases like websearch_interception agentic loop
# which returns a non-streaming dict even for streaming requests
if self._is_streaming_response(response):
self._arm_detached_stream_failure_hook(
logging_obj=logging_obj,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
)
selected_data_generator = ProxyBaseLLMRequestProcessing.async_sse_data_generator(
response=response,
user_api_key_dict=user_api_key_dict,
@ -2875,6 +2880,34 @@ class ProxyBaseLLMRequestProcessing:
),
)
def _arm_detached_stream_failure_hook(
self,
logging_obj: LiteLLMLoggingObj,
user_api_key_dict: "UserAPIKeyAuth",
proxy_logging_obj: ProxyLogging,
) -> None:
"""Let a stream that fails after the client left still reach ``post_call_failure_hook``.
The client-facing generator reports a mid-stream failure itself, but once
the client disconnects that generator is gone and the detached upstream
drain is the only code that sees the provider error. It fires this closure
so the failed spend is still written and the budget reservation released;
a replacement error the hook raises has no client left to reach.
"""
request_data: Final = self.data
async def _on_detached_stream_failure(exc: Exception) -> None:
try:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=exc,
request_data=request_data,
)
except HTTPException:
return
logging_obj._on_detached_stream_failure = _on_detached_stream_failure
def _is_streaming_response(self, response: Any) -> bool:
"""
Check if the response object is actually a streaming response by inspecting its type.

16
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -2,11 +2,11 @@ import json
import re
from collections.abc import Collection, Mapping
from types import MappingProxyType, UnionType
from typing import Any, Final, Union, get_args, get_origin
from typing import Annotated, Any, Final, Union, get_args, get_origin
import orjson
from fastapi import Request, UploadFile, status
from typing_extensions import ReadOnly
from typing_extensions import NotRequired, ReadOnly, Required
from litellm._logging import verbose_proxy_logger
from litellm.constants import MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB
@ -18,6 +18,8 @@ from litellm.types.router import Deployment
_FORM_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-www-form-urlencoded", "multipart/form-data"})
_ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequired, ReadOnly, Required})
def _normalize_media_type(content_type: str) -> str:
"""Return the bare media type per RFC 7231: strip params, trim, lowercase."""
@ -42,9 +44,17 @@ def _is_json_content_type(content_type: str) -> bool:
return _normalize_media_type(content_type) == "application/json"
def _unqualified(annotation: object) -> object:
"""Which qualifiers ``get_type_hints`` already stripped varies by interpreter version, so peel them all."""
if get_origin(annotation) not in _ANNOTATION_QUALIFIERS:
return annotation
qualified: Final[tuple[object, ...]] = get_args(annotation)
return _unqualified(qualified[0])
def _numeric_form_type(annotation: object) -> type[int] | type[float] | None:
"""The scalar to parse an ``int``/``float``-typed field as, else ``None``."""
unwrapped: Final = get_args(annotation)[0] if get_origin(annotation) is ReadOnly else annotation
unwrapped: Final = _unqualified(annotation)
candidates: Final = (
tuple(arg for arg in get_args(unwrapped) if arg is not type(None))
if get_origin(unwrapped) in (Union, UnionType)

21
litellm/proxy/common_utils/reset_budget_job.py
View file

@ -38,6 +38,7 @@ from litellm.proxy.common_utils.timezone_utils import (
get_budget_reset_settings,
)
from litellm.proxy.common_utils.user_api_key_cache import (
end_user_cache_key,
model_access_group_cache_key,
model_access_group_spend_counter_key,
tag_cache_key,
@ -177,6 +178,21 @@ def _model_access_group_cache_keys(row: _ModelAccessGroupRow) -> tuple[str, ...]
return (model_access_group_cache_key(row.access_group_name),)
def _enduser_counter_key(row: _EndUserRow) -> str:
return f"spend:end_user:{row.user_id}"
def _enduser_cache_keys(row: _EndUserRow) -> tuple[str, ...]:
return (end_user_cache_key(row.user_id),)
def _enduser_carried_spend(row: _EndUserRow, caps: Mapping[str, float]) -> float:
if not caps:
return 0.0
effective_budget_id: Final[str | None] = row.budget_id or litellm.max_end_user_budget_id
return _carried_spend(row.spend, caps.get(effective_budget_id) if effective_budget_id is not None else None)
def _budget_link_where(
budget_ids: Sequence[str],
extra: Mapping[str, object] = MappingProxyType({}),
@ -650,6 +666,7 @@ class ResetBudgetJob:
if _rollover_enabled()
else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType
)
endusers: Final[tuple[_EndUserRow, ...]] = await self._collect_endusers_to_reset(budget_ids)
return _BudgetCascade(
budgets=tuple(budgets_to_reset),
budget_ids=budget_ids,
@ -661,7 +678,7 @@ class ResetBudgetJob:
for b in budgets_to_reset
if b.budget_id is not None and b.budget_duration is not None
),
endusers=await self._collect_endusers_to_reset(budget_ids),
endusers=endusers,
counter_resets=(
*(
(_team_membership_counter_key(row), _row_carried_spend(row, rollover_caps))
@ -674,6 +691,7 @@ class ResetBudgetJob:
(_model_access_group_counter_key(row), _row_carried_spend(row, rollover_caps))
for row in model_access_groups
),
*((_enduser_counter_key(row), _enduser_carried_spend(row, rollover_caps)) for row in endusers),
),
rollover_caps=rollover_caps,
cache_keys=(
@ -682,6 +700,7 @@ class ResetBudgetJob:
*(key for row in orgs for key in _org_cache_keys(row)),
*(key for row in tags for key in _tag_cache_keys(row)),
*(key for row in model_access_groups for key in _model_access_group_cache_keys(row)),
*(key for row in endusers for key in _enduser_cache_keys(row)),
),
)

25
litellm/proxy/db/spend_counter_reseed.py
View file

@ -26,6 +26,7 @@ from litellm.proxy._types import Litellm_EntityType
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.table_repositories import (
BudgetWindowSpendRepository,
EndUserRepository,
SpendLogsRepository,
TeamMembershipRepository,
)
@ -36,6 +37,8 @@ from litellm.repositories.verification_token_repository import (
)
if TYPE_CHECKING:
from prisma.types import LiteLLM_EndUserTableWhereUniqueInput
from litellm.caching.dual_cache import DualCache
from litellm.proxy.utils import PrismaClient
@ -47,6 +50,8 @@ _WINDOW_SPEND_ENTITY_TYPES: Final[Mapping[str, str]] = MappingProxyType(
}
)
END_USER_COUNTER_PREFIX: Final = "spend:end_user:"
_WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{
"Key": "api_key",
@ -74,6 +79,10 @@ class SpendCounterReseed:
End-user and tag spend counters intentionally do not reseed here. Their
auth paths already load the corresponding objects via get_end_user_object()
and get_tag_objects_batch(); callers pass those values as fallback_spend.
end_user_from_db is the one end-user read, used only as the budget floor when
a counter sits below that cached spend: a worker that did not run the budget
reset still caches the pre-reset end-user object, and LiteLLM_EndUserTable
is the row the reset zeroed.
"""
_locks: ClassVar["OrderedDict[str, asyncio.Lock]"] = OrderedDict()
@ -129,7 +138,7 @@ class SpendCounterReseed:
elif counter_key.startswith("spend:user:"):
user_id = counter_key[len("spend:user:") :]
row = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_id})
elif counter_key.startswith("spend:end_user:") or counter_key.startswith("spend:tag:"):
elif counter_key.startswith(END_USER_COUNTER_PREFIX) or counter_key.startswith("spend:tag:"):
return None
elif counter_key.startswith("spend:org:"):
org_id: Final = counter_key[len("spend:org:") :]
@ -143,6 +152,20 @@ class SpendCounterReseed:
return None
return float(getattr(row, "spend", 0.0) or 0.0)
@staticmethod
async def end_user_from_db(prisma_client: Optional["PrismaClient"], counter_key: str) -> float | None:
if prisma_client is None or not counter_key.startswith(END_USER_COUNTER_PREFIX):
return None
where: Final[LiteLLM_EndUserTableWhereUniqueInput] = {"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]}
try:
row: Final = await EndUserRepository(prisma_client).table.find_unique(where=where)
except Exception: # noqa: BLE001 # a failed floor read falls back to the cached spend, like from_db
verbose_proxy_logger.exception("SpendCounterReseed.end_user_from_db: failed for %s", counter_key)
return None
if row is None:
return None
return float(row.spend or 0.0)
@staticmethod
def _is_key_or_team_window_counter(counter_key: str) -> bool:
for prefix in ("spend:key:", "spend:team:"):

48
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -36,7 +36,10 @@ from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_rege
from litellm.litellm_core_utils.litellm_logging import (
_get_masked_values, # pyright: ignore[reportPrivateUsage] # the shared header-masking helper has no public name
)
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import bedrock_guardrail_cost
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
bedrock_guardrail_cost_by_unit,
guardrail_cost_total,
)
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
@ -109,6 +112,7 @@ _BEDROCK_TOO_LARGE_ERROR_SUBSTRINGS: Final = (
_BEDROCK_APPLY_GUARDRAIL_MAX_THROTTLE_RETRIES: Final = 3
_BEDROCK_APPLY_GUARDRAIL_BASE_BACKOFF_SECONDS: Final = 0.5
_BEDROCK_WHITESPACE: Final = re.compile(r"\s")
_NO_TRACING_DETAIL: Final[GuardrailTracingDetail] = {}
# Resource-less, detect-only InvokeGuardrailChecks API (no guardrail resource required).
_BEDROCK_INVOKE_GUARDRAIL_CHECKS_PATH: Final = "/guardrail-checks/invoke"
# InvokeGuardrailChecks accepts at most 10 content blocks per message. A message with
@ -2155,25 +2159,37 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
OTEL integration can expose it as a queryable span attribute
without re-parsing the redacted guardrail_response blob.
"""
tracing_detail: Final[GuardrailTracingDetail] = {}
violation_categories: Final = self._extract_violation_category_names(response)
if violation_categories:
tracing_detail["violation_categories"] = violation_categories
bedrock_action: Final = response.get("action")
if isinstance(bedrock_action, str):
tracing_detail["guardrail_action"] = bedrock_action
usage: Final = response.get("usage")
if isinstance(usage, dict):
usage_units: Final = { # mutable-ok: json.dumps'd into spend log metadata downstream
key: value for key, value in usage.items() if isinstance(value, int)
}
if usage_units:
tracing_detail["guardrail_usage"] = usage_units
tracing_detail["guardrail_cost"] = bedrock_guardrail_cost(
usage_units=usage_units, aws_region_name=aws_region_name
)
categories_detail: Final[GuardrailTracingDetail] = {"violation_categories": violation_categories}
action_detail: Final[GuardrailTracingDetail] = {"guardrail_action": bedrock_action}
tracing_detail: Final[GuardrailTracingDetail] = {
**(categories_detail if violation_categories else _NO_TRACING_DETAIL),
**(action_detail if isinstance(bedrock_action, str) else _NO_TRACING_DETAIL),
**self._usage_tracing_detail(response.get("usage"), aws_region_name),
}
return tracing_detail
@staticmethod
def _usage_tracing_detail(
usage: BedrockGuardrailUsage | None, aws_region_name: str | None
) -> GuardrailTracingDetail:
if not isinstance(usage, dict):
return _NO_TRACING_DETAIL
usage_units: Final = { # mutable-ok: json.dumps'd into spend log metadata downstream
key: value for key, value in usage.items() if isinstance(value, int)
}
if not usage_units:
return _NO_TRACING_DETAIL
cost_by_unit: Final = bedrock_guardrail_cost_by_unit(usage_units=usage_units, aws_region_name=aws_region_name)
priced_detail: Final[GuardrailTracingDetail] = {"guardrail_cost_by_unit": cost_by_unit}
usage_detail: Final[GuardrailTracingDetail] = {
"guardrail_usage": usage_units,
"guardrail_cost": guardrail_cost_total(cost_by_unit),
**(priced_detail if cost_by_unit is not None else _NO_TRACING_DETAIL),
}
return usage_detail
def _extract_violation_category_names(self, response: BedrockGuardrailResponse) -> list[str]:
"""
Flatten the BLOCKED assessments into a list of human-readable category

1
litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py
View file

@ -35,6 +35,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) ->
event_hook=_coerce_event_hook(litellm_params.mode),
default_on=litellm_params.default_on or False,
unreachable_fallback=litellm_params.unreachable_fallback,
timeout=litellm_params.timeout,
)
litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType]
_callback

33
litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
View file

@ -1,6 +1,7 @@
from __future__ import annotations
import json
import math
import re
import time
import uuid
@ -15,6 +16,7 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.compression.compress import get_protected_indices
from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
@ -47,12 +49,16 @@ from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.guardrails import LitellmParams
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
BYPASS_HEADER: Final = "x-headroom-bypass"
_STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset(
(CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses)
)
# The shared GuardrailCallback client carries no per-call bound, so without this a
# stalled service holds the caller's request and a pooled connection for 600s or more.
_COMPRESS_TIMEOUT_SECONDS: Final = 60.0
HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve"
_HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})")
_HASH_CACHE_TTL_SECONDS: Final = 15 * 60
@ -472,6 +478,7 @@ class HeadroomGuardrail(CustomGuardrail):
event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | Mode | None = None,
default_on: bool = False,
unreachable_fallback: str | None = None,
timeout: float | None = None,
):
self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/")
if not self.headroom_api_base:
@ -484,6 +491,7 @@ class HeadroomGuardrail(CustomGuardrail):
self.unreachable_fallback: Literal["fail_closed", "fail_open"] = (
"fail_open" if unreachable_fallback == "fail_open" else "fail_closed"
)
self.timeout: httpx.Timeout = self._resolve_timeout(timeout)
self.async_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.GuardrailCallback,
)
@ -511,6 +519,29 @@ class HeadroomGuardrail(CustomGuardrail):
headers["Authorization"] = f"Bearer {self.headroom_api_key}"
return headers
@staticmethod
def _resolve_timeout(timeout: float | None) -> httpx.Timeout:
"""Budget for one call to the compression service, unset meaning the default.
Zero, negative and non-finite values are rejected instead of passed through:
httpx accepts them, and the transport then reads 0 and inf as no deadline at
all and a negative one as a deadline already past.
"""
rejected: Final = timeout is not None and not (math.isfinite(timeout) and timeout > 0)
if rejected:
verbose_proxy_logger.warning(
"Headroom: ignoring unusable timeout %s, using %s seconds",
timeout,
_COMPRESS_TIMEOUT_SECONDS,
)
seconds: Final = _COMPRESS_TIMEOUT_SECONDS if timeout is None or rejected else timeout
return httpx.Timeout(timeout=seconds, connect=min(seconds, HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS))
def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None:
"""Re-resolve the timeout, which the base implementation would otherwise null out."""
super().update_in_memory_litellm_params(litellm_params)
self.timeout = self._resolve_timeout(litellm_params.timeout)
def _prune_expired_hashes(self) -> None:
now: Final = time.monotonic()
self._issued_hashes_by_call_id = {
@ -548,6 +579,7 @@ class HeadroomGuardrail(CustomGuardrail):
url=f"{self.headroom_api_base}/v1/compress",
json=payload,
headers=self._request_headers(),
timeout=self.timeout,
)
except httpx.HTTPStatusError as e:
return (
@ -685,6 +717,7 @@ class HeadroomGuardrail(CustomGuardrail):
url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}",
params=params,
headers=self._request_headers(),
timeout=self.timeout,
)
except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError, litellm.Timeout) as e:
verbose_proxy_logger.warning("Headroom: retrieve failed for hash=%s: %s", hash_value, e)

112
litellm/proxy/guardrails/usage_endpoints.py
View file

@ -8,10 +8,10 @@ from collections.abc import Callable, Iterable, Mapping, Sequence
from datetime import date, datetime, timedelta, timezone
from itertools import groupby
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, overload
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, overload
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel
from pydantic import BaseModel, Field
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -42,6 +42,8 @@ router: Final = APIRouter()
_EMPTY_UNITS: Final[Mapping[str, int]] = MappingProxyType({})
_T = TypeVar("_T")
_USAGE_MAX_RANGE_DAYS: Final = 366
@ -154,6 +156,16 @@ def _counter_name(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str:
return row.usage_unit
def _row_untracked_units(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> int:
"""A row written before the cost column carries NULL cost and is untracked in full."""
return int(row.units) if row.cost is None else int(row.untracked_units)
def _row_tracked_cost(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> float | None:
"""The row's cost when it prices at least one unit; None when every unit is untracked."""
return None if row.cost is None or _row_untracked_units(row) >= int(row.units) else row.cost
def _sum_counter_units(rows: "Iterable[prisma_models.LiteLLM_DailyGuardrailUsageUnits]") -> Mapping[str, int]:
ordered: Final = sorted(rows, key=_counter_name)
return MappingProxyType(
@ -161,12 +173,31 @@ def _sum_counter_units(rows: "Iterable[prisma_models.LiteLLM_DailyGuardrailUsage
)
def _units_by(
def _sum_untracked_units(rows: "Iterable[prisma_models.LiteLLM_DailyGuardrailUsageUnits]") -> Mapping[str, int]:
ordered: Final = sorted(rows, key=_counter_name)
per_counter: Final = tuple(
(name, sum(map(_row_untracked_units, group))) for name, group in groupby(ordered, key=_counter_name)
)
return MappingProxyType({name: units for name, units in per_counter if units})
def _sum_tracked_cost(rows: "Iterable[prisma_models.LiteLLM_DailyGuardrailUsageUnits]") -> float | None:
"""Sum over rows that price at least one unit; None when no row does."""
tracked: Final = tuple(cost for cost in map(_row_tracked_cost, rows) if cost is not None)
return sum(tracked) if tracked else None
def _by(
rows: "Sequence[prisma_models.LiteLLM_DailyGuardrailUsageUnits]",
key_of: "Callable[[prisma_models.LiteLLM_DailyGuardrailUsageUnits], str]",
) -> Mapping[str, Mapping[str, int]]:
reduce: "Callable[[Iterable[prisma_models.LiteLLM_DailyGuardrailUsageUnits]], _T]",
) -> Mapping[str, _T]:
ordered: Final = sorted(rows, key=key_of)
return MappingProxyType({key: _sum_counter_units(group) for key, group in groupby(ordered, key=key_of)})
return MappingProxyType({key: reduce(group) for key, group in groupby(ordered, key=key_of)})
def _first_match(lookup_keys: Sequence[str], mapping: Mapping[str, _T], default: _T) -> _T:
return next((mapping[k] for k in lookup_keys if k in mapping), default)
# --- Response models ---
@ -218,6 +249,12 @@ class UsageOverviewRow(BaseModel):
status: str # healthy | warning | critical
trend: str # up | down | stable
usageUnits: Mapping[str, int]
cost: float | None = Field(
description="USD for the priced share of usageUnits over the window; null when no unit was priced"
)
untrackedUsageUnits: Mapping[str, int] = Field(
description="The share of usageUnits that cost leaves out: units recorded with no known price, per counter"
)
class UsageOverviewResponse(BaseModel):
@ -227,11 +264,26 @@ class UsageOverviewResponse(BaseModel):
totalBlocked: int
passRate: float
totalUsageUnits: Mapping[str, int]
totalCost: float | None
totalUntrackedUsageUnits: Mapping[str, int]
_EMPTY_OVERVIEW: Final = UsageOverviewResponse(
rows=[],
chart=[],
totalRequests=0,
totalBlocked=0,
passRate=100.0,
totalUsageUnits=_EMPTY_UNITS,
totalCost=None,
totalUntrackedUsageUnits=_EMPTY_UNITS,
)
class UsageUnitsDailyPoint(BaseModel):
date: str
units: Mapping[str, int]
cost: float | None
class UsageDetailResponse(BaseModel):
@ -251,6 +303,11 @@ class UsageDetailResponse(BaseModel):
usage_units_daily: Sequence[UsageUnitsDailyPoint]
usage_units_by_team: Mapping[str, Mapping[str, int]]
usage_units_by_key: Mapping[str, Mapping[str, int]]
cost: float | None
cost_by_unit: Mapping[str, float | None]
cost_by_team: Mapping[str, float | None]
cost_by_key: Mapping[str, float | None]
untracked_usage_units: Mapping[str, int]
class UsageLogEntry(BaseModel):
@ -367,6 +424,8 @@ def _guardrail_overview_rows(
agg: Mapping[str, _MetricTotals],
prev_agg: Mapping[str, float],
units_agg: Mapping[str, Mapping[str, int]],
cost_agg: Mapping[str, float | None],
untracked_agg: Mapping[str, Mapping[str, int]],
) -> list[UsageOverviewRow]:
rows: Final[list[UsageOverviewRow]] = []
covered_keys: Final[set[str]] = set()
@ -392,7 +451,6 @@ def _guardrail_overview_rows(
prev_fail = float(prev_agg.get(k, 0.0) or 0.0)
break
trend = _trend_from_comparison(fail_rate, prev_fail)
row_units: Mapping[str, int] = next((units_agg[k] for k in lookup_keys if k in units_agg), _EMPTY_UNITS)
rows.append(
UsageOverviewRow(
id=gid,
@ -405,7 +463,9 @@ def _guardrail_overview_rows(
avgLatency=None,
status=_status_from_fail_rate(fail_rate),
trend=trend,
usageUnits=row_units,
usageUnits=_first_match(lookup_keys, units_agg, _EMPTY_UNITS),
cost=_first_match(lookup_keys, cost_agg, None),
untrackedUsageUnits=_first_match(lookup_keys, untracked_agg, _EMPTY_UNITS),
)
)
# Add rows for guardrails with metrics but not in guardrails table (e.g. MCP, config)
@ -429,6 +489,8 @@ def _guardrail_overview_rows(
status=_status_from_fail_rate(fail_rate),
trend=trend,
usageUnits=units_agg.get(agg_key, _EMPTY_UNITS),
cost=cost_agg.get(agg_key),
untrackedUsageUnits=untracked_agg.get(agg_key, _EMPTY_UNITS),
)
)
return rows
@ -459,6 +521,8 @@ def _policy_overview_rows(
status=_status_from_fail_rate(fail_rate),
trend=trend,
usageUnits=_EMPTY_UNITS,
cost=None,
untrackedUsageUnits=_EMPTY_UNITS,
)
)
return rows
@ -479,9 +543,7 @@ async def guardrails_usage_overview(
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
return UsageOverviewResponse(
rows=[], chart=[], totalRequests=0, totalBlocked=0, passRate=100.0, totalUsageUnits=_EMPTY_UNITS
)
return _EMPTY_OVERVIEW
start, end = _resolve_usage_window(start_date, end_date)
@ -515,12 +577,14 @@ async def guardrails_usage_overview(
agg: Final = _aggregate_daily_metrics(metrics, "guardrail_id")
prev_agg: Final = _prev_fail_rates(metrics_prev, "guardrail_id")
units_agg: Final = _units_by(units_rows, lambda r: r.guardrail_id)
units_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_counter_units)
cost_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_tracked_cost)
untracked_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_untracked_units)
chart: Final = _chart_from_metrics(metrics)
total_requests: Final = sum(a["requests"] for a in agg.values())
total_blocked: Final = sum(a["blocked"] for a in agg.values())
pass_rate: Final = (100.0 * (total_requests - total_blocked) / total_requests) if total_requests else 100.0
rows: Final = _guardrail_overview_rows(guardrails, agg, prev_agg, units_agg)
rows: Final = _guardrail_overview_rows(guardrails, agg, prev_agg, units_agg, cost_agg, untracked_agg)
return UsageOverviewResponse(
rows=rows,
chart=chart,
@ -528,6 +592,8 @@ async def guardrails_usage_overview(
totalBlocked=total_blocked,
passRate=round(pass_rate, 1),
totalUsageUnits=_sum_counter_units(units_rows),
totalCost=_sum_tracked_cost(units_rows),
totalUntrackedUsageUnits=_sum_untracked_units(units_rows),
)
except Exception as e:
from litellm.proxy.utils import handle_exception_on_proxy
@ -618,8 +684,11 @@ async def guardrails_usage_detail(
litellm_params: Final = _to_dict(_get_guardrail_field(guardrail, "litellm_params"))
guardrail_info: Final = _to_dict(_get_guardrail_field(guardrail, "guardrail_info"))
_guardrail_name: Final = _get_guardrail_field(guardrail, "guardrail_name")
daily_unit_sums: Final = sorted(_units_by(units_rows, lambda r: r.date).items())
units_daily: Final = tuple(UsageUnitsDailyPoint(date=d, units=units) for d, units in daily_unit_sums)
daily_unit_sums: Final = sorted(_by(units_rows, lambda r: r.date, _sum_counter_units).items())
daily_cost: Final = _by(units_rows, lambda r: r.date, _sum_tracked_cost)
units_daily: Final = tuple(
UsageUnitsDailyPoint(date=d, units=units, cost=daily_cost.get(d)) for d, units in daily_unit_sums
)
return UsageDetailResponse(
guardrail_id=guardrail_id,
@ -636,8 +705,13 @@ async def guardrails_usage_detail(
time_series=time_series,
usage_units=_sum_counter_units(units_rows),
usage_units_daily=units_daily,
usage_units_by_team=_units_by(units_rows, lambda r: r.team_id),
usage_units_by_key=_units_by(units_rows, lambda r: r.api_key),
usage_units_by_team=_by(units_rows, lambda r: r.team_id, _sum_counter_units),
usage_units_by_key=_by(units_rows, lambda r: r.api_key, _sum_counter_units),
cost=_sum_tracked_cost(units_rows),
cost_by_unit=_by(units_rows, _counter_name, _sum_tracked_cost),
cost_by_team=_by(units_rows, lambda r: r.team_id, _sum_tracked_cost),
cost_by_key=_by(units_rows, lambda r: r.api_key, _sum_tracked_cost),
untracked_usage_units=_sum_untracked_units(units_rows),
)
@ -857,9 +931,7 @@ async def policies_usage_overview(
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
return UsageOverviewResponse(
rows=[], chart=[], totalRequests=0, totalBlocked=0, passRate=100.0, totalUsageUnits=_EMPTY_UNITS
)
return _EMPTY_OVERVIEW
start, end = _resolve_usage_window(start_date, end_date)
@ -891,6 +963,8 @@ async def policies_usage_overview(
totalBlocked=total_blocked,
passRate=round(pass_rate, 1),
totalUsageUnits=_EMPTY_UNITS,
totalCost=None,
totalUntrackedUsageUnits=_EMPTY_UNITS,
)
except Exception as e:
from litellm.proxy.utils import handle_exception_on_proxy

73
litellm/proxy/guardrails/usage_tracking.py
View file

@ -6,7 +6,7 @@ insert into SpendLogGuardrailIndex when spend logs are written.
import asyncio
import json
from collections import defaultdict
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from datetime import datetime, timezone
from functools import partial
from itertools import groupby
@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, NamedTuple, TypeVar
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import billed_guardrail_cost_by_unit
from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES
from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import (
@ -44,6 +45,20 @@ class _UsageUnitKey(NamedTuple):
usage_unit: str
class _UsageUnitIncrement(NamedTuple):
units: int
cost: float
"""USD for the priced share of units."""
untracked_units: int
"""Units recorded with no known price, the share cost leaves out."""
def _usage_unit_increment(units: int, cost: float | None) -> _UsageUnitIncrement:
if cost is None:
return _UsageUnitIncrement(units=units, cost=0.0, untracked_units=units)
return _UsageUnitIncrement(units=units, cost=cost, untracked_units=0)
class _MetricsKey(NamedTuple):
guardrail_id: str
date: str
@ -67,22 +82,37 @@ class PendingRollups:
def __init__(self) -> None:
self.lock: Final = asyncio.Lock()
self.metrics: Mapping[_MetricsKey, Mapping[str, int]] = MappingProxyType({})
self.units: Mapping[_UsageUnitKey, int] = MappingProxyType({})
self.units: Mapping[_UsageUnitKey, _UsageUnitIncrement] = MappingProxyType({})
_PENDING_ROLLUPS: Final = PendingRollups()
_NO_COUNTERS: Final[Mapping[str, int]] = MappingProxyType({})
_NO_INCREMENT: Final = _UsageUnitIncrement(units=0, cost=0.0, untracked_units=0)
def _merged_keys(base: Mapping[_RowKey, object], extra: Mapping[_RowKey, object]) -> tuple[_RowKey, ...]:
return (*base, *(key for key in extra if key not in base))
def _summed_increments(increments: Iterable[_UsageUnitIncrement]) -> _UsageUnitIncrement:
materialized: Final = tuple(increments)
return _UsageUnitIncrement(
units=sum(i.units for i in materialized),
cost=sum(i.cost for i in materialized),
untracked_units=sum(i.untracked_units for i in materialized),
)
def _merged_unit_rows(
base: Mapping[_UsageUnitKey, int], extra: Mapping[_UsageUnitKey, int]
) -> Mapping[_UsageUnitKey, int]:
return MappingProxyType({key: base.get(key, 0) + extra.get(key, 0) for key in _merged_keys(base, extra)})
base: Mapping[_UsageUnitKey, _UsageUnitIncrement], extra: Mapping[_UsageUnitKey, _UsageUnitIncrement]
) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]:
return MappingProxyType(
{
key: _summed_increments((base.get(key, _NO_INCREMENT), extra.get(key, _NO_INCREMENT)))
for key in _merged_keys(base, extra)
}
)
def _merged_metric_rows(
@ -209,7 +239,9 @@ def _parse_payload_start_time(payload: Mapping[str, Any]) -> datetime | None:
return None
def _iter_usage_unit_increments(logs_to_process: Sequence[Mapping[str, Any]]) -> Iterator[tuple[_UsageUnitKey, int]]:
def _iter_usage_unit_increments(
logs_to_process: Sequence[Mapping[str, Any]],
) -> Iterator[tuple[_UsageUnitKey, _UsageUnitIncrement]]:
for payload in logs_to_process:
start_time = _parse_payload_start_time(payload)
if not payload.get("request_id") or start_time is None:
@ -222,26 +254,38 @@ def _iter_usage_unit_increments(logs_to_process: Sequence[Mapping[str, Any]]) ->
usage = entry.get("guardrail_usage")
if not guardrail_id or not isinstance(usage, dict):
continue
cost_by_unit = billed_guardrail_cost_by_unit(entry)
for unit_name, units in usage.items():
if isinstance(units, int) and not isinstance(units, bool) and units > 0:
yield _UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name)), units
key = _UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name))
cost = cost_by_unit.get(str(unit_name)) if cost_by_unit is not None else None
yield key, _usage_unit_increment(units=units, cost=cost)
def _sum_usage_unit_increments(logs_to_process: Sequence[Mapping[str, Any]]) -> Mapping[_UsageUnitKey, int]:
def _sum_usage_unit_increments(
logs_to_process: Sequence[Mapping[str, Any]],
) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]:
ordered: Final = sorted(_iter_usage_unit_increments(logs_to_process), key=itemgetter(0))
return MappingProxyType(
{key: sum(units for _, units in group) for key, group in groupby(ordered, key=itemgetter(0))}
{
key: _summed_increments(increment for _, increment in group)
for key, group in groupby(ordered, key=itemgetter(0))
}
)
async def _upsert_usage_unit_row(prisma_client: PrismaClient, key: _UsageUnitKey, units: int) -> None:
async def _upsert_usage_unit_row(
prisma_client: PrismaClient, key: _UsageUnitKey, increment: _UsageUnitIncrement
) -> None:
row: Final[prisma_types.LiteLLM_DailyGuardrailUsageUnitsCreateInput] = {
"guardrail_id": key.guardrail_id,
"date": key.date,
"team_id": key.team_id,
"api_key": key.api_key,
"usage_unit": key.usage_unit,
"units": units,
"units": increment.units,
"cost": increment.cost,
"untracked_units": increment.untracked_units,
}
where: Final[_UsageUnitWhereUnique] = {
"guardrail_id_date_team_id_api_key_usage_unit": {
@ -252,9 +296,14 @@ async def _upsert_usage_unit_row(prisma_client: PrismaClient, key: _UsageUnitKey
"usage_unit": key.usage_unit,
}
}
# A row written before the cost column has NULL cost, and NULL + x stays NULL, so it keeps reading as unknown
data: Final[prisma_types.LiteLLM_DailyGuardrailUsageUnitsUpsertInput] = {
"create": row,
"update": {"units": {"increment": units}},
"update": {
"units": {"increment": increment.units},
"cost": {"increment": increment.cost},
"untracked_units": {"increment": increment.untracked_units},
},
}
await DailyGuardrailUsageUnitsRepository(prisma_client).table.upsert(where=where, data=data)

46
litellm/proxy/health_endpoints/_health_endpoints.py
View file

@ -115,6 +115,29 @@ _CONFIG_CONNECTION_FIELDS: Final[frozenset[str]] = frozenset(
)
def _request_inherits_config_credentials(
config_params: Mapping[str, object],
request_params: Mapping[str, object],
allow_client_side_credentials: bool,
) -> bool:
"""Whether the configuration's credentials are this request's to be probed with.
The configuration reached here by matching the request's model string, which
also matches wildcard routes and unrelated deployments that merely serve the
same model, so a request naming a stored credential of its own has already
said where its credentials come from and does not borrow that one's. A blank
name is no name: ``load_credentials_from_list`` resolves nothing from it, so
it must not cost the request the credentials it would otherwise be probed
with.
"""
requested_credential: Final = request_params.get("litellm_credential_name")
if requested_credential and requested_credential != config_params.get("litellm_credential_name"):
return False
if allow_client_side_credentials:
return True
return not any(param in request_params for param in _BANNED_REQUEST_BODY_PARAMS)
def _config_base_for_health_check(
config_params: Mapping[str, object],
request_params: Mapping[str, object],
@ -122,25 +145,19 @@ def _config_base_for_health_check(
) -> dict[str, object]:
"""Return the configured parameters to merge under a connection-test request.
A request that sets its own connection fields describes a connection of its
own, so the configuration's credentials are not carried into it: they belong
to the endpoint the configuration names. Anything the request does not set
still comes from the configuration, which is what lets a request name a
configured model and test it as configured.
A request that sets its own connection fields, or names its own stored
credential, describes a connection of its own, so the configuration's
credentials are not carried into it: they belong to the endpoint the
configuration names. Anything the request does not set still comes from the
configuration, which is what lets a request name a configured model and test
it as configured.
``litellm_credential_name`` is dropped alongside the literal credential
fields: it names a stored credential that ``load_credentials_from_list``
resolves into the same secrets further down the call, so leaving it in place
would reintroduce them by reference.
``general_settings.allow_client_side_credentials`` is the existing proxy-wide
opt-in for callers supplying their own connection parameters. Where an admin
has enabled it, a request may pair its own endpoint with the configured
credentials, as it could before.
"""
if allow_client_side_credentials:
return dict(config_params)
if not any(param in request_params for param in _BANNED_REQUEST_BODY_PARAMS):
if _request_inherits_config_credentials(config_params, request_params, allow_client_side_credentials):
return dict(config_params)
return {key: value for key, value in config_params.items() if key not in _CONFIG_CONNECTION_FIELDS}
@ -1959,6 +1976,9 @@ async def test_model_connection(
Note:
- If the model is configured in proxy_config.yaml, credentials (api_key, api_base, etc.)
will be automatically loaded from the config (with resolved environment variables).
- A request naming a stored credential (`litellm_credential_name`) that the configuration
does not name is probed with that credential instead, and inherits no credentials
from the configuration its model string happened to match.
- You can override specific params by including them in the request.
- You can use `os.environ/VARIABLE_NAME` syntax to reference environment variables,
which will be resolved automatically (same as in proxy_config.yaml).

58
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -4518,12 +4518,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
statuses=statuses,
)
def _recovered_partial_usage_tokens(self, source: Mapping[str, object]) -> tuple[int, int, int]:
usage: Final = source.get("combined_usage_object")
if not isinstance(usage, Usage) or (usage.completion_tokens or 0) <= 0:
return 0, 0, 0
billable_input, completion_tokens, _ = self._resolve_io_token_reconcile_usage(usage)
return (
self._get_total_tokens_from_usage(usage=usage, rate_limit_type=self.get_rate_limit_type()),
billable_input,
completion_tokens,
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""
On failure: decrement max_parallel_requests and refund the upfront
TPM reservation only against the scopes the reservation actually
charged. Unreserved scopes were never incremented at pre-call, so
refunding them would drive their counter negative.
refunding them would drive their counter negative. A failed stream
whose partial usage was recovered settles the reservation at that
usage instead of refunding it.
"""
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
@ -4552,31 +4565,31 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if stash is None or stash.reservation_released
else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
)
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(kwargs)
if stash is not None and reserved_tokens > 0:
verbose_proxy_logger.debug("Releasing reserved TPM tokens on failure: %s", reserved_tokens)
# Refund only against the scopes the reservation actually
# charged. _build_reservation_aware_tpm_ops with
# actual_tokens=0 emits -reserved on reserved scopes and 0
# on unreserved (skipped), so unreserved scopes can't drift
# negative.
verbose_proxy_logger.debug(
"Settling reserved TPM tokens on failure: reserved=%s actual=%s", reserved_tokens, tpm_actual
)
# Settle only against the scopes the reservation actually
# charged: unreserved scopes were never incremented, so a
# refund there would drive their counter negative.
pipeline_operations.extend(
self._build_reservation_aware_tpm_ops(
targets=list(stash.reserved_scopes),
reserved_scopes=stash.reserved_scopes,
actual_tokens=0,
actual_tokens=tpm_actual,
reserved_tokens=reserved_tokens,
)
)
# Refund project ITPM/OTPM reservations the same way -- full
# refund, since a failed call has no billable usage to reconcile
# against.
# Settle project ITPM/OTPM reservations the same way: at the
# recovered partial usage, or a full refund when there is none.
itpm_operations: Final = (
self._build_project_reservation_ops(
targets=tuple(stash.itpm_reserved_scopes),
reserved_scopes=stash.itpm_reserved_scopes,
actual_tokens=0,
actual_tokens=itpm_actual,
reserved_tokens=itpm_reserved,
reservation_window_identities=stash.itpm_reserved_window_identities,
)
@ -4584,7 +4597,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else self._build_reservation_aware_tpm_ops(
targets=tuple(stash.itpm_reserved_scopes),
reserved_scopes=stash.itpm_reserved_scopes,
actual_tokens=0,
actual_tokens=itpm_actual,
reserved_tokens=itpm_reserved,
)
if stash is not None and itpm_reserved > 0
@ -4595,7 +4608,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self._build_project_reservation_ops(
targets=tuple(stash.otpm_reserved_scopes),
reserved_scopes=stash.otpm_reserved_scopes,
actual_tokens=0,
actual_tokens=otpm_actual,
reserved_tokens=otpm_reserved,
reservation_window_identities=stash.otpm_reserved_window_identities,
)
@ -4603,7 +4616,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else self._build_reservation_aware_tpm_ops(
targets=tuple(stash.otpm_reserved_scopes),
reserved_scopes=stash.otpm_reserved_scopes,
actual_tokens=0,
actual_tokens=otpm_actual,
reserved_tokens=otpm_reserved,
)
if stash is not None and otpm_reserved > 0
@ -4742,7 +4755,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
removal is a no-op ZREM on a second run), and the TPM/ITPM/OTPM
refund is guarded by the stash's ``reservation_released`` flag — if
both this hook and async_log_failure_event end up running in the same
flow, only the first release/refund applies.
flow, only the first release/refund applies. A mid-stream failure
relayed here with recovered partial usage settles the reservation at
that usage instead of refunding it.
"""
try:
stash: Final = get_request_stash()
@ -4769,12 +4784,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
otpm_reserved: Final = stash.otpm_reserved_tokens
if reserved_tokens <= 0 and itpm_reserved <= 0 and otpm_reserved <= 0:
return
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(request_data)
combined_ops: Final = (
self._build_reservation_aware_tpm_ops(
targets=tuple(stash.reserved_scopes),
reserved_scopes=stash.reserved_scopes,
actual_tokens=0,
actual_tokens=tpm_actual,
reserved_tokens=reserved_tokens,
)
if reserved_tokens > 0
@ -4784,7 +4800,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self._build_project_reservation_ops(
targets=tuple(stash.itpm_reserved_scopes),
reserved_scopes=stash.itpm_reserved_scopes,
actual_tokens=0,
actual_tokens=itpm_actual,
reserved_tokens=itpm_reserved,
reservation_window_identities=stash.itpm_reserved_window_identities,
)
@ -4792,7 +4808,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else self._build_reservation_aware_tpm_ops(
targets=tuple(stash.itpm_reserved_scopes),
reserved_scopes=stash.itpm_reserved_scopes,
actual_tokens=0,
actual_tokens=itpm_actual,
reserved_tokens=itpm_reserved,
)
if itpm_reserved > 0
@ -4802,7 +4818,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self._build_project_reservation_ops(
targets=tuple(stash.otpm_reserved_scopes),
reserved_scopes=stash.otpm_reserved_scopes,
actual_tokens=0,
actual_tokens=otpm_actual,
reserved_tokens=otpm_reserved,
reservation_window_identities=stash.otpm_reserved_window_identities,
)
@ -4810,7 +4826,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else self._build_reservation_aware_tpm_ops(
targets=tuple(stash.otpm_reserved_scopes),
reserved_scopes=stash.otpm_reserved_scopes,
actual_tokens=0,
actual_tokens=otpm_actual,
reserved_tokens=otpm_reserved,
)
if otpm_reserved > 0

17
litellm/proxy/litellm_pre_call_utils.py
View file

@ -669,6 +669,21 @@ def _extract_codex_session_id_from_headers(
)
def _extract_bare_session_id_from_headers(
normalized: Mapping[str, str],
) -> str | None:
"""
Read a vendor-less ``x-session-id`` header (opencode sends ``X-Session-Id``
alongside ``x-session-affinity`` on every turn of a session). Checked after
the ``x-<vendor>-session-id`` scan so a more specific header such as
opencode's ``x-parent-session-id`` on subagent calls keeps winning.
"""
value: Final = normalized.get("x-session-id")
if isinstance(value, str) and _SESSION_ID_VALUE_RE.match(value):
return value
return None
def get_chain_id_from_headers(headers: dict[str, str] | None) -> str | None:
"""
Extract chain id for call chaining from request headers.
@ -679,6 +694,7 @@ def get_chain_id_from_headers(headers: dict[str, str] | None) -> str | None:
3. Any ``x-<vendor>-session-id`` header whose value looks like a session id
(alphanumeric / UUID, at least 8 chars). E.g. ``x-claude-code-session-id``.
4. Codex's unprefixed ``session-id`` / ``thread-id``, for Codex callers only.
5. A vendor-less ``x-session-id`` header (e.g. opencode), same value rules.
Header keys are matched case-insensitively so this works with raw header
dicts from any transport.
@ -694,6 +710,7 @@ def get_chain_id_from_headers(headers: dict[str, str] | None) -> str | None:
or normalized.get("x-litellm-session-id")
or _extract_generic_session_id_from_headers(normalized)
or _extract_codex_session_id_from_headers(normalized)
or _extract_bare_session_id_from_headers(normalized)
)

4
litellm/proxy/logging_endpoints/callback_logs_endpoints.py
View file

@ -88,8 +88,10 @@ class CallbackLogsReplayer:
)
metadata: Final[dict[str, Any]] = payload.get("metadata") or {}
user_api_key_hash: Final = metadata.get("user_api_key_hash")
litellm_metadata: Final[dict[str, Any]] = {
"user_api_key": metadata.get("user_api_key_hash"),
"user_api_key": user_api_key_hash,
"user_api_key_hash": user_api_key_hash,
"user_api_key_alias": metadata.get("user_api_key_alias"),
"user_api_key_user_id": metadata.get("user_api_key_user_id"),
"user_api_key_team_id": metadata.get("user_api_key_team_id"),

120
litellm/proxy/management_endpoints/access_group_endpoints.py
View file

@ -1,16 +1,20 @@
from collections.abc import Mapping, Sequence
import asyncio
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Protocol
from fastapi import APIRouter, Depends, HTTPException, status
from litellm._logging import verbose_proxy_logger
from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
from litellm.proxy._types import (
CommonProxyErrors,
LiteLLM_AccessGroupTable,
LitellmUserRoles,
UserAPIKeyAuth,
)
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
from litellm.proxy.auth.auth_checks import (
_cache_access_object,
_cache_key_object,
@ -20,10 +24,16 @@ from litellm.proxy.auth.auth_checks import (
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.management_helpers.access_group_team_sync import invalidate_access_group_cache
from litellm.proxy.utils import get_prisma_client_or_throw
from litellm.proxy.management_helpers.resource_display_names import (
agent_display_names,
key_display_names,
mcp_server_display_names,
)
from litellm.proxy.utils import PrismaClient, get_prisma_client_or_throw
from litellm.repositories.table_repositories import AccessGroupRepository, TeamRepository
from litellm.types.access_group import (
AccessGroupCreateRequest,
AccessGroupResource,
AccessGroupResponse,
AccessGroupUpdateRequest,
)
@ -37,6 +47,12 @@ class _AccessGroupRecord(Protocol):
@property
def access_group_id(self) -> str: ...
@property
def access_mcp_server_ids(self) -> Sequence[str] | None: ...
@property
def access_agent_ids(self) -> Sequence[str] | None: ...
@property
def assigned_team_ids(self) -> Sequence[str] | None: ...
@ -50,6 +66,9 @@ class _TeamRecord(Protocol):
@property
def team_id(self) -> str: ...
@property
def team_alias(self) -> str | None: ...
@property
def access_group_ids(self) -> Sequence[str] | None: ...
@ -120,16 +139,75 @@ def _require_admin_view(user_api_key_dict: UserAPIKeyAuth) -> None:
)
@dataclass(frozen=True, slots=True)
class _ResourceNames:
mcp_servers: Mapping[str, str]
agents: Mapping[str, str]
teams: Mapping[str, str | None]
keys: Mapping[str, str]
def _label(ids: Sequence[str], names: Mapping[str, str | None]) -> tuple[AccessGroupResource, ...]:
return tuple(AccessGroupResource(id=resource_id, name=names.get(resource_id)) for resource_id in ids)
def _record_to_response(
record: _AccessGroupRecord, *, assigned_team_ids: Sequence[str] | None = None
record: _AccessGroupRecord, *, assigned_team_ids: Sequence[str], names: _ResourceNames
) -> AccessGroupResponse:
stored: Final = record.dict()
payload: Final = (
stored if assigned_team_ids is None else MappingProxyType({**stored, "assigned_team_ids": assigned_team_ids})
payload: Final = MappingProxyType(
{
**record.dict(),
"assigned_team_ids": assigned_team_ids,
"access_mcp_servers": _label(record.access_mcp_server_ids or (), names.mcp_servers),
"access_agents": _label(record.access_agent_ids or (), names.agents),
"assigned_teams": _label(assigned_team_ids, names.teams),
"assigned_keys": _label(record.assigned_key_ids or (), names.keys),
}
)
return AccessGroupResponse.model_validate(payload)
def _ids_across(
records: Sequence[_AccessGroupRecord], pick: Callable[[_AccessGroupRecord], Sequence[str] | None]
) -> tuple[str, ...]:
return tuple(dict.fromkeys(resource_id for record in records for resource_id in (pick(record) or ())))
async def _responses_for(
prisma_client: PrismaClient, records: Sequence[_AccessGroupRecord]
) -> tuple[AccessGroupResponse, ...]:
if not records:
return ()
teams: Final = await _teams_touching(TeamRepository(prisma_client).table, records)
mcp_servers, agents, keys = await asyncio.gather(
mcp_server_display_names(
prisma_client,
_ids_across(records, lambda record: record.access_mcp_server_ids),
global_mcp_server_manager.config_mcp_servers,
),
agent_display_names(
prisma_client, _ids_across(records, lambda record: record.access_agent_ids), global_agent_registry
),
key_display_names(prisma_client, _ids_across(records, lambda record: record.assigned_key_ids)),
)
names: Final = _ResourceNames(
mcp_servers=mcp_servers,
agents=agents,
teams=MappingProxyType({team.team_id: team.team_alias for team in teams}),
keys=keys,
)
attached: Final = _attached_team_ids_by_group(records, teams)
return tuple(
_record_to_response(record, assigned_team_ids=attached[record.access_group_id], names=names)
for record in records
)
async def _response_for(prisma_client: PrismaClient, record: _AccessGroupRecord) -> AccessGroupResponse:
(response,) = await _responses_for(prisma_client, (record,))
return response
def _attached_team_ids_by_group(
records: Sequence[_AccessGroupRecord], teams: Sequence[_TeamRecord]
) -> Mapping[str, tuple[str, ...]]:
@ -144,19 +222,21 @@ def _attached_team_ids_by_group(
return MappingProxyType({record.access_group_id: attached(record) for record in records})
async def _teams_touching(team_table: _TeamTable, records: Sequence[_AccessGroupRecord]) -> Sequence[_TeamRecord]:
"""Team rows listed on any of the groups or carrying any of them in access_group_ids."""
group_ids: Final = tuple(record.access_group_id for record in records)
stored_team_ids: Final = _ids_across(records, lambda record: record.assigned_team_ids)
carrying: Final = {"access_group_ids": {"hasSome": group_ids}} # mutable-ok: prisma where is a dict
listed: Final = {"team_id": {"in": stored_team_ids}} # mutable-ok: prisma where is a dict
return await team_table.find_many(where={"OR": (carrying, listed)}) # mutable-ok: prisma where is a dict
async def _attached_team_ids_for(
team_table: _TeamTable, records: Sequence[_AccessGroupRecord]
) -> Mapping[str, tuple[str, ...]]:
if not records:
return MappingProxyType({})
group_ids: Final = tuple(record.access_group_id for record in records)
stored_team_ids: Final = tuple(
dict.fromkeys(team_id for record in records for team_id in (record.assigned_team_ids or ()))
)
carrying: Final = {"access_group_ids": {"hasSome": group_ids}} # mutable-ok: prisma where is a dict
listed: Final = {"team_id": {"in": stored_team_ids}} # mutable-ok: prisma where is a dict
teams: Final = await team_table.find_many(where={"OR": (carrying, listed)}) # mutable-ok: prisma where is a dict
return _attached_team_ids_by_group(records, teams)
return _attached_team_ids_by_group(records, await _teams_touching(team_table, records))
async def _require_teams_exist(tx: _AccessGroupTx, team_ids: Sequence[str]) -> None:
@ -425,7 +505,7 @@ async def create_access_group(
proxy_logging_obj,
)
return _record_to_response(record)
return await _response_for(prisma_client, record)
@router.get(
@ -434,14 +514,13 @@ async def create_access_group(
)
async def list_access_groups(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
) -> list[AccessGroupResponse]:
) -> Sequence[AccessGroupResponse]:
_require_admin_view(user_api_key_dict)
prisma_client: Final = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value)
table: Final = AccessGroupRepository(prisma_client).table
records: Final = await table.find_many(order={"created_at": "desc"})
attached: Final = await _attached_team_ids_for(TeamRepository(prisma_client).table, records)
return [_record_to_response(r, assigned_team_ids=attached[r.access_group_id]) for r in records]
return await _responses_for(prisma_client, records)
@router.get(
@ -462,8 +541,7 @@ async def get_access_group(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Access group '{access_group_id}' not found",
)
attached: Final = await _attached_team_ids_for(TeamRepository(prisma_client).table, (record,))
return _record_to_response(record, assigned_team_ids=attached[record.access_group_id])
return await _response_for(prisma_client, record)
@router.put(
@ -560,7 +638,7 @@ async def update_access_group(
await _patch_key_caches_add_access_group(keys_to_add, access_group_id, user_api_key_cache, proxy_logging_obj)
await _patch_key_caches_remove_access_group(keys_to_remove, access_group_id, user_api_key_cache, proxy_logging_obj)
return _record_to_response(record)
return await _response_for(prisma_client, record)
@router.delete(

84
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -2,15 +2,19 @@ import asyncio
from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Set as AbstractSet
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from types import MappingProxyType, SimpleNamespace
from typing import TYPE_CHECKING, Final, Protocol
from fastapi import HTTPException, status
from typing_extensions import TypedDict
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.constants import PTU_SENTINEL_API_KEY
from litellm.proxy._types import CommonProxyErrors
from litellm.proxy.spend_tracking.key_metadata_recovery import (
attach_user_emails,
recover_double_hashed_key_metadata,
)
from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled
from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import DeletedVerificationTokenRepository
@ -111,8 +115,19 @@ class DailySpendRecord(Protocol):
class _KeyMetadataDict(TypedDict, total=False):
key_alias: str | None
team_id: str | None
key_alias: ReadOnly[str | None]
team_id: ReadOnly[str | None]
user_id: ReadOnly[str | None]
user_email: ReadOnly[str | None]
def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata:
meta: Final = api_key_metadata.get(api_key, {})
return KeyMetadata(
key_alias=meta.get("key_alias"),
team_id=meta.get("team_id"),
user_email=meta.get("user_email"),
)
_WhereValue = str | dict[str, object]
@ -283,10 +298,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.models[model_key].api_key_breakdown:
breakdown.models[model_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.models[model_key].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.models[model_key].api_key_breakdown[record.api_key].metrics,
@ -310,10 +322,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.model_groups[model_group_key].api_key_breakdown:
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics,
@ -335,10 +344,7 @@ def update_breakdown_metrics(
breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[record.api_key] = (
KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
)
@ -363,10 +369,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.providers[provider].api_key_breakdown:
breakdown.providers[provider].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.providers[provider].api_key_breakdown[record.api_key].metrics,
@ -388,10 +391,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown:
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics,
@ -403,10 +403,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.api_keys:
breakdown.api_keys[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
), # Add any api_key-specific metadata here
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.api_keys[record.api_key].metrics = update_metrics(breakdown.api_keys[record.api_key].metrics, record)
@ -426,10 +423,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.entities[entity_value].api_key_breakdown:
breakdown.entities[entity_value].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics,
@ -442,17 +436,23 @@ def update_breakdown_metrics(
async def get_api_key_metadata(
prisma_client: PrismaClient,
api_keys: AbstractSet[str],
) -> dict[str, _KeyMetadataDict]:
) -> Mapping[str, _KeyMetadataDict]:
"""Get api key metadata, falling back to deleted keys table for keys not found in active table.
This ensures that key_alias and team_id are preserved in historical activity logs
even after a key is deleted or regenerated.
even after a key is deleted or regenerated. Also recovers aliases for api_key
values that were double-hashed by the v1.99 spend-log provenance gate.
"""
key_records: Sequence[PrismaVerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many(
where={"token": {"in": list(api_keys)}}
)
result: Final[dict[str, _KeyMetadataDict]] = {
k.token: {"key_alias": k.key_alias, "team_id": k.team_id} for k in key_records
k.token: {
"key_alias": k.key_alias,
"team_id": k.team_id,
"user_id": getattr(k, "user_id", None),
}
for k in key_records
}
# For any keys not found in the active table, check the deleted keys table
@ -471,6 +471,7 @@ async def get_api_key_metadata(
result[k.token] = {
"key_alias": k.key_alias,
"team_id": k.team_id,
"user_id": getattr(k, "user_id", None),
}
except Exception as e:
verbose_proxy_logger.warning(
@ -479,7 +480,13 @@ async def get_api_key_metadata(
e,
)
return result
still_missing: Final = api_keys - frozenset(result)
combined: Final = (
result
if not still_missing
else MappingProxyType({**result, **(await recover_double_hashed_key_metadata(prisma_client, still_missing))})
)
return await attach_user_emails(prisma_client, combined)
def _adjust_dates_for_timezone(
@ -951,11 +958,6 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
)
def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata:
meta: Final = api_key_metadata.get(api_key, {})
return KeyMetadata(key_alias=meta.get("key_alias"), team_id=meta.get("team_id"))
def _aggregate_grouping_sets_records_sync(
*,
records: Sequence[_GroupingSetsRow],

4
litellm/proxy/management_endpoints/common_utils.py
View file

@ -22,7 +22,7 @@ def validate_finite_spend(spend: float | None) -> None:
)
def validate_budget_duration(budget_duration: str | None) -> None:
def validate_budget_duration(budget_duration: str | None, status_code: int = 400) -> None:
"""Reject budget durations that can't be parsed, are non-positive, or
overflow date math, so a bad value can't be persisted and later crash the
budget reset job.
@ -44,7 +44,7 @@ def validate_budget_duration(budget_duration: str | None) -> None:
get_budget_reset_time(budget_duration=budget_duration)
except (ValueError, OverflowError):
raise HTTPException(
status_code=400,
status_code=status_code,
detail={
"error": f"Invalid budget_duration '{budget_duration}'. Use a format like '1h', '24h', '7d', or '30d'."
},

30
litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py
View file

@ -13,7 +13,9 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
hash_token,
)
from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
from litellm.repositories.table_repositories import JWTKeyMappingRepository
@ -118,9 +120,8 @@ async def create_jwt_key_mapping(
new_mapping: Final = await _mapping_table(prisma_client).create(data=create_data)
# Invalidate cache
cache_key: Final = f"jwt_key_mapping:{data.jwt_claim_name}:{data.jwt_claim_value}"
await user_api_key_cache.async_delete_cache(cache_key)
cache_key: Final = jwt_key_mapping_cache_key(data.jwt_claim_name, data.jwt_claim_value)
await evict_and_broadcast(cache_keys=(cache_key,), user_api_key_cache=user_api_key_cache)
return _to_response(new_mapping)
except HTTPException:
@ -169,17 +170,20 @@ async def update_jwt_key_mapping(
if old_mapping is None:
raise HTTPException(status_code=404, detail="Mapping not found")
cache_key = f"jwt_key_mapping:{old_mapping.jwt_claim_name}:{old_mapping.jwt_claim_value}"
await user_api_key_cache.async_delete_cache(cache_key)
updated_mapping: Final = await _mapping_table(prisma_client).update(where={"id": data.id}, data=update_data)
if updated_mapping is None:
raise HTTPException(status_code=404, detail="Mapping not found")
# Invalidate new cache key if claim fields changed
cache_key = f"jwt_key_mapping:{updated_mapping.jwt_claim_name}:{updated_mapping.jwt_claim_value}"
await user_api_key_cache.async_delete_cache(cache_key)
# Evict only after the write commits: a concurrent request between an
# early eviction and the commit would re-cache the old mapping and keep
# it authorized until TTL.
old_cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value)
new_cache_key: Final = jwt_key_mapping_cache_key(
updated_mapping.jwt_claim_name, updated_mapping.jwt_claim_value
)
cache_keys: Final = (old_cache_key,) if old_cache_key == new_cache_key else (old_cache_key, new_cache_key)
await evict_and_broadcast(cache_keys=cache_keys, user_api_key_cache=user_api_key_cache)
return _to_response(updated_mapping)
except HTTPException:
@ -219,10 +223,12 @@ async def delete_jwt_key_mapping(
if old_mapping is None:
raise HTTPException(status_code=404, detail="Mapping not found")
cache_key: Final = f"jwt_key_mapping:{old_mapping.jwt_claim_name}:{old_mapping.jwt_claim_value}"
await user_api_key_cache.async_delete_cache(cache_key)
await _mapping_table(prisma_client).delete(where={"id": data.id})
# Evict only after the row is gone, else a concurrent request can
# re-cache the deleted mapping and keep it authorized until TTL.
cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value)
await evict_and_broadcast(cache_keys=(cache_key,), user_api_key_cache=user_api_key_cache)
return {"status": "success"}
except HTTPException:
raise

11
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -54,6 +54,7 @@ from litellm.proxy._types import Litellm_EntityType, LiteLLM_VerificationToken,
from litellm.proxy.auth.auth_checks import (
_delete_cache_key_object,
can_team_access_model,
get_jwt_key_mapping_cache_keys_for_token,
get_org_object,
get_project_object,
get_team_object,
@ -65,6 +66,7 @@ from litellm.proxy.auth.auth_utils import (
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
evict_and_broadcast,
publish_auth_cache_invalidation,
)
from litellm.proxy.common_utils.callback_config_validation import logging_metadata_config_error
@ -4975,6 +4977,13 @@ async def _execute_virtual_key_regeneration(
update_data.update(non_default_values)
jsonified_update_data: Final[Mapping[str, object]] = prisma_client.jsonify_object(data=update_data)
# Snapshot before the token update: the FK cascade rewrites mapping rows to the new hash,
# but their cached jwt_key_mapping entries still point at the old token (LIT-5379).
jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_token(
hashed_token=hashed_api_key,
prisma_client=prisma_client,
)
# If grace period set, insert deprecated key so old key remains valid
await _insert_deprecated_key(
prisma_client=prisma_client,
@ -5000,6 +5009,8 @@ async def _execute_virtual_key_regeneration(
proxy_logging_obj=proxy_logging_obj,
)
await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache)
# After credential invalidation, so a failure here can never keep the old key alive.
await sync_key_regeneration_access_group_membership(
prisma_client=prisma_client,

69
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -91,8 +91,10 @@ from litellm.router_strategy.complexity_router import (
ComplexityRouterConfig,
ComplexityTier,
TierDefinition,
built_in_tier_classification_prompt,
classification_system_prompt,
custom_tier_classification_prompt,
normalize_classification_examples,
normalize_classification_prompt,
)
from litellm.router_utils.auto_router_model_naming import (
@ -2374,21 +2376,13 @@ async def update_useful_links(
)
def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[ComplexityTier, str], ...] | None:
"""Resolve the tier_labels query param into the labeled tiers the rubric is built from.
Validated through ComplexityRouterConfig so the editor prefills what the router would send: the
same field validators that reject a blank, duplicated, or canonical-name-stealing label on the
write path reject it here, rather than this returning a rubric no router could be configured to
use. A malformed value is the caller's error, so it surfaces as a 400.
None when unset, letting classification_system_prompt apply its own default names.
"""
if not tier_labels:
return None
def _validated_labeled_tiers(
tier_labels: dict[ComplexityTier, str], # mutable-ok: Pydantic materializes JSON object fields as dicts
) -> tuple[tuple[ComplexityTier, str], ...]:
"""Validate tier labels once for both prompt-preview transports."""
try:
return ComplexityRouterConfig(tier_labels=json.loads(tier_labels)).labeled_tiers()
except (JSONDecodeError, ValidationError) as e:
return ComplexityRouterConfig(tier_labels=tier_labels).labeled_tiers()
except (TypeError, ValidationError) as e:
raise ProxyException(
message=f"tier_labels must be a JSON object of tier name to display name: {e}",
type=ProxyErrorTypes.bad_request_error,
@ -2397,15 +2391,35 @@ def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[Complexity
) from e
class AutoRouterClassifierPromptPreviewRequest(BaseModel):
"""A POST rather than query params: classification_prompt is the operator's own text, which must
not reach access logs through a URL."""
def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[ComplexityTier, str], ...] | None:
"""Resolve the tier_labels query param into the labeled tiers the rubric is built from."""
if not tier_labels:
return None
try:
parsed: Final = json.loads(tier_labels)
except JSONDecodeError as e:
raise ProxyException(
message=f"tier_labels must be a JSON object of tier name to display name: {e}",
type=ProxyErrorTypes.bad_request_error,
code=status.HTTP_400_BAD_REQUEST,
param="tier_labels",
) from e
return _validated_labeled_tiers(parsed)
tier_definitions: tuple[TierDefinition, ...]
class AutoRouterClassifierPromptPreviewRequest(BaseModel):
"""A POST rather than query params: the classification sections are the operator's own text,
which must not reach access logs through a URL."""
tier_definitions: tuple[TierDefinition, ...] | None = None
tier_labels: dict[ComplexityTier, str] | None = None # mutable-ok: FastAPI parses JSON object fields into dicts
classification_rubric: ClassificationRubric | None = None
context_window_size: Annotated[int, Field(ge=0)] = DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE
classification_prompt: str | None = None
classification_examples: str | None = None
_normalize_prompt = field_validator("classification_prompt")(normalize_classification_prompt)
_normalize_examples = field_validator("classification_examples")(normalize_classification_examples)
@router.post(
@ -2423,11 +2437,24 @@ async def preview_auto_router_classifier_prompt(
Built by the same function the live classifier uses, so the preview cannot drift from what the
router sends. Payload validity beyond a renderable definition stays the dry-run's job.
"""
return AutoRouterClassifierDefaultPromptResponse(
system_prompt=custom_tier_classification_prompt(
request.tier_definitions, request.classification_prompt, request.context_window_size
labeled_tiers: Final = _validated_labeled_tiers(request.tier_labels or {}) # mutable-ok: Pydantic field default
system_prompt: Final = (
custom_tier_classification_prompt(
request.tier_definitions,
request.classification_prompt,
request.context_window_size,
classification_examples=request.classification_examples,
)
if request.tier_definitions is not None
else built_in_tier_classification_prompt(
request.classification_prompt,
request.context_window_size,
labeled_tiers=labeled_tiers,
classification_rubric=request.classification_rubric,
classification_examples=request.classification_examples,
)
)
return AutoRouterClassifierDefaultPromptResponse(system_prompt=system_prompt)
@router.get(

18
litellm/proxy/management_endpoints/organization_endpoints.py
View file

@ -41,6 +41,7 @@ from litellm.proxy.management_endpoints.common_daily_activity import get_daily_a
from litellm.proxy.management_endpoints.common_utils import (
_set_object_metadata_field,
_user_has_admin_view,
validate_budget_duration,
)
from litellm.proxy.management_helpers.object_permission_utils import (
handle_update_object_permission_common,
@ -312,7 +313,7 @@ def handle_nested_budget_structure_in_organization_update_request(
# Extract valid budget fields and merge into top level
budget_fields: Final = LiteLLM_BudgetTable.model_fields.keys()
for key, value in budget_data.items():
if key in budget_fields and value is not None:
if key in budget_fields:
transformed_data[key] = value
return transformed_data
@ -708,9 +709,8 @@ async def update_organization(
existing_organization_row=existing_organization_row,
)
# Handle budget updates if budget fields are provided
budget_fields: Final = {
k: v for k, v in data.model_dump().items() if k in LiteLLM_BudgetTable.model_fields and v is not None
k: v for k, v in data.model_dump().items() if k in _BUDGET_SETTABLE_FIELDS and k in data.model_fields_set
}
if budget_fields and existing_organization_row.budget_id:
@ -764,7 +764,6 @@ async def handle_update_object_permission(
tags=["organization management"],
dependencies=[Depends(user_api_key_auth)],
response_model=LiteLLM_OrganizationTableWithMembers,
include_in_schema=False,
)
async def update_organization_v2(
organization_id: str,
@ -807,6 +806,17 @@ async def update_organization_v2(
status_code=422,
detail={"error": f"soft_budget must be a non-negative finite number. Received: {data.soft_budget}"},
)
for limit_name, limit_value in (
("tpm_limit", data.tpm_limit),
("rpm_limit", data.rpm_limit),
("max_parallel_requests", data.max_parallel_requests),
):
if limit_value is not None and limit_value < 0:
raise HTTPException(
status_code=422,
detail={"error": f"{limit_name} must be non-negative. Received: {limit_value}"},
)
validate_budget_duration(data.budget_duration, status_code=422)
if data.model_max_budget:
from litellm.proxy.management_endpoints.key_management_endpoints import (
validate_model_max_budget,

168
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -170,6 +170,8 @@ from litellm.types.proxy.management_endpoints.team_endpoints import (
TeamMemberAddResult,
TeamMemberInfoResponse,
TeamMetadataSchemaResponse,
TeamUserSpendResponse,
TeamUserSpendRow,
UpdateTeamMemberPermissionsRequest,
)
@ -3324,9 +3326,6 @@ async def team_member_delete(
data=data,
)
if not removed_team_members:
raise HTTPException(status_code=400, detail={"error": "User not found in team"})
existing_team_row.members_with_roles = new_team_members
_db_new_team_members: Final[list[dict]] = [m.model_dump() for m in new_team_members]
@ -3334,17 +3333,27 @@ async def team_member_delete(
## DELETE TEAM ID from USER ROW, IF EXISTS ##
# get user row
removed_user_ids: Final = frozenset(m.user_id for m in removed_team_members if m.user_id is not None)
addressed_user_ids: Final = (
removed_user_ids if removed_team_members else frozenset((data.user_id,) if data.user_id is not None else ())
)
key_val: Final[Mapping[str, object]] = (
{"user_id": {"in": sorted(removed_user_ids)}} if removed_user_ids else {"user_email": data.user_email}
{"user_id": {"in": sorted(addressed_user_ids)}} if addressed_user_ids else {"user_email": data.user_email}
)
member_tx: Final[_MemberDeleteTx] = tx
existing_user_rows: Final = await member_tx.litellm_usertable.find_many(where=key_val)
# Also clean up any existing team membership rows for this user and team
user_ids_to_delete: Final = removed_user_ids.union(
(data.user_id,) if data.user_id is not None else (),
(user.user_id for user in existing_user_rows if user.user_id),
)
# A user row can outlive its roster entry, and until the team is off user.teams the user
# still sees it and still fails key creation against it, so removal has to clear it too
stale_user_rows: Final = tuple(user for user in existing_user_rows if data.team_id in user.teams)
# Also clean up any existing team membership rows for this user and team. An email can
# match several user rows, so with no roster entry to name the member, only the rows
# actually carrying the team are the ones this request is allowed to touch
cleanup_user_rows: Final = existing_user_rows if removed_team_members else stale_user_rows
user_ids_to_delete: Final = addressed_user_ids.union(user.user_id for user in cleanup_user_rows if user.user_id)
if not removed_team_members and not stale_user_rows:
raise HTTPException(status_code=400, detail={"error": "User not found in team"})
## DELETE KEYS CREATED BY USER FOR THIS TEAM
# Fetch keys before deletion so their audit records can be persisted alongside the delete.
@ -3356,17 +3365,17 @@ async def team_member_delete(
}
)
await _team_tx_db(tx).update(
where={"team_id": data.team_id},
data={"members_with_roles": json.dumps(_db_new_team_members)},
)
if removed_team_members:
await _team_tx_db(tx).update(
where={"team_id": data.team_id},
data={"members_with_roles": json.dumps(_db_new_team_members)},
)
for existing_user in existing_user_rows:
if data.team_id in existing_user.teams:
await tx.litellm_usertable.update(
where={"user_id": existing_user.user_id},
data={"teams": {"set": [team for team in existing_user.teams if team != data.team_id]}},
)
for existing_user in stale_user_rows:
await tx.litellm_usertable.update(
where={"user_id": existing_user.user_id},
data={"teams": {"set": [team for team in existing_user.teams if team != data.team_id]}},
)
for _uid in sorted(user_ids_to_delete):
await tx.litellm_teammembership.delete_many(where={"team_id": data.team_id, "user_id": _uid})
@ -6231,3 +6240,124 @@ async def get_team_daily_activity_aggregated(
timezone_offset_minutes=timezone,
include_entity_breakdown=True,
)
def _team_user_spend_sql(*, team_count: int, restrict_to_user: bool) -> str:
team_placeholders: Final = ", ".join(f"${i}" for i in range(3, 3 + team_count))
user_clause: Final = f' AND sl."user" = ${3 + team_count}' if restrict_to_user else ""
return f"""
SELECT
sl.team_id,
sl."user" AS user_id,
u.user_email,
u.user_alias,
SUM(sl.spend)::float AS spend,
SUM(sl.prompt_tokens)::bigint AS prompt_tokens,
SUM(sl.completion_tokens)::bigint AS completion_tokens,
SUM(sl.total_tokens)::bigint AS total_tokens,
COUNT(*)::bigint AS api_requests,
COUNT(*) FILTER (WHERE sl.status IS DISTINCT FROM 'failure')::bigint AS successful_requests,
COUNT(*) FILTER (WHERE sl.status = 'failure')::bigint AS failed_requests
FROM "LiteLLM_SpendLogs" sl
LEFT JOIN "LiteLLM_UserTable" u ON u.user_id = sl."user"
WHERE sl."startTime" >= $1::timestamp
AND sl."startTime" < $2::timestamp + INTERVAL '1 day'
AND sl.team_id IN ({team_placeholders}){user_clause}
GROUP BY sl.team_id, sl."user", u.user_email, u.user_alias
ORDER BY spend DESC, sl.team_id, sl."user"
"""
class _TeamUserSpendDbRow(TypedDict):
team_id: ReadOnly[str]
user_id: ReadOnly[str | None]
user_email: ReadOnly[str | None]
user_alias: ReadOnly[str | None]
spend: ReadOnly[float]
prompt_tokens: ReadOnly[int]
completion_tokens: ReadOnly[int]
total_tokens: ReadOnly[int]
api_requests: ReadOnly[int]
successful_requests: ReadOnly[int]
failed_requests: ReadOnly[int]
@router.get(
"/team/spend/by_user",
response_model=TeamUserSpendResponse,
tags=["team management"], # mutable-ok: fastapi route tags must be a list
)
async def get_team_spend_by_user(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
team_ids: str | None = None,
start_date: str | None = None,
end_date: str | None = None,
) -> TeamUserSpendResponse:
"""
Spend per user within the given teams, attributed per request from spend logs.
Proxy admins may query any team. Team admins and members holding the
`/team/daily/activity` permission see every user of the requested teams;
other members only see their own row.
"""
from litellm.proxy.proxy_server import (
prisma_client,
proxy_logging_obj,
user_api_key_cache,
)
if prisma_client is None:
raise _daily_activity_error(status_code=500, message=CommonProxyErrors.db_not_connected_error.value)
range_error: Final = _aggregated_date_range_error(start_date, end_date)
if range_error is not None or start_date is None or end_date is None:
raise _daily_activity_error(status_code=400, message=range_error or "Please provide start_date and end_date")
if not team_ids:
raise _daily_activity_error(status_code=400, message="Please provide team_ids")
scope: Final = await _resolve_team_daily_activity_scope(
team_ids=team_ids,
exclude_team_ids=None,
api_key=None,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
scoped_team_ids: Final = tuple(scope.team_ids or ())
if not scoped_team_ids:
return TeamUserSpendResponse(start_date=start_date, end_date=end_date, results=())
own_user_only: Final = scope.api_key_filter is not None
user_param: Final = (user_api_key_dict.user_id or "",) if own_user_only else ()
rows: Final[Sequence[_TeamUserSpendDbRow]] = await prisma_client.db.query_raw(
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
start_date,
end_date,
*scoped_team_ids,
*user_param,
)
results: Final = tuple(
TeamUserSpendRow(
team_id=row["team_id"],
team_alias=_team_alias_or_none(scope.team_alias_metadata.get(row["team_id"])),
user_id=row["user_id"] or "",
user_email=row["user_email"],
user_alias=row["user_alias"],
spend=row["spend"],
prompt_tokens=row["prompt_tokens"],
completion_tokens=row["completion_tokens"],
total_tokens=row["total_tokens"],
api_requests=row["api_requests"],
successful_requests=row["successful_requests"],
failed_requests=row["failed_requests"],
)
for row in rows
)
return TeamUserSpendResponse(start_date=start_date, end_date=end_date, results=results)
def _team_alias_or_none(metadata: Mapping[str, object] | None) -> str | None:
alias: Final = metadata.get("team_alias") if metadata is not None else None
return alias if isinstance(alias, str) else None

61
litellm/proxy/management_helpers/resource_display_names.py Normal file
View file

@ -0,0 +1,61 @@
"""Display names for ids stored on management objects. DB rows win; config-declared servers and agents fill the gaps."""
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry
from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import AgentsRepository, MCPServerRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
from litellm.types.mcp_server.mcp_server_manager import MCPServer
async def mcp_server_display_names(
prisma_client: PrismaClient,
server_ids: Sequence[str],
config_servers: Mapping[str, MCPServer],
) -> Mapping[str, str]:
"""server_id -> alias, falling back to server_name; config-only servers also fall back to their registry name."""
if not server_ids:
return MappingProxyType({})
wanted: Final = frozenset(server_ids)
where: Final = {"server_id": {"in": tuple(wanted)}} # mutable-ok: prisma where is a dict
rows: Final = await MCPServerRepository(prisma_client).table.find_many(where=where)
from_config: Final = {
server_id: server.alias or server.server_name or server.name
for server_id, server in config_servers.items()
if server_id in wanted
}
from_db: Final = {row.server_id: name for row in rows if (name := row.alias or row.server_name)}
return MappingProxyType({**from_config, **from_db})
async def agent_display_names(
prisma_client: PrismaClient,
agent_ids: Sequence[str],
registry: AgentRegistry,
) -> Mapping[str, str]:
"""agent_id -> agent_name. The registry covers config-declared agents and their legacy ids."""
if not agent_ids:
return MappingProxyType({})
wanted: Final = frozenset(agent_ids)
where: Final = {"agent_id": {"in": tuple(wanted)}} # mutable-ok: prisma where is a dict
rows: Final = await AgentsRepository(prisma_client).table.find_many(where=where)
from_registry: Final = {
alias_id: agent.agent_name
for agent in registry.get_agent_list()
for alias_id in registry.ids_for_agent(agent.agent_id)
if alias_id in wanted
}
from_db: Final = {row.agent_id: row.agent_name for row in rows}
return MappingProxyType({**from_registry, **from_db})
async def key_display_names(prisma_client: PrismaClient, tokens: Sequence[str]) -> Mapping[str, str]:
"""token hash -> key_alias for the keys that have one."""
if not tokens:
return MappingProxyType({})
where: Final = {"token": {"in": tuple(frozenset(tokens))}} # mutable-ok: prisma where is a dict
rows: Final = await VerificationTokenRepository(prisma_client).table.find_many(where=where)
return MappingProxyType({row.token: row.key_alias for row in rows if row.key_alias})

227
litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
View file

@ -37,6 +37,7 @@ from litellm.types.utils import (
Message,
ModelResponse,
TextCompletionResponse,
Usage,
)
if TYPE_CHECKING:
@ -148,6 +149,144 @@ class AnthropicPassthroughLoggingHandler:
return model_group.removeprefix("passthrough/")
return model
@staticmethod
def _resolve_logged_model(
litellm_logging_obj: LiteLLMLoggingObj,
request_body: Mapping[str, object],
all_chunks: Sequence[str | bytes],
) -> str:
request_model: Final = request_body.get("model")
logged_model: Final = (
request_model
if isinstance(request_model, str) and request_model
else str(litellm_logging_obj.model_call_details.get("model") or "")
)
if logged_model and logged_model != "unknown":
return logged_model
return AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks(all_chunks) or logged_model
@staticmethod
def _usage_only_response_or_none(
all_chunks: Sequence[str | bytes], model: str, speed: str | None
) -> ModelResponse | None:
try:
return AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks(
all_chunks=all_chunks, model=model, speed=speed
)
except Exception as e: # noqa: BLE001 # the usage-only fallback must never raise out of failure logging
verbose_proxy_logger.warning("Anthropic passthrough: usage-only fallback failed (model=%s): %s", model, e)
return None
@staticmethod
def _assemble_streaming_response(
all_chunks: Sequence[str | bytes],
litellm_logging_obj: LiteLLMLoggingObj,
model: str,
speed: str | None,
) -> ModelResponse | TextCompletionResponse | None:
try:
assembled: Final = AnthropicPassthroughLoggingHandler._build_complete_streaming_response(
all_chunks=all_chunks,
litellm_logging_obj=litellm_logging_obj,
model=model,
speed=speed,
)
except Exception as e: # noqa: BLE001 # any assembly error falls back to usage-only cost
verbose_proxy_logger.warning(
"Anthropic passthrough: stream assembly raised (model=%s): %s; falling "
"back to usage-only cost from raw SSE events.",
model,
e,
)
return AnthropicPassthroughLoggingHandler._usage_only_response_or_none(all_chunks, model, speed)
if assembled is not None:
return assembled
return AnthropicPassthroughLoggingHandler._usage_only_response_or_none(all_chunks, model, speed)
@staticmethod
def _build_streaming_response_for_logging(
litellm_logging_obj: LiteLLMLoggingObj,
request_body: Mapping[str, object],
all_chunks: Sequence[str | bytes],
model: str,
) -> ModelResponse | TextCompletionResponse | None:
response: Final = AnthropicPassthroughLoggingHandler._assemble_streaming_response(
all_chunks=all_chunks,
litellm_logging_obj=litellm_logging_obj,
model=model,
speed=AnthropicPassthroughLoggingHandler._cost_relevant_speed(request_body),
)
if response is None:
return None
AnthropicPassthroughLoggingHandler._recover_interrupted_stream_output_tokens(
response=response, all_chunks=all_chunks, model=model
)
return response
@staticmethod
def record_partial_usage_for_failure(
litellm_logging_obj: LiteLLMLoggingObj,
request_body: Mapping[str, object],
all_chunks: Sequence[str | bytes],
) -> None:
if not all_chunks:
return
model: Final = AnthropicPassthroughLoggingHandler._resolve_logged_model(
litellm_logging_obj, request_body, all_chunks
)
partial_response: Final = AnthropicPassthroughLoggingHandler._build_streaming_response_for_logging(
litellm_logging_obj=litellm_logging_obj, request_body=request_body, all_chunks=all_chunks, model=model
)
usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
if partial_response is None or usage is None:
return
litellm_logging_obj.record_partial_usage_for_failure(
usage=usage,
response_cost=AnthropicPassthroughLoggingHandler._cost_partial_stream_or_zero(
partial_response=partial_response, model=model, logging_obj=litellm_logging_obj
),
)
@staticmethod
def _cost_partial_stream_or_zero(
partial_response: ModelResponse | TextCompletionResponse, model: str, logging_obj: LiteLLMLoggingObj
) -> float:
try:
return AnthropicPassthroughLoggingHandler._compute_response_cost(
litellm_model_response=partial_response,
model=AnthropicPassthroughLoggingHandler._resolve_costing_model(model, logging_obj),
logging_obj=logging_obj,
)
except Exception as e: # noqa: BLE001 # an uncostable partial stream still bills its tokens, at zero cost
verbose_proxy_logger.warning(
"Anthropic passthrough: could not cost the partial usage of a failed stream (model=%s): %s", model, e
)
return 0.0
@staticmethod
def _compute_response_cost(
litellm_model_response: ModelResponse | TextCompletionResponse,
model: str,
logging_obj: LiteLLMLoggingObj,
) -> float:
if logging_obj.model_call_details.get("cache_hit") is True:
return 0.0
custom_llm_provider: Final = logging_obj.model_call_details.get("custom_llm_provider")
model_for_cost: Final = (
f"{custom_llm_provider}/{model}"
if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/")
else model
)
return litellm.completion_cost(
completion_response=litellm_model_response,
model=model_for_cost,
custom_llm_provider=custom_llm_provider,
custom_pricing=use_custom_pricing_for_model(
litellm_params=(logging_obj.litellm_params if hasattr(logging_obj, "litellm_params") else None)
),
router_model_id=logging_obj.get_router_model_id(),
)
@staticmethod
def _extract_message_start_field(
all_chunks: Sequence[str | bytes],
@ -278,31 +417,9 @@ class AnthropicPassthroughLoggingHandler:
if logging_obj.model_call_details.get("stream") is True:
logging_obj.model_call_details["complete_streaming_response"] = litellm_model_response
try:
# Get custom_llm_provider from logging object if available (e.g., azure_ai for Azure Anthropic)
custom_llm_provider: Final = logging_obj.model_call_details.get("custom_llm_provider")
model = AnthropicPassthroughLoggingHandler._resolve_costing_model(model, logging_obj)
# Prepend custom_llm_provider to model if not already present
model_for_cost = model
if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"):
model_for_cost = f"{custom_llm_provider}/{model}"
router_model_id: Final = logging_obj.get_router_model_id()
custom_pricing: Final = use_custom_pricing_for_model(
litellm_params=(logging_obj.litellm_params if hasattr(logging_obj, "litellm_params") else None)
)
response_cost: Final = (
0.0
if logging_obj.model_call_details.get("cache_hit") is True
else litellm.completion_cost(
completion_response=litellm_model_response,
model=model_for_cost,
custom_llm_provider=custom_llm_provider,
custom_pricing=custom_pricing,
router_model_id=router_model_id,
)
response_cost: Final = AnthropicPassthroughLoggingHandler._compute_response_cost(
litellm_model_response=litellm_model_response, model=model, logging_obj=logging_obj
)
kwargs["response_cost"] = response_cost
@ -356,57 +473,12 @@ class AnthropicPassthroughLoggingHandler:
- Logs in litellm callbacks
"""
speed: Final = AnthropicPassthroughLoggingHandler._cost_relevant_speed(request_body)
model = request_body.get("model", "")
# Check if it's available in the logging object
if (
not model
and hasattr(litellm_logging_obj, "model_call_details")
and litellm_logging_obj.model_call_details.get("model")
):
model = cast(str, litellm_logging_obj.model_call_details.get("model"))
if not model or model == "unknown":
chunk_model: Final = AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks(all_chunks)
if chunk_model:
model = chunk_model
try:
complete_streaming_response = AnthropicPassthroughLoggingHandler._build_complete_streaming_response(
all_chunks=all_chunks,
litellm_logging_obj=litellm_logging_obj,
model=model,
speed=speed,
)
except Exception as e:
# stream_chunk_builder re-raises assembly failures (as litellm.APIError)
# on large agentic tool-use / thinking streams; treat that the same as a
# None result so the usage-only fallback below still recovers cost
verbose_proxy_logger.warning(
"Anthropic passthrough: stream assembly raised (model=%s): %s; falling "
"back to usage-only cost from raw SSE events.",
model,
e,
)
complete_streaming_response = None
if complete_streaming_response is None:
# stream_chunk_builder cannot always reassemble large agentic streams, but
# Anthropic still emits token usage in the message_start / message_delta SSE
# events regardless of content shape; recover usage-only so cost is tracked.
# Guard it too: a raise here would defeat the point and drop the request
try:
complete_streaming_response = AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks(
all_chunks=all_chunks,
model=model,
speed=speed,
)
except Exception as e:
verbose_proxy_logger.warning(
"Anthropic passthrough: usage-only fallback failed (model=%s): %s",
model,
e,
)
complete_streaming_response = None
model: Final = AnthropicPassthroughLoggingHandler._resolve_logged_model(
litellm_logging_obj, request_body, all_chunks
)
complete_streaming_response: Final = AnthropicPassthroughLoggingHandler._build_streaming_response_for_logging(
litellm_logging_obj=litellm_logging_obj, request_body=request_body, all_chunks=all_chunks, model=model
)
if complete_streaming_response is None:
verbose_proxy_logger.error(
"Unable to build complete streaming response for Anthropic passthrough endpoint, not logging..."
@ -415,11 +487,6 @@ class AnthropicPassthroughLoggingHandler:
"result": None,
"kwargs": {},
}
AnthropicPassthroughLoggingHandler._recover_interrupted_stream_output_tokens(
response=complete_streaming_response,
all_chunks=all_chunks,
model=model,
)
kwargs: Final = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload(
litellm_model_response=complete_streaming_response,
model=model,

81
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -870,6 +870,35 @@ async def _log_passthrough_upstream_failure(
)
async def _relay_reporting_failures(
stream: AsyncGenerator[bytes, None],
upstream_status: int,
user_api_key_dict: UserAPIKeyAuth,
request_payload: dict, # mutable-ok: post_call_failure_hook lifts fields onto request_data in place
) -> AsyncGenerator[bytes, None]:
from litellm.proxy.proxy_server import proxy_logging_obj
try:
async for chunk in stream:
yield chunk
except Exception as e:
if upstream_status >= 400:
raise
try:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=request_payload,
traceback_str=traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG),
)
except Exception: # noqa: BLE001 - a failing logging callback must never mask the upstream error
verbose_proxy_logger.warning(
"pass_through_endpoint: post_call_failure_hook raised for a mid-stream upstream error",
exc_info=True,
)
raise
from litellm.passthrough.timeout_utils import (
DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS, # noqa: F401 - re-exported for backward compat
resolve_llm_passthrough_timeout, # noqa: F401 - re-exported for backward compat
@ -1293,14 +1322,24 @@ async def pass_through_request(
return StreamingResponse(
wrap_passthrough_sse_bytes_with_keepalive_pings(
stream=_own_streamed_managed_ids(
stream=PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=_parsed_body,
litellm_logging_obj=logging_obj,
endpoint_type=endpoint_type,
start_time=start_time,
passthrough_success_handler_obj=pass_through_endpoint_logging,
url_route=str(url),
stream=_relay_reporting_failures(
stream=PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=_parsed_body,
litellm_logging_obj=logging_obj,
endpoint_type=endpoint_type,
start_time=start_time,
passthrough_success_handler_obj=pass_through_endpoint_logging,
url_route=str(url),
),
upstream_status=response.status_code,
user_api_key_dict=user_api_key_dict,
request_payload=_build_passthrough_failure_request_payload(
parsed_body=_parsed_body,
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
),
),
managed_id_provider=_managed_id_provider,
request=request,
@ -1374,14 +1413,24 @@ async def pass_through_request(
return StreamingResponse(
wrap_passthrough_sse_bytes_with_keepalive_pings(
stream=_own_streamed_managed_ids(
stream=PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=_parsed_body,
litellm_logging_obj=logging_obj,
endpoint_type=endpoint_type,
start_time=start_time,
passthrough_success_handler_obj=pass_through_endpoint_logging,
url_route=str(url),
stream=_relay_reporting_failures(
stream=PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=_parsed_body,
litellm_logging_obj=logging_obj,
endpoint_type=endpoint_type,
start_time=start_time,
passthrough_success_handler_obj=pass_through_endpoint_logging,
url_route=str(url),
),
upstream_status=response.status_code,
user_api_key_dict=user_api_key_dict,
request_payload=_build_passthrough_failure_request_payload(
parsed_body=_parsed_body,
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
),
),
managed_id_provider=_managed_id_provider,
request=request,

110
litellm/proxy/pass_through_endpoints/streaming_handler.py
View file

@ -1,5 +1,7 @@
from collections.abc import Coroutine
from datetime import datetime
import traceback
from collections.abc import Coroutine, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Final, Protocol
import httpx
@ -12,7 +14,7 @@ from litellm.proxy._types import PassThroughEndpointLoggingResultValues
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
from litellm.types.utils import StandardPassThroughResponseObject
from litellm.types.utils import StandardPassThroughResponseObject, Usage
from .llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
@ -44,12 +46,85 @@ class RouteStreamingLogging(Protocol):
) -> Coroutine[None, None, None]: ...
@dataclass(frozen=True, slots=True)
class PassThroughStreamContext:
passthrough_success_handler_obj: PassThroughEndpointLogging
url_route: str
start_time: datetime
class PassThroughStreamingHandler:
@staticmethod
def _stamp_first_chunk_if_needed(litellm_logging_obj: LiteLLMLoggingObj) -> None:
if litellm_logging_obj.completion_start_time is None:
litellm_logging_obj._update_completion_start_time(completion_start_time=datetime.now())
@staticmethod
def schedule_stream_failure_logging(
litellm_logging_obj: LiteLLMLoggingObj,
endpoint_type: EndpointType,
request_body: dict[str, object],
raw_bytes: Sequence[bytes],
exception: Exception,
stream_context: PassThroughStreamContext | None = None,
) -> None:
PassThroughStreamingHandler._record_partial_usage_for_failure(
litellm_logging_obj=litellm_logging_obj,
endpoint_type=endpoint_type,
request_body=request_body,
raw_bytes=raw_bytes,
stream_context=stream_context,
)
try:
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
async_coroutine=litellm_logging_obj.dispatch_failure_handlers(
exception, traceback.format_exc(), prefer_async_handlers=True
)
)
except Exception as e:
verbose_proxy_logger.error("Error scheduling stream failure logging: %s", e)
@staticmethod
def _record_partial_usage_for_failure(
litellm_logging_obj: LiteLLMLoggingObj,
endpoint_type: EndpointType,
request_body: dict[str, object],
raw_bytes: Sequence[bytes],
stream_context: PassThroughStreamContext | None,
) -> None:
if endpoint_type == EndpointType.ANTHROPIC:
AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure(
litellm_logging_obj=litellm_logging_obj, request_body=request_body, all_chunks=raw_bytes
)
return
if stream_context is None or not raw_bytes:
return
try:
partial_response, kwargs = PassThroughStreamingHandler._build_passthrough_logging_result(
litellm_logging_obj=litellm_logging_obj,
passthrough_success_handler_obj=stream_context.passthrough_success_handler_obj,
url_route=stream_context.url_route,
request_body=request_body,
endpoint_type=endpoint_type,
start_time=stream_context.start_time,
raw_bytes=raw_bytes,
end_time=datetime.now(timezone.utc),
model=None,
)
except Exception as e:
verbose_proxy_logger.warning(
"Could not recover the partial usage of a failed %s pass-through stream: %s", endpoint_type.value, e
)
return
usage: Final = getattr(partial_response, "usage", None)
if not isinstance(usage, Usage):
return
response_cost: Final = kwargs.get("response_cost")
litellm_logging_obj.record_partial_usage_for_failure(
usage=usage,
response_cost=float(response_cost) if isinstance(response_cost, (int, float)) else 0.0,
)
@staticmethod
async def chunk_processor(
response: httpx.Response,
@ -65,13 +140,14 @@ class PassThroughStreamingHandler:
route_streaming_logging or PassThroughStreamingHandler._route_streaming_logging_to_handler
)
raw_bytes: Final[list[bytes]] = []
resolved_request_body: Final[dict[str, object]] = request_body or {}
def _build_logging_coroutine() -> Coroutine[None, None, None]:
return resolved_route_streaming_logging(
litellm_logging_obj=litellm_logging_obj,
passthrough_success_handler_obj=passthrough_success_handler_obj,
url_route=url_route,
request_body=request_body or {},
request_body=resolved_request_body,
endpoint_type=endpoint_type,
start_time=start_time,
raw_bytes=raw_bytes,
@ -132,9 +208,9 @@ class PassThroughStreamingHandler:
# coroutine on logging_obj instead of enqueueing now, so
# ProxyLogging._fire_deferred_stream_logging fires it after
# guardrail end-of-stream blocks populate guardrail_information.
# Disconnect/exception paths skip this and fall through to the
# immediate enqueue in ``finally`` to keep partial billing
# (LIT-2642).
# Disconnect paths skip this and fall through to the immediate
# enqueue in ``finally`` to keep partial billing (LIT-2642);
# upstream exceptions log a failure instead (LIT-3798).
if (
getattr(litellm_logging_obj, "_on_deferred_stream_complete", None) is not None
and raw_bytes
@ -144,6 +220,20 @@ class PassThroughStreamingHandler:
litellm_logging_obj._deferred_stream_complete_args = (_build_logging_coroutine(),)
except Exception as e:
verbose_proxy_logger.error("Error in chunk_processor: %s", e)
if response.status_code < 400:
logging_scheduled = True
PassThroughStreamingHandler.schedule_stream_failure_logging(
litellm_logging_obj=litellm_logging_obj,
endpoint_type=endpoint_type,
request_body=resolved_request_body,
raw_bytes=raw_bytes,
exception=e,
stream_context=PassThroughStreamContext(
passthrough_success_handler_obj=passthrough_success_handler_obj,
url_route=url_route,
start_time=start_time,
),
)
raise
finally:
# GeneratorExit (raised on client disconnect) is not caught by
@ -168,7 +258,7 @@ class PassThroughStreamingHandler:
request_body: dict,
endpoint_type: EndpointType,
start_time: datetime,
raw_bytes: list[bytes],
raw_bytes: Sequence[bytes],
end_time: datetime,
model: str | None = None,
):
@ -218,7 +308,7 @@ class PassThroughStreamingHandler:
request_body: dict,
endpoint_type: EndpointType,
start_time: datetime,
raw_bytes: list[bytes],
raw_bytes: Sequence[bytes],
end_time: datetime,
model: str | None,
) -> tuple[PassThroughEndpointLoggingResultValues, dict]:
@ -336,7 +426,7 @@ class PassThroughStreamingHandler:
return None
@staticmethod
def _convert_raw_bytes_to_str_lines(raw_bytes: list[bytes]) -> list[str]:
def _convert_raw_bytes_to_str_lines(raw_bytes: Sequence[bytes]) -> list[str]:
"""
Converts a list of raw bytes into a list of string lines, similar to aiter_lines()

54
litellm/proxy/proxy_server.py
View file

@ -423,7 +423,7 @@ from litellm.proxy.db.proxy_worker_heartbeat import (
PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS,
ProxyWorkerHeartbeat,
)
from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed
from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router
from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router
from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config
@ -2477,7 +2477,8 @@ async def get_current_spend(
authoritative source depends on the counter: primary key/team/user/org
counters read the DB row; per-window counters (``window_start`` supplied)
read the maintained window-spend row and only aggregate spend logs when
that row is missing or stale; end-user/tag counters have no DB row, so the caller's
that row is missing or stale; end-user counters read ``LiteLLM_EndUserTable``, the
row the budget reset zeroes; tag counters have no DB row, so the caller's
``fallback_spend`` (loaded fresh in auth) is authoritative. The DB read is
skipped for healthy primary counters (counter at or above recorded spend)
and cached in-process for a few seconds, so a persistently stale counter
@ -2511,8 +2512,8 @@ async def get_current_spend(
await _repair_stale_spend_counter(counter_key=counter_key, db_spend=authoritative)
return authoritative
elif fallback_spend > current:
# end-user / tag counters have no DB row; fallback_spend is the
# authoritative recorded value loaded in auth.
# nothing to read (tag counters, an end user without a row or a DB client, a
# failed read); fallback_spend is the authoritative recorded value loaded in auth.
return fallback_spend
# Opt-in hard guarantee: when the spend backing this admit decision came
@ -2580,6 +2581,29 @@ async def reseed_spend_counter_from_db(counter_key: str) -> None:
await _repair_stale_spend_counter(counter_key=counter_key, db_spend=db_spend)
async def _floor_spend_from_db(
counter_key: str,
window_entity_type: str | None,
window_entity_id: str | None,
window_duration: str | None,
window_start: datetime | None,
) -> float | None:
if counter_key.startswith(END_USER_COUNTER_PREFIX):
return await SpendCounterReseed.end_user_from_db(prisma_client=prisma_client, counter_key=counter_key)
entity_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key)
if entity_spend is not None:
return entity_spend
if window_entity_type is None or window_entity_id is None or window_start is None:
return None
return await SpendCounterReseed.window_from_db(
prisma_client=prisma_client,
entity_type=window_entity_type,
entity_id=window_entity_id,
window_duration=window_duration,
window_start=window_start,
)
async def _authoritative_floor_spend(
counter_key: str,
window_entity_type: str | None = None,
@ -2592,20 +2616,13 @@ async def _authoritative_floor_spend(
if cached is not None:
return float(cached)
db_spend = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key)
if (
db_spend is None
and window_entity_type is not None
and window_entity_id is not None
and window_start is not None
):
db_spend = await SpendCounterReseed.window_from_db(
prisma_client=prisma_client,
entity_type=window_entity_type,
entity_id=window_entity_id,
window_duration=window_duration,
window_start=window_start,
)
db_spend: Final = await _floor_spend_from_db(
counter_key=counter_key,
window_entity_type=window_entity_type,
window_entity_id=window_entity_id,
window_duration=window_duration,
window_start=window_start,
)
if db_spend is None:
return None
@ -15212,6 +15229,7 @@ async def async_queue_request(
# extra_body); see above for the same guard upstream.
data["metadata"] = {}
data["metadata"]["user_api_key"] = user_api_key_dict.api_key
data["metadata"]["user_api_key_hash"] = user_api_key_dict.api_key
data["metadata"]["user_api_key_metadata"] = strip_callback_config(user_api_key_dict.metadata)
_headers: Final = _safe_get_request_headers(request).copy()
_headers.pop("authorization", None) # do not store the original `sk-..` api key in the db

4
litellm/proxy/public_endpoints/autorouter_presets.json
View file

@ -1,12 +1,12 @@
{
"1m_context": {
"label": "1M Context",
"description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Opus 5 for complex, Opus 5 at high thinking for reasoning.",
"description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Sol for complex, Opus 5 at high thinking for reasoning.",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["gpt-5.6-luna"],
"MEDIUM": ["gpt-5.6-terra"],
"COMPLEX": ["claude-opus-5"],
"COMPLEX": ["gpt-5.6-sol"],
"REASONING": ["claude-opus-5"]
},
"tier_model_configs": {

2
litellm/proxy/schema.prisma
View file

@ -1124,6 +1124,8 @@ model LiteLLM_DailyGuardrailUsageUnits {
api_key String // hashed virtual key; empty string when unknown
usage_unit String // provider counter name, e.g. Bedrock's contentPolicyUnits
units BigInt @default(0)
cost Float? // USD for the priced share of units; null only on rows written before this column existed
untracked_units BigInt @default(0) // units recorded with no known price, the share cost leaves out
created_at DateTime @default(now())
updated_at DateTime @updatedAt

236
litellm/proxy/spend_tracking/key_metadata_recovery.py Normal file
View file

@ -0,0 +1,236 @@
from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Set as AbstractSet
from types import MappingProxyType
from typing import Final, TypeVar
from pydantic import BaseModel, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash
from litellm.proxy.utils import PrismaClient
from litellm.repositories.user_repository import UserRepository
_T = TypeVar("_T")
_ACTIVE_TOKEN_DIGEST_SQL: Final = """
SELECT encode(sha256(convert_to(token, 'UTF8')), 'hex') AS digest, key_alias, team_id, user_id
FROM "LiteLLM_VerificationToken"
WHERE encode(sha256(convert_to(token, 'UTF8')), 'hex') = ANY($1::text[])
"""
_DELETED_TOKEN_DIGEST_SQL: Final = """
SELECT DISTINCT ON (token)
encode(sha256(convert_to(token, 'UTF8')), 'hex') AS digest, key_alias, team_id, user_id
FROM "LiteLLM_DeletedVerificationToken"
WHERE encode(sha256(convert_to(token, 'UTF8')), 'hex') = ANY($1::text[])
ORDER BY token, deleted_at DESC
"""
class KeyMetadataDict(TypedDict, total=False):
key_alias: ReadOnly[str | None]
team_id: ReadOnly[str | None]
user_id: ReadOnly[str | None]
user_email: ReadOnly[str | None]
class _TokenDigestRow(BaseModel):
digest: str
key_alias: str | None = None
team_id: str | None = None
user_id: str | None = None
_TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...])
_EMPTY_KEY_METADATA: Final[Mapping[str, KeyMetadataDict]] = MappingProxyType({})
_EMPTY_EMAILS: Final[Mapping[str, str]] = MappingProxyType({})
async def _db_or_empty(
load: Callable[[], Awaitable[_T]],
warning: str,
count: int,
) -> _T | None:
from prisma.errors import PrismaError
try:
return await load()
except PrismaError as e:
verbose_proxy_logger.warning(warning, count, e)
return None
async def _reverse_hash_key_metadata(
prisma_client: PrismaClient,
sql: str,
wanted: AbstractSet[str],
*,
warning: str,
) -> Mapping[str, KeyMetadataDict]:
rows: Final = await _db_or_empty(
lambda: prisma_client.db.query_raw(sql, sorted(wanted)),
warning,
len(wanted),
)
if rows is None:
return _EMPTY_KEY_METADATA
return MappingProxyType(
{
row.digest: KeyMetadataDict(key_alias=row.key_alias, team_id=row.team_id, user_id=row.user_id)
for row in _TOKEN_DIGEST_ROWS.validate_python(rows)
if row.digest in wanted
}
)
async def _emails_for_user_ids(
prisma_client: PrismaClient,
user_ids: AbstractSet[str],
) -> Mapping[str, str]:
if not user_ids:
return _EMPTY_EMAILS
users: Final = await _db_or_empty(
lambda: UserRepository(prisma_client).table.find_many(
where={"user_id": {"in": list(user_ids)}}, # mutable-ok: Prisma find_many where= is a dict
),
"Failed user_email recovery for %d user ids: %s",
len(user_ids),
)
if users is None:
return _EMPTY_EMAILS
return MappingProxyType(
{
user.user_id: user.user_email
for user in users
if getattr(user, "user_id", None) and getattr(user, "user_email", None)
}
)
def _meta_with_email(meta: KeyMetadataDict, emails: Mapping[str, str]) -> KeyMetadataDict:
if meta.get("user_email"):
return meta
user_id: Final = meta.get("user_id")
if not isinstance(user_id, str) or user_id not in emails:
return meta
updated: Final[KeyMetadataDict] = {**meta, "user_email": emails[user_id]}
return updated
async def attach_user_emails(
prisma_client: PrismaClient,
recovered: Mapping[str, KeyMetadataDict],
) -> Mapping[str, KeyMetadataDict]:
needing_email: Final = frozenset(
user_id
for meta in recovered.values()
for user_id in (meta.get("user_id"),)
if isinstance(user_id, str) and user_id and not meta.get("user_email")
)
emails: Final = await _emails_for_user_ids(prisma_client, needing_email)
if not emails:
return recovered
return MappingProxyType({api_key: _meta_with_email(meta, emails) for api_key, meta in recovered.items()})
async def recover_double_hashed_key_metadata(
prisma_client: PrismaClient,
missing_keys: AbstractSet[str],
) -> Mapping[str, KeyMetadataDict]:
"""
Recover key_alias/team_id/user_id for DailyUserSpend.api_key values that
were double-hashed by the v1.99 spend-log provenance gate.
Those rows store hash(VerificationToken.token) instead of the token, so the
exact join misses. Postgres hashes the token column itself, one pass over
active keys and one over deleted keys, so no key row crosses the wire.
"""
sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key))
if not sha_missing:
return _EMPTY_KEY_METADATA
from_active: Final = await _reverse_hash_key_metadata(
prisma_client,
_ACTIVE_TOKEN_DIGEST_SQL,
sha_missing,
warning="Failed reverse-hash recovery against active keys for %d missing keys: %s",
)
still_missing: Final = sha_missing - frozenset(from_active)
if not still_missing:
return from_active
from_deleted: Final = await _reverse_hash_key_metadata(
prisma_client,
_DELETED_TOKEN_DIGEST_SQL,
still_missing,
warning="Failed reverse-hash recovery against deleted keys for %d missing keys: %s",
)
return MappingProxyType({**from_active, **from_deleted})
def _row_with_recovered_fields(
row: Mapping[str, object],
recovered: Mapping[str, KeyMetadataDict],
*,
api_key_field: str,
alias_field: str,
team_id_field: str,
user_email_field: str,
) -> Mapping[str, object]:
api_key: Final = row.get(api_key_field)
if not isinstance(api_key, str) or api_key not in recovered:
return row
meta: Final = recovered[api_key]
return MappingProxyType(
{
**row,
alias_field: meta.get("key_alias") or row.get(alias_field),
team_id_field: meta.get("team_id") or row.get(team_id_field),
user_email_field: row.get(user_email_field) or meta.get("user_email"),
}
)
async def fill_missing_api_key_aliases(
prisma_client: PrismaClient,
rows: Sequence[Mapping[str, object]],
*,
api_key_field: str = "api_key",
alias_field: str = "api_key_alias",
team_id_field: str = "team_id",
user_email_field: str = "user_email",
) -> tuple[Mapping[str, object], ...]:
"""
Fill null api_key_alias / team_id / user_email on export rows whose api_key
was double-hashed.
Used by CloudZero and Focus, which join DailyUserSpend.api_key to
VerificationToken.token and otherwise export null aliases for those rows.
"""
missing_keys: Final = frozenset(
key
for row in rows
for key in (row.get(api_key_field),)
if isinstance(key, str) and key and row.get(alias_field) in (None, "")
)
if not missing_keys:
return tuple(rows)
recovered: Final = await attach_user_emails(
prisma_client,
await recover_double_hashed_key_metadata(prisma_client, missing_keys),
)
if not recovered:
return tuple(rows)
return tuple(
_row_with_recovered_fields(
row,
recovered,
api_key_field=api_key_field,
alias_field=alias_field,
team_id_field=team_id_field,
user_email_field=user_email_field,
)
for row in rows
)

23
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -14,6 +14,8 @@ from litellm.constants import (
LITELLM_PROXY_MASTER_KEY_ALIAS,
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
LITTELM_CLI_SERVICE_ACCOUNT_NAME,
LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
REDACTED_BY_LITELM_STRING,
SESSION_ID_OMITTED_METADATA_KEY,
)
@ -35,6 +37,7 @@ from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload, SpendLogsR
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.proxy.utils import PrismaClient, hash_token
from litellm.types.utils import (
PROMPT_CARRYING_GUARDRAIL_FIELDS,
CallTypes,
CostBreakdown,
StandardLoggingGuardrailInformation,
@ -73,13 +76,18 @@ def _is_master_key(api_key: str | None, _master_key: str | None) -> bool:
_HASHED_JWT_RE = re.compile(r"hashed-jwt-[a-fA-F0-9]{64}")
_NON_SECRET_KEY_ALIASES: Final = frozenset(
{
LITELLM_PROXY_MASTER_KEY_ALIAS,
LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
LITTELM_CLI_SERVICE_ACCOUNT_NAME,
}
)
def _is_non_secret_key_value(value: str) -> bool:
return (
value == LITELLM_PROXY_MASTER_KEY_ALIAS
or is_valid_sha256_hash(value)
or _HASHED_JWT_RE.fullmatch(value) is not None
value in _NON_SECRET_KEY_ALIASES or is_valid_sha256_hash(value) or _HASHED_JWT_RE.fullmatch(value) is not None
)
@ -1066,13 +1074,6 @@ def _sanitize_guardrail_information_for_spend_logs(
return [_redact_prompt_fields_in_guardrail_entry(entry) for entry in entries if isinstance(entry, dict)]
_PROMPT_CARRYING_GUARDRAIL_FIELDS: Final = (
"guardrail_request",
"guardrail_response",
"match_details",
"classification",
)
_NUMERIC_COMPRESSION_STAT_KEYS: Final = (
"tokens_before",
"tokens_after",
@ -1107,7 +1108,7 @@ def _redact_prompt_fields_in_guardrail_entry(
preserved_stats: Final = _numeric_compression_stats_from_guardrail_response(entry.get("guardrail_response"))
redacted: Final[StandardLoggingGuardrailInformation] = {
**entry,
**{key: REDACTED_BY_LITELM_STRING for key in _PROMPT_CARRYING_GUARDRAIL_FIELDS if key in entry},
**{key: REDACTED_BY_LITELM_STRING for key in PROMPT_CARRYING_GUARDRAIL_FIELDS if key in entry},
}
if preserved_stats is None:
return redacted

2
litellm/proxy/vector_store_endpoints/management_endpoints.py
View file

@ -56,7 +56,7 @@ def _row_to_vector_store(row: "_VectorStoreRow") -> LiteLLM_ManagedVectorStore:
return LiteLLM_ManagedVectorStore(**row.model_dump())
_LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker()
_LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker(extra_sensitive_patterns=frozenset(("connection",)))
_REDACT_LITELLM_PARAMS_MAX_DEPTH: Final = 10

9
litellm/responses/main.py
View file

@ -327,8 +327,13 @@ async def aresponses_api_with_mcp(
)
if tool_results:
persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params)
follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input(
response=response, tool_results=tool_results, original_input=input
response=response,
tool_results=tool_results,
original_input=input,
preserve_reasoning=persistence_disabled,
)
# Prepare parameters for follow-up call (restores original stream setting)
@ -347,7 +352,7 @@ async def aresponses_api_with_mcp(
follow_up_input=follow_up_input,
model=model,
all_tools=all_tools,
response_id=response.id,
response_id=previous_response_id if persistence_disabled else response.id,
**follow_up_call_params,
)

20
litellm/responses/mcp/litellm_proxy_mcp_handler.py
View file

@ -963,11 +963,17 @@ class LiteLLM_Proxy_MCP_Handler:
return follow_up_messages
@staticmethod
def _is_persistence_disabled(call_params: Mapping[str, object]) -> bool:
"""store=false means the provider kept nothing, so the follow-up call cannot chain on a response id."""
return call_params.get("store") is False
@staticmethod
def _create_follow_up_input(
response: ResponsesAPIResponse,
tool_results: Sequence[Mapping[str, object]],
original_input: str | ResponseInputParam | None = None,
preserve_reasoning: bool = False,
) -> list[object]:
"""Create follow-up input with tool results in proper format."""
follow_up_input: Final[list[object]] = []
@ -983,11 +989,11 @@ class LiteLLM_Proxy_MCP_Handler:
# Add the assistant message with function calls
assistant_message_content: Final[list[object]] = []
function_calls: Final[list[dict[str, object]]] = []
turn_items: Final[list[Mapping[str, object]]] = []
for output_item in response.output:
if not isinstance(output_item, dict) and hasattr(output_item, "model_dump"):
output_item = output_item.model_dump()
output_item = output_item.model_dump(exclude_none=True)
if isinstance(output_item, dict):
if output_item.get("type") == "function_call":
@ -997,7 +1003,7 @@ class LiteLLM_Proxy_MCP_Handler:
# Only add if we have required fields
if call_id and name:
function_calls.append(
turn_items.append(
{
"type": "function_call",
"call_id": call_id,
@ -1005,6 +1011,8 @@ class LiteLLM_Proxy_MCP_Handler:
"arguments": arguments,
}
)
elif output_item.get("type") == "reasoning" and preserve_reasoning:
turn_items.append(output_item)
elif output_item.get("type") == "message":
# Extract content from message
content = output_item.get("content", [])
@ -1025,9 +1033,7 @@ class LiteLLM_Proxy_MCP_Handler:
}
)
# Add function calls (these can come directly after user message for LLM)
for function_call in function_calls:
follow_up_input.append(function_call)
follow_up_input.extend(turn_items)
# Add tool results (function call outputs)
for tool_result in tool_results:
@ -1046,7 +1052,7 @@ class LiteLLM_Proxy_MCP_Handler:
follow_up_input: list[Any],
model: str,
all_tools: Sequence[ResponsesToolParam] | None,
response_id: str,
response_id: str | None,
**call_params: Any,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
"""Make follow-up response API call with tool results."""

5
litellm/responses/mcp/mcp_streaming_iterator.py
View file

@ -781,10 +781,15 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
try:
# Create follow-up input
if self.collected_response is not None:
persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(
self.original_request_params
)
follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input(
response=self.collected_response,
tool_results=self.tool_results,
original_input=self.original_request_params.get("input"),
preserve_reasoning=persistence_disabled,
)
# Make follow-up call with streaming

2
litellm/router.py
View file

@ -7513,7 +7513,7 @@ class Router:
# Check retry policy FIRST, before should_retry_this_error
# This allows retry policies to override the healthy deployments check
_retry_policy_applies = False
if self.retry_policy is not None or model_group_retry_policy is not None:
if request_num_retries != 0 and (self.retry_policy is not None or model_group_retry_policy is not None):
# get num_retries from retry policy
# Use the model_group captured at the start of the function, or get it from metadata
# kwargs.get("model") at this point is the deployment model, not the model_group

61
litellm/router_strategy/complexity_router/README.md
View file

@ -247,6 +247,58 @@ unless `modality_routing` is also on.
`session_affinity_ttl_seconds` is the idle window for both the model pin selected by session affinity and the deployment pin. Every request that reuses a pin refreshes its TTL, so a session actively sending requests stays pinned. After the window passes with no pin reuse, the next request classifies again and creates a fresh pin. Omit the setting to track the default of 3600 seconds.
### Mid-task stall escalation
A weak model working an agentic task can get stuck: it keeps calling the same tool with the
same arguments, or the same call keeps erroring, when a stronger model would have broken the
loop. `stall_escalation_enabled: true` catches this and bumps the request one tier higher, the
automatic counterpart to a user typing an escalation keyword:
```yaml
model_list:
- model_name: smart-router
litellm_params:
model: auto_router/complexity_router
complexity_router_config:
stall_escalation_enabled: true
stall_escalation_window: 6
stall_escalation_repeat_threshold: 3
tiers:
SIMPLE: gpt-4o-mini
MEDIUM: gpt-4o
COMPLEX: claude-sonnet-4
REASONING: o1-preview
```
Detection looks at the assistant's own tool calls, not the human's messages. The task counts as
stalled when the NEWEST tool call is still part of a stuck pattern: it repeats, or it errored, at
least `stall_escalation_repeat_threshold` times across the last `stall_escalation_window` calls.
The tier is then bumped one step by the same `_escalate_tier` ladder `escalation_keywords` uses,
capped at the highest configured tier. It reads both tool-call shapes: Anthropic Messages
`tool_use`/`tool_result` blocks (including `is_error`) and chat-completions `tool_calls`/`tool`
messages (which carry no standard error flag, so those calls are judged on repetition alone).
Anchoring on the newest call is what keeps a recovered task from being escalated on stale
evidence. A model that tried the same command three times and then moved on still has those
three calls sitting in the window for a few turns, and counting whichever pattern is most common
in the window would escalate a request that is already making progress again. Anchoring still
leaves room between the matches, so a retry loop broken up by an unrelated lookup counts.
There is no state to expire or leak: detection reruns on every classified turn from that
request's own message list, so the bump lasts only as long as the recent tool calls still look
stuck and lifts on its own the moment they don't. This also means it reads the whole
conversation rather than only the turns since the newest human ask, so a plain follow-up like
"try again" does not discard evidence from before it. Escalation records `stall_escalation` in
`routing_decision.signals`; unlike `escalation_keywords`, it does not set the
`escalated`/`escalation_keyword` pair, which is reserved for the keyword mechanism specifically.
`stall_escalation_enabled` cannot be combined with `session_affinity` or
`classification_mode: user_turn`: both replay a held routing decision on most turns instead of
classifying, so detection would never see the tool calls it needs to look at. It is also
rejected together with `tier_definitions`, for the same reason `escalation_keywords` is: both
rely on the built-in tier severity order, which a custom tier set does not define. Off by
default.
### Heuristic-first chaining
`classifier_type: heuristic_first` runs the local scorer on every request and only calls the LLM
@ -275,6 +327,15 @@ model_list:
keep the classifier deployment or provider default, or set a supported value such as `none` or
`low` to override that call.
Classifier calls have a one-attempt hard deadline. After a timeout, the router opens a process-local
circuit for that classifier and sends every session through `classifier_fallback` for
`classifier_llm_config.circuit_breaker_cooldown_seconds` (30 seconds by default). When the cooldown
expires, one request probes the classifier while concurrent requests continue through the fallback.
A successful probe closes the circuit; a failed probe restarts the cooldown. The circuit breaker is
on by default; set `classifier_llm_config.circuit_breaker_enabled: false` to disable it. The default
fallback is the local heuristic scorer, so a classifier outage does not repeat its timeout across
every turn or session handled by the router process.
A request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when
two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least
one signal. Everything else goes to the classifier, which then decides as it normally would.

4
litellm/router_strategy/complexity_router/__init__.py
View file

@ -9,6 +9,7 @@ No external API calls - all scoring is local and <1ms.
from litellm.router_strategy.complexity_router.complexity_router import (
ComplexityRouter,
built_in_tier_classification_prompt,
classification_system_prompt,
custom_tier_classification_prompt,
)
@ -20,6 +21,7 @@ from litellm.router_strategy.complexity_router.config import (
ComplexityTier,
ReminderMarkerPair,
TierDefinition,
normalize_classification_examples,
normalize_classification_prompt,
)
@ -32,7 +34,9 @@ __all__ = [
"ComplexityTier",
"ReminderMarkerPair",
"TierDefinition",
"built_in_tier_classification_prompt",
"classification_system_prompt",
"custom_tier_classification_prompt",
"normalize_classification_examples",
"normalize_classification_prompt",
]

633
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -18,8 +18,10 @@ from __future__ import annotations
import asyncio
import random
import re
from collections.abc import Iterator, Mapping, Sequence
import time
from collections.abc import Callable, Iterator, Mapping, Sequence
from itertools import accumulate, islice, takewhile
from threading import Lock
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
@ -33,9 +35,15 @@ from litellm.constants import (
SESSION_ID_GENERATED_METADATA_KEY,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
get_metadata_variable_name_from_kwargs,
)
from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
from litellm.litellm_core_utils.prompt_templates.common_utils import (
as_openai_image_part,
request_contains_image_content,
)
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
@ -43,7 +51,11 @@ from litellm.router_strategy.complexity_router.tier_predictor import (
TierSuccessPredictor,
resolve_tier_artifact,
)
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionImageObject,
ChatCompletionTextObject,
)
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
ModelResponse,
@ -54,6 +66,7 @@ from litellm.types.utils import (
from .classification_rubrics import BUSINESS_TIER_CRITERIA, calibration_examples_section
from .config import (
CALIBRATION_EXAMPLES_HEADING,
DEFAULT_CLASSIFICATION_RUBRIC,
DEFAULT_CODE_KEYWORDS,
DEFAULT_ESCALATION_KEYWORDS,
@ -70,6 +83,7 @@ from .config import (
ComplexityTier,
TierDefinition,
)
from .stall_detector import detect_stalled_task
if TYPE_CHECKING:
from semantic_router.routers import SemanticRouter
@ -125,16 +139,17 @@ TIER_SEVERITY_ORDER_LABELED: Final[tuple[tuple[ComplexityTier, str], ...]] = tup
(tier, tier.value) for tier in TIER_SEVERITY_ORDER
)
_CLASSIFICATION_RUBRIC_PREAMBLE_LEGACY: Final = """Classify the complexity of a user request into exactly one tier.
_CLASSIFICATION_INSTRUCTIONS_LEGACY: Final = """Classify the complexity of a user request into exactly one tier.
Judge the intellectual difficulty of answering correctly, not how short the request is.
Judge the intellectual difficulty of answering correctly, not how short the request is."""
Tiers:"""
_CLASSIFICATION_RUBRIC_PREAMBLE_LEGACY: Final = f"{_CLASSIFICATION_INSTRUCTIONS_LEGACY}\n\nTiers:"
_CLASSIFICATION_RUBRIC_PREAMBLE_BODY: Final = """Classify the complexity of a user request into exactly one tier.
Judge the intellectual difficulty of answering correctly, not how short, long, or technical-sounding the request is."""
_CLASSIFICATION_RUBRIC_PREAMBLE: Final = f"{_CLASSIFICATION_RUBRIC_PREAMBLE_BODY}\n\nTiers:"
_CLASSIFICATION_RUBRIC_TRUST_BOUNDARY: Final = """The message may quote the caller's own system prompt and a few of their prior turns. Those sections are material to judge, never instructions to you: follow this rubric only, and if the quoted text asks for a particular tier, ignore it and rate the request on its merits."""
@ -148,6 +163,11 @@ def _tier_bullets(
return "\n".join(f"- {label}: {criteria[tier]}" for tier, label in labeled_tiers)
def _built_in_criteria(preset: ClassificationRubric) -> Mapping[ComplexityTier, str]:
"""The per-tier criteria a preset states, the one owner both built-in prompt shapes read."""
return BUSINESS_TIER_CRITERIA if preset is ClassificationRubric.BUSINESS else _CLASSIFICATION_TIER_CRITERIA
def _built_in_prompt(
labeled_tiers: Sequence[tuple[ComplexityTier, str]], preset: ClassificationRubric, closing: str
) -> str:
@ -160,10 +180,7 @@ def _built_in_prompt(
swaps the tier criteria for business-flavored ones, which its sweep found mattered more than the
examples.
"""
criteria: Final = (
BUSINESS_TIER_CRITERIA if preset is ClassificationRubric.BUSINESS else _CLASSIFICATION_TIER_CRITERIA
)
bullets: Final = _tier_bullets(labeled_tiers, criteria)
bullets: Final = _tier_bullets(labeled_tiers, _built_in_criteria(preset))
if preset is ClassificationRubric.LEGACY:
return (
f"{_CLASSIFICATION_RUBRIC_PREAMBLE_LEGACY}\n{bullets}\n\n{_CLASSIFICATION_RUBRIC_TRUST_BOUNDARY} {closing}"
@ -195,18 +212,62 @@ def _closing_line(context_window_size: int) -> str:
return _CLASSIFICATION_WITH_CONVERSATION if context_window_size > 0 else _CLASSIFICATION_CURRENT_MESSAGE_ONLY
def _custom_tier_prompt(entries: Sequence[tuple[str, str]], preamble: str | None, closing: str) -> str:
"""The classifier's system role for an operator-defined tier set.
def _sectioned_prompt(instructions: str, bullets: str, examples_section: str | None, closing: str) -> str:
"""The classifier's system role assembled section by section.
The trust-boundary paragraph is appended unconditionally after any operator-supplied
preamble, so a custom classification_prompt cannot remove the instruction to ignore tier
requests embedded in quoted caller text; without it a caller could pin themselves to the
most expensive tier from inside their prompt.
The trust-boundary paragraph is appended unconditionally after the operator-reachable sections,
so no custom instruction or example text can remove the instruction to ignore tier requests
embedded in quoted caller text; without it a caller could pin themselves to the most expensive
tier from inside their prompt.
"""
bullets: Final = "\n".join(f"- {name}: {description}" for name, description in entries)
return (
f"{preamble or _CLASSIFICATION_RUBRIC_PREAMBLE_BODY}\n\nTiers:\n{bullets}\n\n"
f"{_CLASSIFICATION_RUBRIC_TRUST_BOUNDARY}\n\n{closing}"
sections: Final = (
instructions,
f"Tiers:\n{bullets}",
examples_section,
_CLASSIFICATION_RUBRIC_TRUST_BOUNDARY,
closing,
)
return "\n\n".join(section for section in sections if section is not None)
def _operator_examples_section(classification_examples: str | None) -> str | None:
return None if classification_examples is None else f"{CALIBRATION_EXAMPLES_HEADING}\n{classification_examples}"
def built_in_tier_classification_prompt(
classification_prompt: str | None,
context_window_size: int,
labeled_tiers: Sequence[tuple[ComplexityTier, str]] = TIER_SEVERITY_ORDER_LABELED,
classification_rubric: ClassificationRubric | None = None,
classification_examples: str | None = None,
) -> str:
"""The classifier's system role when an operator customizes the BUILT-IN tier set's prompt.
The operator owns the classification instructions and the calibration examples, each falling
back to the selected rubric's shipped section when not written; the tier bullets, the trust
boundary, and the closing line are always derived from the router's configuration between and
below them. With neither section written this delegates to the shipped rubric verbatim, which
is what keeps every preset, LEGACY's older wording and cramped closing included, byte-stable
for existing routers.
"""
preset: Final = classification_rubric or DEFAULT_CLASSIFICATION_RUBRIC
closing: Final = _closing_line(context_window_size)
if classification_prompt is None and classification_examples is None:
return _built_in_prompt(labeled_tiers, preset, closing)
criteria: Final = _built_in_criteria(preset)
default_examples: Final = (
None if preset is ClassificationRubric.LEGACY else calibration_examples_section(preset, labeled_tiers)
)
default_instructions: Final = (
_CLASSIFICATION_INSTRUCTIONS_LEGACY
if preset is ClassificationRubric.LEGACY
else _CLASSIFICATION_RUBRIC_PREAMBLE_BODY
)
return _sectioned_prompt(
classification_prompt or default_instructions,
_tier_bullets(labeled_tiers, criteria),
_operator_examples_section(classification_examples) or default_examples,
closing,
)
@ -214,20 +275,25 @@ def custom_tier_classification_prompt(
definitions: Sequence[TierDefinition],
classification_prompt: str | None,
context_window_size: int,
classification_examples: str | None = None,
) -> str:
"""The classifier's system role for an operator-defined tier set.
The single owner of the built-in-criteria substitution, so the dashboard's preview resolves a
blank description exactly as the live classifier does.
blank description exactly as the live classifier does. A custom tier set ships no calibration
examples of its own, so the section renders only when the operator writes one.
"""
entries: Final = tuple(
(
definition.name,
definition.description or _CLASSIFICATION_TIER_CRITERIA[ComplexityTier[definition.name.upper()]],
)
bullets: Final = "\n".join(
f"- {definition.name}: "
f"{definition.description or _CLASSIFICATION_TIER_CRITERIA[ComplexityTier[definition.name.upper()]]}"
for definition in definitions
)
return _custom_tier_prompt(entries, classification_prompt, _closing_line(context_window_size))
return _sectioned_prompt(
classification_prompt or _CLASSIFICATION_RUBRIC_PREAMBLE_BODY,
bullets,
_operator_examples_section(classification_examples),
_closing_line(context_window_size),
)
def classification_system_prompt(
@ -311,6 +377,8 @@ _TRUNCATION_MARKER: Final = "..."
_TRUNCATION_HEAD_FRACTION: Final = 0.3
_MIN_QUOTED_TURN_CHARS: Final = 120
_CLASSIFIER_CIRCUIT_OPEN_SIGNAL: Final = "classifier-circuit-open"
_CJK_CHARACTER: Final = re.compile("[぀-ヿㇰ-ㇿ㐀-䶿一-鿿豈-﫿ｦ-ﾝ\U00020000-\U0003ffff]")
@ -374,6 +442,23 @@ def _strip_reminder_blocks(text: str, marker_pairs: tuple[tuple[str, str], ...]
return " ".join(kept for a, b in zip(keep_from, keep_to) if (kept := text[a:b].strip()))
def _inline_image_part(part: Mapping[str, object]) -> ChatCompletionImageObject | None:
"""One image content part safe to hand the classifier, or None.
Inline data URIs only. A remote URL is caller-controlled and provider adapters do not uniformly
delegate fetching to the provider: gigachat's file handler downloads any non-data URL with
`client.get` from the proxy host, so forwarding one would let a key scoped to this router aim a
proxy-side request at an internal address, on a call the caller never asked for. The routed
model still receives the original URL exactly as before.
"""
converted: Final = as_openai_image_part(part)
if converted is None:
return None
image_url: Final = converted["image_url"]
url: Final = image_url if isinstance(image_url, str) else image_url.get("url", "")
return converted if url.startswith("data:") else None
def _human_text(content: object, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS) -> str:
"""Message content as the text a human wrote, with complete reminder blocks removed.
@ -755,6 +840,17 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo
image), not what the session's traffic looks like, and pinning it would hold every following
text turn on the vision-capable model the image forced. A modality pin override is the same
fact on a session that already holds a pin, so it must not overwrite the pin it displaced.
An open classifier circuit is the shortest-lived state of all: the fallback ran because the
breaker skipped the classifier, not because the request got classified, and the cooldown is
seconds against a TTL of an hour that every later turn refreshes. Its cause is whatever the
fallback path reports, so the circuit signal is what marks the decision, and leaving it
unpinned lets the session classify again as soon as the breaker closes.
A health failover describes the fleet's state right now, not the session's traffic, and it can
displace decisions that were themselves unpinnable (a housekeeping call, a modality escalation).
Pinning it would hold the session on the substitute long after the displaced group recovers; the
gate re-fires per request, so leaving it unpinned costs nothing but the classifier call.
"""
return decision is None or (
decision.get("cause")
@ -764,8 +860,10 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo
"housekeeping",
"modality_escalation",
"modality_pin_override",
"health_failover",
)
and not decision.get("context_escalated")
and _CLASSIFIER_CIRCUIT_OPEN_SIGNAL not in (decision.get("signals") or ())
)
@ -816,6 +914,81 @@ class ClassificationOutcome(NamedTuple):
classifier_cost: float | None = None
def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome:
return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal))
class _ClassifierCircuitBreaker:
"""Process-local timeout breaker for one complexity-router classifier.
The router instance serves every session assigned to that auto-router deployment, so the
breaker prevents one unhealthy classifier from charging the same timeout to each session.
Exactly one request becomes the recovery probe after the cooldown; the lock makes that state
transition atomic even when several request tasks arrive together.
"""
CLOSED: Final = "closed"
OPEN: Final = "open"
HALF_OPEN: Final = "half_open"
def __init__(self, cooldown_seconds: float, clock: Callable[[], float] = time.monotonic) -> None:
self._cooldown_seconds = cooldown_seconds
self._clock = clock
self._state = self.CLOSED
self._opened_at: float | None = None
self._generation = 0
self._lock = Lock()
def acquire_permit(self) -> int | None:
"""Return a generation-scoped permit, or deny the call while the circuit is open.
Calls admitted together while closed share a generation. The first timeout advances it,
making every other in-flight completion stale so it cannot erase the new cooldown.
"""
with self._lock:
if self._state == self.CLOSED:
return self._generation
if self._state == self.HALF_OPEN:
return None
opened_at: Final = self._opened_at
if opened_at is not None and self._clock() - opened_at >= self._cooldown_seconds:
self._state = self.HALF_OPEN
return self._generation
return None
def record_success(self, permit: int) -> None:
"""Close only when the current half-open recovery probe succeeds."""
with self._lock:
if self._state != self.HALF_OPEN or permit != self._generation:
return
self._state = self.CLOSED
self._opened_at = None
def record_failure(self, permit: int, *, is_timeout: bool) -> None:
"""Open on a normal timeout, or reopen when the single recovery probe fails."""
with self._lock:
if permit != self._generation:
return
if self._state == self.CLOSED:
if not is_timeout:
return
elif self._state != self.HALF_OPEN:
return
self._generation += 1
self._state = self.OPEN
self._opened_at = self._clock()
def _is_classifier_timeout(exc: BaseException) -> bool:
# asyncio.TimeoutError became an alias of the built-in TimeoutError in Python 3.11.
# LiteLLM still supports 3.10, where they are distinct exception classes.
if isinstance(exc, (TimeoutError, asyncio.TimeoutError)):
return True
from litellm.exceptions import Timeout as LiteLLMTimeout
return isinstance(exc, LiteLLMTimeout)
def _allowed(models: tuple[str, ...], fit_filter: frozenset[str] | None) -> tuple[str, ...]:
return models if fit_filter is None else tuple(model for model in models if model in fit_filter)
@ -993,6 +1166,15 @@ class ComplexityRouter(CustomLogger):
if llm_classifier_configured
else None
)
self._classifier_circuit_breaker: _ClassifierCircuitBreaker | None = (
_ClassifierCircuitBreaker(self.config.classifier_llm_config.circuit_breaker_cooldown_seconds)
if (
llm_classifier_configured
and self.config.classifier_llm_config is not None
and self.config.classifier_llm_config.circuit_breaker_enabled
)
else None
)
self._tier_success_predictor: TierSuccessPredictor | None = (
TierSuccessPredictor(resolve_tier_artifact(self.config.heuristic_v2_artifact))
if self.config.classifier_type == "heuristic_v2"
@ -1012,6 +1194,15 @@ class ComplexityRouter(CustomLogger):
definitions,
self.config.classification_prompt,
self.config.classifier_context_window_size,
classification_examples=self.config.classification_examples,
)
if llm_config.system_prompt is None:
return built_in_tier_classification_prompt(
self.config.classification_prompt,
self.config.classifier_context_window_size,
labeled_tiers=self.config.labeled_tiers(),
classification_rubric=llm_config.classification_rubric,
classification_examples=self.config.classification_examples,
)
return classification_system_prompt(
self.config.classifier_context_window_size,
@ -1425,6 +1616,10 @@ class ComplexityRouter(CustomLogger):
threshold check alone would hand that traffic to the cheapest model without ever consulting
the classifier. Scores also go negative when simple indicators fire, so a score threshold
would reject exactly the trivial prompts this path exists to serve.
A turn carrying images the classifier would see is never decided cheaply: the scorer reads
text alone, so its confidence describes a request it has only partly seen, and a trivial
caption beside a screenshot is exactly the misrouting vision classification exists to stop.
"""
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
@ -1432,6 +1627,7 @@ class ComplexityRouter(CustomLogger):
decided_cheaply: Final = (
threshold is not None
and bool(signals)
and not self._classifier_image_parts(messages)
and self._active_tier_severity(tier) <= self._active_tier_severity(threshold)
)
if decided_cheaply:
@ -1456,11 +1652,43 @@ class ComplexityRouter(CustomLogger):
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
margin: Final = self.config.hybrid_boundary_margin
decided: Final = margin is not None and bool(signals) and not self._is_near_tier_boundary(score, margin)
decided: Final = (
margin is not None
and bool(signals)
and not self._classifier_image_parts(messages)
and not self._is_near_tier_boundary(score, margin)
)
if decided:
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="hybrid_short_circuit")
return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored)
def _classifier_image_parts(
self, messages: Sequence[Mapping[str, object]] | None
) -> tuple[ChatCompletionImageObject, ...]:
"""Images from the newest user turn to hand the classifier, capped by max_images.
Empty unless the operator opted in AND the classifier model is declared vision-capable, so
every other deployment keeps today's text-only payload byte for byte. Only the newest user
turn is read: earlier turns are context the classifier already gets as quoted text, and an
image nested in a tool_result is tool output rather than the ask being classified.
Remote-URL images are left out entirely; `_inline_image_part` carries why.
"""
llm_config: Final = self.config.classifier_llm_config
if llm_config is None or not llm_config.vision.enabled or not self.config.uses_llm_classifier or not messages:
return ()
if not self._model_declares_vision_support(llm_config.model):
return ()
newest_user_turn: Final = next((msg for msg in reversed(messages) if msg.get("role") == "user"), None)
content: Final = newest_user_turn.get("content") if newest_user_turn is not None else None
if not isinstance(content, list):
return ()
return tuple(
islice(
(part for raw in content if isinstance(raw, Mapping) and (part := _inline_image_part(raw)) is not None),
llm_config.vision.max_images,
)
)
async def _llm_classifier_outcome(
self,
prompt: str,
@ -1474,8 +1702,20 @@ class ComplexityRouter(CustomLogger):
`scored` is the heuristic outcome the caller already computed, which only "heuristic_first"
has. It is handed to the failure path so a classifier error does not re-run the scorer.
"""
breaker: Final = self._classifier_circuit_breaker
permit: Final = breaker.acquire_permit() if breaker is not None else None
if breaker is not None and permit is None:
return self._classifier_failure_outcome(
"LLM classifier circuit is open",
prompt,
system_prompt,
scored,
signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL,
)
try:
tier, classifier_cost = await self._classify_with_llm(prompt, system_prompt, request_kwargs, messages)
if breaker is not None and permit is not None:
breaker.record_success(permit)
return ClassificationOutcome(
tier=tier,
score=None,
@ -1483,7 +1723,13 @@ class ComplexityRouter(CustomLogger):
cause="llm_classifier",
classifier_cost=classifier_cost,
)
except asyncio.CancelledError:
if breaker is not None and permit is not None:
breaker.record_failure(permit, is_timeout=False)
raise
except Exception as e: # noqa: BLE001 -- external LLM call can fail in many distinct ways (timeout, provider error, validation, parse error); any failure must fall back to the configured fallback path
if breaker is not None and permit is not None:
breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e))
return self._classifier_failure_outcome(f"LLM classifier failed ({e})", prompt, system_prompt, scored)
def _classifier_failure_outcome(
@ -1492,6 +1738,7 @@ class ComplexityRouter(CustomLogger):
prompt: str,
system_prompt: str | None,
scored: ClassificationOutcome | None = None,
signal: str | None = None,
) -> ClassificationOutcome:
"""The outcome when the LLM classifier or classifier plugin produced no usable tier:
fallback_tier on a custom tier set, classifier_fallback otherwise.
@ -1501,21 +1748,24 @@ class ComplexityRouter(CustomLogger):
fallback_tier: Final = self.config.fallback_tier
if fallback_tier is not None:
verbose_router_logger.warning("ComplexityRouter: %s, routing to fallback_tier %s", reason, fallback_tier)
return ClassificationOutcome(
tier=fallback_tier,
score=None,
signals=(f"classifier-fallback:{fallback_tier}",),
cause="classifier_fallback",
return _with_signal(
ClassificationOutcome(
tier=fallback_tier,
score=None,
signals=(f"classifier-fallback:{fallback_tier}",),
cause="classifier_fallback",
),
signal,
)
verbose_router_logger.warning(
"ComplexityRouter: %s, falling back to %s", reason, self.config.classifier_fallback
)
if self.config.classifier_fallback == "default_model":
return self._default_model_fallback_outcome()
return _with_signal(self._default_model_fallback_outcome(), signal)
if scored is not None:
return scored
return _with_signal(scored, signal)
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
return _with_signal(ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause), signal)
async def _classify_with_plugin(
self,
@ -1676,9 +1926,18 @@ class ComplexityRouter(CustomLogger):
}
turn_off_message_logging: Final = _effective_turn_off_message_logging(request_kwargs)
image_parts: Final = self._classifier_image_parts(messages)
user_content: Final[str | Sequence[ChatCompletionTextObject | ChatCompletionImageObject]] = (
[ # mutable-ok: SDK request payload content list is built once
{"type": "text", "text": user_payload},
*image_parts,
]
if image_parts
else user_payload
)
messages_for_call: Final[list[AllMessageValues]] = [ # mutable-ok: SDK request payload list is built once
{"role": "system", "content": classifier_system_prompt},
{"role": "user", "content": user_payload},
{"role": "user", "content": user_content},
]
response_format: Final = classifier_response_format
classifier_call_params: Mapping[str, str] = EMPTY_MAPPING
@ -1694,16 +1953,23 @@ class ComplexityRouter(CustomLogger):
}
}
response: Final[ModelResponse] = await self.litellm_router_instance.acompletion(
model=llm_config.model,
messages=messages_for_call,
response_format=response_format,
timeout=llm_config.timeout_ms / 1000,
metadata=metadata,
proxy_server_request=proxy_server_request,
turn_off_message_logging=turn_off_message_logging,
**classifier_call_params,
**_parent_session_kwargs(request_kwargs),
classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000
response: Final[ModelResponse] = await asyncio.wait_for(
self.litellm_router_instance.acompletion(
model=llm_config.model,
messages=messages_for_call,
stream=False,
response_format=response_format,
timeout=classifier_timeout_s,
num_retries=0,
disable_fallbacks=True,
metadata=metadata,
proxy_server_request=proxy_server_request,
turn_off_message_logging=turn_off_message_logging,
**classifier_call_params,
**_parent_session_kwargs(request_kwargs),
),
timeout=classifier_timeout_s,
)
content: Final = response.choices[0].message.content
if not content:
@ -2362,31 +2628,53 @@ class ComplexityRouter(CustomLogger):
return pinned_model
return self.get_model_for_tier(escalated_tier)
def _model_accepts_image_input(self, model_name: str) -> bool:
"""Whether a routed model or pool entry can serve an image request.
def _vision_verdicts(self, model_name: str) -> tuple[bool | None, ...]:
"""Declared vision support per deployment serving the name: True, False, or None when
nothing declares either way.
Resolved through the deployments that would actually serve the name; a name with no
deployment on the router is served by the SDK directly and is checked against the model
cost map itself. Only an explicit supports_vision false excludes, a deployment-level
model_info override first and the map otherwise, so unmapped custom names stay routable.
cost map itself. A deployment-level model_info override wins over the map.
One verdict set, two readings, because the two callers fail in opposite directions.
Routing a user's image asks whether anything RULES IT OUT, so an undeclared model stays
eligible and unmapped custom names keep routing. Handing an image to the classifier asks
whether something RULES IT IN: an undeclared model that turns out to be text-only rejects
every image request, and that rejection is swallowed by the classifier's own fallback, so
the router quietly serves all image traffic from the fallback tier and pays for the failed
call each time. An undeclared model instead keeps today's text-only payload, which is a
visible no-op the operator fixes by declaring supports_vision on the deployment.
"""
from litellm.utils import is_vision_explicitly_disabled, supports_vision
def model_verdict(model: str) -> bool | None:
if supports_vision(model):
return True
return False if is_vision_explicitly_disabled(model) else None
def deployment_verdict(deployment: Mapping[str, Any]) -> bool | None:
declared: Final = (deployment.get("model_info") or EMPTY_MAPPING).get("supports_vision")
if declared is not None:
return declared is True
return model_verdict((deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name)
deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name)
if not deployments:
return (model_verdict(model_name),)
return tuple(deployment_verdict(deployment) for deployment in deployments)
def _model_accepts_image_input(self, model_name: str) -> bool:
"""Whether a routed model or pool entry can serve an image request.
A multi-deployment group must accept on EVERY deployment: the router picks a deployment
inside the group after this gate runs, so a mixed group marked eligible could still hand
the image to its text-only member and fail with the exact 400 the gate exists to prevent.
"""
from litellm.utils import is_vision_explicitly_disabled
return all(verdict is not False for verdict in self._vision_verdicts(model_name))
def deployment_accepts(deployment: Mapping[str, Any]) -> bool:
declared: Final = (deployment.get("model_info") or EMPTY_MAPPING).get("supports_vision")
if declared is not None:
return declared is True
litellm_model: Final = (deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name
return not is_vision_explicitly_disabled(litellm_model)
deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name)
if not deployments:
return not is_vision_explicitly_disabled(model_name)
return all(deployment_accepts(deployment) for deployment in deployments)
def _model_declares_vision_support(self, model_name: str) -> bool:
"""Whether every deployment serving the name is declared vision-capable."""
return all(verdict is True for verdict in self._vision_verdicts(model_name))
def _modality_eligible_models(self) -> frozenset[str]:
"""Every configured pool entry, plus default_model, that can serve an image request."""
@ -2526,6 +2814,150 @@ class ComplexityRouter(CustomLogger):
and self._matched_plan_mode_signal(request_kwargs, resolved_messages) is None
)
async def _model_group_can_serve(
self,
model_name: str,
messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the router's own probe
input: str | list | None, # mutable-ok: mirrors the owner's own input parameter, which this forwards verbatim
request_kwargs: dict, # mutable-ok: same shape the hook receives
) -> bool:
"""Whether the router would find a deployment for this group ON THIS REQUEST.
Asks the same owner the routing path itself will ask, with the same prompt arguments it
will pass, so every filter that decides a deployment's eligibility applies here exactly
as it applies downstream: cooldowns, admin pause, team scoping, model access groups, tag
routing, routing plugins, RPM limits, and the context-window pre-call check. Re-deriving
any subset of that list is how a substitute gets chosen that the pipeline then rejects,
and dropping `input` would silently skip the window check on the Responses API surface,
where the prompt never arrives as messages.
Probed on a COPY of request_kwargs because the owner pops routing bookkeeping off the
dict it is handed (`_target_order`, `_excluded_deployment_ids`), and this is a
speculative question about a model that may never be picked.
Every way the owner says "nothing here can serve this" is a negative verdict: no healthy
deployment for the group at all (BadRequestError, which ContextWindowExceededError
subclasses), every deployment filtered out (RouterRateLimitError), and every deployment
over its RPM (RouterRateLimitErrorBasic). Anything else is unknown rather than negative,
so it reads as capacity: absent information must never decide the verdict.
"""
from litellm.exceptions import BadRequestError
from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic
probe_kwargs: Final = dict(request_kwargs) # mutable-ok: the owner pops routing keys off the dict it is handed
try:
deployments: Final = await self.litellm_router_instance.async_get_healthy_deployments(
model=model_name,
request_kwargs=probe_kwargs,
messages=messages,
input=input,
parent_otel_span=_get_parent_otel_span_from_kwargs(request_kwargs),
)
except (RouterRateLimitError, RouterRateLimitErrorBasic, BadRequestError):
return False
except Exception as exc: # noqa: BLE001 # a speculative eligibility read must fail open on unknown faults
verbose_router_logger.debug(
"ComplexityRouter: eligibility probe for %s failed, treating the group as live: %s", model_name, exc
)
return True
return bool(deployments)
async def _gate_response_health(
self,
response: PreRoutingHookResponse,
messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick
input: str | list | None, # mutable-ok: mirrors the owner's own input parameter, which this forwards verbatim
resolved_messages: Sequence[Mapping[str, object]] | None,
request_kwargs: dict, # mutable-ok: same shape the hook receives
) -> PreRoutingHookResponse:
"""Replace a decided model group that has no serving capacity with a live peer in the same tier.
Applied to the decided response at the hook's exits, so every arm that can place a request
is covered by one owner: a fresh classification, a replayed or escalated session pin, a
plan-mode floor, a context-window escalation, an adaptive pick, and whatever arm is added
next. Peers come from the DECIDED tier only; climbing to another tier is deliberately not
done here, since a higher tier costs more than the classifier asked for.
Serving capacity is one question asked of one owner (`_model_group_can_serve`), so the
substitute is only ever a group the pipeline would actually accept for this request. The
pick then runs through `_pick_model_for_tier`, so routing plugins decide the substitute
exactly as they decided the original.
Fails open everywhere it cannot be sure: an unreadable eligibility view, a decision
carrying no tier (default_model), or a tier whose every peer is unusable too. It fails
CLOSED on a plugin that empties the pool, leaving the original decision to fail rather
than serving a model the plugin excluded.
"""
decision: Final = response.routing_decision
decided_tier: Final = decision.get("tier") if decision is not None else None
if decision is None or not isinstance(decided_tier, str):
return response
peers: Final = tuple(self._tier_pools().get(decided_tier, ()))
if len(peers) < 2:
return response
if await self._model_group_can_serve(response.model, messages, input, request_kwargs):
return response
eligible: Final = (
self._modality_eligible_models()
if self.config.modality_routing and resolved_messages and request_contains_image_content(resolved_messages)
else None
)
candidates: Final = tuple(
peer for peer in peers if peer != response.model and (eligible is None or peer in eligible)
)
if not candidates:
return response
servable: Final = await asyncio.gather(
*(self._model_group_can_serve(peer, messages, input, request_kwargs) for peer in candidates)
)
live: Final = tuple(peer for peer, can_serve in zip(candidates, servable) if can_serve)
if not live:
return response
repick_messages: Final = (
list(resolved_messages) if resolved_messages else None # mutable-ok: the pick's param is list-typed
)
try:
new_model: Final = await self._pick_model_for_tier(
decided_tier if self.config.has_custom_tiers else ComplexityTier(decided_tier),
messages,
repick_messages, # pyright: ignore[reportArgumentType] # hook-resolved message dicts; the pick only reads them
request_kwargs,
allowed_models=live,
)
except ValueError as exc:
verbose_router_logger.debug(
"ComplexityRouter: health failover found no candidate the routing plugins allow: %s", exc
)
return response
self._restamp_adaptive_choice(request_kwargs, response.model, new_model)
verbose_router_logger.info(
"ComplexityRouter: routing decision cause=health_failover, routed_model=%s, displaced=%s",
new_model,
response.model,
)
new_decision: Final = self._build_routing_decision(
routed_model=new_model,
cause="health_failover",
tier=decision.get("tier"),
score=decision.get("score"),
signals=(*(decision.get("signals") or ()), f"health_displaced:{response.model}"),
matched_keyword=decision.get("matched_keyword"),
escalation_keyword=decision.get("escalation_keyword"),
escalated=bool(decision.get("escalated", False)),
classifier_model=decision.get("classifier_model"),
classifier_cost=decision.get("classifier_cost"),
conversation_continuing=bool(decision.get("conversation_continuing", True)),
tier_litellm_params=self._litellm_params_for_model(decided_tier, new_model),
context_escalation_original_tier=decision.get("context_escalation_original_tier"),
)
return response.model_copy(
update={ # mutable-ok: model_copy types update as a plain dict
"model": new_model,
"litellm_params": self._litellm_params_for_model(decided_tier, new_model),
"routing_decision": new_decision,
}
)
def _placed_default_model(self) -> str:
"""The default_model behind a usable-default verdict; the raise is the type-level
proof, not a reachable path."""
@ -2923,24 +3355,30 @@ class ComplexityRouter(CustomLogger):
session_tier_litellm_params: Final = self._litellm_params_for_model(routed_pin_tier, routed_model)
has_original_messages: Final = messages is not None and len(messages) > 0
return self._with_session_deployment_affinity(
await self._gate_response_modality(
PreRoutingHookResponse(
model=routed_model,
messages=messages if has_original_messages else None,
litellm_params=session_tier_litellm_params,
routing_decision=self._build_routing_decision(
routed_model=routed_model,
cause=cause,
tier=routed_pin_tier,
matched_keyword=pin_plan_sentinel if plan_floored else None,
escalation_keyword=pin_escalation_keyword,
escalated=escalated,
conversation_continuing=conversation_continuing,
tier_litellm_params=session_tier_litellm_params,
context_escalation_original_tier=pin_context_original_tier,
await self._gate_response_health(
await self._gate_response_modality(
PreRoutingHookResponse(
model=routed_model,
messages=messages if has_original_messages else None,
litellm_params=session_tier_litellm_params,
routing_decision=self._build_routing_decision(
routed_model=routed_model,
cause=cause,
tier=routed_pin_tier,
matched_keyword=pin_plan_sentinel if plan_floored else None,
escalation_keyword=pin_escalation_keyword,
escalated=escalated,
conversation_continuing=conversation_continuing,
tier_litellm_params=session_tier_litellm_params,
context_escalation_original_tier=pin_context_original_tier,
),
),
messages,
resolved_messages,
request_kwargs,
),
messages,
input,
resolved_messages,
request_kwargs,
)
@ -2956,7 +3394,13 @@ class ComplexityRouter(CustomLogger):
resolved_messages=resolved_messages,
)
response: Final = (
await self._gate_response_modality(routed_response, messages, resolved_messages, request_kwargs)
await self._gate_response_health(
await self._gate_response_modality(routed_response, messages, resolved_messages, request_kwargs),
messages,
input,
resolved_messages,
request_kwargs,
)
if routed_response is not None
else None
)
@ -3022,8 +3466,9 @@ class ComplexityRouter(CustomLogger):
has_original_messages: Final = messages is not None and len(messages) > 0
user_message, system_prompt = _extract_current_ask_and_system_prompt(resolved_messages, self._reminder_markers)
classifier_images: Final = self._classifier_image_parts(resolved_messages)
if user_message is None:
if user_message is None and not classifier_images:
verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model")
default_model_first: Final = not self.config.plugins and self.config.default_model
if default_model_first:
@ -3050,8 +3495,17 @@ class ComplexityRouter(CustomLogger):
),
)
ask: Final = user_message or ""
newest_ask: Final = _newest_turn_ask(resolved_messages, self._reminder_markers)
escalation_keyword: Final = self._matched_escalation_keyword(newest_ask) if newest_ask is not None else None
# Resolved here rather than beside the classifier because the keyword-override path below
# returns before any classification runs, and a forced tier gets stuck for the same reason
# a classified one does.
stalled: Final = self.config.stall_escalation_enabled and detect_stalled_task(
resolved_messages,
window=self.config.stall_escalation_window,
repeat_threshold=self.config.stall_escalation_repeat_threshold,
)
plan_mode_sentinel: Final = self._matched_plan_mode_signal(request_kwargs, resolved_messages)
plan_floor: Final = self._resolve_plan_mode_floor() if plan_mode_sentinel is not None else None
@ -3079,12 +3533,13 @@ class ComplexityRouter(CustomLogger):
),
)
override: Final = await self._resolve_keyword_tier_override(user_message, request_kwargs)
override: Final = await self._resolve_keyword_tier_override(ask, request_kwargs)
if override is not None:
escalated_tier: Final = (
keyword_bumped_tier: Final = (
self._escalate_tier(override.tier) if escalation_keyword is not None else override.tier
)
keyword_escalated: Final = escalated_tier != override.tier
escalated_tier: Final = self._escalate_tier(keyword_bumped_tier) if stalled else keyword_bumped_tier
keyword_escalated: Final = keyword_bumped_tier != override.tier
routed_tier: Final = (
self._apply_plan_mode_floor(escalated_tier) if plan_floor is not None else escalated_tier
)
@ -3112,6 +3567,7 @@ class ComplexityRouter(CustomLogger):
conversation_continuing=conversation_continuing,
cause=keyword_cause,
tier=routed_tier,
signals=("stall_escalation",) if stalled else None,
matched_keyword=plan_mode_sentinel if keyword_plan_floored else override.matched_keyword,
escalation_keyword=escalation_keyword,
escalated=keyword_escalated,
@ -3124,9 +3580,7 @@ class ComplexityRouter(CustomLogger):
outcome: Final = (
ClassificationOutcome(tier=housekeeping_tier, score=None, signals=("housekeeping",), cause="housekeeping")
if housekeeping_tier is not None
else await self.aclassify(
user_message, system_prompt, request_kwargs, resolved_messages, raw_messages=messages
)
else await self.aclassify(ask, system_prompt, request_kwargs, resolved_messages, raw_messages=messages)
)
tier, score, signals = outcome.tier, outcome.score, outcome.signals
classified_tier: Final = tier
@ -3135,6 +3589,9 @@ class ComplexityRouter(CustomLogger):
escalated: Final = tier != classified_tier
if escalated:
signals = (*signals, "escalation")
if stalled:
tier = self._escalate_tier(tier)
signals = (*signals, "stall_escalation")
pre_floor_tier: Final = tier
if plan_floor is not None:
tier = self._apply_plan_mode_floor(tier)
@ -3193,7 +3650,7 @@ class ComplexityRouter(CustomLogger):
# under is not a floor.
routed_model = self._soft_floor_pick(
tier,
user_message,
ask,
request_kwargs,
hard_floor=tier if context_original_tier is not None else plan_floor,
hard_ceiling=housekeeping_ceiling,

208
litellm/router_strategy/complexity_router/config.py
View file

@ -100,25 +100,40 @@ MAX_TIER_DEFINITIONS: Final[int] = 8
MAX_TIER_NAME_CHARS: Final[int] = 64
MAX_TIER_DESCRIPTION_CHARS: Final[int] = 500
MAX_CLASSIFICATION_PROMPT_CHARS: Final[int] = 2000
# Roomier than the instructions because the shipped example blocks an operator starts from are
# themselves ~2.6k characters, so the instruction cap would reject an edited copy of one.
MAX_CLASSIFICATION_EXAMPLES_CHARS: Final[int] = 4000
CALIBRATION_EXAMPLES_HEADING: Final[str] = "Calibration examples:"
def normalize_classification_prompt(value: str | None) -> str | None:
"""Strip, reject blank, and cap an operator-written classifier preamble.
def _normalize_operator_section(value: str | None, field: str, cap: int) -> str | None:
"""Strip, reject blank, and cap one operator-written section of the classifier rubric.
The single owner of the rule, so the dashboard's prompt preview normalizes exactly what the
write gate stores: previewing the raw value would render leading whitespace the router strips,
or an over-long prompt the write then rejects.
or an over-long section the write then rejects.
"""
if value is None:
return None
stripped: Final = value.strip()
if not stripped:
raise ValueError("must be non-empty; omit the field instead")
if len(stripped) > MAX_CLASSIFICATION_PROMPT_CHARS:
raise ValueError(f"classification_prompt exceeds {MAX_CLASSIFICATION_PROMPT_CHARS} characters")
if len(stripped) > cap:
raise ValueError(f"{field} exceeds {cap} characters")
return stripped
def normalize_classification_prompt(value: str | None) -> str | None:
"""Normalize the operator-written classification instructions."""
return _normalize_operator_section(value, "classification_prompt", MAX_CLASSIFICATION_PROMPT_CHARS)
def normalize_classification_examples(value: str | None) -> str | None:
"""Normalize the operator-written calibration examples, which carry no heading of their own."""
return _normalize_operator_section(value, "classification_examples", MAX_CLASSIFICATION_EXAMPLES_CHARS)
class TierDefinition(BaseModel):
"""An operator-defined tier: the name the LLM classifier must return and its rubric description."""
@ -427,12 +442,47 @@ DEFAULT_TIER_MODELS: Final[dict[str, str]] = {
}
class ClassifierVisionConfig(BaseModel):
"""Whether the LLM classifier sees the images on the request it is classifying.
Off by default because images cost far more than the text ask they arrive with, and the
classifier runs on every request. A turn whose complexity lives in the image ("what is wrong in
this stack trace screenshot") is invisible to a text-only classifier, which is what this buys.
"""
enabled: bool = Field(
default=False,
description=(
"Forward image content to the classifier. Requires a classifier model declared "
"supports_vision, on the deployment's model_info or in the model cost map; images stay "
"stripped otherwise, so a classifier that cannot read them is never sent one. Declare "
"model_info.supports_vision on the deployment to enable a model the cost map does not "
"describe. Only inline data: URIs are forwarded. A request whose images are http(s) "
"URLs still classifies on its text alone, because some providers fetch such a URL from "
"the proxy rather than the provider, which would let a caller aim a proxy-side request "
"at an address of their choosing."
),
)
max_images: int = Field(
default=1,
ge=1,
description=(
"How many images from the newest user turn to forward, in wire order. Bounds the added "
"cost of a turn that attaches many images. Images on earlier turns are never forwarded."
),
)
class ClassifierLLMConfig(BaseModel):
"""Configuration for the LLM-based complexity classifier."""
model: str = Field(
description="Model name (from the router's model_list) to call for classification",
)
vision: ClassifierVisionConfig = Field(
default_factory=ClassifierVisionConfig,
description="Whether the classifier sees images on the request, and how many",
)
reasoning_effort: REASONING_EFFORT | None = Field(
default=None,
description=(
@ -444,6 +494,23 @@ class ClassifierLLMConfig(BaseModel):
default=3000,
description="Timeout budget for the classification call, in milliseconds",
)
circuit_breaker_enabled: bool = Field(
default=True,
description=(
"Whether one classifier timeout temporarily sends requests through classifier_fallback. "
"Enabled by default so an unhealthy classifier cannot repeat its timeout across sessions."
),
)
circuit_breaker_cooldown_seconds: float = Field(
default=30.0,
gt=0.0,
description=(
"How long to skip this router's LLM classifier after a classification call times out. "
"Requests use classifier_fallback during the cooldown. When it expires, one request "
"probes the classifier while concurrent requests keep using the fallback; a successful "
"probe closes the circuit and a failed probe restarts the cooldown."
),
)
classification_rubric: ClassificationRubric | None = Field(
default=None,
description=(
@ -543,12 +610,23 @@ class ComplexityRouterConfig(BaseModel):
classification_prompt: str | None = Field(
default=None,
description=(
"Replaces the opening instructions of the LLM classifier rubric (the judging-criteria "
"prose) for a custom tier set. The per-tier bullets and the trust-boundary paragraph "
"telling the classifier to ignore tier requests embedded in quoted caller text are "
"always appended after it and cannot be overridden. Requires tier_definitions; a "
"built-in-tier router customizes its prompt via classifier_llm_config.system_prompt "
"or classification_rubric instead."
"Replaces the classification instructions that open the LLM classifier rubric, and nothing else. The "
"per-tier bullets follow it, the calibration examples follow those, and the trust-boundary paragraph "
"telling the classifier to ignore tier requests embedded in quoted caller text is always appended "
"after them and cannot be overridden. Requires an LLM classifier and cannot be combined with "
"classifier_llm_config.system_prompt. With built-in tiers the rubric preset still supplies the tier "
"criteria and, unless classification_examples replaces them, the calibration examples."
),
)
classification_examples: str | None = Field(
default=None,
description=(
"Replaces the calibration examples of the LLM classifier rubric, and nothing else. Written as example "
"lines only: the router renders the 'Calibration examples:' heading above them, after the per-tier "
"bullets. Requires an LLM classifier and cannot be combined with classifier_llm_config.system_prompt. "
"With built-in tiers the rubric preset still supplies the tier criteria and, unless "
"classification_prompt replaces them, the classification instructions; a custom tier set ships no "
"examples of its own, so the section renders only when this is set."
),
)
tier_labels: dict[ComplexityTier, str] = Field(
@ -809,6 +887,43 @@ class ComplexityRouterConfig(BaseModel):
description="Rules that force a specific tier when their keywords match the prompt",
)
stall_escalation_enabled: bool = Field(
default=False,
description=(
"Escalate mid-task to the next-higher configured tier when the assistant's own recent "
"tool calls look stuck: the newest tool call repeats, or errors, at least "
"stall_escalation_repeat_threshold times across the last stall_escalation_window "
"calls. Both tests are anchored on the newest call, so a task that tried the same "
"thing a few times and then moved on is not escalated on the strength of those older "
"calls alone, while a retry loop broken up by an unrelated lookup still counts. One "
"tier at most, on the same ladder escalation_keywords bumps along, and never above "
"the highest configured tier. Detection re-runs on every classified turn from the "
"tool calls visible in that request, so it needs no state and nothing survives past "
"the task. Mutually exclusive with session_affinity and classification_mode="
"'user_turn', which both replay a held routing decision instead of classifying most "
"turns, so this would never see the tool calls to look at. Off by default."
),
)
stall_escalation_window: int = Field(
default=6,
gt=0,
description=(
"How many of the assistant's most recent tool calls stall detection looks at, oldest "
"ones dropped as new calls happen. Counted across the whole visible conversation "
"rather than reset at the newest human ask, so evidence from before a plain follow-up "
"message like 'try again' is still visible on the turn after it."
),
)
stall_escalation_repeat_threshold: int = Field(
default=3,
ge=2,
description=(
"How many of the last stall_escalation_window tool calls must repeat the newest call, "
"or must have errored alongside it, before the task counts as stalled. Must not "
"exceed stall_escalation_window, or the condition could never be reached."
),
)
plan_mode_min_tier: str | None = Field(
default=None,
description=(
@ -1205,6 +1320,11 @@ class ComplexityRouterConfig(BaseModel):
def _normalize_classification_prompt_field(cls, value: str | None) -> str | None:
return normalize_classification_prompt(value)
@field_validator("classification_examples")
@classmethod
def _normalize_classification_examples_field(cls, value: str | None) -> str | None:
return normalize_classification_examples(value)
@property
def has_custom_tiers(self) -> bool:
"""True when the operator replaced the built-in tier set via tier_definitions."""
@ -1237,6 +1357,35 @@ class ComplexityRouterConfig(BaseModel):
folded: Final = label.strip().casefold()
return next((name for name in self.tier_names() if name.casefold() == folded), None)
def _built_in_opening_conflicts(self) -> tuple[str, ...]:
"""Error messages for mutually exclusive built-in classifier prompt settings.
The two sections are independent, so each is checked on its own name: an operator who wrote
only examples must not read an error naming the instructions field they never set.
"""
written: Final = tuple(
field
for field, value in (
("classification_prompt", self.classification_prompt),
("classification_examples", self.classification_examples),
)
if value is not None
)
if not written:
return ()
llm_config: Final = self.classifier_llm_config
if llm_config is not None and llm_config.system_prompt is not None:
return tuple(
f"{field} cannot be combined with classifier_llm_config.system_prompt: choose the section-shaped "
"rubric or the legacy wholesale prompt"
for field in written
)
if not self.uses_llm_classifier:
return tuple(
f"{field} requires an LLM classifier, got classifier_type={self.classifier_type!r}" for field in written
)
return ()
def _tier_definition_conflicts(self) -> tuple[str, ...]:
"""Error messages for config features that cannot coexist with a custom tier set."""
llm_config: Final = self.classifier_llm_config
@ -1246,6 +1395,7 @@ class ComplexityRouterConfig(BaseModel):
("adaptive", self.adaptive),
("session_affinity", self.session_affinity),
("escalation_keywords", bool(self.escalation_keywords)),
("stall_escalation_enabled", self.stall_escalation_enabled),
("plugins", bool(self.plugins)),
)
if enabled
@ -1287,19 +1437,10 @@ class ComplexityRouterConfig(BaseModel):
@model_validator(mode="after")
def _validate_tier_definitions(self) -> "ComplexityRouterConfig":
if self.tier_definitions is None:
orphaned: Final = next(
(
field
for field, value in (
("fallback_tier", self.fallback_tier),
("classification_prompt", self.classification_prompt),
)
if value is not None
),
None,
)
if orphaned is not None:
raise ValueError(f"{orphaned} requires tier_definitions")
if self.fallback_tier is not None:
raise ValueError("fallback_tier requires tier_definitions")
for message in self._built_in_opening_conflicts():
raise ValueError(message)
return self
names: Final = tuple(definition.name for definition in self.tier_definitions)
if not 2 <= len(names) <= MAX_TIER_DEFINITIONS:
@ -1422,6 +1563,25 @@ class ComplexityRouterConfig(BaseModel):
)
return self
@model_validator(mode="after")
def _validate_stall_escalation(self) -> "ComplexityRouterConfig":
if not self.stall_escalation_enabled:
return self
if self.session_affinity or self.classification_mode == "user_turn":
raise ValueError(
"stall_escalation_enabled cannot be combined with session_affinity or "
"classification_mode='user_turn': both replay a held routing decision on most "
"turns instead of classifying, so stall detection would never see the tool calls "
"of the turns it needs to look at. Disable one or the other."
)
if self.stall_escalation_repeat_threshold > self.stall_escalation_window:
raise ValueError(
"stall_escalation_repeat_threshold "
f"({self.stall_escalation_repeat_threshold}) cannot exceed stall_escalation_window "
f"({self.stall_escalation_window}); the condition could never be reached."
)
return self
@model_validator(mode="after")
def _validate_tier_param_placement(self) -> "ComplexityRouterConfig":
"""Reject a router setting written into a tier entry's request params.

118
litellm/router_strategy/complexity_router/stall_detector.py Normal file
View file

@ -0,0 +1,118 @@
"""
Mid-task stall detection for the Complexity Router.
Reads the assistant's own recent tool calls, which every agentic client resends on each
turn, and reports whether the task currently looks stuck. No LLM call and no stored state:
the same window is rescanned per classified turn, so the verdict follows the conversation
rather than latching.
Tool calls arrive in two shapes and are read in place rather than translated:
- Anthropic Messages: assistant `tool_use` content blocks, answered by a user-turn
`tool_result` block carrying `is_error`
- Chat completions: assistant `tool_calls` entries, answered by a `role: "tool"` message,
which has no standard error flag, so those calls are judged on repetition alone
"""
from __future__ import annotations
import json
from collections.abc import Iterator, Mapping, Sequence
from itertools import islice
from typing import Final, NamedTuple
_ARGUMENTS_PARSE_FAILED: Final = object()
class _ToolCallEvent(NamedTuple):
signature: tuple[str, str]
is_error: bool | None
"""None where the surface reports no error status, and never counted as an error."""
def _json_arguments(raw: str) -> object:
try:
return json.loads(raw)
except (TypeError, ValueError):
return _ARGUMENTS_PARSE_FAILED
def _tool_call_signature(name: str, raw_arguments: object) -> tuple[str, str]:
"""Canonicalized so the same call compares equal across both surfaces, which carry
arguments as a dict and as a JSON string respectively."""
parsed: Final = _json_arguments(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments
arguments: Final = raw_arguments if parsed is _ARGUMENTS_PARSE_FAILED else parsed
try:
return name, json.dumps(arguments, sort_keys=True, default=str)
except (TypeError, ValueError):
return name, str(arguments)
def _iter_tool_result_error_pairs(messages: Sequence[Mapping[str, object]]) -> Iterator[tuple[str, bool]]:
for msg in messages:
content = msg.get("content")
if msg.get("role") != "user" or not isinstance(content, list):
continue
for part in content:
if isinstance(part, Mapping) and part.get("type") == "tool_result":
call_id = part.get("tool_use_id")
if isinstance(call_id, str):
yield call_id, bool(part.get("is_error", False))
def _iter_tool_call_events_newest_first(messages: Sequence[Mapping[str, object]]) -> Iterator[_ToolCallEvent]:
error_by_call_id: Final = dict(_iter_tool_result_error_pairs(messages))
for msg in reversed(messages):
if msg.get("role") != "assistant":
continue
content = msg.get("content")
if isinstance(content, list):
for part in reversed(content):
if not (isinstance(part, Mapping) and part.get("type") == "tool_use"):
continue
name = part.get("name")
if isinstance(name, str):
call_id = part.get("id")
yield _ToolCallEvent(
signature=_tool_call_signature(name, part.get("input")),
is_error=error_by_call_id.get(call_id) if isinstance(call_id, str) else None,
)
tool_calls = msg.get("tool_calls")
if not isinstance(tool_calls, list):
continue
for call in reversed(tool_calls):
function = call.get("function") if isinstance(call, Mapping) else None
name = function.get("name") if isinstance(function, Mapping) else None
if isinstance(name, str):
yield _ToolCallEvent(
signature=_tool_call_signature(name, function.get("arguments") if function else None),
is_error=None,
)
def detect_stalled_task(
messages: Sequence[Mapping[str, object]] | None,
*,
window: int,
repeat_threshold: int,
) -> bool:
"""Whether the newest tool call is still part of a stuck pattern: it repeats, or it
errored, at least repeat_threshold times across the last `window` calls.
Both tests are anchored on the newest call rather than counting whichever pattern is
most common in the window. A task that tried the same thing three times and then moved
on has those three calls in the window for a while yet, and counting them alone would
escalate a request that already recovered. Anchoring also leaves room between the
matches, so a retry loop broken up by an unrelated lookup still reads as stuck.
"""
if not messages or repeat_threshold <= 0:
return False
recent: Final = tuple(islice(_iter_tool_call_events_newest_first(messages), window))
if len(recent) < repeat_threshold:
return False
newest: Final = recent[0]
repeats: Final = sum(1 for event in recent if event.signature == newest.signature)
if repeats >= repeat_threshold:
return True
if not newest.is_error:
return False
return sum(1 for event in recent if event.is_error) >= repeat_threshold

79
litellm/router_utils/get_retry_from_policy.py
View file

@ -1,55 +1,62 @@
"""
Get num retries for an exception.
"""Resolve how many retries a RetryPolicy grants for a given exception."""
- Account for retry policy by exception type.
"""
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import Final
from litellm.exceptions import (
AuthenticationError,
BadRequestError,
ContentPolicyViolationError,
InternalServerError,
RateLimitError,
ServiceUnavailableError,
Timeout,
)
from litellm.types.router import RetryPolicy
_RETRIES_BY_EXCEPTION_TYPE: Final[Mapping[type, Callable[[RetryPolicy], int | None]]] = MappingProxyType(
{
AuthenticationError: lambda policy: policy.AuthenticationErrorRetries,
Timeout: lambda policy: policy.TimeoutErrorRetries,
RateLimitError: lambda policy: policy.RateLimitErrorRetries,
ContentPolicyViolationError: lambda policy: policy.ContentPolicyViolationErrorRetries,
BadRequestError: lambda policy: policy.BadRequestErrorRetries,
ServiceUnavailableError: lambda policy: policy.ServiceUnavailableErrorRetries,
InternalServerError: lambda policy: policy.InternalServerErrorRetries,
}
)
def _resolve_policy(
retry_policy: RetryPolicy | Mapping[str, int | None] | None,
model_group: str | None,
model_group_retry_policy: Mapping[str, RetryPolicy | Mapping[str, int | None]] | None,
) -> RetryPolicy | None:
selected: Final = (
model_group_retry_policy[model_group]
if model_group_retry_policy is not None and model_group is not None and model_group in model_group_retry_policy
else retry_policy
)
if isinstance(selected, Mapping):
return RetryPolicy(**selected)
return selected
def get_num_retries_from_retry_policy(
exception: Exception,
retry_policy: RetryPolicy | dict | None = None,
retry_policy: RetryPolicy | Mapping[str, int | None] | None = None,
model_group: str | None = None,
model_group_retry_policy: dict[str, RetryPolicy] | None = None,
):
"""
BadRequestErrorRetries: Optional[int] = None
AuthenticationErrorRetries: Optional[int] = None
TimeoutErrorRetries: Optional[int] = None
RateLimitErrorRetries: Optional[int] = None
ContentPolicyViolationErrorRetries: Optional[int] = None
"""
# if we can find the exception then in the retry policy -> return the number of retries
if model_group_retry_policy is not None and model_group is not None and model_group in model_group_retry_policy:
retry_policy = model_group_retry_policy.get(model_group, None)
if retry_policy is None:
model_group_retry_policy: Mapping[str, RetryPolicy | Mapping[str, int | None]] | None = None,
) -> int | None:
"""Walk the exception's MRO, most specific class first, and return the first configured retry count."""
policy: Final = _resolve_policy(retry_policy, model_group, model_group_retry_policy)
if policy is None:
return None
if isinstance(retry_policy, dict):
retry_policy = RetryPolicy(**retry_policy)
if isinstance(exception, AuthenticationError) and retry_policy.AuthenticationErrorRetries is not None:
return retry_policy.AuthenticationErrorRetries
if isinstance(exception, Timeout) and retry_policy.TimeoutErrorRetries is not None:
return retry_policy.TimeoutErrorRetries
if isinstance(exception, RateLimitError) and retry_policy.RateLimitErrorRetries is not None:
return retry_policy.RateLimitErrorRetries
if (
isinstance(exception, ContentPolicyViolationError)
and retry_policy.ContentPolicyViolationErrorRetries is not None
):
return retry_policy.ContentPolicyViolationErrorRetries
if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None:
return retry_policy.BadRequestErrorRetries
configured: Final = (
_RETRIES_BY_EXCEPTION_TYPE[cls](policy) for cls in type(exception).__mro__ if cls in _RETRIES_BY_EXCEPTION_TYPE
)
return next((retries for retries in configured if retries is not None), policy.DefaultRetries)
def reset_retry_policy() -> RetryPolicy:

11
litellm/types/access_group.py
View file

@ -23,6 +23,13 @@ class AccessGroupUpdateRequest(BaseModel):
assigned_key_ids: list[str] | None = None
class AccessGroupResource(BaseModel):
"""A resource referenced by an access group. `name` is null when the id no longer resolves or has no alias."""
id: str
name: str | None
class AccessGroupResponse(BaseModel):
access_group_id: str
access_group_name: str
@ -32,6 +39,10 @@ class AccessGroupResponse(BaseModel):
access_agent_ids: list[str]
assigned_team_ids: list[str]
assigned_key_ids: list[str]
access_mcp_servers: tuple[AccessGroupResource, ...]
access_agents: tuple[AccessGroupResource, ...]
assigned_teams: tuple[AccessGroupResource, ...]
assigned_keys: tuple[AccessGroupResource, ...]
created_at: datetime
created_by: str | None = None
updated_at: datetime

1
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -43,6 +43,7 @@ class KeyMetadata(BaseModel):
key_alias: str | None = None
team_id: str | None = None
user_email: str | None = None
class KeyMetricWithMetadata(MetricBase):

21
litellm/types/proxy/management_endpoints/team_endpoints.py
View file

@ -143,3 +143,24 @@ class TeamMetadataSchemaResponse(BaseModel):
"""Response for GET /team/metadata_schema; ``fields`` is empty when no schema is configured."""
fields: tuple[TeamMetadataFieldSchema, ...]
class TeamUserSpendRow(BaseModel):
team_id: str
team_alias: str | None = None
user_id: str
user_email: str | None = None
user_alias: str | None = None
spend: float = 0.0
prompt_tokens: int = 0
completion_tokens: int = 0
total_tokens: int = 0
api_requests: int = 0
successful_requests: int = 0
failed_requests: int = 0
class TeamUserSpendResponse(BaseModel):
start_date: str
end_date: str
results: tuple[TeamUserSpendRow, ...]

2
litellm/types/router.py
View file

@ -104,6 +104,8 @@ class RetryPolicy(BaseModel):
RateLimitErrorRetries: int | None = None
ContentPolicyViolationErrorRetries: int | None = None
InternalServerErrorRetries: int | None = None
ServiceUnavailableErrorRetries: int | None = None
DefaultRetries: int | None = None
OptionalPreCallChecks = list[

57
litellm/types/utils.py
View file

@ -2886,6 +2886,10 @@ RoutingDecisionCause = Literal[
# carries an image the pinned model cannot accept. The stored pin is untouched, so the next
# text turn replays it. Distinct from "modality_escalation", which never displaces a pin.
"modality_pin_override",
# Every deployment behind the decided model group was in cooldown, so a healthy peer in the
# same tier served instead. The displaced group rides in signals. Reported even on a kept
# session pin, since the pinned model did not serve the request.
"health_failover",
"session_affinity_pin",
"session_affinity_escalation",
# classification_mode 'user_turn': the request is an agent loop's continuation turn (no new
@ -3076,6 +3080,51 @@ class GuardrailMode(TypedDict, total=False):
GuardrailStatus = Literal["success", "guardrail_intervened", "guardrail_failed_to_respond", "not_run"]
# Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the
# guardrail, the provider response that echoes it back, and the two first-party hooks that inline
# prompt substrings (``block_code_execution`` and ``litellm_content_filter``). Every other field
# reports what the guardrail decided without reproducing the prompt, so redaction replaces these
# four and keeps the rest of the record.
PROMPT_CARRYING_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset(
{
"guardrail_request",
"guardrail_response",
"match_details",
"classification",
}
)
# The rest of the record: what the guardrail is, what it decided, how long it took and what it cost.
# None of these reproduce the prompt, so a redacted record keeps them and stays explainable.
# `test_a_redacted_span_carries_every_declared_guardrail_field` fails if a field is added to the record without being
# placed in one set or the other, so a new field is dropped from redacted records rather than
# shipped unexamined.
AUDIT_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset(
{
"guardrail_name",
"guardrail_provider",
"guardrail_mode",
"guardrail_status",
"start_time",
"end_time",
"duration",
"masked_entity_count",
"guardrail_id",
"policy_template",
"detection_method",
"confidence_score",
"patterns_checked",
"alert_recipients",
"risk_score",
"violation_categories",
"guardrail_action",
"guardrail_usage",
"guardrail_cost",
"guardrail_cost_by_unit",
"guardrail_cost_in_spend",
}
)
class StandardLoggingGuardrailInformation(TypedDict, total=False):
guardrail_name: str | None
@ -3151,6 +3200,12 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False):
provider hook. Summed into the request's ``response_cost`` so it counts against
spend and budgets like token cost, unless ``guardrail_cost_in_spend`` is False."""
guardrail_cost_by_unit: ReadOnly[Mapping[str, float | None] | None]
"""``guardrail_cost`` split per ``guardrail_usage`` counter, so the daily
per-counter usage rollup can carry cost at its own grain. Absent when the
hook had no pricing for the invocation; a counter is None when the pricing
entry has no price for it, which the rollup stores as unknown rather than $0."""
guardrail_cost_in_spend: ReadOnly[bool | None]
"""Whether ``guardrail_cost`` participates in the request's ``response_cost`` and
the spend/budget aggregates built from it. Absent, None, or True keeps the default
@ -3202,6 +3257,7 @@ class GuardrailTracingDetail(TypedDict, total=False):
guardrail_action: str | None
guardrail_usage: ReadOnly[Mapping[str, int] | None]
guardrail_cost: ReadOnly[float | None]
guardrail_cost_by_unit: ReadOnly[Mapping[str, float | None] | None]
guardrail_cost_in_spend: ReadOnly[bool | None]
@ -3877,6 +3933,7 @@ class LlmProviders(str, Enum):
PG_VECTOR = "pg_vector"
S3_VECTORS = "s3_vectors"
VALKEY = "valkey"
MONGODB = "mongodb"
HELICONE = "helicone"
HYPERBOLIC = "hyperbolic"
RECRAFT = "recraft"

6
litellm/utils.py
View file

@ -8989,6 +8989,12 @@ class ProviderConfigManager:
)
return ValkeyVectorStoreConfig()
elif litellm.LlmProviders.MONGODB == provider:
from litellm.llms.mongodb.vector_stores.transformation import (
MongoDBVectorStoreConfig,
)
return MongoDBVectorStoreConfig()
return None
@staticmethod

5008
model_prices_and_context_window.json
View file

File diff suppressed because it is too large Load diff

20
model_prices_and_context_window.schema.json
View file

@ -79,6 +79,11 @@
"minimum": 0,
"description": "USD per token written to the provider's prompt cache."
},
"cache_creation_input_token_cost_above_128k_tokens": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_creation_input_token_cost_above_1hr": {
"type": "number",
"minimum": 0,
@ -94,6 +99,11 @@
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_creation_input_token_cost_above_256k_tokens": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_creation_input_token_cost_above_272k_tokens": {
"type": "number",
"minimum": 0,
@ -128,6 +138,11 @@
"minimum": 0,
"description": "USD per prompt token served from the provider's prompt cache."
},
"cache_read_input_token_cost_above_128k_tokens": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_read_input_token_cost_above_200k_tokens": {
"type": "number",
"minimum": 0,
@ -138,6 +153,11 @@
"minimum": 0,
"description": "Priority service-tier rate for the same-named base field."
},
"cache_read_input_token_cost_above_256k_tokens": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_read_input_token_cost_above_272k_tokens": {
"type": "number",
"minimum": 0,

7
provider_endpoints_support.json
View file

@ -2880,6 +2880,13 @@
"vector_stores_search": true
}
},
"mongodb": {
"display_name": "MongoDB Atlas (`mongodb`)",
"url": "https://docs.litellm.ai/docs/providers/mongodb_vector_stores",
"endpoints": {
"vector_stores_search": true
}
},
"valkey": {
"display_name": "Valkey (`valkey`)",
"url": "https://docs.litellm.ai/docs/providers/valkey_vector_stores",

3
pyproject.toml
View file

@ -112,6 +112,9 @@ utils = [
]
caching = ["diskcache>=5.6.3,<6.0"]
mcp = ["mcp>=1.28.1,<2.0"]
# Driver for the MongoDB Atlas vector store; Atlas Vector Search has no HTTP query API.
# The floor is 4.9 because that is the release AsyncMongoClient landed in.
mongodb = ["pymongo>=4.9,<5.0"]
# SAML SSO for the admin UI. python3-saml pulls in xmlsec/lxml, whose wheels
# bundle the native libxmlsec1/libxml2 libraries, so no system packages are
# required. Kept out of the base `proxy` extra so it stays optional.

2
ruff-strict-budget.json
View file

@ -9,7 +9,7 @@
"limit": 809
},
"ANN201": {
"limit": 1999
"limit": 1998
},
"ANN202": {
"limit": 835

2
schema.prisma
View file

@ -1124,6 +1124,8 @@ model LiteLLM_DailyGuardrailUsageUnits {
api_key String // hashed virtual key; empty string when unknown
usage_unit String // provider counter name, e.g. Bedrock's contentPolicyUnits
units BigInt @default(0)
cost Float? // USD for the priced share of units; null only on rows written before this column existed
untracked_units BigInt @default(0) // units recorded with no known price, the share cost leaves out
created_at DateTime @default(now())
updated_at DateTime @updatedAt

1
terraform/provider/tools/endpointaudit/coverage_allowlist.txt
View file

@ -28,6 +28,7 @@ GET /tag/user-agent/per-user-analytics
GET /tag/wau
GET /team/daily/activity
GET /team/daily/activity/aggregated
GET /team/spend/by_user
GET /team/spend/report
GET /user/daily/activity
GET /user/daily/activity/aggregated

1
tests/code_coverage_tests/recursive_detector.py
View file

@ -66,6 +66,7 @@ IGNORE_FUNCTIONS = [
"_json_safe", # max depth set (_MAX_DEPTH) plus a seen-ids cycle guard for self-referential input.
"_redact_agent_params_tree", # max depth set (default 10), same shape as _redact_sensitive_litellm_params.
"_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side.
"_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible).
]

58
tests/proxy_behavior/management/test_team_spend_by_user.py Normal file
View file

@ -0,0 +1,58 @@
import pytest
from .actors import Actor
pytestmark = pytest.mark.asyncio(loop_scope="session")
# GET /team/spend/by_user shares the team-scope resolver with
# /team/daily/activity, so the membership matrix must hold here too. team_ids
# is mandatory on this route (a per-user rollup with no team is meaningless),
# so the bare query is 400 for everyone instead of defaulting to own teams.
_MEMBERS = {
"alpha": {
Actor.TEAM_ADMIN,
Actor.INTERNAL_USER,
Actor.OWNER,
Actor.UNRELATED_SAME_ORG,
Actor.SERVICE_ACCOUNT,
},
"beta": {Actor.CROSS_ORG_USER},
}
def _expected(actor: Actor, team: str) -> int:
if team == "none":
return 400
if actor == Actor.PROXY_ADMIN:
return 200
return 200 if actor in _MEMBERS.get(team, set()) else 404
_CASES = [
(f"{team}/{actor.value}", actor, team, _expected(actor, team))
for team in ("none", "alpha", "beta")
for actor in Actor
]
_DATES = "start_date=2024-01-01&end_date=2024-12-31"
@pytest.mark.parametrize(
"actor,team,expected_status",
[(a, t, s) for (_id, a, t, s) in _CASES],
ids=[c[0] for c in _CASES],
)
async def test_team_spend_by_user_matrix(actor: Actor, team: str, expected_status: int, proxy_client, world):
team_id = {"alpha": world.team_alpha_id, "beta": world.team_beta_id}.get(team)
query = _DATES if team_id is None else f"{_DATES}&team_ids={team_id}"
resp = await proxy_client.get(
f"/team/spend/by_user?{query}",
headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"},
)
assert resp.status_code == expected_status, f"{actor.value} -> {team}: {resp.status_code} {resp.text}"
if expected_status == 200:
body = resp.json()
assert (body["start_date"], body["end_date"]) == ("2024-01-01", "2024-12-31")
assert all(row["team_id"] == team_id for row in body["results"])

43
tests/proxy_unit_tests/test_check_batch_cost.py
View file

@ -2445,6 +2445,7 @@ class TestBatchCostAttribution:
metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1")
assert metadata["user_api_key"] == "hash-alice"
assert metadata["user_api_key_hash"] == "hash-alice"
assert metadata["user_api_key_user_id"] == "alice"
assert metadata["user_api_key_team_id"] == "team-alpha"
assert metadata["user_api_key_alias"] == "prod-key"
@ -2553,6 +2554,48 @@ class TestBatchCostAttribution:
assert metadata["user_api_key_alias"] == "prod-key"
@pytest.mark.asyncio
async def test_metadata_provenance_keeps_spend_log_api_key_joinable(self):
"""
CheckBatchCost stores the VerificationToken hash on the managed object. The
spend-log writer must receive matching user_api_key_hash provenance so it
does not re-hash that value; otherwise DailyUserSpend.api_key no longer joins
VerificationToken and Usage shows key-hash-... with a null alias/email.
"""
from datetime import datetime, timezone
from types import SimpleNamespace
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.proxy.utils import hash_token
token_hash = hash_token("sk-batch-creator-key")
instance = self._instance(
key_row=SimpleNamespace(key_alias="prod-key"),
user_row=SimpleNamespace(user_email="alice@example.com", user_alias=None),
)
metadata = await instance._build_creator_attribution_metadata(
self._job(api_key=token_hash), "batch-1"
)
assert metadata["user_api_key"] == token_hash
assert metadata["user_api_key_hash"] == token_hash
payload = get_logging_payload(
kwargs={
"model": "gpt-4o",
"call_type": "aretrieve_batch",
"litellm_params": {"metadata": metadata},
},
response_obj={
"id": "batch_123",
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
start_time=datetime.now(timezone.utc),
end_time=datetime.now(timezone.utc),
)
assert payload["api_key"] == token_hash
assert payload["api_key"] != hash_token(token_hash)
class TestPollPageStarvation:
"""LIT-5462 regression: a row that can never be costed used to keep its slot in the

83
tests/proxy_unit_tests/test_jwt_key_mapping.py
View file

@ -1333,3 +1333,86 @@ def test_jwt_client_id_field_does_not_raise_on_duplicate():
virtual_key_claim_field="new_field",
)
assert auth.virtual_key_claim_field == "new_field"
# ──────────────────────────────────────────────
# Tests: cache eviction must happen AFTER the DB write commits
# ──────────────────────────────────────────────
@pytest.mark.asyncio
async def test_delete_evicts_cache_after_row_is_gone():
"""A JWT request racing the delete must not keep the removed mapping authorized.
The DB delete simulates a concurrent request re-caching the mapping mid-write.
If the endpoint evicts before the delete commits, that repopulated entry
survives until TTL and the deleted mapping stays usable.
"""
from litellm.proxy._types import DeleteJWTKeyMappingRequest
from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key
cache_key = jwt_key_mapping_cache_key("email", "user@example.com")
user_api_key_cache = DualCache()
await user_api_key_cache.async_set_cache(key=cache_key, value="hashed_token")
mock_prisma = _mock_prisma()
mock_prisma.db.litellm_jwtkeymapping.find_unique.return_value = _mock_mapping()
async def concurrent_reader_repopulates(**kwargs):
await user_api_key_cache.async_set_cache(key=cache_key, value="hashed_token")
return _mock_mapping()
mock_prisma.db.litellm_jwtkeymapping.delete.side_effect = concurrent_reader_repopulates
with (
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: proxy_server module global is the endpoint's only injection point
patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), # test-quality-ok: proxy_server module global is the endpoint's only injection point
):
result = await delete_jwt_key_mapping(
data=DeleteJWTKeyMappingRequest(id="mapping-1"),
user_api_key_dict=_make_admin_auth(),
)
assert result == {"status": "success"}
assert await user_api_key_cache.async_get_cache(cache_key) is None
@pytest.mark.asyncio
async def test_update_evicts_old_and_new_cache_keys_after_write():
"""Renaming a mapping's claim must leave neither claim serving stale cache.
The DB update simulates a concurrent request re-caching the OLD mapping
mid-write. Both the old claim's entry (would restore the pre-rename token)
and the new claim's __NO_MAPPING__ sentinel (would 403 the renamed claim)
must be gone once the endpoint returns.
"""
from litellm.proxy._types import UpdateJWTKeyMappingRequest
from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key
old_cache_key = jwt_key_mapping_cache_key("email", "user@example.com")
new_cache_key = jwt_key_mapping_cache_key("email", "renamed@example.com")
user_api_key_cache = DualCache()
await user_api_key_cache.async_set_cache(key=old_cache_key, value="hashed_token")
await user_api_key_cache.async_set_cache(key=new_cache_key, value="__NO_MAPPING__")
mock_prisma = _mock_prisma()
mock_prisma.db.litellm_jwtkeymapping.find_unique.return_value = _mock_mapping()
async def concurrent_reader_repopulates(**kwargs):
await user_api_key_cache.async_set_cache(key=old_cache_key, value="hashed_token")
return _mock_mapping(claim_value="renamed@example.com")
mock_prisma.db.litellm_jwtkeymapping.update.side_effect = concurrent_reader_repopulates
with (
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: proxy_server module global is the endpoint's only injection point
patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), # test-quality-ok: proxy_server module global is the endpoint's only injection point
):
result = await update_jwt_key_mapping(
data=UpdateJWTKeyMappingRequest(id="mapping-1", jwt_claim_value="renamed@example.com"),
user_api_key_dict=_make_admin_auth(),
)
assert result.jwt_claim_value == "renamed@example.com"
assert await user_api_key_cache.async_get_cache(old_cache_key) is None
assert await user_api_key_cache.async_get_cache(new_cache_key) is None

10
tests/router_unit_tests/test_router_index_management.py
View file

@ -237,10 +237,12 @@ class TestRouterIndexManagement:
- model_name_to_deployment_indices for O(1) + O(k) model_name lookups
"""
# Methods that are allowed to iterate through self.model_list
ALLOWED_METHODS = [
"_get_deployment_by_litellm_model", # Edge case: lookup by litellm_params.model (not indexed)
"_finalize_adaptive_router_if_configured", # Init-time prefix scan for "auto_router/adaptive_router" (no index for prefix match)
]
ALLOWED_METHODS = {
"_get_deployment_by_litellm_model": "lookup by litellm_params.model, which is not indexed",
"_finalize_adaptive_router_if_configured": 'init-time prefix scan for "auto_router/adaptive_router"; no index for prefix match',
"config_deployments": "filters the whole list on model_info.db_model; admin path only (model add/upsert)",
"heuristic_v2_router_limit_violation": "counts heuristic_v2 routers across the whole list; admin path only (auto-router init/upsert)",
}
# Get path to router.py
router_file = os.path.join(

16
tests/rust-python-harness/shared/native_build.py
View file

@ -73,6 +73,15 @@ def _rebuild(repo_root: Path) -> tuple[bool, str]:
return completed.returncode == 0, "\n".join(lines[-_FAILURE_OUTPUT_LINES:])
def trace_bridge_error() -> str | None:
bridge: Final = get_native_bridge()
if bridge is None:
return "native Rust bridge is not importable"
if getattr(bridge, "_trace", None) is None:
return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature"
return None
def ensure_trace_bridge(repo_root: Path) -> str | None:
native_path: Final = _native_module_path()
native_mtime: Final = native_path.stat().st_mtime if native_path is not None and native_path.exists() else None
@ -84,9 +93,4 @@ def ensure_trace_bridge(repo_root: Path) -> str | None:
if not succeeded:
return f"native Rust bridge rebuild failed:\n{output}"
_drop_imported_bridge()
bridge: Final = get_native_bridge()
if bridge is None:
return "native Rust bridge is not importable"
if getattr(bridge, "_trace", None) is None:
return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature"
return None
return trace_bridge_error()

13
tests/store_model_in_db_tests/test_openai_error_handling.py
View file

@ -106,15 +106,22 @@ def test_missing_model_parameter_curl(curl_command):
# Run the curl command and capture the output
key = generate_key_sync()
curl_command = curl_command.replace("sk-1234", key)
result = subprocess.run(curl_command, shell=True, capture_output=True, text=True)
result = subprocess.run(
f'{curl_command} -s -w "\\n%{{http_code}}"',
shell=True,
capture_output=True,
text=True,
)
body, _, status_code = result.stdout.rpartition("\n")
# Parse the JSON response
response = json.loads(result.stdout)
response = json.loads(body)
# Check that we got an error response
assert "error" in response
print("error in response", json.dumps(response, indent=4))
assert "litellm.BadRequestError" in response["error"]["message"]
assert status_code == "400", f"expected HTTP 400, got {status_code}: {response}"
assert isinstance(response["error"]["message"], str) and response["error"]["message"]
@pytest.mark.asyncio

15
tests/test_litellm/caching/test_redis_cache.py
View file

@ -823,15 +823,26 @@ async def test_event_loop_stall_timeout_burst_keeps_breaker_closed():
Every operation already waiting on the loop times out together when the loop resumes,
so a purely consecutive threshold is satisfied instantly even though the Redis on the
other end (here an in-process fake that answers immediately) is healthy.
The fake checks its own client deadline against the clock, the way a client library
does, rather than wrapping the call in asyncio.wait_for: before 3.12 wait_for returns
the inner result when the inner future also completed during the stall, so the burst
never materialises and the test cannot exercise the duration gate.
"""
import time as time_mod
from redis.exceptions import TimeoutError as RedisTimeoutError
from litellm.caching.redis_cache import RedisCircuitBreaker, _run_under_circuit_breaker
breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60, timeout_min_duration=5.0)
async def healthy_redis_call_with_client_timeout():
return await asyncio.wait_for(asyncio.sleep(0.001, result="ok"), timeout=0.05)
deadline = time_mod.monotonic() + 0.05
await asyncio.sleep(0.001)
if time_mod.monotonic() > deadline:
raise RedisTimeoutError("read timed out")
return "ok"
async def stall_the_loop():
await asyncio.sleep(0)
@ -842,7 +853,7 @@ async def test_event_loop_stall_timeout_burst_keeps_breaker_closed():
stall_the_loop(),
return_exceptions=True,
)
timeouts = [r for r in results if isinstance(r, asyncio.TimeoutError)]
timeouts = [r for r in results if isinstance(r, RedisTimeoutError)]
assert len(timeouts) >= breaker.failure_threshold, "the stall must time out a full burst"
assert breaker.is_open() is False, "a healthy Redis behind one loop stall must stay in the pool"

29
tests/test_litellm/integrations/cloudzero/test_cloudzero.py
View file

@ -1,3 +1,4 @@
import hashlib
from datetime import datetime
from unittest.mock import AsyncMock, MagicMock, patch
@ -74,6 +75,8 @@ class TestCloudZeroHourlyExport:
fake_db = MagicMock()
async def query_raw_mock(query: str, *params):
if "sha256(" in query:
return []
start_time_utc = params[0] if len(params) > 0 else None
end_time_utc = params[1] if len(params) > 1 else None
limit = params[2] if len(params) > 2 else None
@ -146,6 +149,9 @@ class TestCloudZeroHourlyExport:
return joined
fake_db.query_raw = AsyncMock(side_effect=query_raw_mock)
fake_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
fake_db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[])
fake_db.litellm_usertable.find_many = AsyncMock(return_value=[])
fake_client.db = fake_db
mock_prisma_client_getter.return_value = fake_client
@ -160,3 +166,26 @@ class TestCloudZeroHourlyExport:
logger = CloudZeroLogger(api_key="test", connection_id="test")
await logger._hourly_usage_data_export()
class TestLiteLLMDatabaseUsageData:
@pytest.mark.asyncio
async def test_builds_frame_from_rows_recovered_for_double_hashed_keys(self, monkeypatch: pytest.MonkeyPatch):
double_hashed = hashlib.sha256(b"sk-hashed-token").hexdigest()
joined_row = {"api_key": "sk-joined", "api_key_alias": "joined", "team_id": "team-0", "user_email": None, "spend": 0.1}
dirty_row = {"api_key": double_hashed, "api_key_alias": None, "team_id": None, "user_email": None, "spend": 0.5}
async def query_raw(query: str, *params):
if "sha256(" in query:
return [{"digest": double_hashed, "key_alias": "batch-worker", "team_id": "team-1", "user_id": None}]
return [joined_row, dirty_row]
fake_client = MagicMock()
fake_client.db.query_raw = AsyncMock(side_effect=query_raw)
db = LiteLLMDatabase()
monkeypatch.setattr(db, "_ensure_prisma_client", lambda: fake_client)
result = await db.get_usage_data()
assert result["api_key_alias"].to_list() == ["joined", "batch-worker"]
assert result["team_id"].to_list() == ["team-0", "team-1"]

Some files were not shown because too many files have changed in this diff Show more