scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-08 08:08:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

Merge remote-tracking branch 'origin/main' into litellm_pr48393_head

Browse source 
# Conflicts:
#	tests/unit/litellm_core_utils/test_streaming_handler.py
This commit is contained in:
mateo-berri 2026-10-30 05:29:08 -07:00
commit c7f26fa79c
225 changed files with 17200 additions and 1490 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/test-unit.yml vendored
View file

@ -227,6 +227,7 @@ jobs:
tests/unit/llms/test_oss_decision.py
tests/unit/llms/test_polling_url_origin_match.py
tests/unit/llms/test_predibase_transformation.py
tests/unit/llms/test_subscription_accounts.py
tests/unit/llms/a2a
tests/unit/llms/aiml
tests/unit/llms/aiohttp_openai

1
backend/routes/allowlist.py
View file

@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/cache_settings",
"/coordination_redis/",
"/cost_tracking",
"/subscription_accounts/",
"/cost_optimization/",
"/cost/",
"/credentials",

21
litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql Normal file
View file

@ -0,0 +1,21 @@
SET LOCAL lock_timeout = '10s';
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
"subscription_account_id" TEXT NOT NULL,
"custom_llm_provider" TEXT NOT NULL,
"account_id" TEXT NOT NULL,
"label" TEXT,
"monthly_fee" DOUBLE PRECISION NOT NULL,
"currency" TEXT NOT NULL,
"billing_period_start" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT NOT NULL,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT NOT NULL,
CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
);
-- CreateIndex
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");

16
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
@@id([event_id, target, version])
@@index([target, state, next_attempt_at])
}
model LiteLLM_SubscriptionAccountTable {
subscription_account_id String @id @default(uuid())
custom_llm_provider String
account_id String
label String?
monthly_fee Float
currency String
billing_period_start String
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
@@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
}

5
litellm/__init__.py
View file

@ -872,6 +872,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "together_ai":
if _is_past_deprecation_date(value):
together_ai_models.discard(key)
elif not key.startswith("together-ai-"):
together_ai_models.add(key)
elif value.get("litellm_provider") == "baseten":
if _is_past_deprecation_date(value):
baseten_models.discard(key)

10
litellm/constants.py
View file

@ -7,6 +7,7 @@ from typing import Final, Literal
from litellm.litellm_core_utils.env_utils import (
get_env_float,
get_env_float_in_range,
get_env_float_or_none,
get_env_int,
get_env_int_in_range,
@ -67,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"retry_after",
"fallbacks",
"context_window_fallbacks",
"content_policy_fallbacks",
"retry_policy",
"model_group_retry_policy",
"model_group_alias",
@ -149,6 +151,9 @@ DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float(
"DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5
) # default cooldown a deployment if 50% of requests fail in a given minute
DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096)
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION: Final = get_env_float_in_range(
"NO_MAX_TOKENS_TPM_RESERVATION_FRACTION", 0.25, 0.01, 1.0
)
DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3)
DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1)
DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5)
@ -416,6 +421,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
)
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
@ -896,6 +903,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
}
)
SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
@ -1273,7 +1281,7 @@ empower_models: Final = set(
]
)
together_ai_models: Final[set] = set(
together_ai_models: Final[set[str]] = set(
[
# llama llms - chat
"togethercomputer/llama-2-70b-chat",

21
litellm/cost_calculator.py
View file

@ -1544,6 +1544,12 @@ def _response_time_ms_for_cost(
return litellm_logging_obj.get_response_ms()
def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None:
breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown
total: Final[object] = None if breakdown is None else breakdown.get("total_cost")
return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
def _adjusted_provider_billed_cost(
reported_cost: float,
custom_llm_provider: str | None,
@ -1561,9 +1567,9 @@ def _adjusted_provider_billed_cost(
)
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=None,
completion_tokens_cost_usd_dollar=None,
cost_for_built_in_tools_cost_usd_dollar=None,
prompt_tokens_cost_usd_dollar=0.0,
completion_tokens_cost_usd_dollar=adjusted_cost,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=adjusted_cost,
original_cost=reported_cost,
discount_percent=discount_percent,
@ -2376,6 +2382,15 @@ def response_cost_calculator(
)
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
if provider_response_cost is not None and not custom_pricing:
if _breakdown_total(litellm_logging_obj) != provider_response_cost:
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=0.0,
completion_tokens_cost_usd_dollar=provider_response_cost,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=provider_response_cost,
original_cost=provider_response_cost,
)
return provider_response_cost
response_cost = completion_cost(

93
litellm/exceptions.py
View file

@ -9,6 +9,7 @@
## LiteLLM versions of the OpenAI Exception Types
import datetime
import enum
from collections.abc import Mapping, Sequence
from typing import Any, Final
@ -115,6 +116,54 @@ def validate_rate_limit_type(value: object) -> str | None:
return None
def _upstream_body(response: object) -> bytes:
if not isinstance(response, httpx.Response):
return b""
try:
return response.content
except httpx.ResponseNotRead:
return b""
def _request_without_credentials(upstream: object, fallback_url: str) -> httpx.Request:
if not isinstance(upstream, httpx.Response):
return httpx.Request(method="POST", url=fallback_url)
try:
upstream_request: Final = upstream.request
except RuntimeError:
return httpx.Request(method="POST", url=fallback_url)
return httpx.Request(
method=upstream_request.method,
url=upstream_request.url.copy_with(userinfo=b"", query=None, fragment=None),
)
def _upstream_elapsed(upstream: object) -> datetime.timedelta | None:
if not isinstance(upstream, httpx.Response):
return None
try:
return upstream.elapsed
except RuntimeError:
return None
def error_response_without_credentials(status_code: int, upstream: object, fallback_url: str) -> httpx.Response:
response: Final = httpx.Response(
status_code=status_code,
stream=httpx.ByteStream(_upstream_body(upstream)),
request=_request_without_credentials(upstream, fallback_url),
)
_ = response.read()
response.headers = httpx.Headers(upstream.headers if isinstance(upstream, httpx.Response) else None)
elapsed: Final = _upstream_elapsed(upstream)
if elapsed is not None:
response.elapsed = elapsed
return response
_NO_UPSTREAM_REQUEST_URL: Final = "https://cloud.google.com/vertex-ai/"
_MINIMAL_ERROR_RESPONSE: httpx.Response | None = None
@ -519,20 +568,11 @@ class RateLimitError(openai.RateLimitError):
# headers stay reachable on `e.response.headers` for callers that
# explicitly want them; only the proxy-supplied `headers=` kwarg
# makes it onto `self.headers`.
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.headers: dict[str, str] | None = {k: str(v) for k, v in headers.items()} if headers else None
# Mirrors FastAPI HTTPException.detail so the same instance can be
# serialized through both the ProxyException and HTTPException paths.
self.detail = detail if detail is not None else self.message
self.response = httpx.Response(
status_code=429,
headers=_response_headers,
content=response.content if response is not None else None,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(429, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message,
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
@ -757,15 +797,7 @@ class ServiceUnavailableError(openai.APIStatusError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.response = httpx.Response(
status_code=self.status_code,
headers=_response_headers,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message, response=self.response, body=None
) # Call the base class constructor with the parameters it needs
@ -808,15 +840,7 @@ class BadGatewayError(openai.APIStatusError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.response = httpx.Response(
status_code=self.status_code,
headers=_response_headers,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message,
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
@ -862,15 +886,7 @@ class InternalServerError(openai.InternalServerError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
_response_headers: Final = getattr(response, "headers", None) if response is not None else None
self.response = httpx.Response(
status_code=self.status_code,
headers=_response_headers,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
)
self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL)
super().__init__(
self.message,
response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime
@ -908,6 +924,7 @@ class APIError(openai.APIError):
max_retries: int | None = None,
num_retries: int | None = None,
body: object | None = None,
response: httpx.Response | None = None,
):
self.status_code = status_code
self.message = f"litellm.APIError: {message}"
@ -916,6 +933,8 @@ class APIError(openai.APIError):
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
if response is not None:
self.response = response
if request is None:
request = httpx.Request(method="POST", url="https://api.openai.com/v1")
super().__init__(self.message, request=request, body=body)

43
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -23,6 +23,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
with_prompt_cache_breakpoint,
@ -115,12 +116,12 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
def _served_provider_for_model(model: str) -> str | None:
def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
try:
_, provider, _, _ = get_llm_provider(model=model)
_, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
except BadRequestError:
return None
return provider
@ -175,7 +176,10 @@ def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[
def _hosted_openai_dialect_flag(
model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None]
model: str,
custom_llm_provider: str | None,
resolve_provider: Callable[[str], str | None],
model_cost: Mapping[str, Mapping[str, object]] | None = None,
) -> bool | None:
"""
Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@ -194,14 +198,19 @@ def _hosted_openai_dialect_flag(
"""
import litellm
exact_entry: Final = litellm.model_cost.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None
cost_map: Final = (
cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries
if model_cost is None
else model_cost
)
exact_entry: Final = cost_map.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
return None
provider: Final = custom_llm_provider or resolve_provider(model)
if provider is None or provider == "openai":
return None
if isinstance(exact_entry, dict) and exact_provider == provider:
if isinstance(exact_entry, Mapping) and exact_provider == provider:
return _hosted_entry_flag(exact_entry, lambda: provider)
from litellm.utils import get_potential_model_names
@ -214,8 +223,8 @@ def _hosted_openai_dialect_flag(
names["stripped_model_name"],
names["provider_prefixed_model_name"],
)
entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict))
entries: Final = (cost_map.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
return next((flag for flag in flags if flag is not None), None)
@ -527,7 +536,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if model is None:
return False
hosted_flag: Final = _hosted_openai_dialect_flag(
model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider
model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
)
if hosted_flag is not None:
return hosted_flag
@ -538,8 +547,20 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return prompt_cache_options is not None or targets_openai_api(api_base)
@staticmethod
def _resolve_provider(model: str) -> str | None:
return _served_provider_for_model(model)
def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
return (
_hosted_openai_dialect_flag(
model,
custom_llm_provider,
AnthropicCacheControlHook._resolve_provider,
GetModelCostMap.loaded_model_cost_map(),
)
is True
)
@staticmethod
def _resolve_provider(model: str, api_base: object = None) -> str | None:
return _served_provider_for_model(model, api_base)
@staticmethod
def count_request_cache_breakpoints(

32
litellm/integrations/custom_guardrail.py
View file

@ -9,6 +9,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
import httpx
from pydantic import TypeAdapter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger
@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
)
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]:
if not isinstance(metadata, dict):
return []
return metadata.get("guardrails") or []
class CustomGuardrail(CustomLogger):
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
use_native_during_call_hook: ClassVar[bool] = False
@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger):
return True
raise
def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]:
def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]:
"""
Returns the guardrail(s) to be run from the metadata or root
Returns the guardrail(s) to be run from the root, else from both metadata fields together
"""
if "guardrails" in data:
return data["guardrails"]
# Check both metadata locations. For regular endpoints move_guardrails_to_metadata
# writes to "metadata"; for thread/assistant endpoints it writes to
# "litellm_metadata". We check the one that actually contains the "guardrails"
# key so that a non-empty litellm_metadata without guardrails does not shadow
# the merged list stored in metadata (which would cause team guardrails to be
# silently skipped while default_on=True policy guardrails still fire).
for meta_key in ("metadata", "litellm_metadata"):
meta = data.get(meta_key) or {}
if isinstance(meta, dict) and "guardrails" in meta:
return meta.get("guardrails") or []
return []
request_fields: Final = _REQUEST_FIELDS.validate_python(data)
return [
*_guardrails_listed_in(request_fields.get("metadata")),
*_guardrails_listed_in(request_fields.get("litellm_metadata")),
]
def _guardrail_is_in_requested_guardrails(
self,
requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]],
requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]],
) -> bool:
for _guardrail in requested_guardrails:
if isinstance(_guardrail, dict):

1
litellm/integrations/otel/model/spans.py
View file

@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_LensReview",
"LiteLLM_LensIngestionKey",
"LiteLLM_LensWorker",
"LiteLLM_SubscriptionAccountTable",
"LiteLLM_LensSignalConfig",
"LiteLLM_LensTraceSignal",
)

15
litellm/integrations/websearch_interception/handler.py
View file

@ -398,14 +398,17 @@ class WebSearchInterceptionLogger(CustomLogger):
(t for t in tools if is_anthropic_native_web_search_tool(t)),
None,
)
emit_native_blocks: Final = native_tool is not None or (
kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY))
)
outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs)
search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome)
content: Final[list[dict[str, object]]] = []
if native_tool is not None:
if emit_native_blocks:
tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
tool_name: Final = native_tool.get("name") or "web_search"
tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search"
content.append(
{
"type": "server_tool_use",
@ -415,10 +418,10 @@ class WebSearchInterceptionLogger(CustomLogger):
}
)
content.append(
WebSearchTransformation.build_web_search_outcome_block(tool_use_id=tool_use_id, outcome=outcome)
WebSearchTransformation.build_web_search_outcome_block(
tool_use_id=tool_use_id, outcome=outcome, with_snippets=False
)
)
# Keep the text block so non-native short-circuit callers (Claude Code,
# github_copilot, etc.) see the same payload they always have.
content.append({"type": "text", "text": search_result_text})
response: Final[dict[str, object]] = {
@ -435,7 +438,7 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
"WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)",
len(search_result_text),
native_tool is not None,
emit_native_blocks,
)
return response

10
litellm/integrations/websearch_interception/transformation.py
View file

@ -412,6 +412,7 @@ class WebSearchTransformation:
def build_web_search_tool_result_block(
tool_use_id: str,
search_response: SearchResponse | None,
with_snippets: bool = True,
) -> dict[str, object]:
"""
Build an Anthropic-native ``web_search_tool_result`` content block.
@ -445,6 +446,8 @@ class WebSearchTransformation:
``litellm.asearch()``. If None or empty, the block is still
emitted with an empty result list (signals "search ran, no
results" rather than "search did not run").
with_snippets: False when a text block in the same reply already
carries the snippets.
"""
items: Final[list[dict[str, object]]] = []
if search_response is not None:
@ -460,7 +463,7 @@ class WebSearchTransformation:
"title": title,
"page_age": page_age,
"encrypted_content": "",
"snippet": getattr(r, "snippet", "") or "",
**({"snippet": getattr(r, "snippet", "") or ""} if with_snippets else {}),
}
)
return {
@ -481,12 +484,15 @@ class WebSearchTransformation:
}
@staticmethod
def build_web_search_outcome_block(tool_use_id: str, outcome: SearchOutcome) -> dict[str, object]:
def build_web_search_outcome_block(
tool_use_id: str, outcome: SearchOutcome, with_snippets: bool = True
) -> dict[str, object]:
match outcome:
case SearchSucceeded(response=response):
return WebSearchTransformation.build_web_search_tool_result_block(
tool_use_id=tool_use_id,
search_response=response,
with_snippets=with_snippets,
)
case SearchFailed(error_code=error_code):
return WebSearchTransformation.build_web_search_tool_result_error_block(

37
litellm/litellm_core_utils/core_helpers.py
View file

@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object:
return getattr(chunk, "usage", None)
USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"})
UPSTREAM_USAGE_COST: Final = "upstream_usage_cost"
def _numeric(value: object) -> float | None:
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None:
"""Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own
``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of
reading litellm's bill back as if the upstream had reported it."""
hidden: Final[object] = getattr(response, "_hidden_params", None)
if isinstance(hidden, dict):
hidden_params: Final = cast(dict[str, object], hidden) # cast-ok: hidden params ride on the response as a dict
upstream: Final = (
hidden_params[UPSTREAM_USAGE_COST]
if UPSTREAM_USAGE_COST in hidden_params
else _numeric(getattr(usage, "cost", None))
)
hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream})
setattr(usage, "cost", bill)
def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
"""The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own
usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal
event do), the upstream's figure is read from ``upstream_usage_cost`` instead."""
if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS:
return None
cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None))
cost: Final = (
_numeric(hidden[UPSTREAM_USAGE_COST])
if hidden is not None and UPSTREAM_USAGE_COST in hidden
else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
)
return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None

10
litellm/litellm_core_utils/env_utils.py
View file

@ -71,7 +71,15 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int)
Values outside the range fall back to the default and warn, so a misconfigured knob can
neither crash the caller nor silently change the meaning of what it computes.
"""
value: Final = get_env_int(env_var, default)
return _within_range(env_var, get_env_int(env_var, default), default, minimum, maximum)
def get_env_float_in_range(env_var: str, default: float, minimum: float, maximum: float) -> float:
"""Float counterpart of `get_env_int_in_range`."""
return _within_range(env_var, get_env_float(env_var, default), default, minimum, maximum)
def _within_range(env_var: str, value: _NumberT, default: _NumberT, minimum: _NumberT, maximum: _NumberT) -> _NumberT:
if minimum <= value <= maximum:
return value
logging.getLogger("LiteLLM").warning(

57
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -37,6 +37,7 @@ from ..exceptions import (
ServiceUnavailableError,
Timeout,
UnprocessableEntityError,
error_response_without_credentials,
)
@ -305,6 +306,16 @@ def _litellm_proxy_response(
)
def _without_credentials(response: httpx.Response | None) -> httpx.Response | None:
if not isinstance(response, httpx.Response):
return response
return error_response_without_credentials(response.status_code, response, "https://docs.litellm.ai/docs")
def _upstream_reply(response: httpx.Response | None) -> httpx.Response | None:
return response if response is not None and (response.headers or response.content) else None
def _map_openai_exception(
*,
model: str,
@ -315,7 +326,7 @@ def _map_openai_exception(
exception_provider: str,
extra_information: str,
) -> None:
response: Final = _litellm_proxy_response(original_exception, custom_llm_provider)
response: Final = _without_credentials(_litellm_proxy_response(original_exception, custom_llm_provider))
if (cyber_policy_body := _openai_cyber_policy_body(original_exception)) is not None:
raise ContentPolicyViolationError(
message=f"ContentPolicyViolationError: {exception_provider} - {original_exception}",
@ -378,6 +389,7 @@ def _map_openai_exception(
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
elif (
("invalid_request_error" in error_str and "content_policy_violation" in error_str)
@ -449,6 +461,7 @@ def _map_openai_exception(
model=model,
llm_provider=custom_llm_provider,
body=getattr(original_exception, "body", None),
response=_upstream_reply(response),
)
elif "Request too large" in error_str:
raise RateLimitError(
@ -479,6 +492,7 @@ def _map_openai_exception(
model=model,
request=_request,
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
elif hasattr(original_exception, "status_code"):
if original_exception.status_code == 400:
@ -512,6 +526,7 @@ def _map_openai_exception(
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
elif original_exception.status_code == 422:
raise BadRequestError(
@ -563,6 +578,7 @@ def _map_openai_exception(
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
exception_status_code=original_exception.status_code,
response=_upstream_reply(response),
)
else:
raise APIError(
@ -572,6 +588,7 @@ def _map_openai_exception(
model=model,
request=getattr(original_exception, "request", None),
litellm_debug_info=extra_information,
response=_upstream_reply(response),
)
else:
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
@ -2019,6 +2036,7 @@ def _map_azure_exception(
exception_provider: str,
extra_information: str,
) -> None:
response: Final = _without_credentials(getattr(original_exception, "response", None))
message = get_error_message(error_obj=original_exception)
if message is None:
if hasattr(original_exception, "message"):
@ -2054,7 +2072,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif "DeploymentNotFound" in error_str:
raise NotFoundError(
@ -2062,7 +2080,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif azure_error_code == "content_policy_violation" or ExceptionCheckers.is_azure_content_policy_violation_error(
error_str
@ -2076,6 +2094,7 @@ def _map_azure_exception(
model=model,
extra_information=extra_information,
original_exception=original_exception,
response=response,
)
elif azure_error_code == "invalid_encrypted_content" or "could not be verified" in error_str:
helpful_message: Final = (
@ -2094,7 +2113,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif getattr(original_exception, "status_code", None) in (None, 400) and (
@ -2106,7 +2125,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
@ -2115,7 +2134,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif "The api_key client option must be set either by passing api_key to the client or by setting" in error_str:
@ -2124,7 +2143,7 @@ def _map_azure_exception(
llm_provider=custom_llm_provider,
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif "Connection error" in error_str:
raise APIConnectionError(
@ -2140,7 +2159,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
body=getattr(original_exception, "body", None),
)
elif original_exception.status_code == 401:
@ -2149,7 +2168,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 404:
raise NotFoundError(
@ -2157,7 +2176,7 @@ def _map_azure_exception(
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 408:
raise Timeout(
@ -2165,6 +2184,7 @@ def _map_azure_exception(
model=model,
litellm_debug_info=extra_information,
llm_provider="azure",
response=_upstream_reply(response),
)
elif original_exception.status_code == 422:
raise BadRequestError(
@ -2172,7 +2192,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 429:
raise RateLimitError(
@ -2180,7 +2200,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 502:
raise BadGatewayError(
@ -2188,7 +2208,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 503:
raise ServiceUnavailableError(
@ -2196,7 +2216,7 @@ def _map_azure_exception(
model=model,
llm_provider="azure",
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
)
elif original_exception.status_code == 504: # gateway timeout error
raise Timeout(
@ -2205,6 +2225,7 @@ def _map_azure_exception(
litellm_debug_info=extra_information,
llm_provider="azure",
exception_status_code=original_exception.status_code,
response=_upstream_reply(response),
)
else:
raise APIError(
@ -2214,6 +2235,7 @@ def _map_azure_exception(
litellm_debug_info=extra_information,
model=model,
request=httpx.Request(method="POST", url="https://openai.com/"),
response=_upstream_reply(response),
)
else:
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
@ -2589,17 +2611,20 @@ def exception_type(
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=_upstream_reply(
_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider))
),
)
if (
custom_llm_provider == "litellm_proxy"
): # handle special case where calling litellm proxy + exception str contains error message
extract_and_raise_litellm_exception(
response=_litellm_proxy_response(mappable_exception, custom_llm_provider),
response=_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)),
error_str=error_str,
model=model,
custom_llm_provider=custom_llm_provider,
body=getattr(original_exception, "body", None),
body=getattr(mappable_exception, "body", None),
)
if (
custom_llm_provider == "openai"

1
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_user_session",
"fireworks_forward_user_id",
"responses_api_version",
"allowed_openai_params",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
}
)

80
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -3,6 +3,8 @@ Helper functions for health check calls.
"""
import base64
import itertools
import re
from collections.abc import Awaitable, Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
@ -26,6 +28,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
)
ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT: Final = 2
WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT: Final = 3
BEDROCK_ANY_MODEL_ROUTES: Final = frozenset({"converse", "invoke"})
EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
{"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})}
)
@ -90,6 +95,62 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document
return native(model, custom_llm_provider)
def _provider_base_model(custom_llm_provider: str, model: str) -> str | None:
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
try:
model_info: Final = ProviderConfigManager.get_provider_model_info(
model=None, provider=LlmProviders(custom_llm_provider)
)
return None if model_info is None else model_info.get_base_model(model)
except (ValueError, NotImplementedError):
return None
def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segment: str) -> bool:
if custom_llm_provider == "bedrock":
from litellm.llms.bedrock.common_utils import get_all_bedrock_regions
return segment in BEDROCK_ANY_MODEL_ROUTES or segment in get_all_bedrock_regions()
base_model: Final = _provider_base_model(custom_llm_provider, f"{segment}/{model_after_segment}")
return base_model is not None and not base_model.startswith(f"{segment}/")
def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
provider_prefix: Final = f"{custom_llm_provider}/"
segments: Final = wildcard.removeprefix(provider_prefix).split("/")
route_segments: Final = tuple(
itertools.takewhile(
lambda segment_index: _is_route_segment(
custom_llm_provider, segments[segment_index], "/".join(segments[segment_index + 1 :])
),
range(len(segments) - 1),
)
)
route_prefix: Final = "".join(f"{segments[segment_index]}/" for segment_index in route_segments)
model_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex("/".join(segments[len(route_segments) :])))
ranked_model_ids: Final = tuple(
(ranked_model.model.removeprefix(provider_prefix), ranked_model)
for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)
)
served_model_ids: Final = sorted(
(ranked for ranked in ranked_model_ids if model_regex.fullmatch(ranked[0])),
key=lambda ranked: (ranked[1].is_past_deprecation_date, ranked[1].route_rank, ranked[1].cost, ranked[0]),
)
served_probes: Final = itertools.islice(
dict.fromkeys(f"{provider_prefix}{route_prefix}{model_id}" for model_id, _ in served_model_ids),
WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT,
)
provider_probes: Final = (
f"{provider_prefix}{model_id}" for model_id, _ in ranked_model_ids[:WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT]
)
return tuple(dict.fromkeys((*served_probes, *provider_probes)))
class HealthCheckHelpers:
@staticmethod
async def ahealth_check_wildcard_models(
@ -99,23 +160,16 @@ class HealthCheckHelpers:
litellm_logging_obj: "Logging",
) -> dict:
from litellm import acompletion
from litellm.litellm_core_utils.llm_request_utils import (
pick_cheapest_chat_models_from_llm_provider,
)
# this is a wildcard model, we need to pick a random model from the provider
cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3)
if len(cheapest_models) == 0:
probe_models: Final = wildcard_health_check_models(wildcard=model, custom_llm_provider=custom_llm_provider)
if len(probe_models) == 0:
raise Exception(
f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
f"Unable to health check wildcard model {model} for provider {custom_llm_provider}: the cost map has "
"no chat model for this provider. Set model_info.health_check_model to a model this wildcard serves"
)
if len(cheapest_models) > 1:
fallback_models = cheapest_models[1:] # Pick the last 2 models from the shuffled list
else:
fallback_models = None
model_params["model"] = cheapest_models[0]
model_params["model"] = probe_models[0]
model_params["litellm_logging_obj"] = litellm_logging_obj
model_params["fallbacks"] = fallback_models
model_params["fallbacks"] = list(probe_models[1:]) or None
model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16
await acompletion(**model_params)
return {}

58
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -1,6 +1,11 @@
import datetime
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass
from typing import Final, cast
from pydantic import TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
@ -151,19 +156,52 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
Returns:
list[str]: A list of the n cheapest chat models.
"""
if custom_llm_provider not in litellm.models_by_provider:
return []
return [ranked_model.model for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)[:n]]
@dataclass(frozen=True, slots=True)
class RankedChatModel:
model: str
route_rank: int
cost: float
is_past_deprecation_date: bool
def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]:
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
ranked_rows: Final = (
(route_rank, cost, model)
for model, cost in _chat_row_costs(known_models, custom_llm_provider)
ranked_models: Final = (
RankedChatModel(
model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date
)
for model, cost, is_past_deprecation_date in _chat_row_costs(known_models, custom_llm_provider)
if (route_rank := _route_rank(model, custom_llm_provider)) is not None
)
return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
return tuple(sorted(ranked_models, key=lambda ranked_model: (ranked_model.route_rank, ranked_model.cost)))
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
class _CostMapDeprecation(TypedDict):
deprecation_date: NotRequired[ReadOnly[object]]
_COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
def _is_past_deprecation_date(cost_map_key: str) -> bool:
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
).get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return (
datetime.date.fromisoformat(deprecation_date)
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
)
except ValueError:
return False
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
for model in models:
try:
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
@ -171,7 +209,11 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
continue
if model_info.get("mode") != "chat":
continue
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
yield (
model,
(model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0),
_is_past_deprecation_date(model_info["key"]),
)
def _route_rank(model: str, custom_llm_provider: str) -> int | None:

6
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
"""
from litellm.litellm_core_utils.token_counter import get_image_type
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
# If we have a valid content type that's not generic, use it
if current_content_type and current_content_type not in [
if media_type and media_type not in [
"binary/octet-stream",
"application/octet-stream",
]:
return current_content_type
return media_type
# Extension to MIME type mapping
# Supports images, documents, and other common file types

63
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -912,9 +912,9 @@ def create_anthropic_image_param(
image_url_input: str | dict,
format: str | None = None,
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam:
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
"""
Create an AnthropicMessagesImageParam from an image URL input.
Create an Anthropic content block from an image URL input.
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
@ -933,14 +933,7 @@ def create_anthropic_image_param(
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url: Final = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
else:
# HTTPS URL - pass directly for regular Anthropic
return AnthropicMessagesImageParam(
@ -953,14 +946,7 @@ def create_anthropic_image_param(
else:
# Convert to base64 for data URIs or other formats
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
anthropic_content_list.append(
cast(
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
_anthropic_image_param,
)
)
elif content["type"] == "tool_reference":
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":
@ -5878,38 +5869,36 @@ def function_call_prompt(
]
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str:
"""
Decides if a user-defined custom prompt or default needs to be used
Returns the prompt str that's passed to the model as a user message
"""
custom_prompt_details: dict | None = None
response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}]
if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict:
custom_prompt_details = litellm.custom_prompt_dict[
f"{model}/response_schema_prompt"
] # allow user to define custom response schema prompt by model
elif "response_schema_prompt" in litellm.custom_prompt_dict:
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
if custom_prompt_details is not None:
return render_custom_prompt_template(
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
)
else:
prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict
model_prompt_key: Final = f"{model}/response_schema_prompt"
custom_prompt_details: Final = prompts.get(
model_prompt_key if model_prompt_key in prompts else "response_schema_prompt"
)
if custom_prompt_details is None:
return default_response_schema_prompt(response_schema=response_schema)
return render_custom_prompt_template(
template=custom_prompt_details,
messages=[{"role": "user", "content": f"{response_schema}"}],
model=model,
llm_provider=llm_provider,
)
def default_response_schema_prompt(response_schema: dict) -> str:
def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str:
"""
Used if provider/model doesn't support 'response_schema' param.
This is the default prompt. Allow user to override this with a custom_prompt.
"""
prompt_str: Final = f"""Use this JSON schema:
```json
{response_schema}
```json
{json.dumps(response_schema, default=str)}
```"""
return prompt_str

330
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1,5 +1,6 @@
import asyncio
import json
import time
import traceback
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
REALTIME_METERED_USAGE_KEY,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
from litellm.llms.base_llm.realtime.transformation import (
BaseRealtimeConfig,
RealtimeBackend,
RealtimeSessionUpdateRejection,
)
from litellm.types.llms.openai import (
OpenAIRealtimeEvents,
OpenAIRealtimeOutputItemDone,
@ -125,6 +132,65 @@ class _LoggingWorker(Protocol):
_DECODED_JSON: Final = TypeAdapter(object)
_SESSION_FIELDS: Final = TypeAdapter(dict[str, object])
_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_MERGE_MAX_DEPTH: Final = 8
class _SessionUpdateFrame(TypedDict):
type: ReadOnly[str]
session: ReadOnly[Mapping[str, object]]
def _merge_session_fields(
base: Mapping[str, object], update: Mapping[str, object], depth: int = 0
) -> Mapping[str, object]:
def merged_value(key: str) -> object:
if key not in update:
return base[key]
value: Final = update[key]
base_value: Final = base.get(key)
if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping):
return _merge_session_fields(
_SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1
)
return update[key]
return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))})
def _session_update_frame(session: Mapping[str, object]) -> str:
frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session}
return json.dumps(frame, default=_SESSION_FIELDS.validate_python)
_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection(
code="session_update_too_large",
message=(
"The session.update frames sent before the first conversation content add up to more than 10 MB, "
"so this session.update was not applied"
),
param="session",
)
def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]:
return {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": "audio_input_not_screenable",
"message": (
"A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it "
"starts answering it, so audio input was not sent. Send text with conversation.item.create instead"
),
"param": None,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
def _decode_json_object(payload: str) -> Mapping[str, object]:
"""Decode a realtime frame into its top-level field mapping."""
return json.loads(payload)
@ -243,8 +309,17 @@ class RealTimeStreaming:
event_normalizer: RealtimeEventNormalizer | None = None,
usage_meter: RealtimeUsageMeter | None = None,
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
clock: Callable[[], float] = time.monotonic,
):
self.websocket: _ClientWebSocket = websocket
self._clock: Final = clock
self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds
self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds
self._held_client_content: tuple[str, ...] = ()
self._held_client_content_bytes: int = 0
self._held_client_content_deadline: float | None = None
self._replay_scope: Final = client_websocket_scope(websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
@ -300,6 +375,10 @@ class RealTimeStreaming:
# Gemini Live rejects a follow-up BidiGenerateContentSetup once any
# content (realtimeInput / clientContent / toolResponse) has been sent.
self._content_sent_after_setup: bool = False
self._held_session_update: Mapping[str, object] | None = None
self._applied_session_fields: Mapping[str, object] | None = None
self._held_session_update_bytes: int = 0
self._unscreened_audio_refused: bool = False
# Whether this is a transcription-only session (session.type == "transcription",
# e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
# their input_audio_transcription.completed usage drives duration-based cost.
@ -337,6 +416,17 @@ class RealTimeStreaming:
]
)
_CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"])
_CLIENT_CONTENT_TYPES = frozenset(
[
"conversation.item.create",
"response.create",
"input_audio_buffer.append",
"input_audio_buffer.commit",
"input_audio_buffer.end",
]
)
_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"])
_DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
_AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
"pcm16": {"type": "audio/pcm", "rate": 24000},
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
@ -377,7 +467,7 @@ class RealTimeStreaming:
message_obj: dict[str, Any] = cast(dict[str, Any], message)
else:
message_obj = cast(dict[str, Any], json.loads(message))
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
self._collect_tool_calls_from_response_done(message_obj)
if not self._should_store_message(message_obj):
return
try:
@ -502,7 +592,7 @@ class RealTimeStreaming:
self.store_message(flush_event)
self._capture_transcription_usage(flush_event)
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None:
"""Extract function_call items from response.done events for spend logging."""
try:
if event_obj.get("type") != "response.done":
@ -607,6 +697,167 @@ class RealTimeStreaming:
self._usage_meter.observe_client_event(sent_event)
return True
def _builds_setup_lazily(self) -> bool:
if self.provider_config is None:
return False
return self.provider_config.builds_setup_from_session_update()
def _holds_session_updates_until_first_content(self) -> bool:
return self.session_configuration_request is None and self._builds_setup_lazily()
async def _rejected_setup_session_update(self, message: str) -> bool:
provider_config: Final = self.provider_config
if provider_config is None or not provider_config.builds_setup_from_session_update():
return False
client_event: Final = _decode_json_object(message)
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
if self._leaves_applied_setup_unchanged(update):
await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
return True
rejection: Final = provider_config.setup_session_update_rejection(
self.model, update, setup_sent=self.session_configuration_request is not None
)
if rejection is None:
return False
await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id"))))
return True
def _held_session_update_message(self) -> str:
return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
async def _hold_session_update(self, message: str) -> None:
client_event: Final = _decode_json_object(message)
if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
await self.websocket.send_text(
json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
)
return
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
self._held_session_update = held
self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
await self._acknowledge_session_setup(held)
def _setup_preview(self, session: Mapping[str, object]) -> str | None:
if self.provider_config is None:
return None
preview: Final = self.provider_config.transform_realtime_request(
_session_update_frame(session), self.model, None
)
return next(
(frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
)
def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
applied: Final = self._applied_session_fields
if self.session_configuration_request is None or applied is None:
return False
applied_setup: Final = self._setup_preview(applied)
merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
return (
applied_setup is not None
and merged_setup is not None
and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
)
async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
if self.provider_config is None:
return
setup: Final = self._setup_preview(session)
if setup is None:
return
acknowledged: Final = self.provider_config.transform_session_created_event(
self.model, str(self.logging_obj.litellm_trace_id), setup
)
if acknowledged is None:
return
event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"}
event_str: Final = json.dumps(event)
self.store_message(event_str)
await self._send_event_to_client(event, event_str)
async def _send_held_session_update_as_setup(self) -> None:
held: Final = self._held_session_update
if held is None:
return
message: Final = self._held_session_update_message()
self._held_session_update = None
self._held_session_update_bytes = 0
if await self._send_to_backend(message):
self._applied_session_fields = held
await self._send_held_client_content()
def _holds_client_content(self, msg_type: str | None, message: str) -> bool:
held_types: Final = (
RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES
if self._uses_deferred_backend_setup()
else RealTimeStreaming._HELD_CONTENT_TYPES
)
return (
msg_type in held_types
and self._held_session_update is not None
and self._holds_session_updates_until_first_content()
and not self._pending_messages_until_setup
and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES
and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES
)
def _hold_client_content(self, message: str) -> None:
if self._held_client_content_deadline is None:
self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds
self._held_client_content = (*self._held_client_content, message)
self._held_client_content_bytes += len(message.encode("utf-8"))
async def _send_held_client_content(self) -> None:
held: Final = self._held_client_content
self._held_client_content = ()
self._held_client_content_bytes = 0
self._held_client_content_deadline = None
for message in held:
await self._send_to_backend(message)
def _setup_deadline(self) -> float:
content_deadline: Final = self._held_client_content_deadline
return (
self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline)
)
async def _send_setup_before_content(self, msg_type: str | None) -> None:
if self._holds_session_updates_until_first_content():
await self._send_held_session_update_as_setup()
await self._send_default_setup_before_first_client_message(msg_type)
async def _receive_client_message_before_setup_deadline(self) -> str | None:
if not self._holds_session_updates_until_first_content():
return await self._receive_client_message()
receive: Final = asyncio.ensure_future(self._receive_client_message())
try:
done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock())
if not done:
await self._send_setup_before_content(None)
return await receive
finally:
receive.cancel()
async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None:
if self.session_configuration_request is not None or self.provider_config is None:
return
if msg_type == "session.update" or (
self._uses_deferred_backend_setup()
and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup)
):
return
default_setup: Final = self.provider_config.session_configuration_request(self.model)
if not default_setup:
return
setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
await self.backend_ws.send(setup)
self._cache_session_configuration_request(setup)
self._applied_session_fields = _NO_SESSION_FIELDS
await self._send_held_client_content()
def _enforce_transcription_session_model(self, message: str) -> str:
"""Force client transcription session updates to the authorized model.
@ -851,6 +1102,20 @@ class RealTimeStreaming:
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None:
if pre_block_backend_message is None and self._is_transcription_session:
return
await self._send_setup_before_content("conversation.item.create")
# Deliver any caller-supplied backend message FIRST so that
# protocol contracts requiring a specific ordering (e.g.
# Gemini Live's mandatory ``toolResponse`` after a
# ``toolCall``) are honored before the guardrail's own
# clientContent / cancel messages are sent.
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._withdraw_blocked_turn(item_id)
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
await self._send_guardrail_cancel()
if item_id is not None and self._deletes_conversation_items:
@ -958,6 +1223,21 @@ class RealTimeStreaming:
"""Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
return not self._is_transcription_session and self._has_audio_transcription_guardrails()
def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
return (
msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES
and self._builds_setup_lazily()
and self._should_disable_vad_auto_response()
)
async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None:
if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES:
return
self._unscreened_audio_refused = True
await self.websocket.send_text(
json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id")))
)
async def run_realtime_guardrails(
self,
transcript: str,
@ -1030,15 +1310,7 @@ class RealTimeStreaming:
# Use realtime_violation_message if configured; fall back to guardrail error text.
error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
# Deliver any caller-supplied backend message FIRST so that
# protocol contracts requiring a specific ordering (e.g.
# Gemini Live's mandatory ``toolResponse`` after a
# ``toolCall``) are honored before the guardrail's own
# clientContent / cancel messages are sent.
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._withdraw_blocked_turn(item_id)
await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
@ -1136,6 +1408,9 @@ class RealTimeStreaming:
break
finally:
self._flushing_pending_messages_until_setup = False
if self.session_created_sent_to_client and self._builds_setup_lazily():
await self._maybe_send_guardrail_turn_detection_update()
continue
if self.session_created_sent_to_client:
# A synthetic session.created (with placeholder defaults) was
# already forwarded to the client when we connected. The
@ -1295,7 +1570,11 @@ class RealTimeStreaming:
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
def _backend_refused_session(self, close: BackendClose) -> bool:
return close.code != 1000 and not self._backend_sent_frames
return (
close.code != 1000
and not self._backend_sent_frames
and not self._holds_session_updates_until_first_content()
)
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
@ -1488,7 +1767,7 @@ class RealTimeStreaming:
client_event: _ClientEventFrame
try:
while True:
message = await self._receive_client_message()
message = await self._receive_client_message_before_setup_deadline()
if message is None:
return ClientLoopExit.CLIENT_DISCONNECTED
@ -1600,6 +1879,7 @@ class RealTimeStreaming:
if (
msg_type == "session.update"
and self.session_configuration_request is None
and self._held_session_update is None
and not self._guardrail_turn_detection_update_sent
and self._should_disable_vad_auto_response()
and (
@ -1663,12 +1943,25 @@ class RealTimeStreaming:
except (json.JSONDecodeError, AttributeError):
pass
if self._refuses_unscreened_audio(msg_type):
await self._refuse_unscreened_audio(msg_type, message)
continue
## LOGGING
# Log after any in-place modifications (GA remap, guardrail
# turn_detection injection) so audit logs reflect what we
# actually forward to the backend.
self.store_input(message=message)
if self._holds_client_content(msg_type, message):
self._hold_client_content(message)
continue
if self._held_client_content and msg_type != "session.update":
await self._send_setup_before_content(msg_type)
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup():
await self._send_setup_before_content(msg_type)
if self._should_buffer_client_message_until_setup(message):
self._buffer_pending_message_until_setup(message)
continue
@ -1697,6 +1990,15 @@ class RealTimeStreaming:
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
if msg_type == "session.update" and self.provider_config is None:
await self._send_session_gate_once()
if msg_type == "session.update" and await self._rejected_setup_session_update(message):
continue
if msg_type == "session.update" and self._holds_session_updates_until_first_content():
await self._hold_session_update(message)
if self._pending_messages_until_setup:
await self._send_held_session_update_as_setup()
continue
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES:
await self._send_setup_before_content(msg_type)
try:
sent = await self._send_to_backend(message)
except RealtimeTranscriptionProtocolError as e:

23
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -14,9 +14,10 @@ import litellm
from litellm._logging import verbose_logger
from litellm.constants import VLLM_CHOICE_FIELDS
from litellm.litellm_core_utils.core_helpers import (
USAGE_COST_BILLED_PROVIDERS,
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
chunk_usage,
provider_reported_usage_cost,
stamp_litellm_bill_on_usage,
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
@ -1385,9 +1386,6 @@ def stream_chunk_builder_text_completion(
return TextCompletionResponse(**response)
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
if logging_obj is None:
return None
@ -1395,10 +1393,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object:
return provider
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
def _stream_builder_response_cost(
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
) -> float | None:
@ -1409,9 +1403,6 @@ def _stream_builder_response_cost(
for chunk in reversed(chunks)
)
return next((cost for cost in reported if cost is not None), None)
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
return float(usage_cost)
if logging_obj is not None:
return None
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
@ -1458,13 +1449,15 @@ def _set_stream_builder_response_cost(
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if isinstance(getattr(usage, "cost", None), (int, float)):
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
getattr(usage, "cost", None), (int, float)
):
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0:
return
setattr(usage, "cost", computed_cost)
_set_hidden_response_cost(response, float(computed_cost))
if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
stamp_litellm_bill_on_usage(response, usage, float(computed_cost))
def _field_of(value: object, name: str) -> object:

8
litellm/litellm_core_utils/streaming_handler.py
View file

@ -42,6 +42,7 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
CHOICE_LOGPROBS_OR_JSON,
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
Delta,
@ -1271,7 +1272,10 @@ class CustomStreamWrapper:
choice_json.pop(
"finish_reason", None
) # for mistral etc. which return a value in their last chunk (not-openai compatible).
choices.append(StreamingChoices(**choice_json))
choice_json.pop("logprobs", None)
choices.append(
StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None))
)
except Exception:
choices.append(StreamingChoices())
setattr(model_response, "choices", choices)
@ -1709,7 +1713,7 @@ class CustomStreamWrapper:
model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint
self.system_fingerprint = response_obj["original_chunk"].system_fingerprint
if response_obj["logprobs"] is not None:
model_response.choices[0].logprobs = response_obj["logprobs"]
model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"])
if response_obj["usage"] is not None:
if isinstance(response_obj["usage"], dict):

43
litellm/litellm_core_utils/token_counter.py
View file

@ -17,6 +17,7 @@ import anyio
import anyio.lowlevel
import httpx
import tiktoken
from pydantic import TypeAdapter
from tiktoken.model import MODEL_TO_ENCODING
from typing_extensions import ParamSpec, TypeVar
@ -69,6 +70,7 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesToolResultParam,
AnthropicMessagesToolUseParam,
AnthropicSearchResultParam,
CompactionBlock,
)
from litellm.types.llms.openai import (
AllMessageValues,
@ -937,6 +939,8 @@ def _count_messages(
default_token_count,
params.pdf_pricing,
)
elif key == "provider_specific_fields":
num_tokens += sum(map(params.count_function, _compaction_summaries(value)))
elif key == "search_results" and isinstance(value, list):
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_search_results_text,
@ -951,6 +955,25 @@ def _count_messages(
return num_tokens
_OBJECT_LIST: Final = TypeAdapter(list[object])
def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]:
fields: Final = as_str_mapping(provider_specific_fields)
blocks: Final = fields.get("compaction_blocks") if fields is not None else None
block_fields: Final = (
tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else ()
)
return tuple(
content
for block in block_fields
if block is not None
and block.get("type") == "compaction"
and isinstance(content := block.get("content"), str)
and content
)
def _count_extra(
count_function: TokenCounterFunction,
tools: list[ChatCompletionToolParam] | None,
@ -1648,6 +1671,11 @@ def _count_anthropic_content(
return tokens
_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
{"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"}
)
def _count_content_list(
count_function: TokenCounterFunction,
content_list: str
@ -1659,6 +1687,7 @@ def _count_content_list(
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
| AnthropicSearchResultParam
| CompactionBlock
],
use_default_image_token_count: bool,
default_token_count: int | None,
@ -1670,8 +1699,10 @@ def _count_content_list(
for c in content_list:
if isinstance(c, str):
num_tokens += count_function(c)
elif c["type"] == "text":
num_tokens += count_function(str(c.get("text", "")))
elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE:
block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")
if block_text:
num_tokens += count_function(block_text)
elif c["type"] == "image_url":
image_url = c.get("image_url")
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
@ -1704,12 +1735,6 @@ def _count_content_list(
default_token_count,
pdf_pricing,
)
elif c["type"] in ("thinking", "redacted_thinking"):
# Claude extended thinking content block
# Count the thinking text and skip the opaque blobs (signature, redacted data)
thinking_text = str(c.get("thinking", ""))
if thinking_text:
num_tokens += count_function(thinking_text)
elif c["type"] == "tool_reference":
# Anthropic tool-search reference block: a lightweight pointer to
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
@ -1727,7 +1752,7 @@ def _count_content_list(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
f"tool_reference, search_result)."
f"compaction, tool_reference, search_result)."
)
return num_tokens
except Exception as e:

335
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -24,7 +24,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.messages.adapters.transformation import (
OPENAI_MAX_TOOL_NAME_LENGTH,
@ -61,6 +61,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
unappliable_request_rewrite,
)
from litellm.types.llms.anthropic import (
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
AllAnthropicToolsValues,
AnthropicMessagesRequest,
)
@ -153,6 +154,7 @@ class _TextAttrBlock(Protocol):
text: str
@runtime_checkable
class _WritableMessage(Protocol):
@overload
def get(self, key: str, /) -> object | None: ...
@ -195,6 +197,140 @@ class _ToolCallShape:
arguments: str
@dataclass(frozen=True, slots=True)
class _WebSearchString:
block_idx: int
result_idx: int | None
field: str
value: str
def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]:
choice: Final = response.choices[0] if response.choices else None
fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None
blocks: Final = (
(as_str_mapping(item) for item in iter_items(fields.get("web_search_results")))
if fields is not None and isinstance(fields.get("web_search_results"), list)
else ()
)
return tuple(
deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result"
)
def _web_search_results(block: Mapping[str, object]) -> Iterable[object]:
return iter_items(block.get("content")) if isinstance(block.get("content"), list) else ()
def _web_search_result_strings(block_idx: int, result_idx: int, result: object) -> tuple[_WebSearchString, ...]:
fields: Final = as_str_mapping(result)
if fields is None or fields.get("type") != "web_search_result":
return ()
values: Final = ((field, fields.get(field)) for field in ("title", "url"))
return tuple(
_WebSearchString(block_idx, result_idx, field, value)
for field, value in values
if isinstance(value, str) and value
)
def _is_server_tool_use(block: Mapping[str, object]) -> bool:
block_id: Final = block.get("id")
match block.get("type"):
case "server_tool_use":
return True
case "tool_use":
return isinstance(block_id, str) and block_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
case _:
return False
def _web_search_block_strings(
block_idx: int, block: Mapping[str, object], *, with_query: bool
) -> tuple[_WebSearchString, ...]:
"""The strings a client reads off a native web search block: the ``server_tool_use`` query and each
``web_search_tool_result`` result's title and url."""
match block.get("type"):
case "server_tool_use" | "tool_use" if with_query and _is_server_tool_use(block):
tool_input: Final = as_str_mapping(block.get("input"))
query: Final = None if tool_input is None else tool_input.get("query")
return (_WebSearchString(block_idx, None, "query", query),) if isinstance(query, str) and query else ()
case "web_search_tool_result":
return tuple(
chain.from_iterable(
_web_search_result_strings(block_idx, result_idx, result)
for result_idx, result in enumerate(_web_search_results(block))
)
)
case _:
return ()
def _rewritten_web_search_result(
result_idx: int, result: object, rewrites: Mapping[tuple[int | None, str], str]
) -> object:
fields: Final = as_str_mapping(result)
if fields is None:
return result
return {
**fields,
**{field: rewrites[(result_idx, field)] for field in ("title", "url") if (result_idx, field) in rewrites},
}
def _rewritten_web_search_block(
block: Mapping[str, object], rewrites: Sequence[tuple[_WebSearchString, str]]
) -> Mapping[str, object]:
by_location: Final = MappingProxyType({(target.result_idx, target.field): value for target, value in rewrites})
if _is_server_tool_use(block):
query: Final = by_location.get((None, "query"))
tool_input: Final = as_str_mapping(block.get("input")) or MappingProxyType({})
return block if query is None else {**block, "input": {**tool_input, "query": query}}
if not isinstance(block.get("content"), list):
return block
return {
**block,
"content": [
_rewritten_web_search_result(result_idx, result, by_location)
for result_idx, result in enumerate(_web_search_results(block))
],
}
def _write_back_web_search_block(content_block: object, rewrites: Sequence[tuple[_WebSearchString, str]]) -> None:
block: Final = as_str_mapping(content_block)
if block is None or not isinstance(content_block, _WritableMessage):
return
rewritten: Final = _rewritten_web_search_block(block, rewrites)
writable: Final = _as_writable(content_block)
for key in ("input", "content"):
if key in rewritten:
writable[key] = rewritten[key]
def _split_web_search_texts(
guardrailed: Sequence[str], web_search_count: int, reply_count: int
) -> tuple[Sequence[str], Sequence[str]]:
split_at: Final = web_search_count if len(guardrailed) == web_search_count + reply_count else 0
return guardrailed[:split_at], guardrailed[split_at:]
def _web_search_rewrites_by_block(
strings: Sequence[_WebSearchString], guardrailed: Sequence[object]
) -> Mapping[int, tuple[tuple[_WebSearchString, str], ...]]:
rewrites: Final = tuple(
(target, value)
for target, value in zip(strings, guardrailed)
if isinstance(value, str) and value != target.value
)
return MappingProxyType(
{
block_idx: tuple(rewrite for rewrite in rewrites if rewrite[0].block_idx == block_idx)
for block_idx in {target.block_idx for target, _ in rewrites}
}
)
def _is_client_tool_use(block: Mapping[str, object]) -> bool:
return (
block.get("type") == "tool_use"
@ -289,6 +425,7 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge
_TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object])
_DECODED_SSE_DATA: Final = TypeAdapter(object)
_REBUILT_TOOL_CALL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use"})
_RELEASED_TOOL_USE_STOP: Final = (
b"event: message_delta\n"
b'data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, '
@ -315,6 +452,51 @@ def _write_back_tool_use(
block["name"] = shape.name
def _streamed_query_strings(
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], tool_calls: Sequence[_ToolCallShape]
) -> tuple[_WebSearchString, ...]:
if len(tool_call_blocks) != len(tool_calls):
return ()
return tuple(
chain.from_iterable(
_web_search_block_strings(
index, {**block, "input": _rewritten_tool_use_input(tool_call.arguments)}, with_query=True
)
for (index, block), tool_call in zip(tool_call_blocks, tool_calls)
)
)
def _query_rewrites(strings: Sequence[_WebSearchString], guardrailed: Sequence[object]) -> Mapping[int, str]:
return MappingProxyType(
{
target.block_idx: value
for target, value in zip(strings, guardrailed)
if isinstance(value, str) and value != target.value
}
)
def _with_query(tool_call: _ToolCallShape, query: str) -> _ToolCallShape:
arguments: Final = _rewritten_tool_use_input(tool_call.arguments)
if arguments is None:
return tool_call
return _ToolCallShape(name=tool_call.name, arguments=json.dumps({**arguments, "query": query}))
def _with_query_rewrites(
tool_calls: tuple[_ToolCallShape, ...],
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]],
query_rewrites: Mapping[int, str],
) -> tuple[_ToolCallShape, ...]:
if not query_rewrites or len(tool_call_blocks) != len(tool_calls):
return tool_calls
return tuple(
_with_query(tool_call, query_rewrites[index]) if index in query_rewrites else tool_call
for (index, _), tool_call in zip(tool_call_blocks, tool_calls)
)
@dataclass(frozen=True, slots=True)
class _SSEFieldRewrite:
"""One field of one nested section of a buffered SSE event, rewritten."""
@ -402,7 +584,12 @@ class AnthropicMessagesHandler(BaseTranslation):
def post_call_hook_response(self, response: object) -> object:
if not isinstance(response, ModelResponse):
return response
return self.adapter.translate_openai_response_to_anthropic(response)
translated: Final = self.adapter.translate_openai_response_to_anthropic(response)
result_blocks: Final = _streamed_web_search_result_blocks(response)
if not result_blocks:
return translated
content: Final = translated.get("content") or ()
return {**translated, "content": [*result_blocks, *content]}
@staticmethod
def _build_streaming_usage_response(
@ -1273,9 +1460,17 @@ class AnthropicMessagesHandler(BaseTranslation):
task_mappings,
tool_calls_to_check,
)
block_fields: Final = tuple(map(self._output_block_fields, response_content))
web_search_strings: Final = tuple(
chain.from_iterable(
_web_search_block_strings(content_idx, fields[1], with_query=True)
for content_idx, fields in enumerate(block_fields)
if fields is not None
)
)
# Step 2: Apply guardrail to all texts in batch
if texts_to_check or tool_calls_to_check:
if texts_to_check or tool_calls_to_check or web_search_strings:
request_data = self._prepare_request_data(
request_data,
response,
@ -1284,7 +1479,7 @@ class AnthropicMessagesHandler(BaseTranslation):
)
inputs: Final = self._build_guardrail_inputs(
texts_to_check,
[*(target.value for target in web_search_strings), *texts_to_check],
images_to_check,
tool_calls_to_check,
response,
@ -1297,14 +1492,19 @@ class AnthropicMessagesHandler(BaseTranslation):
logging_obj=litellm_logging_obj,
)
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
web_search_rewrites, reply_rewrites = _split_web_search_texts(
guardrailed_inputs.get("texts", []), len(web_search_strings), len(texts_to_check)
)
# Step 3: Map guardrail responses back to original response structure
await self._apply_guardrail_responses_to_output(
response=response,
responses=guardrailed_texts,
responses=list(reply_rewrites),
task_mappings=task_mappings,
)
self._write_back_web_search_strings(
response_content, _web_search_rewrites_by_block(web_search_strings, web_search_rewrites)
)
verbose_proxy_logger.debug("Anthropic Messages: Processed output response: %s", response)
@ -1350,8 +1550,14 @@ class AnthropicMessagesHandler(BaseTranslation):
string_so_far = first_choice.message.content
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
guardrail_inputs: Final = GenericGuardrailAPIInputs()
if string_so_far:
guardrail_inputs["texts"] = [string_so_far]
leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else ()
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls)
streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far)
streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings)
texts_to_check: Final = [*(target.value for target in streamed_web_search_strings), *leading_texts]
if texts_to_check:
guardrail_inputs["texts"] = texts_to_check
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
@ -1374,29 +1580,42 @@ class AnthropicMessagesHandler(BaseTranslation):
responses_so_far, request_data
)
raise
guardrailed_texts: Final = _guardrailed_inputs.get("texts")
web_search_rewrites, reply_rewrites = _split_web_search_texts(
_guardrailed_inputs.get("texts") or [], len(streamed_web_search_strings), len(leading_texts)
)
if (
deliver_ended_stream_rewrites
and isinstance(string_so_far, str)
and string_so_far
and guardrailed_texts
and guardrailed_texts[0] != string_so_far
and reply_rewrites
and reply_rewrites[0] != string_so_far
):
self._write_ended_stream_text_rewrite(
responses_so_far,
guardrailed_texts[0],
reply_rewrites[0],
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
if deliver_ended_stream_rewrites:
self._write_ended_stream_web_search_rewrites(
responses_so_far,
_web_search_rewrites_by_block(streamed_result_strings, web_search_rewrites),
)
if deliver_ended_stream_rewrites:
returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls")
self._write_ended_stream_tool_call_rewrites(
responses_so_far,
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
post_guardrail_tool_calls=_tool_call_shapes(
returned_tool_calls
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
else tool_calls_list or ()
post_guardrail_tool_calls=_with_query_rewrites(
_tool_call_shapes(
returned_tool_calls
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
else tool_calls_list or ()
),
tool_call_blocks,
_query_rewrites(
streamed_query_strings, web_search_rewrites[len(streamed_result_strings) :]
),
),
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
@ -1451,6 +1670,14 @@ class AnthropicMessagesHandler(BaseTranslation):
request_data["litellm_metadata"] = user_metadata
return request_data
@staticmethod
def _write_back_web_search_strings(
response_content: Sequence[object],
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
) -> None:
for block_idx, rewrites in rewrites_by_block.items():
_write_back_web_search_block(_content_block_at(response_content, block_idx), rewrites)
@staticmethod
def _get_response_content(response: object) -> Sequence[object]:
"""Extract content list from a dict or object response."""
@ -1573,30 +1800,22 @@ class AnthropicMessagesHandler(BaseTranslation):
) -> None:
"""Deliver ended-stream guardrail tool-call rewrites by rewriting the
buffered chunks in place: the rebuilt response lists tool calls in the
order of the stream's ``tool_use`` blocks, so the nth rewritten call lands
on the nth block, its first ``input_json_delta`` carrying the full rewritten
order of the stream's ``tool_use`` and ``server_tool_use`` blocks, so the nth
rewritten call lands on the nth block, its first ``input_json_delta`` carrying the full rewritten
arguments, every later one blanked, and ``content_block_start`` carrying the
rewritten name. Blocks that do not line up with the rebuilt tool calls make
the rewrite undeliverable, so the pipeline executor discards it and releases
the original chunks."""
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
return
block_indices: Final = tuple(
index
for item in responses_so_far
for event in cls._iter_sse_events(item)
if event.get("type") == "content_block_start"
and isinstance(block := event.get("content_block"), Mapping)
and block.get("type") == "tool_use"
and isinstance(index := event.get("index"), int)
)
block_indices: Final = tuple(index for index, _ in cls._streamed_tool_call_blocks(responses_so_far))
if len(block_indices) != len(post_guardrail_tool_calls):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
raise UndeliverableStreamRewrite(
guardrail_name,
f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried "
f"{len(block_indices)} tool_use blocks",
f"{len(block_indices)} tool_use or server_tool_use blocks",
)
rewrites_by_block: Final = MappingProxyType(
{
@ -1636,6 +1855,60 @@ class AnthropicMessagesHandler(BaseTranslation):
),
)
@classmethod
def _streamed_web_search_result_strings(cls, responses_so_far: Sequence[object]) -> tuple[_WebSearchString, ...]:
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
result_blocks: Final = (
(index, block)
for event in events
if event.get("type") == "content_block_start"
and isinstance(index := event.get("index"), int)
and (block := as_str_mapping(event.get("content_block"))) is not None
)
return tuple(
chain.from_iterable(
_web_search_block_strings(index, block, with_query=False) for index, block in result_blocks
)
)
@classmethod
def _write_ended_stream_web_search_rewrites(
cls,
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
) -> None:
"""Deliver ended-stream guardrail rewrites of web search result titles and urls into the
``content_block_start`` that carries each whole ``web_search_tool_result`` block."""
if not rewrites_by_block:
return
def rewrite_result_block(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
index: Final = event.get("index")
block: Final = as_str_mapping(event.get("content_block"))
rewrites: Final = rewrites_by_block.get(index) if isinstance(index, int) else None
if event.get("type") != "content_block_start" or block is None or rewrites is None:
return None
return _SSEFieldRewrite(
"content_block", "content", _rewritten_web_search_block(block, rewrites).get("content")
)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_result_block)
@classmethod
def _streamed_tool_call_blocks(
cls, responses_so_far: Sequence[object]
) -> tuple[tuple[int, Mapping[str, object]], ...]:
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
return tuple(
(index, block)
for event in events
if event.get("type") == "content_block_start"
and (block := as_str_mapping(event.get("content_block"))) is not None
and isinstance(block_type := block.get("type"), str)
and block_type in _REBUILT_TOOL_CALL_BLOCK_TYPES
and isinstance(index := event.get("index"), int)
)
@classmethod
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
"""The id of each buffered ``tool_use`` block, by its content block index."""
@ -1738,8 +2011,10 @@ class AnthropicMessagesHandler(BaseTranslation):
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
stream_ended: Final = self._check_streaming_has_ended(responses_so_far)
tool_use_fingerprints: Final = self._streamed_tool_use_fingerprints(responses_so_far)
streamed_text: Final = self.get_streaming_string_so_far(responses_so_far)
result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) if stream_ended else ()
return StreamingScanKey(
texts=(self.get_streaming_string_so_far(responses_so_far),),
texts=(*(target.value for target in result_strings), streamed_text),
tool_calls=tool_use_fingerprints if stream_ended else (),
stream_ended=stream_ended,
tool_calls_in_flight=bool(tool_use_fingerprints) and not stream_ended,

74
litellm/llms/anthropic/common_utils.py
View file

@ -8,12 +8,13 @@ import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
from urllib.parse import quote, urlparse
import httpx
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -24,6 +25,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
is_encrypted_reasoning_block,
@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import (
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
@ -118,6 +121,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_DECODED_JSON: Final = TypeAdapter(object)
_JSON_VALUE: Final = TypeAdapter(JsonValue)
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
@ -177,24 +181,68 @@ def is_anthropic_messages_url(url: str) -> bool:
return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages")
def _message_blocks(message: object) -> Sequence[object]:
fields: Final = as_str_mapping(message)
blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None
return blocks or ()
def _is_signed_compaction_block(block: object) -> bool:
fields: Final = as_str_mapping(block)
if fields is None or fields.get("type") != "compaction":
return False
signature: Final = fields.get("signature")
return isinstance(signature, str) and bool(signature)
def carries_signed_compaction_block(messages: Sequence[object]) -> bool:
return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages))))
def requires_native_compaction_beta(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
optional_params.get("compaction") is not None
or any(
isinstance(block, Mapping)
and block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and bool(block.get("signature"))
for message in messages
if isinstance(message, Mapping)
for content in (message.get("content"),)
if isinstance(content, (list, tuple))
for block in content
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
)
def anthropic_context_management(context_management: JsonValue) -> JsonValue:
if not isinstance(context_management, list):
return context_management
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic(
[entry for entry in context_management if isinstance(entry, dict)]
)
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
mapping: Final = as_str_mapping(spec)
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
spec: Final = anthropic_context_management(context_management)
return spec if _context_management_edits(spec) else None
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
edits: Final = _context_management_edits(anthropic_context_management(context_management))
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
return tuple(
beta
for beta, used in (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types),
(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
any(edit_type != "compact_20260112" for edit_type in edit_types),
),
)
if used
)
@ -1367,8 +1415,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if is_vertex_request is True:
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(all_betas) > 0:
headers["anthropic-beta"] = ",".join(all_betas)

4
litellm/llms/anthropic/count_tokens/handler.py
View file

@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]:
"""
@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools=tools,
system=system,
optional_params=optional_params,
context_management=context_management,
drop_params=drop_params,
)
@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
verbose_logger.debug("Making request to: %s", endpoint_url)
# Get required headers
headers: Final = self.get_count_tokens_headers(auth_header)
headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management)
# Use LiteLLM's async httpx client
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)

1
litellm/llms/anthropic/count_tokens/token_counter.py
View file

@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
tools=tools,
system=system,
optional_params=count_options,
context_management=context_management.spec if context_management is not None else None,
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
)

31
litellm/llms/anthropic/count_tokens/transformation.py
View file

@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import (
carries_signed_compaction_block,
context_management_betas,
context_management_with_edits,
is_claude_model_target,
merge_anthropic_beta_headers,
tools_without_custom_field,
)
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@ -376,6 +380,7 @@ class AnthropicCountTokensConfig:
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
"""
@ -408,6 +413,7 @@ class AnthropicCountTokensConfig:
drop_custom=drop_params,
),
),
("context_management", context_management_with_edits(context_management)),
)
if value is not None
}
@ -419,18 +425,33 @@ class AnthropicCountTokensConfig:
)
)
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
def get_count_tokens_headers(
self,
auth_header: Mapping[str, str],
messages: Sequence[Mapping[str, JsonValue]],
context_management: JsonValue = None,
) -> dict[str, str]:
"""The count-tokens headers around a resolved Anthropic auth header
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
merges with the token-counting beta instead of replacing it."""
merges with the token-counting beta instead of replacing it. A history that carries a
signed compaction block also needs the compaction beta, or Anthropic rejects the block
as an unknown content type, and ``context_management`` edits need the betas inference sends."""
compaction_betas: Final = (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,)
if carries_signed_compaction_block(messages)
else ()
)
request_betas: Final = (
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
*compaction_betas,
*context_management_betas(context_management),
)
return {
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
**auth_header,
"anthropic-beta": merge_anthropic_beta_headers(
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
),
"anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas),
}
def validate_request(

2
litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -647,12 +647,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
request_data["output_config"] = extra_kwargs["output_config"]
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
api_base: Final = extra_kwargs.get("api_base")
(
openai_request,
tool_name_mapping,
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
request_data,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
)

30
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
TOOL_NAME_HASH_LENGTH: Final = 8
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"})
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
@ -255,7 +256,12 @@ class AnthropicAdapter:
return result
def translate_completion_input_params_with_tool_mapping(
self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True
self,
kwargs,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
drop_params: bool = True,
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
"""
Translate Anthropic request params to OpenAI format, returning tool name mapping.
@ -289,6 +295,7 @@ class AnthropicAdapter:
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=request_body,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
drop_params=drop_params,
)
@ -1001,7 +1008,9 @@ class LiteLLMAnthropicMessagesAdapter:
)
@staticmethod
def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool:
def _supports_prompt_cache_key(
prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None
) -> bool:
if not model or not custom_llm_provider:
return False
if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND:
@ -1009,7 +1018,17 @@ class LiteLLMAnthropicMessagesAdapter:
supported_params: Final = litellm.get_supported_openai_params(
model=model, custom_llm_provider=custom_llm_provider
)
return "prompt_cache_key" in (supported_params or ())
if "prompt_cache_key" not in (supported_params or ()):
return False
if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG:
return True
_, _, _, resolved_api_base = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
)
mistral_config: Final = litellm.MistralConfig()
return mistral_config.takes_prompt_cache_key_value(
prompt_cache_key
) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base)
@staticmethod
def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool:
@ -1042,6 +1061,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs: ChatCompletionRequest,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
) -> None:
"""Translate metadata fields from Anthropic request to OpenAI request."""
if "metadata" in anthropic_message_request:
@ -1050,7 +1070,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs["user"] = metadata["user_id"]
prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"])
if prompt_cache_key is not None and self._supports_prompt_cache_key(
anthropic_message_request.get("model"), custom_llm_provider
prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base
):
new_kwargs["prompt_cache_key"] = prompt_cache_key
@ -1244,6 +1264,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request: AnthropicMessagesRequest,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
preserve_midturn_system: bool = False,
drop_params: bool = True,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
@ -1282,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request=anthropic_message_request,
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
## CONVERT TOOL CHOICE
self._translate_tool_choice_to_openai(

56
litellm/llms/anthropic/messages/native/fake_stream_iterator.py
View file

@ -12,11 +12,20 @@ import json
from collections.abc import Mapping, Sequence
from typing import Final, cast
from pydantic import TypeAdapter
from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
_CITATIONS: Final = TypeAdapter(tuple[object, ...])
def _citations_of(block: Mapping[str, object]) -> tuple[object, ...]:
citations: Final = block.get("citations")
return _CITATIONS.validate_python(citations) if isinstance(citations, list) else ()
class FakeAnthropicMessagesStreamIterator:
"""
@ -47,16 +56,30 @@ class FakeAnthropicMessagesStreamIterator:
def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]:
"""Build SSE chunks for a single content block."""
chunks: Final = []
chunks: Final = list[bytes]()
block_type: Final = block_dict.get("type")
if block_type == "text":
citations: Final = _citations_of(block_dict)
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
"content_block": {"type": "text", "text": "", "citations": ()}
if citations
else {"type": "text", "text": ""},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
citation_deltas: Final = (
{
"type": "content_block_delta",
"index": index,
"delta": {"type": "citations_delta", "citation": citation},
}
for citation in citations
)
chunks.extend(
f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode() for delta in citation_deltas
)
text: Final = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
@ -89,36 +112,23 @@ class FakeAnthropicMessagesStreamIterator:
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode())
elif block_type == "redacted_thinking":
content_block_start = {
elif block_type in ("tool_use", "server_tool_use"):
empty_input: Final[Mapping[str, object]] = {}
tool_use_start: Final = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
"content_block": {**block_dict, "input": empty_input},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
elif block_type == "tool_use":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
input_data: Final = block_dict.get("input", {})
content_block_delta = {
chunks.append(f"event: content_block_start\ndata: {json.dumps(tool_use_start)}\n\n".encode())
tool_use_delta: Final = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "input_json_delta",
"partial_json": json.dumps(input_data),
"partial_json": json.dumps(block_dict.get("input", {})),
},
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
chunks.append(f"event: content_block_delta\ndata: {json.dumps(tool_use_delta)}\n\n".encode())
else:
passthrough_start: Final = {

26
litellm/llms/anthropic/messages/native/transformation.py
View file

@ -43,6 +43,7 @@ from ...common_utils import (
OBJECT_LIST_ADAPTER,
AnthropicError,
AnthropicModelInfo,
context_management_betas,
is_claude_model_target,
merge_anthropic_beta_headers,
optionally_handle_anthropic_oauth,
@ -745,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
)
if transformed_context_management is not None:
anthropic_messages_optional_request_params["context_management"] = transformed_context_management
elif drops_unsupported_params(litellm_params.drop_params):
anthropic_messages_optional_request_params.pop("context_management")
####### get required params for all anthropic messages requests ######
# Lazy %s: the f-string previously stringified the entire messages
@ -873,28 +876,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", ())
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management")))
# Check for structured outputs. Anthropic's newer request shape nests
# the schema under output_config.format; the older top-level

12
litellm/llms/azure/azure.py
View file

@ -42,6 +42,7 @@ from ..openai.common_utils import (
build_output_token_limit_response,
is_output_token_limit_error,
model_dump_provider_response,
upstream_error_response,
)
from .common_utils import (
AzureOpenAIError,
@ -399,6 +400,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
response=upstream_error_response(e),
headers=error_headers,
body=error_body,
)
@ -662,6 +664,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
raise AzureOpenAIError(
status_code=status_code,
message=message,
response=upstream_error_response(e),
headers=error_headers,
body=error_body,
)
@ -855,7 +858,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
error_text = error_response.text
raise AzureOpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
async def make_async_azure_httpx_request(
self,
@ -1190,7 +1198,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
except Exception as e:
error_code: Final[object] = getattr(e, "status_code", None)
if error_code is not None:
raise AzureOpenAIError(status_code=error_code, message=str(e))
raise AzureOpenAIError(status_code=error_code, message=str(e), response=upstream_error_response(e))
else:
raise AzureOpenAIError(status_code=500, message=str(e))

22
litellm/llms/azure/completion/handler.py
View file

@ -8,6 +8,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory
from litellm.utils import CustomStreamWrapper, ModelResponse, TextCompletionResponse
from ...openai.common_utils import upstream_error_response
from ...openai.completion.transformation import OpenAITextCompletionConfig
from ..common_utils import AzureOpenAIError, BaseAzureLLM
@ -183,7 +184,12 @@ class AzureTextCompletion(BaseAzureLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
headers=error_headers,
response=upstream_error_response(e),
)
async def acompletion(
self,
@ -244,7 +250,12 @@ class AzureTextCompletion(BaseAzureLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
headers=error_headers,
response=upstream_error_response(e),
)
def streaming(
self,
@ -355,4 +366,9 @@ class AzureTextCompletion(BaseAzureLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers)
raise AzureOpenAIError(
status_code=status_code,
message=str(e),
headers=error_headers,
response=upstream_error_response(e),
)

5
litellm/llms/azure/exception_mapping.py
View file

@ -1,5 +1,7 @@
from typing import Final
import httpx
from litellm.exceptions import ContentPolicyViolationError
@ -14,6 +16,7 @@ class AzureOpenAIExceptionMapping:
model: str,
extra_information: str,
original_exception: Exception,
response: httpx.Response | None,
) -> ContentPolicyViolationError:
"""
Create a content policy violation error
@ -39,7 +42,7 @@ class AzureOpenAIExceptionMapping:
llm_provider="azure",
model=model,
litellm_debug_info=extra_information,
response=getattr(original_exception, "response", None),
response=response,
provider_specific_fields={
# Preserve legacy key for backward compatibility.
"innererror": inner_error,

7
litellm/llms/azure/responses/transformation.py
View file

@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import *
from litellm.types.responses.main import *
@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)
def deployment_names_wrap_openai_models(self) -> bool:
return True
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None
def get_supported_openai_params(self, model: str) -> list:
"""
Azure Responses API does not support context_management (compaction).

111
litellm/llms/azure_ai/chat/transformation.py
View file

@ -6,15 +6,19 @@ from typing import TYPE_CHECKING, Final, cast
import httpx
from httpx import Response
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.prompt_templates.common_utils import (
content_parts,
convert_content_list_to_str,
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
is_pdf_file_part,
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure_ai.common_utils import (
@ -27,7 +31,7 @@ from litellm.llms.openai.common_utils import drop_params_from_unprocessable_enti
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
from litellm.llms.xai.chat.transformation import XAIChatConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
from litellm.utils import add_path_to_api_base, supports_tool_choice
@ -62,6 +66,72 @@ def _has_list_only_content_part(message: Mapping[str, object]) -> bool:
return any(_is_list_only_content_part(part) for part in content_parts(message))
_CONTENT_PARTS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
def _text_parts(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
content: Final = message.get("content")
if not isinstance(content, list):
return ()
try:
parts: Final = _CONTENT_PARTS_ADAPTER.validate_python(content)
except ValidationError:
return ()
return tuple(part for part in parts if part.get("type") == "text")
def _with_only_text_parts(message: AllMessageValues) -> AllMessageValues:
if _has_list_only_content_part(message) or not (text_parts := _text_parts(message)):
return message
text_only: Final = {**message, "content": list(text_parts)}
return cast(AllMessageValues, text_only) # cast-ok: the same message, text parts only
_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _foundry_breakpoint(marker: object) -> PromptCacheBreakpoint | None:
if prompt_cache_breakpoint_for_wire(marker, drop_params=True) is None:
return None
return PromptCacheBreakpoint(mode="explicit")
def _part_with_foundry_breakpoint(part: object) -> object:
if not isinstance(part, Mapping):
return part
fields: Final = _FIELDS.validate_python(part)
if "prompt_cache_breakpoint" not in fields:
return fields
unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
return with_prompt_cache_breakpoint(unmarked, _foundry_breakpoint(fields["prompt_cache_breakpoint"]))
def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
renamed: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(_with_only_text_parts(message))
parts: Final = content_parts(renamed) if drop_params else ()
if not parts:
return renamed
foundry_message: Final = {**renamed, "content": [_part_with_foundry_breakpoint(part) for part in parts]}
return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape
def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
base_model: Final = litellm_params.get("base_model")
named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,)
model_info: Final = litellm_params.get("model_info")
if not isinstance(model_info, Mapping):
return any(
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models
)
deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint")
if isinstance(deployment_flag, bool):
return deployment_flag
return any(
AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai")
for name in named_models
)
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
usage: Final = chunk.get("usage")
@ -251,6 +321,8 @@ class AzureAIStudioConfig(OpenAIConfig):
self,
messages: list[AllMessageValues],
model: str,
reads_breakpoints: bool | None = None,
drop_params: bool = False,
) -> list:
"""
- Azure AI Studio doesn't support content as a list. This handles:
@ -260,13 +332,31 @@ class AzureAIStudioConfig(OpenAIConfig):
Azure AI Foundry backends set additionalProperties=false and reject
these with "Extra inputs are not permitted", which breaks multi-turn
Anthropic-format clients that echo thinking blocks back as history.
2. Transforms list content to a string.
2. Transforms list content to a string, except on deployments that read OpenAI explicit
prompt cache breakpoints (the deployment's own model_info opt-in, else the cost-map entry
of its base_model or name, as loaded and without a sibling deployment's opt-in): only a
content part can carry the breakpoint (a message-level one is silently ignored), so those
keep their text parts, with each Anthropic `cache_control` mark renamed to a breakpoint the
way the openai provider renames it. Every other part is dropped, exactly as the string
conversion drops it, before the marks are renamed, so a message-level mark lands on the
last text part. Under drop_params each breakpoint goes out as `{"mode": "explicit"}`, the only
shape Foundry takes, and a malformed one is dropped.
3. If message contains an image, audio, or PDF file part, send as is (user-intended)
Operates on a deep copy so the caller's messages keep their thinking blocks
and provider metadata, which a fallback to another provider still needs.
"""
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
deployment_reads_breakpoints: Final = (
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, "azure_ai")
if reads_breakpoints is None
else reads_breakpoints
)
copied_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
stripped_messages: Final = (
[_with_breakpoints_for_foundry(message, drop_params) for message in copied_messages]
if deployment_reads_breakpoints
else copied_messages
)
for message in stripped_messages:
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
@ -275,6 +365,10 @@ class AzureAIStudioConfig(OpenAIConfig):
if _has_list_only_content_part(message):
continue
if deployment_reads_breakpoints and (text_parts := _text_parts(message)):
message_dict["content"] = list(text_parts)
continue
texts = convert_content_list_to_str(message=message)
if texts:
message["content"] = texts
@ -329,7 +423,16 @@ class AzureAIStudioConfig(OpenAIConfig):
if extra_body and isinstance(extra_body, dict):
optional_params.update(extra_body)
optional_params.pop("max_retries", None)
return super().transform_request(model, messages, optional_params, litellm_params, headers)
return {
"model": model,
"messages": self._transform_messages(
messages,
model,
_deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)),
bool(litellm_params.get("drop_params") or litellm.drop_params),
),
**optional_params,
}
def transform_response(
self,

32
litellm/llms/base_llm/realtime/transformation.py
View file

@ -1,10 +1,12 @@
import dataclasses
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from types import TracebackType
from typing import TYPE_CHECKING, Any, Protocol
from typing import TYPE_CHECKING, Any, Literal, Protocol
import httpx
from litellm._uuid import uuid
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
from litellm.types.realtime import (
RealtimeInputAudioTranscriptionUsage,
@ -22,6 +24,26 @@ else:
LiteLLMLoggingObj = Any
@dataclasses.dataclass(frozen=True, slots=True)
class RealtimeSessionUpdateRejection:
code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"]
message: str
param: str
def error_event(self, client_event_id: object) -> dict[str, object]:
return {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": self.code,
"message": self.message,
"param": self.param,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
class RealtimeBackend(Protocol):
async def __aenter__(self) -> object: ...
@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC):
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
return None
def builds_setup_from_session_update(self) -> bool:
return False
def setup_session_update_rejection(
self, model: str, session: Mapping[str, object], *, setup_sent: bool
) -> RealtimeSessionUpdateRejection | None:
return None
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
return None

28
litellm/llms/base_llm/responses/transformation.py
View file

@ -1,7 +1,7 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args
import httpx
@ -27,6 +27,20 @@ else:
BaseLLMException = Any
ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None]
REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{"summary": REASONING_SUMMARY_VALUES}
)
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})}
)
STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None}
)
ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None}
)
class BaseResponsesAPIConfig(ABC):
@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC):
"""The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
return "detailed"
def reasoning_field_values(self) -> ReasoningFieldValues | None:
return None
def deployment_names_wrap_openai_models(self) -> bool:
return False
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
return None
def ignores_unsupported_reasoning_fields(self) -> bool:
return False
@abstractmethod
def map_openai_params(
self,

20
litellm/llms/bedrock/realtime/handler.py
View file

@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import redact_string, verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
from litellm.types.llms.bedrock import AwsAuthParams
from litellm.types.llms.openai import OpenAIRealtimeEvents
from litellm.types.realtime import RealtimeResponseTransformInput
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
from ..common_utils import BedrockError
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
from .transformation import BedrockRealtimeConfig, requested_modalities
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
@ -256,23 +256,11 @@ async def _ack_session_update(
async def _reject_session_update(
client_ws: RealtimeClientWebSocket,
parsed_client_message: Mapping[str, JsonValue],
rejection: BedrockSessionUpdateRejection,
rejection: RealtimeSessionUpdateRejection,
) -> bool:
"""Send an error event for a session.update Bedrock cannot apply; False means the client is gone."""
client_event_id: Final = parsed_client_message.get("event_id")
error_event: Final = {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": rejection.code,
"message": rejection.message,
"param": rejection.param,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
try:
await client_ws.send_text(json.dumps(error_event))
await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id"))))
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
return False

19
litellm/llms/bedrock/realtime/transformation.py
View file

@ -11,7 +11,7 @@ import math
import uuid as uuid_lib
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, Literal
from typing import Final
import httpx
from pydantic import ConfigDict, JsonValue, TypeAdapter
@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
from litellm.types.llms.base import LiteLLMBaseModel
@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[
return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockSessionUpdateRejection:
code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
message: str
param: str
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockLockedSessionSettings:
"""The session settings Bedrock Nova Sonic locks in once the prompt starts."""
@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
max_tokens=max_tokens,
)
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None:
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None:
"""The error for a session.update this session cannot apply, or None when it applies."""
session_config: Final = json_message.get("session")
if not isinstance(session_config, Mapping):
@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
malformed: Final = () if self.drop_params else _malformed_settings(session_config)
if malformed:
name, expected = malformed[0]
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="invalid_type",
message=(
f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
)
unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
if unplayable_voice is not None:
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
if not self.prompt_started:
return None
if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings:
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="session_update_not_applied",
message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied",
param="session",

4
litellm/llms/bigmodel/chat/transformation.py
View file

@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig):
def custom_llm_provider(self) -> str:
return "bigmodel"
@property
def prompts_unsupported_json_schema(self) -> bool:
return False
def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)

8
litellm/llms/chatgpt/authenticator.py
View file

@ -122,6 +122,12 @@ class Authenticator:
return tokens["access_token"]
def get_account_id(self, access_token: str | None = None) -> str | None:
return self._account_id(access_token, persist_derived=True)
def read_account_id(self, access_token: str | None = None) -> str | None:
return self._account_id(access_token, persist_derived=False)
def _account_id(self, access_token: str | None, persist_derived: bool) -> str | None:
auth_data: Final = self._read_auth_file()
if access_token and (not auth_data or auth_data.get("access_token") != access_token):
return extract_chatgpt_account_id(access_token)
@ -133,7 +139,7 @@ class Authenticator:
id_token: Final = auth_data.get("id_token")
saved_access_token: Final = auth_data.get("access_token")
derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
if derived:
if derived and persist_derived:
self._write_auth_file({**auth_data, "account_id": derived})
return derived

4
litellm/llms/chatgpt/responses/transformation.py
View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
safe_convert_created_field,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""ChatGPT does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_REASONING_FIELD_VALUES

11
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler:
),
)
# Auto-send session setup if the provider requires it (e.g.
# Gemini/Vertex AI Live needs a `setup` before any realtime_input).
# Auto-send session setup at connect unless the provider builds it from the
# client's first session.update (Gemini/Vertex AI Live accept exactly one
# setup, so RealTimeStreaming sends theirs on the first client message).
# Build the streaming handler first so a transcription guardrail's
# auto-response disable can be folded into this one setup: Gemini
# rejects a second setup, so a follow-up disable would be dropped
# and the guardrail bypassed.
_session_config: str | None = None
if provider_config.requires_session_configuration():
sends_setup_at_connect: Final = (
provider_config.requires_session_configuration()
and not provider_config.builds_setup_from_session_update()
)
if sends_setup_at_connect:
_session_config = provider_config.session_configuration_request(model)
if _session_config:
_session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable(

10
litellm/llms/fireworks_ai/responses/transformation.py
View file

@ -6,6 +6,10 @@ from urllib.parse import unquote
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import (
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
ReasoningFieldValues,
)
from litellm.llms.fireworks_ai.common_utils import (
FIREWORKS_FORWARD_USER_ID_PARAM,
get_fireworks_forwarded_user_id,
@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_file_search(self) -> bool:
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

317
litellm/llms/gemini/realtime/transformation.py
View file

@ -7,6 +7,7 @@ from collections import OrderedDict
from collections.abc import Mapping, Sequence
from typing import Final, cast
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, Required, TypedDict
import litellm
@ -15,8 +16,9 @@ from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
OPENAI_TO_GEMINI_TTS_VOICES,
VertexGeminiConfig,
)
from litellm.responses.litellm_completion_transformation.transformation import (
@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
# Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped.
_KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"})
OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
)
# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
{
"achernar",
"achird",
"algenib",
"algieba",
"alnilam",
"aoede",
"autonoe",
"callirrhoe",
"charon",
"despina",
"enceladus",
"erinome",
"fenrir",
"gacrux",
"iapetus",
"kore",
"laomedeia",
"leda",
"orus",
"puck",
"pulcherrima",
"rasalgethi",
"sadachbia",
"sadaltager",
"schedar",
"sulafat",
"umbriel",
"vindemiatrix",
"zephyr",
"zubenelgenubi",
}
)
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
def _is_undocumented_live_voice(voice: object) -> bool:
return (
isinstance(voice, str)
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
)
def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None:
"""Build the Gemini Live speechConfig for a client-requested voice.
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
the session on an unknown voice, so they are dropped with a warning and
the model keeps its default voice. Every other name is forwarded verbatim.
OpenAI stock voice names have no Gemini equivalent, so they are always
dropped, and so is any name outside Google's documented Live voices when
``drop_undocumented`` is set, keeping the model's default voice. Every
other name is forwarded verbatim.
"""
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
if not isinstance(voice, str):
return VertexGeminiConfig().map_audio_params({"voice": voice})
if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
verbose_logger.warning(
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
"dropping it so the session keeps the model's default voice.",
voice,
)
return None
if drop_undocumented and _is_undocumented_live_voice(voice):
verbose_logger.warning(
"Gemini Realtime: voice %s is not a documented Gemini Live voice; "
"drop_params is set, so it is dropped and the session keeps the model's default voice.",
voice,
)
return None
return VertexGeminiConfig().map_audio_params({"voice": voice})
@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int:
return max(len(data) * 3 // 4 - padding, 0)
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
def _chat_completion_tool(tool: object) -> dict[str, object]:
fields: Final = as_str_mapping(tool) or {}
if fields.get("type") != "function" or "function" in fields:
return {**fields}
return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}}
def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None:
parts: Final = system_instruction.get("parts", []) if system_instruction else []
texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str))
return "\n".join(texts) if texts else None
class GeminiRealtimeConfig(BaseRealtimeConfig):
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
def __init__(self):
def __init__(self, drop_params: bool = False):
super().__init__()
self.drop_params = drop_params
self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()
# Gemini Live sometimes emits usageMetadata in a standalone frame between
# turns; buffer it here so the next response.done carries the token counts.
self._pending_usage_metadata: dict | None = None
self._unbilled_input_audio_bytes: int = 0
self._manual_activity_open: bool = False
def is_setup_message(self, msg_obj: dict) -> bool:
return "setup" in msg_obj
@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"""Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
return True
def _rejects_undocumented_voices(self, model: str) -> bool:
"""Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models."""
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
@staticmethod
def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]:
detail_mapping: Final = as_str_mapping(details)
@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
except (json.JSONDecodeError, TypeError, AttributeError):
return False
def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]:
if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request):
return ()
self._manual_activity_open = True
activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}}
return (json.dumps({"realtimeInput": activity_start}),)
def _end_manual_activity(self) -> tuple[str, ...]:
if not self._manual_activity_open:
return ()
self._manual_activity_open = False
activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}}
return (json.dumps({"realtimeInput": activity_end}),)
def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
"""Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
if self._manual_turn_detection_enabled(session_configuration_request):
realtime_input_dict: BidiGenerateContentRealtimeInput = {
"activityEnd": True,
}
verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
else:
realtime_input_dict = {"audioStreamEnd": True}
verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend")
return [json.dumps({"realtimeInput": realtime_input_dict})]
return list(self._end_manual_activity())
audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True}
return [json.dumps({"realtimeInput": audio_stream_end})]
def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection:
"""Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``.
@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"voice",
]
def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict:
def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict:
if "generationConfig" not in optional_params:
optional_params["generationConfig"] = {}
for key, value in non_default_params.items():
@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}])
elif key == "temperature":
optional_params["generationConfig"]["temperature"] = value
elif key == "max_response_output_tokens":
elif key == "max_response_output_tokens" and isinstance(value, int):
optional_params["generationConfig"]["maxOutputTokens"] = value
elif key == "modalities":
optional_params["generationConfig"]["responseModalities"] = [
modality.upper() for modality in cast(list[str], value)
]
optional_params["generationConfig"]["responseModalities"] = list(
GeminiRealtimeConfig._requested_response_modalities(cast(object, value))
)
elif key == "tools":
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
vertex_gemini_config = VertexGeminiConfig()
# Tools should be at the top level of setup, not inside generationConfig
optional_params["tools"] = vertex_gemini_config.map_function(
value=value, optional_params=optional_params
value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)],
optional_params=optional_params,
)
elif key == "input_audio_transcription" and value is not None:
optional_params["inputAudioTranscription"] = {}
elif key == "turn_detection" and value is not None:
value_typed = cast(OpenAIRealtimeTurnDetection, value)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
continue
transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
if transformed_audio_activity_config:
optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
automaticActivityDetection=transformed_audio_activity_config
)
elif key == "turn_detection":
realtime_input_config = self._realtime_input_config(cast(object, value))
if realtime_input_config is not None:
optional_params["realtimeInputConfig"] = realtime_input_config
elif key == "voice":
speech_config = _gemini_live_speech_config(value)
speech_config = _gemini_live_speech_config(
value, self.drop_params and self._rejects_undocumented_voices(model)
)
if speech_config:
optional_params["generationConfig"]["speechConfig"] = speech_config
if len(optional_params["generationConfig"]) == 0:
optional_params.pop("generationConfig")
return optional_params
def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
if turn_detection is None:
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
return None
transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
if not transformed_audio_activity_config:
return None
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
@staticmethod
def _extract_turn_detection(session: dict) -> dict | None:
"""Extract turn_detection from a session.update payload.
@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if isinstance(input_cfg, dict):
if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
normalized["input_audio_transcription"] = input_cfg["transcription"]
if "turn_detection" not in normalized and "turn_detection" in input_cfg:
normalized["turn_detection"] = input_cfg["turn_detection"]
output_cfg: Final = audio.get("output")
if isinstance(output_cfg, dict) and output_cfg.get("voice"):
normalized["voice"] = output_cfg["voice"]
@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return normalized
@staticmethod
def _model_cost_entry(model: str) -> dict:
def _model_cost_entry(model: str) -> Mapping[str, object]:
entry = litellm.model_cost.get(model)
if entry is None:
stripped: Final = model.split("/", 1)[-1]
@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
normalized: Final = tuple(
modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
def _requested_response_modalities(value: object) -> tuple[str, ...]:
if isinstance(value, str):
return (value.upper(),)
if not isinstance(value, (list, tuple)):
return ()
return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value))
@staticmethod
def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]:
unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES)
if unknown:
return unknown
if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)):
return ("TEXT",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)):
return ("AUDIO",)
return ()
def setup_session_update_rejection(
self, model: str, session: Mapping[str, object], *, setup_sent: bool
) -> RealtimeSessionUpdateRejection | None:
if self.drop_params:
return None
if setup_sent:
return RealtimeSessionUpdateRejection(
code="session_update_not_applied",
message=(
"Gemini Live accepts one session configuration per connection, and this session already sent "
"its configuration, so this session.update was not applied. Put every setting in session.update "
"before the first conversation content, or set drop_params to drop it instead"
),
param="session",
)
return self._modality_rejection(model, session) or self._voice_rejection(model, session)
@staticmethod
def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
param: Final = "output_modalities" if "output_modalities" in session else "modalities"
unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities(
model, GeminiRealtimeConfig._requested_response_modalities(session.get(param))
)
if not unsupported:
return None
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.{param}': {model} cannot produce "
f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead"
),
param=f"session.{param}",
)
def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {}
output_voice: Final = audio_output.get("voice")
voice: Final = output_voice or session.get("voice")
if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)):
return None
param: Final = "session.audio.output.voice" if output_voice else "session.voice"
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. "
"Set drop_params to drop it instead"
),
param=param,
)
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
"""Drop modalities Gemini Live cannot produce at all, swap the ones this
model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for
text-only ones such as transcribe-live), then keep a single modality
because Gemini Live rejects a setup naming two."""
return GeminiRealtimeConfig._single_response_modality(
GeminiRealtimeConfig._swap_unsupported_response_modalities(
model, GeminiRealtimeConfig._known_response_modalities(model, modalities)
)
)
@staticmethod
def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
normalized: Final = tuple(str(modality).upper() for modality in modalities)
known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES)
if len(known) < len(normalized):
verbose_logger.warning(
"Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s",
model,
", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES),
)
return known or (GeminiRealtimeConfig._default_response_modality(model),)
@staticmethod
def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]:
if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
verbose_logger.warning(
"Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO",
model,
)
return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
return normalized
@staticmethod
def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]:
if len(modalities) <= 1:
return modalities
chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0]
verbose_logger.warning(
"Gemini Live: at most one response modality is allowed per session; using %s for the requested %s",
chosen,
", ".join(modalities),
)
return (chosen,)
@staticmethod
def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]:
generation_config: Final = setup.get("generationConfig")
@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# would be silently dropped because ``map_openai_params`` only
# recognises the flat OpenAI-beta key names.
session_payload = self._normalize_session_payload_for_mapping(session_payload)
new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload)
new_overrides: Final = self.map_openai_params(
optional_params={}, non_default_params=session_payload, model=model
)
if session_configuration_request is None:
generation_config: Final = new_overrides.setdefault("generationConfig", {})
@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
message_str = str(message)
raise ValueError(f"Invalid JSON message: {message_str}")
messages: Final[list[str]] = []
msg_type: Final = json_message.get("type")
if msg_type == "session.update":
return self._handle_session_update(json_message, model, session_configuration_request)
if msg_type == "response.create":
return [] # Gemini responds automatically; nothing to forward
return list(self._end_manual_activity())
if msg_type == "conversation.item.create":
return self._handle_conversation_item(json_message)
@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict})
verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
messages.append(gemini_msg)
return messages
return [*self._start_manual_activity(session_configuration_request), gemini_msg]
if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"):
return self._handle_input_audio_buffer_commit_or_end(session_configuration_request)
@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
logging_session_id: str,
session_configuration_request: str | None = None,
) -> OpenAIRealtimeStreamSessionEvents:
if session_configuration_request:
session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
else:
session_configuration_request_dict = {}
session_configuration_request_dict: Final = _parse_setup(
session_configuration_request or self.session_configuration_request(model)
)
_model: Final = session_configuration_request_dict.get("model") or model
generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {}
gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"])
_modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)]
_system_instruction: Final = session_configuration_request_dict.get("systemInstruction")
instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction"))
session: Final = OpenAIRealtimeStreamSession(
id=logging_session_id,
modalities=_modalities,
)
if _system_instruction is not None and isinstance(_system_instruction, str):
session["instructions"] = _system_instruction
if instructions is not None:
session["instructions"] = instructions
if _model is not None and isinstance(_model, str):
# Strip Vertex/AI Studio path prefixes to expose the bare model name.
if "/models/" in _model:
@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# Deferred setup opt-in: litellm.gemini_live_defer_setup = True
return not litellm.gemini_live_defer_setup
def builds_setup_from_session_update(self) -> bool:
return True
def session_configuration_request(self, model: str) -> str:
"""

7
litellm/llms/github_copilot/responses/transformation.py
View file

@ -24,6 +24,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import (
ResponseInputParam,
@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True
def supports_native_websocket(self) -> bool:
"""GitHub Copilot does not support native WebSocket for Responses API"""
return False

4
litellm/llms/litellm_proxy/responses/transformation.py
View file

@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import LlmProviders
@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.LITELLM_PROXY
def reasoning_field_values(self) -> ReasoningFieldValues:
return ANY_REASONING_FIELD_VALUES
def get_complete_url(
self,
api_base: str | None,

153
litellm/llms/mistral/chat/transformation.py
View file

@ -9,11 +9,16 @@ Docs - https://docs.mistral.ai/api/
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from itertools import accumulate, count
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
from urllib.parse import urlparse
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
@ -30,7 +35,15 @@ from litellm.router_utils.reasoning_effort_capability import (
nearest_declared_reasoning_effort,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage
from litellm.types.llms.mistral import (
MistralCacheKeyParams,
MistralFilePart,
MistralReasoningEffortParams,
MistralRequestRouting,
MistralTextBlock,
MistralThinkingBlock,
MistralToolCallMessage,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject
from litellm.types.utils import ModelResponse, ModelResponseStream
from litellm.utils import convert_to_model_response_object, supports_reasoning
@ -57,6 +70,17 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
requested: Final = as_str_mapping(reasoning_effort)
if requested is None:
return reasoning_effort, None
return requested.get("effort"), requested.get("summary")
def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list
@ -88,6 +112,8 @@ class MistralConfig(OpenAIGPTConfig):
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
"""
_is_base_class = False
temperature: int | None = None
top_p: int | None = None
max_tokens: int | None = None
@ -131,6 +157,7 @@ class MistralConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list[str]:
is_magistral: Final = "magistral" in model.lower()
accepts_reasoning_effort: Final = self._accepts_reasoning(model)
accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral"
return [
"stream",
"temperature",
@ -145,6 +172,7 @@ class MistralConfig(OpenAIGPTConfig):
"parallel_tool_calls",
*(("thinking",) if is_magistral else ()),
*(("reasoning_effort",) if accepts_reasoning_effort else ()),
*(("prompt_cache_key",) if accepts_prompt_cache_key else ()),
]
def _map_tool_choice(self, tool_choice: str) -> str:
@ -211,15 +239,84 @@ class MistralConfig(OpenAIGPTConfig):
optional_params["extra_body"] = {"random_seed": value}
if param == "response_format":
optional_params["response_format"] = value
if param == "reasoning_effort" and "magistral" not in model.lower():
optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider)
if (
param == "reasoning_effort"
and "magistral" not in model.lower()
and (
reasoning_level := self._reasoning_level_to_send(
model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params
)
)
is not None
):
optional_params["reasoning_effort"] = reasoning_level
if param in ("reasoning_effort", "thinking") and "magistral" in model.lower():
# Flag that we need to add reasoning system prompt
optional_params["_add_reasoning_prompt"] = True
if param == "parallel_tool_calls":
optional_params["parallel_tool_calls"] = value
if param in ("parallel_tool_calls", "prompt_cache_key"):
optional_params[param] = value
return optional_params
def _reasoning_level_to_send(
self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
) -> str | None:
level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort"))
sendable: Final = (
_accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
)
if sendable is not None and summary is None:
return sendable
if drop_params or litellm.drop_params is True:
return sendable
raise UnsupportedParamsError(
message=(
f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, "
"since the Mistral API takes it only as a level such as 'high', with no reasoning summary. "
"To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=self.custom_llm_provider,
model=model,
)
def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool:
resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None)
hostname: Final = urlparse(resolved_api_base).hostname or ""
return hostname == "mistral.ai" or hostname.endswith(".mistral.ai")
@staticmethod
def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool:
return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii()
def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None:
if not self.accepts_prompt_cache_key_at(api_base):
return "since only the Mistral API (*.mistral.ai) takes it"
if not self.takes_prompt_cache_key_value(prompt_cache_key):
return "since the Mistral API takes it only as a string of ASCII characters"
return None
def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool:
requested_api_base: Final = routing.get("api_base")
api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None
allowed_openai_params: Final = routing.get("allowed_openai_params")
if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params:
return False
rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base)
if rejection is None:
return False
if litellm.drop_params is True or routing.get("drop_params") is True:
return True
requested_provider: Final = routing.get("custom_llm_provider")
provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider
raise UnsupportedParamsError(
message=(
f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. "
"To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set "
"`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=provider,
model=model,
)
def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
# mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
api_base = (
@ -359,24 +456,31 @@ class MistralConfig(OpenAIGPTConfig):
messages = self._handle_message_with_file(messages)
return messages
@staticmethod
def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]:
file_part: Final = block.get("file") if block.get("type") == "file" else None
file_id: Final = (
_MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None
)
if not file_id:
return block
return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id}
@classmethod
def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues:
content: Final = message.get("content")
if not isinstance(content, list):
return message
blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content)
return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes
AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]}
)
def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]:
"""
Mistral API supports only 'file_id' in message content with type 'file'.
"""
for m in messages:
_content_block = m.get("content")
if _content_block and isinstance(_content_block, list):
if any(c.get("type") == "file" for c in _content_block):
# If file content is present, we get file_id from 'file' attribute of content block
# then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
file_contents = [c for c in _content_block if c.get("type") == "file"]
for file_content in file_contents:
file_id = file_content.get("file", {}).get("file_id")
if file_id:
# Replace 'file' with 'file_id'
file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape
file_content.pop("file", None)
return messages
return [self._with_mistral_file_ids(m) for m in messages]
def _add_reasoning_system_prompt_if_needed(
self, messages: list[AllMessageValues], optional_params: dict
@ -794,7 +898,16 @@ class MistralConfig(OpenAIGPTConfig):
if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False):
messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params)
upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"}
cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params)
withheld_params: Final = (
("client_metadata", "prompt_cache_key")
if "prompt_cache_key" in cache_key_params
and self._prompt_cache_key_withheld(
model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params)
)
else ("client_metadata",)
)
upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params}
# Call parent transform_request which handles _transform_messages
return super().transform_request(

9
litellm/llms/openai/common_utils.py
View file

@ -29,6 +29,7 @@ if TYPE_CHECKING:
from aiohttp import ClientSession
import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
@ -148,6 +149,14 @@ class OpenAIError(BaseLLMException):
)
def upstream_error_response(error: Exception) -> httpx.Response | None:
if not isinstance(error, (openai.APIStatusError, httpx.HTTPStatusError)):
return None
return error_response_without_credentials(
error.response.status_code, error.response, "https://docs.litellm.ai/docs"
)
_DECODED_JSON: Final = TypeAdapter(object)

23
litellm/llms/openai/completion/handler.py
View file

@ -10,7 +10,7 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse
from litellm.utils import ProviderConfigManager
from ..common_utils import BaseOpenAILLM, OpenAIError
from ..common_utils import BaseOpenAILLM, OpenAIError, upstream_error_response
from ..workload_identity import build_async_openai_client, build_openai_client
from .transformation import OpenAITextCompletionConfig
@ -164,7 +164,12 @@ class OpenAITextCompletion(BaseLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
async def acompletion(
self,
@ -220,7 +225,12 @@ class OpenAITextCompletion(BaseLLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
def streaming(
self,
@ -261,7 +271,12 @@ class OpenAITextCompletion(BaseLLM):
error_response: object = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=response,
model=model,

16
litellm/llms/openai/image_variations/handler.py
View file

@ -14,7 +14,7 @@ from litellm.utils import ProviderConfigManager
from ...base_llm.image_variations.transformation import BaseImageVariationConfig
from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, upstream_error_response
from ..workload_identity import build_async_openai_client, build_openai_client
@ -129,7 +129,12 @@ class OpenAIImageVariationsHandler:
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
def image_variations(
self,
@ -245,4 +250,9 @@ class OpenAIImageVariationsHandler:
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)

29
litellm/llms/openai/openai.py
View file

@ -27,6 +27,7 @@ import litellm
from litellm import LlmProviders
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.exceptions import UnsupportedParamsError
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -64,6 +65,7 @@ from .common_utils import (
is_openai_backed_api_base,
is_output_token_limit_error,
model_dump_provider_response,
upstream_error_response,
)
from .workload_identity import (
build_async_openai_client,
@ -970,8 +972,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
litellm.remove_index_from_tool_calls(messages=messages)
else:
raise e
except OpenAIError as e:
raise e
except (OpenAIError, UnsupportedParamsError):
raise
except Exception as e:
status_code: Final = getattr(e, "status_code", 500)
error_headers = getattr(e, "headers", None)
@ -983,6 +985,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
body=error_body,
)
@ -1110,6 +1113,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
raise OpenAIError(
status_code=status_code,
message=message,
response=upstream_error_response(e),
headers=error_headers,
body=exception_body,
)
@ -1282,6 +1286,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
raise OpenAIError(
status_code=getattr(e, "status_code", 500),
message=str(e),
response=upstream_error_response(e),
headers=error_headers,
body=exception_body,
)
@ -1409,7 +1414,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
def embedding(
self,
@ -1499,7 +1509,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
error_response: Final[object] = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers)
raise OpenAIError(
status_code=status_code,
message=error_text,
response=upstream_error_response(e),
headers=error_headers,
)
async def aimage_generation(
self,
@ -1666,7 +1681,11 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
original_response=str(e),
)
if hasattr(e, "status_code"):
raise OpenAIError(status_code=getattr(e, "status_code", 500), message=str(e))
raise OpenAIError(
status_code=getattr(e, "status_code", 500),
message=str(e),
response=upstream_error_response(e),
)
else:
raise OpenAIError(status_code=500, message=str(e))

11
litellm/llms/perplexity/responses/transformation.py
View file

@ -9,6 +9,7 @@ The only provider quirks:
Ref: https://docs.perplexity.ai/api-reference/responses-post
"""
from types import MappingProxyType
from typing import Any, Final
import httpx
@ -17,6 +18,7 @@ from pydantic import TypeAdapter
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
_DECODED_JSON: Final = TypeAdapter(object)
PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType(
{**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})}
)
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""Perplexity does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return PERPLEXITY_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

14
litellm/llms/sagemaker/chat/transformation.py
View file

@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import LlmProviders
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
from ..common_utils import AWSEventStreamDecoder, SagemakerError
from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
inference_component_name: Final = optional_params.get("model_id")
if not isinstance(inference_component_name, str):
return headers
return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
other_headers: Final = {
name: value
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower()
}
return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name}
def transform_request(
self,
@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
litellm_params=litellm_params,
headers=headers,
)
container_request: Final = {
key: value for key, value in request.items() if key not in self.aws_authentication_params
}
header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id"))
container_request: Final = {key: value for key, value in request.items() if key not in header_only_params}
served_model_name: Final = litellm_params.get("hf_model_name")
if not isinstance(served_model_name, str):
return container_request

33
litellm/llms/sagemaker/common_utils.py
View file

@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict
from litellm import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import StreamingChatCompletionChunk
from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk
STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape():
return _load_sagemaker_response_stream_shape()
INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component"
_SAGEMAKER_PROVIDERS: Final = frozenset(
provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA)
)
def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None:
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
if isinstance(custom_llm_provider, str) and custom_llm_provider:
return custom_llm_provider
model: Final = litellm_params.get("model")
return model.split("/", 1)[0] if isinstance(model, str) else None
def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None:
headers: Final = litellm_params.get("extra_headers")
if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping):
return None
return next(
(
value
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str)
),
None,
)
def with_inference_component_hint(error_message: str) -> str:
if "Inference Component Name header is required" in error_message:
return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`"

6
litellm/llms/sagemaker/completion/transformation.py
View file

@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
from litellm.utils import token_counter
from ..common_utils import SagemakerError
from ..common_utils import STR_KEYED_MAPPING, SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
inference_params: Final = optional_params.copy()
inference_params: Final = {
key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id"
}
stream: Final = inference_params.pop("stream", False)
data: Final[dict] = {"parameters": inference_params}
if stream is True:

37
litellm/llms/subscription_accounts.py Normal file
View file

@ -0,0 +1,37 @@
"""
Providers whose requests are covered by a flat subscription, each with the way to read
the account id of the login a deployment is signed into.
"""
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import Final
from litellm.llms.chatgpt.authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
from litellm.secret_managers.main import UnresolvableSecretReference, resolve_local_oidc_reference
from litellm.utils import with_named_credential
def _chatgpt_account_id(litellm_params: Mapping[str, object]) -> str | None:
params: Final = with_named_credential(litellm_params)
auth_file: Final = get_chatgpt_auth_file(params)
if auth_file:
return get_cached_authenticator(auth_file).read_account_id()
api_key: Final = params.get("api_key")
if not isinstance(api_key, str) or not api_key:
return Authenticator().read_account_id()
match resolve_local_oidc_reference(api_key):
case UnresolvableSecretReference():
return None
case str() as access_token:
return Authenticator().read_account_id(access_token)
SUBSCRIPTION_ACCOUNT_ID_RESOLVERS: Final[Mapping[str, Callable[[Mapping[str, object]], str | None]]] = MappingProxyType(
{"chatgpt": _chatgpt_account_id}
)
def resolve_subscription_account_id(custom_llm_provider: str, litellm_params: Mapping[str, object]) -> str | None:
resolver: Final = SUBSCRIPTION_ACCOUNT_ID_RESOLVERS.get(custom_llm_provider)
return None if resolver is None else resolver(litellm_params)

17
litellm/llms/vertex_ai/realtime/transformation.py
View file

@ -39,8 +39,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
(they require async I/O) and injected at construction time.
"""
def __init__(self, access_token: str, project: str, location: str) -> None:
super().__init__()
def __init__(self, access_token: str, project: str, location: str, drop_params: bool = False) -> None:
super().__init__(drop_params=drop_params)
self._access_token = access_token
self._project = project
self._location = location
@ -51,6 +51,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
def _rejects_vad_tuning_with_detection_off(self) -> bool:
return False
def _rejects_undocumented_voices(self, model: str) -> bool:
return True
# ------------------------------------------------------------------
# URL
# ------------------------------------------------------------------
@ -163,7 +166,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
# settings would be silently dropped because ``map_openai_params`` only
# recognises the flat OpenAI-beta key names.
session_params = self._normalize_session_payload_for_mapping(session_params)
setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params)
setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params, model=model)
# Use full Vertex AI model path
setup_config["model"] = self._vertex_model_path(model)
@ -177,15 +180,16 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
# ``silence_duration_ms``). ``map_automatic_turn_detection`` sets
# ``disabled=True`` whenever ``create_response`` is absent or
# ``False``. Force ``disabled=False`` only when the client did
# not explicitly request ``create_response: False`` — that path
# not explicitly request ``create_response: False`` or
# ``turn_detection: null`` — the first of those paths
# is how transcription guardrails suppress automatic responses,
# and overriding it here would silently bypass the guardrail.
# Vertex Live has no "VAD on, no auto-response" mode, so callers
# that need that behaviour must accept that VAD is off.
client_turn_detection: Final = session_params.get("turn_detection")
client_disabled_auto_response: Final = (
isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False
)
"turn_detection" in session_params and client_turn_detection is None
) or (isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False)
realtime_input_config: Final = setup_config.setdefault("realtimeInputConfig", {})
automatic_detection: Final = realtime_input_config.setdefault("automaticActivityDetection", {})
if not client_disabled_auto_response:
@ -261,4 +265,5 @@ def vertex_realtime_config(
access_token=access_token,
project=project,
location=VertexBase.get_vertex_region(vertex_region=location, model=model),
drop_params=drop_params,
)

25
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py
View file

@ -3,7 +3,7 @@ from typing import Final
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
messages_carry_output_config,
@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management")))
# Check for web search tool
for tool in tools:

10
litellm/llms/xai/responses/transformation.py
View file

@ -10,6 +10,10 @@ from litellm._logging import verbose_logger
from litellm.constants import XAI_API_BASE
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import (
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
ReasoningFieldValues,
)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd
from litellm.secret_managers.main import get_secret_str
@ -290,3 +294,9 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""XAI does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

94
litellm/llms/zai/chat/transformation.py
View file

@ -1,11 +1,43 @@
from collections.abc import Mapping
from typing import Final
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import response_schema_prompt
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam, ChatCompletionUserMessage
from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
ZAI_API_BASE: Final = "https://api.z.ai/api/paas/v4"
TOKEN_LIMIT_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens"})
def _is_json_object(
value: object,
) -> TypeIs[Mapping[str, object]]: # guard-ok: cost-map rows and request params are JSON
return isinstance(value, Mapping)
def _json_field(value: object, key: str) -> object:
return value.get(key) if _is_json_object(value) else None
def _exceeds_output_limit(model: str, max_tokens: object) -> bool:
info: Final = lookup_model_info_for_registration(model=model, custom_llm_provider="zai")
limit: Final = None if isinstance(info, ModelInfoLookupMiss) else info.get("max_output_tokens")
return isinstance(max_tokens, int) and isinstance(limit, int) and max_tokens > limit
def _schema_without_native_support(model: str, response_format: object) -> Mapping[str, object] | None:
if _json_field(response_format, "type") != "json_schema":
return None
if litellm.supports_response_schema(model=model, custom_llm_provider="zai"):
return None
schema: Final = _json_field(_json_field(response_format, "json_schema"), "schema")
return schema if _is_json_object(schema) else None
class ZAIChatConfig(OpenAIGPTConfig):
@ -13,6 +45,10 @@ class ZAIChatConfig(OpenAIGPTConfig):
def custom_llm_provider(self) -> str | None:
return "zai"
@property
def prompts_unsupported_json_schema(self) -> bool:
return True
def _get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None
) -> tuple[str | None, str | None]:
@ -40,9 +76,11 @@ class ZAIChatConfig(OpenAIGPTConfig):
# GLM/ZAI supports cache_control, so return messages and tools unchanged
return messages, tools
def get_supported_openai_params(self, model: str) -> list:
def get_supported_openai_params(self, model: str) -> list[str]:
base_params: Final = [
"max_tokens",
"max_completion_tokens",
"response_format",
"stream",
"stream_options",
"temperature",
@ -52,8 +90,6 @@ class ZAIChatConfig(OpenAIGPTConfig):
"tool_choice",
]
import litellm
try:
if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider):
base_params.append("thinking")
@ -61,3 +97,53 @@ class ZAIChatConfig(OpenAIGPTConfig):
pass
return base_params
def map_openai_params(
self,
non_default_params: Mapping[str, object],
optional_params: Mapping[str, object],
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body
supported_params: Final = self.get_supported_openai_params(model)
sent: Final = {param: value for param, value in non_default_params.items() if param in supported_params}
max_completion_tokens: Final = sent.get("max_completion_tokens")
falls_back_to_max_tokens: Final = max_completion_tokens is None or (
(litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_completion_tokens)
)
max_tokens: Final = sent.get("max_tokens") if falls_back_to_max_tokens else max_completion_tokens
return {
**optional_params,
**{param: value for param, value in sent.items() if param not in TOKEN_LIMIT_PARAMS},
**({} if max_tokens is None else {"max_tokens": max_tokens}),
}
def transform_request(
self,
model: str,
messages: list[AllMessageValues], # mutable-ok: OpenAIGPTConfig.transform_request signature
optional_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature
) -> dict[str, object]: # mutable-ok: inherited contract returns the request body
schema: Final = (
_schema_without_native_support(model, optional_params.get("response_format"))
if self.prompts_unsupported_json_schema
else None
)
schema_message: Final = (
()
if schema is None
else (
ChatCompletionUserMessage(
role="user", content=response_schema_prompt(model=model, response_schema=schema, llm_provider="zai")
),
)
)
return super().transform_request(
model,
[*messages, *schema_message],
optional_params if schema is None else {**optional_params, "response_format": {"type": "json_object"}},
litellm_params,
headers,
)

67
litellm/main.py
View file

@ -681,7 +681,7 @@ async def acompletion(
- The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
- If `stream` is True, the function returns an async generator that yields completion lines.
"""
fallbacks = kwargs.get("fallbacks", None)
fallbacks: Final = kwargs.get("fallbacks", None) or litellm.model_fallbacks
mock_timeout = kwargs.get("mock_timeout", None)
if mock_timeout is True:
@ -697,6 +697,7 @@ async def acompletion(
loop: Final = asyncio.get_event_loop()
custom_llm_provider = kwargs.get("custom_llm_provider", None)
_ = _resolve_control_options(kwargs, model)
caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs
## PROMPT MANAGEMENT HOOKS ##
#########################################################
@ -715,15 +716,34 @@ async def acompletion(
custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
api_base=base_url or kwargs.get("api_base"),
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=kwargs.get("prompt_id", None),
prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None)) # cast-ok: read from untyped kwargs
prompt_management_logger: Final = (
litellm_logging_obj.get_custom_logger_for_prompt_management(
model=model,
non_default_params=kwargs,
tools=tools,
prompt_id=prompt_id,
dynamic_callback_params=litellm_logging_obj.standard_callback_dynamic_params,
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj)
and litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=prompt_id,
non_default_params=kwargs,
tools=tools,
)
else None
)
each_fallback_writes_its_own_cache_dialect: Final = fallbacks is not None and isinstance(
prompt_management_logger, AnthropicCacheControlHook
)
if (
isinstance(litellm_logging_obj, LiteLLMLoggingObj)
and prompt_management_logger is not None
and not each_fallback_writes_its_own_cache_dialect
):
(
model,
@ -733,8 +753,9 @@ async def acompletion(
model=model,
messages=messages,
non_default_params=kwargs,
prompt_id=kwargs.get("prompt_id", None),
prompt_id=prompt_id,
prompt_variables=kwargs.get("prompt_variables", None),
prompt_management_logger=prompt_management_logger,
tools=tools,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
@ -826,9 +847,10 @@ async def acompletion(
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
)
fallbacks = fallbacks or litellm.model_fallbacks
if fallbacks is not None:
response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs})
response = await async_completion_with_fallbacks(
**completion_kwargs, kwargs={"fallbacks": fallbacks, **caller_kwargs}
)
if response is None:
raise Exception(
"No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details."
@ -5765,7 +5787,7 @@ def completion(
custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
api_base=base_url or kwargs.get("api_base"),
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
@ -6199,6 +6221,7 @@ def completion(
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
"fireworks_forward_user_id",
"responses_api_version",
"allowed_openai_params",
)
if key in kwargs
},
@ -6266,6 +6289,9 @@ def completion(
# reasoningSummary/reasoning_summary without tools (AI SDK) that the first
# (early) check doesn't cover.
_reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params)
typed_deployment_model_info: Final = str_keyed_mapping(
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
)
if responses_api_model_info.get("mode") != "responses":
responses_api_model_info, model = responses_api_bridge_check(
model=model,
@ -6275,9 +6301,7 @@ def completion(
reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")),
reasoning_summary=_reasoning_summary_for_bridge,
api_base=api_base,
deployment_model_info=str_keyed_mapping(
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
),
deployment_model_info=typed_deployment_model_info,
)
# Use base_model (the true underlying model) for Azure model-type
@ -6289,19 +6313,28 @@ def completion(
optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict):
from litellm.responses.utils import openai_bridge_reasoning
if isinstance(resolved_reasoning_effort, dict):
from litellm.responses.utils import (
RefusedReasoningFields,
bridge_reasoning,
raise_refused_reasoning_fields,
)
optional_params["reasoning_effort"] = openai_bridge_reasoning(
bridged_reasoning: Final = bridge_reasoning(
requested=resolved_reasoning_effort,
effort_kept="reasoning_effort" in optional_params,
summary_alias=rs_val,
model=model,
base_model=base_model if isinstance(base_model, str) else None,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
deployment_model_info=typed_deployment_model_info,
drop_params=drop_unsupported_reasoning,
additional_drop_params=routing_kwargs.get("additional_drop_params"),
)
elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort:
optional_params["reasoning_effort"] = resolved_reasoning_effort
if isinstance(bridged_reasoning, RefusedReasoningFields):
raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider)
optional_params["reasoning_effort"] = bridged_reasoning
elif rs_val is not None:
eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort)
if isinstance(eff, dict):

41
litellm/model_prices_and_context_window_backup.json
View file

@ -6014,11 +6014,14 @@
"deprecation_date": "2027-04-14",
"cache_read_input_token_cost": 1.25e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_batches": 1.25e-06,
"litellm_provider": "azure",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 1e-05,
"output_cost_per_token_batches": 5e-06,
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
@ -6313,16 +6316,19 @@
"deprecation_date": "2027-04-14",
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"litellm_provider": "azure",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 6e-07,
"output_cost_per_token_batches": 3e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.014,
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
@ -11303,16 +11309,19 @@
"deprecation_date": "2026-11-19",
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "azure",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 8e-06,
"output_cost_per_token_batches": 4e-06,
"search_context_cost_per_query": {
"search_context_size_high": 0.014,
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -11411,11 +11420,14 @@
"deprecation_date": "2026-11-19",
"cache_read_input_token_cost": 5.5e-07,
"input_cost_per_token": 1.1e-06,
"input_cost_per_token_batches": 5.5e-07,
"litellm_provider": "azure",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"output_cost_per_token_batches": 2.2e-06,
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -11513,16 +11525,19 @@
"deprecation_date": "2026-11-19",
"cache_read_input_token_cost": 2.75e-07,
"input_cost_per_token": 1.1e-06,
"input_cost_per_token_batches": 5.5e-07,
"litellm_provider": "azure",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"output_cost_per_token_batches": 2.2e-06,
"search_context_cost_per_query": {
"search_context_size_high": 0.014,
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -54512,6 +54527,7 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"zai/glm-5.3-flash": {
@ -54526,10 +54542,26 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing",
"supports_vision": true
},
"zai/glm-5v-turbo": {
"cache_read_input_token_cost": 2.4e-07,
"input_cost_per_token": 1.2e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://vercel.com/ai-gateway/models/glm-5v-turbo",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"zai/glm-5.1": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2.6e-07,
@ -81059,5 +81091,14 @@
"supports_response_schema": true,
"supports_tool_choice": false,
"supports_vision": true
},
"baseten/decision-models/baseten-joey-2b-latest": {
"input_cost_per_token": 1e-08,
"litellm_provider": "baseten",
"max_input_tokens": 16384,
"mode": "decisions",
"output_cost_per_token": 0.0,
"source": "https://inference.baseten.co/v1/models",
"supports_vision": true
}
}

2
litellm/models/__init__.py
View file

@ -30,6 +30,7 @@ from litellm.models.organization_membership import LiteLLM_OrganizationMembershi
from litellm.models.project import LiteLLM_ProjectTable
from litellm.models.skills import LiteLLM_SkillsTable
from litellm.models.spend_logs import LiteLLM_ErrorLogs, LiteLLM_SpendLogs
from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
from litellm.models.tag import LiteLLM_TagTable
from litellm.models.team import LiteLLM_TeamTable
from litellm.models.team_membership import LiteLLM_TeamMembership
@ -59,6 +60,7 @@ __all__ = [
"LiteLLM_ProxyModelTable",
"LiteLLM_SkillsTable",
"LiteLLM_SpendLogs",
"LiteLLM_SubscriptionAccountTable",
"LiteLLM_TagTable",
"LiteLLM_TeamMemberTable",
"LiteLLM_TeamMembership",

28
litellm/models/subscription_account.py Normal file
View file

@ -0,0 +1,28 @@
"""
Subscription account table model.
Canonical definition for ``litellm_subscriptionaccounttable``: the operator-entered
recurring fee of a provider account whose requests are covered by a flat subscription.
"""
from datetime import datetime
from pydantic import ConfigDict
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_SubscriptionAccountTable(LiteLLMPydanticObjectBase):
model_config = ConfigDict(protected_namespaces=(), frozen=True)
subscription_account_id: str
custom_llm_provider: str
account_id: str
label: str | None = None
monthly_fee: float
currency: str
billing_period_start: str
created_at: datetime
created_by: str
updated_at: datetime
updated_by: str

2
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -1648,7 +1648,7 @@
},
"/access_group/{access_group}/update": {
"put": {
"description": "Update an access group's model names.\n\nThis will:\n1. Remove the access group from all current deployments\n2. Add the access group to all deployments for the new model_names list\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found",
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s",
"operationId": "update_access_group_access_group__access_group__update_put",
"parameters": [
{

4
litellm/proxy/_types.py
View file

@ -667,6 +667,10 @@ class LiteLLMRoutes(enum.Enum):
"/team/spend/by_user",
# gateway request counts (SGR); deployment-wide, admin-only
"/gateway/daily/activity",
"/subscription_accounts/usage",
"/subscription_accounts/new",
"/subscription_accounts/update",
"/subscription_accounts/delete",
# model
"/model/new",
"/model/update",

18
litellm/proxy/anthropic_endpoints/endpoints.py
View file

@ -8,7 +8,7 @@ from typing import Final
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import JSONResponse
from pydantic import JsonValue, ValidationError
from pydantic import JsonValue, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -54,6 +54,7 @@ from litellm.proxy.litellm_pre_call_utils import (
should_auto_drop_params_for_agentic_cli,
)
from litellm.router import Router
from litellm.types.utils import TokenCountResponse
router: Final = APIRouter()
@ -334,6 +335,15 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
}
_PROVIDER_COUNT_REPLY: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
def _provider_context_management(response: TokenCountResponse) -> Mapping[str, object]:
original: Final = _PROVIDER_COUNT_REPLY.validate_python(response.original_response)
context_management: Final = original.get("context_management") if original is not None else None
return {} if context_management is None else {"context_management": context_management}
@router.post(
"/v1/messages/count_tokens",
tags=["[beta] Anthropic Messages Token Counting"],
@ -413,7 +423,11 @@ async def count_tokens(
log_llm_api_exception(e, litellm_call_id)
return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
else:
return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)}
return {
"input_tokens": token_response.total_tokens,
**_provider_context_management(token_response),
**count_source_fields(token_response),
}
@router.post(

10
litellm/proxy/auth/auth_checks.py
View file

@ -446,7 +446,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request,
from litellm.proxy.litellm_pre_call_utils import (
_trace_id_from_traceparent, # pyright: ignore[reportPrivateUsage] # reuse W3C traceparent parser
get_chain_id_from_headers,
metadata_variable_name_for_route,
metadata_variable_name_for_request,
)
if get_chain_id_from_headers(dict(request.headers)):
@ -460,7 +460,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request,
traceparent: Final[str | None] = request.headers.get("traceparent")
if isinstance(traceparent, str) and _trace_id_from_traceparent(traceparent):
return True
metadata_variable_name: Final = metadata_variable_name_for_route(get_request_route(request))
metadata_variable_name: Final = metadata_variable_name_for_request(get_request_route(request), request_body)
selected_metadata: Final[object] = request_body.get(metadata_variable_name)
if isinstance(selected_metadata, dict):
selected_metadata_dict: Final = _TRACE_ID_METADATA_ADAPTER.validate_python(selected_metadata)
@ -1367,11 +1367,15 @@ async def common_checks(
if not skip_all_budget_checks:
# Key metadata.tags are injected into request_body here so the tag budget
# check can read them; this mutation must run before the gathered checks.
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request
if valid_token is not None:
for tag_target in LiteLLMProxyRequestSetup.pre_auth_tag_merge_targets(request, request_body):
LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(request_data=tag_target, route=route)
LiteLLMProxyRequestSetup.merge_caller_body_tags(
request_data=tag_target,
metadata_variable_name=metadata_variable_name_for_request(route, tag_target),
)
LiteLLMProxyRequestSetup.apply_key_tags_pre_auth(request_data=tag_target, user_api_key_dict=valid_token)
LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, request_body)

23
litellm/proxy/auth/handle_jwt.py
View file

@ -2199,15 +2199,19 @@ class JWTAuthManager:
# not mistaken for claimless and left with stale DB memberships the fallback
# could later attribute. With the flag off, keep the upstream plural-only
# reconciliation so existing deployments are unchanged.
jwt_team_ids: Final = set(
jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
if jwt_handler.litellm_jwtauth.fallback_to_db_teams
else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
claim_team_ids: Final = tuple(
dict.fromkeys(
jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
if jwt_handler.litellm_jwtauth.fallback_to_db_teams
else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
)
)
existing_teams: Final = set(user_object.teams or [])
teams_to_add: Final = jwt_team_ids - existing_teams
jwt_team_ids: Final = frozenset(claim_team_ids)
existing_team_order: Final = tuple(dict.fromkeys(user_object.teams or []))
existing_teams: Final = frozenset(existing_team_order)
teams_to_add: Final = tuple(team_id for team_id in claim_team_ids if team_id not in existing_teams)
preserve_db_teams_without_claims: Final = jwt_handler.litellm_jwtauth.fallback_to_db_teams and not jwt_team_ids
teams_to_remove: Final = set() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
teams_to_remove: Final = frozenset() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
if teams_to_add or teams_to_remove:
from litellm.proxy.management_endpoints.scim.scim_v2 import (
patch_team_membership,
@ -2218,7 +2222,10 @@ class JWTAuthManager:
teams_ids_to_add_user_to=list(teams_to_add),
teams_ids_to_remove_user_from=list(teams_to_remove),
)
user_object.teams = list(jwt_team_ids)
user_object.teams = [
*(team_id for team_id in existing_team_order if team_id not in teams_to_remove),
*teams_to_add,
]
if user_api_key_cache is not None:
await user_api_key_cache.async_set_cache(
key=user_object.user_id,

34
litellm/proxy/auth/model_checks.py
View file

@ -14,14 +14,16 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.responses.main import strip_responses_routing_prefix
from litellm.router import Router
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models
from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
_PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value})
_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
@ -331,6 +333,25 @@ def _routes_back_to(
)
def _resolves_to_catalog_model(model: str, provider: str, provider_model_ids: frozenset[str]) -> bool:
provider_id_prefix: Final = f"{provider}/"
resolved_model: Final = (
f"{provider_id_prefix}{strip_responses_routing_prefix(model.removeprefix(provider_id_prefix))}"
)
return any(
name.removeprefix(provider_id_prefix) in provider_model_ids
for name in potential_model_names_in_lookup_order(resolved_model, provider)
)
def _routes_to_catalog_model(
routes_to_model: Callable[[str, str], bool], provider: str, provider_model_ids: frozenset[str]
) -> Callable[[str, str], bool]:
return lambda listed_id, model: (
routes_to_model(listed_id, model) and _resolves_to_catalog_model(model, provider, provider_model_ids)
)
def _ids_routing_to_catalog_models(
name_pattern: str,
deployment_pattern: str,
@ -344,8 +365,15 @@ def _ids_routing_to_catalog_models(
return ()
provider_id_prefix: Final = f"{provider}/"
provider_model_ids: Final = tuple(catalog_id.removeprefix(provider_id_prefix) for catalog_id in catalog_ids)
deployment_model_pattern_without_provider: Final = deployment_pattern.removeprefix(provider_id_prefix)
catalog_pattern: Final = _without_routing_segments(
deployment_pattern.removeprefix(provider_id_prefix), provider, name_pattern, provider_model_ids
deployment_model_pattern_without_provider, provider, name_pattern, provider_model_ids
)
routing_prefix: Final = deployment_model_pattern_without_provider.removesuffix(catalog_pattern)
reaches_catalog_model: Final = (
_routes_to_catalog_model(routes_to_model, provider, frozenset(provider_model_ids))
if routing_prefix and provider in _PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID
else routes_to_model
)
deployment_model_pattern: Final = f"{provider_id_prefix}{catalog_pattern}"
deployment_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex(deployment_model_pattern))
@ -369,7 +397,7 @@ def _ids_routing_to_catalog_models(
for listed_id, catalog_id in listed_ids
if listed_id
and _routes_back_to(
listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, routes_to_model
listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, reaches_catalog_model
)
)
)

16
litellm/proxy/common_request_processing.py
View file

@ -2401,9 +2401,10 @@ class ProxyBaseLLMRequestProcessing:
version=version,
proxy_config=proxy_config,
)
request_fields: Final = _REQUEST_FIELDS.validate_python(self.data)
metadata_variable_name: Final = get_metadata_variable_name(request, request_fields)
if logging_auth is not None:
metadata_variable_name: Final = get_metadata_variable_name(request)
request_metadata: Final = self.data.get(metadata_variable_name)
request_metadata: Final = request_fields.get(metadata_variable_name)
if isinstance(request_metadata, dict):
self.data[metadata_variable_name] = {
**TypeAdapter(dict[str, object]).validate_python(request_metadata),
@ -2431,12 +2432,11 @@ class ProxyBaseLLMRequestProcessing:
# Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved
if queue_time_seconds is not None:
_metadata_variable_name: Final = get_metadata_variable_name(request)
if _metadata_variable_name not in self.data:
self.data[_metadata_variable_name] = {}
if not isinstance(self.data[_metadata_variable_name], dict):
self.data[_metadata_variable_name] = {}
self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds
if metadata_variable_name not in self.data:
self.data[metadata_variable_name] = {}
if not isinstance(self.data[metadata_variable_name], dict):
self.data[metadata_variable_name] = {}
self.data[metadata_variable_name]["queue_time_seconds"] = queue_time_seconds
if isinstance(model, str):
reject_url_valued_destination("model", model)

136
litellm/proxy/common_utils/config_sync_pubsub.py
View file

@ -2,9 +2,12 @@ import asyncio
import json
import random
import time
from collections.abc import Awaitable, Callable
from collections.abc import AsyncIterator, Awaitable, Callable
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # untyped prisma/redis boundary needs cast
from itertools import chain
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast # noqa: TID251 # untyped prisma/redis boundary
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.repositories.prisma_protocols import RowT_co, TableActions
@ -106,24 +109,69 @@ _pubsub_capable_client: Final = pubsub_capable_client
@dataclass(frozen=True, slots=True)
class _ConfigChangeMessage:
object_type: str
model_ids: tuple[str, ...] = ()
def _config_change_message_json(object_type: str) -> str:
return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type)))
@dataclass(frozen=True, slots=True)
class _PubSubEnvelope:
data: str | bytes
async def publish_config_change(redis_cache: "RedisCache | None", object_type: str) -> None:
class _WrittenModelRow(BaseModel):
model_config = ConfigDict(from_attributes=True, frozen=True)
model_id: str
_CONFIG_CHANGE_MESSAGE: Final = TypeAdapter(_ConfigChangeMessage)
_PUBSUB_ENVELOPE: Final = TypeAdapter(_PubSubEnvelope)
_MODEL_TABLE_NAME: Final = "litellm_proxymodeltable"
_TARGETED_WRITE_ACTION_NAMES: Final[frozenset[str]] = frozenset({"create", "update", "upsert"})
PublishConfigChange: TypeAlias = Callable[[str, tuple[str, ...]], Awaitable[None]]
def _config_change_message_json(object_type: str, model_ids: tuple[str, ...]) -> str:
return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type, model_ids=model_ids)))
def parse_config_change(message: object) -> _ConfigChangeMessage | None:
try:
payload: Final = (
message if isinstance(message, (str, bytes)) else _PUBSUB_ENVELOPE.validate_python(message).data
)
return _CONFIG_CHANGE_MESSAGE.validate_json(payload)
except ValidationError:
return None
def written_model_ids(object_type: str, action_name: str, result: object) -> tuple[str, ...]:
if object_type != _MODEL_TABLE_NAME or action_name not in _TARGETED_WRITE_ACTION_NAMES:
return ()
try:
return (_WrittenModelRow.model_validate(result).model_id,)
except ValidationError:
return ()
async def publish_config_change(
redis_cache: "RedisCache | None", object_type: str, model_ids: tuple[str, ...] = ()
) -> None:
if redis_cache is None:
return
try:
client: Final = pubsub_capable_client(redis_cache)
await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type))
await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type, model_ids))
except Exception as e: # noqa: BLE001 # best-effort publish; writes must never fail on redis errors
verbose_proxy_logger.warning("config sync publish for %s failed: %s", object_type, e)
async def publish_config_change_for_object_type(object_type: str) -> None:
await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type)
async def publish_config_change_for_object_type(object_type: str, model_ids: tuple[str, ...] = ()) -> None:
await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type, model_ids=model_ids)
async def publish_model_rows_written(model_ids: tuple[str, ...]) -> None:
await publish_config_change_for_object_type(_MODEL_TABLE_NAME, model_ids)
async def publish_config_param_change(param_name: str) -> None:
@ -139,7 +187,7 @@ async def publish_config_param_change(param_name: str) -> None:
class _PublishOnWriteActions:
__slots__ = ("_actions", "_object_type", "_publish")
def __init__(self, actions: object, object_type: str, publish: Callable[[str], Awaitable[None]]) -> None:
def __init__(self, actions: object, object_type: str, publish: PublishConfigChange) -> None:
self._actions = actions
self._object_type = object_type
self._publish = publish
@ -157,7 +205,7 @@ class _PublishOnWriteActions:
**kwargs: object, # kwargs-ok: transparent passthrough to untyped prisma action
) -> object:
result: Final = await write_action(*args, **kwargs)
await publish(object_type)
await publish(object_type, written_model_ids(object_type, name, result))
return result
return _write_then_publish
@ -166,7 +214,7 @@ class _PublishOnWriteActions:
def wrap_table_actions_for_config_sync(
actions: "TableActions[RowT_co]",
table_name: str,
publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type,
publish: PublishConfigChange = publish_config_change_for_object_type,
) -> "TableActions[RowT_co]":
if table_name not in _CONFIG_SYNCED_TABLE_NAMES:
return actions
@ -174,8 +222,14 @@ def wrap_table_actions_for_config_sync(
return cast("TableActions[RowT_co]", wrapped) # cast-ok: dynamic write-through proxy keeps the wrapped row type
def _published_model_ids(messages: tuple[object, ...]) -> tuple[str, ...]:
changes: Final = (parse_config_change(message) for message in messages)
return tuple(dict.fromkeys(chain.from_iterable(change.model_ids for change in changes if change is not None)))
class ConfigSyncSubscriber:
__slots__ = (
"_apply_model_ids",
"_backoff_initial_seconds",
"_backoff_max_seconds",
"_debounce_seconds",
@ -195,6 +249,7 @@ class ConfigSyncSubscriber:
self,
redis_cache: "RedisCache",
resync_callbacks: tuple[Callable[[], Awaitable[None]], ...],
apply_model_ids: Callable[[tuple[str, ...]], Awaitable[None]] | None = None,
debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS,
jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS,
min_resync_interval_seconds: float = CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS,
@ -206,6 +261,7 @@ class ConfigSyncSubscriber:
) -> None:
self._redis_cache = redis_cache
self._resync_callbacks = resync_callbacks
self._apply_model_ids = apply_model_ids
self._debounce_seconds = debounce_seconds
self._jitter_max_seconds = jitter_max_seconds
self._min_resync_interval_seconds = min_resync_interval_seconds
@ -266,32 +322,64 @@ class ConfigSyncSubscriber:
message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS)
if message is None:
continue
await self._apply(pubsub, message)
await self._debounce_and_resync(pubsub)
async def _debounce_and_resync(self, pubsub: _ConfigSyncPubSub) -> None:
await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds))
await self._wait_for_min_resync_interval()
async def _debounce_and_resync(self, pubsub: ConfigSyncPubSub) -> None:
await self._apply_until(
pubsub, self._monotonic() + self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)
)
await self._apply_until(pubsub, self._next_resync_at())
await self._drain_pending(pubsub)
await self._run_resync_callbacks()
self._last_resync_at = self._monotonic()
async def _wait_for_min_resync_interval(self) -> None:
def _next_resync_at(self) -> float:
if self._last_resync_at is None:
return
seconds_until_next_resync = self._min_resync_interval_seconds - (self._monotonic() - self._last_resync_at)
if seconds_until_next_resync <= 0:
return
verbose_proxy_logger.debug(
"config sync resync throttled for %.1fs to cap fleet-wide reload rate",
seconds_until_next_resync,
)
await self._sleep(seconds_until_next_resync)
return self._monotonic()
next_resync_at: Final = self._last_resync_at + self._min_resync_interval_seconds
throttled_for: Final = next_resync_at - self._monotonic()
if throttled_for > 0:
verbose_proxy_logger.debug(
"config sync resync throttled for %.1fs to cap fleet-wide reload rate",
throttled_for,
)
return next_resync_at
async def _apply_until(self, pubsub: ConfigSyncPubSub, deadline: float) -> None:
while True:
remaining = deadline - self._monotonic()
if remaining <= 0:
return
message = await pubsub.get_message(
ignore_subscribe_messages=True, timeout=min(remaining, _POLL_TIMEOUT_SECONDS)
)
if message is not None:
await self._apply(pubsub, message)
@staticmethod
async def _drain_pending(pubsub: ConfigSyncPubSub) -> None:
while await pubsub.get_message(ignore_subscribe_messages=True, timeout=0) is not None:
pass
@staticmethod
async def _ready_messages(pubsub: ConfigSyncPubSub) -> AsyncIterator[object]:
while (message := await pubsub.get_message(ignore_subscribe_messages=True, timeout=0)) is not None:
yield message
async def _apply(self, pubsub: ConfigSyncPubSub, message: object) -> None:
apply: Final = self._apply_model_ids
if apply is None:
return
ready: Final = (message, *[ready_message async for ready_message in self._ready_messages(pubsub)])
model_ids: Final = _published_model_ids(ready)
if not model_ids:
return
try:
await apply(model_ids)
except Exception as e: # noqa: BLE001 # the debounced full resync retries what a targeted apply could not
verbose_proxy_logger.warning("config sync targeted apply of %s failed: %s", model_ids, e)
async def _run_resync_callbacks(self) -> None:
for callback in self._resync_callbacks:
try:

126
litellm/proxy/common_utils/registry_read_through.py
View file

@ -15,12 +15,13 @@ or the first routed request on this replica would pick a tier it has not loaded.
import asyncio
import time
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Final
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
if TYPE_CHECKING:
@ -33,8 +34,14 @@ if TYPE_CHECKING:
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.repositories.prisma_protocols import TableActions
from litellm.types.agents import AgentResponse
_ModelTable: TypeAlias = TableActions[prisma_models.LiteLLM_ProxyModelTable]
_ModelRows: TypeAlias = Sequence[prisma_models.LiteLLM_ProxyModelTable]
ModelLoadOutcome: TypeAlias = Literal["loaded", "missing", "no_router", "auto_router"]
READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0
READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0
READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20
@ -123,47 +130,114 @@ async def _resync_model_deployments(model_name: str) -> bool:
return False
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
table: Final = ModelRepository(prisma_client).table
name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
rows: Final = await _find_rows_by_name_or_id(model_name, ModelRepository(prisma_client).table)
if not rows:
return False
router: Final = proxy_server.llm_router
if router is None:
await proxy_server.proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
)
return proxy_server.llm_router is not None
if _holds_an_auto_router(rows):
await proxy_server.proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
)
return _model_is_loaded(model_name)
async with proxy_server.MODEL_RECONCILE_LOCK:
outcome: Final = await _install_model_rows(rows, _always)
if outcome in ("no_router", "auto_router"):
await _reload_every_model()
return outcome == "loaded" or _model_is_loaded(model_name)
async def resync_model_deployments_by_ids(model_ids: tuple[str, ...]) -> None:
from litellm.proxy import proxy_server
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
from litellm.repositories.model_repository import ModelRepository
if not _db_backed_registries_enabled("models"):
return
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
writer_table: Final = ModelRepository(WriterPinnedClient(prisma_client.db)).table
async with proxy_server.MODEL_RECONCILE_LOCK:
rows: Final = await _find_rows_by_ids(model_ids, writer_table)
_ = await _install_model_rows(_without_auto_routers(rows), _names_an_unloaded_credential)
async def _find_rows_by_name_or_id(model_name: str, table: "_ModelTable") -> "_ModelRows":
name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
return await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
async def _find_rows_by_ids(model_ids: tuple[str, ...], table: "_ModelTable") -> "_ModelRows":
from litellm.repositories.chunked_in import find_many_in
return list(await find_many_in(table, "model_id", model_ids))
async def _install_model_rows(
rows: "_ModelRows",
needs_credentials: Callable[["_ModelRows"], bool],
) -> ModelLoadOutcome:
from litellm.proxy import proxy_server
router: Final = proxy_server.llm_router
if not rows:
return "missing"
if router is None:
return "no_router"
if _holds_an_auto_router(rows):
return "auto_router"
if needs_credentials(rows):
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client)
proxy_server.proxy_config._add_deployment(db_models=rows)
proxy_server.llm_model_list = router.registry().deployments()
proxy_server.proxy_config._add_deployment(db_models=rows)
proxy_server.llm_model_list = router.registry().deployments()
return "loaded"
async def _reload_every_model() -> None:
from litellm.proxy import proxy_server
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
await proxy_server.proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
)
def _always(rows: "_ModelRows") -> bool:
return True
def _holds_an_auto_router(rows: Sequence["prisma_models.LiteLLM_ProxyModelTable"]) -> bool:
def _holds_an_auto_router(rows: "_ModelRows") -> bool:
return any(_names_an_auto_router(row.litellm_params) for row in rows)
def _without_auto_routers(rows: "_ModelRows") -> "_ModelRows":
return list(row for row in rows if not _names_an_auto_router(row.litellm_params))
def _names_an_unloaded_credential(rows: "_ModelRows") -> bool:
return any(_names_an_unloaded_credential_param(row.litellm_params) for row in rows)
_STORED_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
def _names_an_auto_router(stored_litellm_params: object) -> bool:
def _decrypted_param(stored_litellm_params: object, key: str) -> str | None:
try:
params: Final = _STORED_LITELLM_PARAMS.validate_python(stored_litellm_params)
except ValidationError:
return False
stored_model: Final = params.get("model")
if not isinstance(stored_model, str):
return False
model: Final = decrypt_value_helper(value=stored_model, key="model", return_original_value=True)
return isinstance(model, str) and model.startswith("auto_router/")
return None
stored_value: Final = params.get(key)
if not isinstance(stored_value, str):
return None
value: Final = decrypt_value_helper(value=stored_value, key=key, return_original_value=True)
return value if isinstance(value, str) else None
def _names_an_auto_router(stored_litellm_params: object) -> bool:
model: Final = _decrypted_param(stored_litellm_params, "model")
return model is not None and model.startswith("auto_router/")
def _names_an_unloaded_credential_param(stored_litellm_params: object) -> bool:
credential_name: Final = _decrypted_param(stored_litellm_params, "litellm_credential_name")
return credential_name is not None and CredentialAccessor.find_credential(credential_name) is None
async def _resync_guardrails(guardrail_name: str) -> bool:

8
litellm/proxy/health_check.py
View file

@ -32,7 +32,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import (
classify_strategy_router_model,
strategy_router_dependencies,
)
from litellm.router_utils.common_utils import is_model_capability_field
from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field
from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params
# Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy
@ -855,6 +855,8 @@ def update_litellm_params_for_health_check(
`deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model`
in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept,
and `modalities` set there without "audio" turns the audio fields off
- sends a SageMaker deployment's `X-Amzn-SageMaker-Inference-Component` extra header as `model_id` when the
deployment sets none, so legacy `sagemaker/` probes reach that component the way routed requests do
"""
incoming_model: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model")
base_model_owner: Final = deployment_model or (incoming_model if isinstance(incoming_model, str) else None)
@ -935,9 +937,11 @@ def update_litellm_params_for_health_check(
audio_output_params: Final = _health_check_audio_output_params(
_MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner
)
model_id_pin: Final = deployment_model_id_pin(probe_params)
model_id_params: Final = {} if model_id_pin is None else {"model_id": model_id_pin}
return {
key: value
for key, value in {**audio_output_params, **probe_params}.items()
for key, value in {**audio_output_params, **model_id_params, **probe_params}.items()
if not is_model_capability_field(key)
}

20
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -42,7 +42,11 @@ from litellm.caching.redis_batch import (
active_request_redis_batch,
)
from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.constants import (
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION,
)
from litellm.cost_calculator import get_usage_object
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
@ -508,11 +512,6 @@ REDIS_NODE_HASHTAG_NAME: Final = "all_keys"
# *some* output budget; these define that fallback estimate.
DEFAULT_MAX_TOKENS_ESTIMATE: Final = 4096
DEFAULT_CHARS_PER_TOKEN: Final = 4
# Fraction of the available output budget reserved as the upfront floor when
# the request omits max_tokens. Applied to both DEFAULT_MAX_TOKENS_ESTIMATE
# (baseline floor) and to the smallest configured TPM limit (capped floor for
# small per-tenant TPM caps).
_TPM_FLOOR_FRACTION: Final = 4
# Both embeddings and the Responses API put their prompt in data["input"],
# but only embeddings have no output tokens. Every "is this an embedding"
# check on data["input"] must exclude these call types, or a Responses call
@ -1003,10 +1002,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
per-tenant cap can't be tripped by the floor alone. Returns the
baseline floor when no limit is provided.
"""
baseline: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION
baseline: Final = int(DEFAULT_MAX_TOKENS_ESTIMATE * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)
if min_configured_tpm_limit is None:
return baseline
return min(baseline, max(1, min_configured_tpm_limit // _TPM_FLOOR_FRACTION))
return min(baseline, max(1, int(min_configured_tpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)))
@staticmethod
def _is_embedding_request(data: object, call_type: str | None) -> bool:
@ -1138,7 +1137,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if call_type in RESPONSES_API_CALL_TYPES
else base_capped_floor
)
baseline_floor: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION
baseline_floor: Final = int(DEFAULT_MAX_TOKENS_ESTIMATE * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)
is_embedding: Final = PROXY_MaxParallelRequestsHandler_v3._is_embedding_request(data, call_type)
if (
capped_floor >= baseline_floor
@ -3313,7 +3312,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if key_agent_id:
return key_agent_id
metadata: Final = data.get("metadata") or {}
return metadata.get("agent_id")
litellm_metadata: Final = data.get("litellm_metadata") or {}
return metadata.get("agent_id") or litellm_metadata.get("agent_id")
def _get_session_id_from_data(self, data: dict) -> str | None:
"""Extract session_id from request metadata or litellm_session_id."""

37
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -37,7 +37,7 @@ from litellm.proxy.db.db_spend_update_writer import (
get_llm_router,
)
from litellm.proxy.hooks.search_free_tier import apply_search_free_tier
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope
from litellm.proxy.spend_tracking.spend_event import (
@ -137,19 +137,34 @@ _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset(
_OBJECT_MAPPING: Final = TypeAdapter(ObjectMapping)
def _proxy_stamped_used_client_oauth_token(
request_data: Mapping[str, object], request_route: str | None
) -> bool | None:
def _proxy_stamped_metadata(request_data: Mapping[str, object], request_route: str | None) -> Mapping[str, object]:
proxy_bucket: Final = (
get_metadata_variable_name_from_kwargs(request_data)
if request_route is None
else metadata_variable_name_for_route(request_route)
else metadata_variable_name_for_request(request_route, request_data)
)
proxy_metadata: Final = request_data.get(proxy_bucket)
stamped: Final = proxy_metadata.get("used_client_oauth_token") if isinstance(proxy_metadata, dict) else None
return _str_keyed_fields(request_data.get(proxy_bucket))
def _proxy_stamped_used_client_oauth_token(
request_data: Mapping[str, object], request_route: str | None
) -> bool | None:
stamped: Final = _proxy_stamped_metadata(request_data, request_route).get("used_client_oauth_token")
return stamped if isinstance(stamped, bool) else None
_PROXY_STAMPED_REQUEST_FIELDS: Final = ("requester_ip_address", "user_agent")
def _proxy_stamped_request_fields(
request_data: Mapping[str, object], request_route: str | None
) -> Mapping[str, object]:
if request_route is None:
return {}
proxy_metadata: Final = _proxy_stamped_metadata(request_data, request_route)
return {field: proxy_metadata[field] for field in _PROXY_STAMPED_REQUEST_FIELDS if field in proxy_metadata}
def _str_keyed_fields(value: object) -> Mapping[str, object]:
return value if is_str_keyed_mapping(value) else {}
@ -455,7 +470,13 @@ class _ProxyDBLogger(CustomLogger):
_metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route)
existing_metadata: Final[dict] = request_data.get("metadata", None) or {}
existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata})
existing_metadata.update(
{
**_routed_metadata_fallback(request_fields),
**_proxy_stamped_request_fields(request_fields, request_route),
**_metadata,
}
)
litellm_metadata_bucket: Final = request_data.get("litellm_metadata")
litellm_metadata: Final[ObjectMapping] = (

105
litellm/proxy/litellm_pre_call_utils.py
View file

@ -379,6 +379,8 @@ LITELLM_METADATA_ROUTES: Final = (
"interactions",
)
CURSOR_CHAT_COMPLETIONS_ROUTE: Final = "/cursor/chat/completions"
LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
{
"mask_input",
@ -636,7 +638,7 @@ def _has_tag_budget(tag: "LiteLLM_TagTable | None") -> bool:
return tag.litellm_budget_table.max_budget is not None
def _normalized_metadata_slot(
def normalized_metadata_slot(
request_data: MutableMapping[str, object], metadata_variable_name: str
) -> dict[str, object]:
"""Return the request's metadata slot as a dict, normalising it in place first.
@ -653,6 +655,12 @@ def _normalized_metadata_slot(
return normalized
def _holds_metadata_object(value: object) -> bool:
if value is None or isinstance(value, dict):
return True
return isinstance(value, str) and isinstance(_DECODED_JSON.validate_python(safe_json_loads(value)), dict)
def _metadata_object_or_error(field: str, value: object) -> Mapping[str, object] | ProxyException:
if value is None:
return {}
@ -863,7 +871,7 @@ def _strip_router_reserved_metadata(
)
def get_metadata_variable_name(request: Request) -> str:
def get_metadata_variable_name(request: Request, data: Mapping[str, object] | None = None) -> str:
"""
Helper to return what the "metadata" field should be called in the request data
@ -874,7 +882,10 @@ def get_metadata_variable_name(request: Request) -> str:
# Inline imports — auth_utils/route_checks participate in a proxy import cycle.
from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415
return metadata_variable_name_for_route(get_request_route(request))
route: Final = get_request_route(request)
if data is None:
return metadata_variable_name_for_route(route)
return metadata_variable_name_for_request(route, data)
_get_metadata_variable_name: Final = get_metadata_variable_name
@ -890,6 +901,25 @@ def metadata_variable_name_for_route(route: str) -> Literal["metadata", "litellm
return "metadata"
def is_chat_completions_body(data: Mapping[str, object]) -> bool:
messages: Final = data.get("messages")
if isinstance(messages, list) and messages:
return True
return "messages" in data and "input" not in data
def is_cursor_responses_request(route: str, body: Mapping[str, object]) -> bool:
return CURSOR_CHAT_COMPLETIONS_ROUTE in route and not is_chat_completions_body(body)
def metadata_variable_name_for_request(
route: str, body: Mapping[str, object]
) -> Literal["metadata", "litellm_metadata"]:
if is_cursor_responses_request(route, body):
return "litellm_metadata"
return metadata_variable_name_for_route(route)
def _promoted_trace_control_fields(
requester_metadata: Mapping[str, object],
litellm_metadata: Mapping[str, object],
@ -2309,7 +2339,9 @@ class LiteLLMProxyRequestSetup:
``_tag_max_budget_check`` resolve to ``litellm_metadata``, leaving
header tags invisible to per-tag budget enforcement.
"""
if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES):
if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES) or is_cursor_responses_request(
route, request_data
):
request_data.setdefault("litellm_metadata", {})
@staticmethod
@ -2380,7 +2412,7 @@ class LiteLLMProxyRequestSetup:
return
tags: Final = _STRING_TAGS_ADAPTER.validate_python(raw_tags)
slot: Final = metadata_key if metadata_key is not None else get_metadata_variable_name_from_kwargs(request_data)
metadata: Final = _normalized_metadata_slot(request_data, slot)
metadata: Final = normalized_metadata_slot(request_data, slot)
existing: Final = metadata.get("tags")
if existing and not isinstance(existing, list):
return
@ -2393,6 +2425,34 @@ class LiteLLMProxyRequestSetup:
tags=tuple(tag for tag in tags if tag not in sent_tags)
)
@staticmethod
def merge_caller_body_tags(
request_data: MutableMapping[str, object], # mutable-ok: the request body the tag merges write into
metadata_variable_name: str | None = None,
) -> None:
slot: Final = (
metadata_variable_name
if metadata_variable_name is not None
else get_metadata_variable_name_from_kwargs(request_data)
)
if slot == "metadata" or not _holds_metadata_object(request_data.get(slot)):
return
raw_caller_metadata: Final = request_data.get("metadata")
caller_metadata: Final = (
_DECODED_JSON.validate_python(safe_json_loads(raw_caller_metadata))
if isinstance(raw_caller_metadata, str)
else raw_caller_metadata
)
caller_tags: Final = caller_metadata.get("tags") if isinstance(caller_metadata, dict) else None
if not isinstance(caller_tags, list) or not caller_tags:
return
metadata: Final = normalized_metadata_slot(request_data, slot)
existing_tags: Final = metadata.get("tags")
metadata["tags"] = LiteLLMProxyRequestSetup.merge_tags(
request_tags=existing_tags if isinstance(existing_tags, list) else None,
tags_to_add=caller_tags,
)
@staticmethod
def apply_key_tags_pre_auth(
request_data: dict,
@ -2408,7 +2468,7 @@ class LiteLLMProxyRequestSetup:
return
_metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_data)
metadata: Final = _normalized_metadata_slot(request_data, _metadata_variable_name)
metadata: Final = normalized_metadata_slot(request_data, _metadata_variable_name)
existing_tags: Final = metadata.get("tags")
metadata["tags"] = LiteLLMProxyRequestSetup.merge_tags(
@ -2460,7 +2520,7 @@ class LiteLLMProxyRequestSetup:
# from (litellm_metadata vs metadata) so the merged tags are visible
# to _tag_max_budget_check.
_metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_data)
metadata: Final = _normalized_metadata_slot(request_data, _metadata_variable_name)
metadata: Final = normalized_metadata_slot(request_data, _metadata_variable_name)
existing_tags: Final = metadata.get("tags")
metadata["tags"] = LiteLLMProxyRequestSetup.merge_tags(
@ -2637,7 +2697,7 @@ async def add_litellm_data_to_request(
}
safe_add_api_version_from_query_params(data, request)
_metadata_variable_name: Final = get_metadata_variable_name(request)
_metadata_variable_name: Final = get_metadata_variable_name(request, _REQUEST_FIELDS.validate_python(data))
if data.get(_metadata_variable_name, None) is None:
data[_metadata_variable_name] = {}
@ -2974,17 +3034,7 @@ async def add_litellm_data_to_request(
tags_to_add=tags,
)
_caller_body_metadata: Final = data.get("metadata") if _metadata_variable_name != "metadata" else None
_caller_body_tags: Final = (
_caller_body_metadata.get("tags")
if isinstance(_caller_body_metadata, dict) and isinstance(_caller_body_metadata.get("tags"), list)
else None
)
if _caller_body_tags:
data[_metadata_variable_name]["tags"] = LiteLLMProxyRequestSetup.merge_tags( # rebind-ok: matches file idiom
request_tags=data[_metadata_variable_name].get("tags"),
tags_to_add=_caller_body_tags,
)
LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=data, metadata_variable_name=_metadata_variable_name)
# caller_tags: the tags the request's headers and body carry (x-litellm-tags, body "tags", and body
# "metadata.tags" / "litellm_metadata.tags" read before the mapped tags are added below),
@ -3138,9 +3188,9 @@ def _fallback_lists(
) -> tuple[Sequence[object], ...]:
"""Every fallback list the router can route this request through: the three
router-level lists, the same three keys from the request body (which replace
the router's when present, so the union is the safe superset), and the
key/team ``router_settings.fallbacks`` the proxy prefers over router-level
fallbacks in ``_configured_fallbacks``."""
the router's when present, so the union is the safe superset), and the same
three keys from key/team ``router_settings``, which the proxy forwards into
the request after this discovery runs."""
lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found
configured: Final = llm_router.settings()
for router_list in (
@ -3155,11 +3205,12 @@ def _fallback_lists(
if isinstance(entries, list):
lists.append(cast("Sequence[object]", entries)) # cast-ok: request fallback lists hold mixed entry shapes
for settings in (router_settings, team_router_settings):
key_fallbacks: object = settings.get("fallbacks") if settings is not None else None
if isinstance(key_fallbacks, list):
lists.append(
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
)
for key in _REQUEST_FALLBACK_KEYS:
key_fallbacks: object = settings.get(key) if settings is not None else None
if isinstance(key_fallbacks, list):
lists.append(
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
)
return tuple(lists)

35
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -12,7 +12,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm import constants
from litellm._logging import verbose_proxy_logger
from litellm.constants import PTU_SENTINEL_API_KEY
from litellm.constants import PTU_SENTINEL_API_KEY, SUBSCRIPTION_BACKED_PROVIDERS
from litellm.proxy._types import CommonProxyErrors
from litellm.proxy.spend_tracking.key_metadata_recovery import (
attach_user_details,
@ -295,6 +295,12 @@ def _entity_metadata(
return stored if stored is not None else {}
def provider_bucket_metadata(provider: str, base: Mapping[str, object]) -> Mapping[str, object]:
if provider in SUBSCRIPTION_BACKED_PROVIDERS:
return {**base, "subscription_covered": True}
return dict(base)
def update_breakdown_metrics(
breakdown: BreakdownMetrics,
record: DailySpendRecord,
@ -395,7 +401,7 @@ def update_breakdown_metrics(
if provider not in breakdown.providers:
breakdown.providers[provider] = MetricWithMetadata(
metrics=SpendMetrics(),
metadata=provider_metadata.get(provider, {}), # Add any provider-specific metadata here
metadata=provider_bucket_metadata(provider, provider_metadata.get(provider, {})),
)
breakdown.providers[provider].metrics = update_metrics(breakdown.providers[provider].metrics, record)
@ -795,12 +801,23 @@ def _aggregate_grouping_sets_records_sync(
grouped_data[date_str] = bucket
return bucket
def assign_metric_with_metadata(target: dict[str, MetricWithMetadata], key: str, metrics: SpendMetrics) -> None:
def assign_metric_with_metadata(
target: dict[str, MetricWithMetadata],
key: str,
metrics: SpendMetrics,
metadata: Mapping[str, object] | None = None,
) -> None:
existing: Final = target.get(key)
if existing is None:
target[key] = MetricWithMetadata(metrics=metrics, metadata={})
else:
existing.metrics = sum_spend_metrics((existing.metrics, metrics))
target[key] = (
MetricWithMetadata(metrics=metrics, metadata=dict(metadata or {}))
if existing is None
else existing.model_copy(
update={
"metrics": sum_spend_metrics((existing.metrics, metrics)),
"metadata": {**existing.metadata, **(metadata or {})},
}
)
)
def assign_api_key_breakdown(
target: dict[str, MetricWithMetadata],
@ -864,7 +881,9 @@ def _aggregate_grouping_sets_records_sync(
# provider the base build reported.
provider_metrics = metrics.model_copy(update={"flat_cost": 0.0})
provider = record.custom_llm_provider or "unknown"
assign_metric_with_metadata(breakdown.providers, provider, provider_metrics)
assign_metric_with_metadata(
breakdown.providers, provider, provider_metrics, provider_bucket_metadata(provider, {})
)
elif level == _GROUP_DATE_PROVIDER_API_KEY:
if not is_ptu_sentinel:
provider = record.custom_llm_provider or "unknown"

95
litellm/proxy/management_endpoints/fallback_management_endpoints.py
View file

@ -14,10 +14,12 @@ import json
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Annotated, Final, Literal
from pydantic import Field, JsonValue, TypeAdapter
from pydantic import Field, JsonValue, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import fallback_target_model_name
from litellm.proxy.auth.model_checks import get_all_fallbacks
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.utils import PrismaClient, invalidate_config_param
@ -26,6 +28,7 @@ if TYPE_CHECKING:
from fastapi import APIRouter, Depends, HTTPException, status
from litellm.proxy.proxy_server import ProxyConfig
from litellm.router import Router
else:
try:
from fastapi import APIRouter, Depends, HTTPException, status
@ -41,6 +44,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import (
FallbackGetResponse,
FallbackResponse,
)
from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbackRules, FallbackTargets
router: Final = APIRouter()
@ -50,12 +54,83 @@ StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_m
_STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback])
_JSON_RULES: Final = TypeAdapter(list[JsonValue])
_STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue])
_OPTIONAL_JSON_RULES: Final[TypeAdapter[Sequence[JsonValue] | None]] = TypeAdapter(Sequence[JsonValue] | None)
_CONTENT_POLICY_FALLBACK_RULE: Final[TypeAdapter[ContentPolicyFallbackRule]] = TypeAdapter(ContentPolicyFallbackRule)
_FALLBACK_TARGETS: Final[TypeAdapter[FallbackTargets]] = TypeAdapter(FallbackTargets)
def _rule_covers(entry: StoredFallback, model: str) -> bool:
return isinstance(entry, dict) and model in entry
def _layered_over_config_rules(
runtime_rules: ContentPolicyFallbackRules, config_rules: Sequence[Mapping[str, JsonValue]]
) -> Sequence[Mapping[str, object]]:
covered: Final = frozenset[str]().union(*runtime_rules)
return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))]
def _config_file_rules(rules: object) -> Sequence[Mapping[str, JsonValue]]:
try:
candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
except ValidationError:
return ()
return tuple(rule for rule in candidates if isinstance(rule, Mapping))
def _log_ignored_content_policy_rules(rules: object, error: ValidationError) -> None:
verbose_proxy_logger.error(
"Ignoring invalid content_policy_fallbacks %r from the DB, every other rule still applies. "
"Fix it through POST /fallback or /config/update: %s",
rules,
error,
)
def _checked_content_policy_rule(rule: JsonValue) -> ContentPolicyFallbackRule | ValidationError:
try:
return _CONTENT_POLICY_FALLBACK_RULE.validate_python(rule)
except ValidationError as invalid_rule:
return invalid_rule
def _checked_content_policy_rules(
rules: object,
) -> Sequence[tuple[object, ContentPolicyFallbackRule | ValidationError]]:
try:
candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
except ValidationError as not_a_rule_list:
return ((rules, not_a_rule_list),)
return tuple((candidate, _checked_content_policy_rule(candidate)) for candidate in candidates)
def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None:
checked_rules: Final = _checked_content_policy_rules(rules)
runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError))
layered_rules: Final = (
_layered_over_config_rules(
runtime_rules,
_config_file_rules(
litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list
),
)
or None
)
for rule, result in checked_rules:
if log_skipped_rules and isinstance(result, ValidationError):
_log_ignored_content_policy_rules(rule, result)
if layered_rules == llm_router.settings().content_policy_fallbacks: # pyright: ignore[reportUnknownMemberType] # untyped RouterSettings list
return
llm_router.apply(ChangeSet(settings={"content_policy_fallbacks": layered_rules}))
def _apply_fallback_rules(llm_router: "Router", fallback_key: str, rules: Sequence[StoredFallback]) -> None:
if fallback_key == "content_policy_fallbacks":
apply_content_policy_fallbacks(llm_router, rules, log_skipped_rules=True)
return
llm_router.apply(ChangeSet(assign={fallback_key: rules}))
async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None:
await proxy_config.get_config()
@ -83,6 +158,7 @@ def _editable_rules(
async def _write_rules(
prisma_client: PrismaClient,
proxy_config: "ProxyConfig",
llm_router: "Router",
stored_settings: Mapping[str, JsonValue],
fallback_key: str,
rules: Sequence[StoredFallback],
@ -91,8 +167,10 @@ async def _write_rules(
section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)}
)
if proxy_config.router_settings.owned_by_config(fallback_key):
llm_router.apply(ChangeSet(assign={fallback_key: rules}))
return
await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules})
_apply_fallback_rules(llm_router, fallback_key, rules)
async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None:
@ -218,10 +296,7 @@ async def create_fallback(
# Add new fallback
existing_fallbacks.append({data.model: data.fallback_models})
await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks)
# Update the in-memory router configuration
llm_router.apply(ChangeSet(assign={fallback_key: existing_fallbacks}))
await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, existing_fallbacks)
verbose_proxy_logger.info(
"Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type
@ -277,7 +352,10 @@ async def get_fallback(
)
# Get fallbacks using the existing utility function
fallback_models: Final = get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
fallback_targets: Final = _FALLBACK_TARGETS.validate_python(
get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
)
fallback_models: Final = [fallback_target_model_name(target) or model for target in fallback_targets]
if not fallback_models:
raise HTTPException(
@ -367,10 +445,7 @@ async def delete_fallback(
detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"},
)
await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks)
# Update the in-memory router configuration
llm_router.apply(ChangeSet(assign={fallback_key: updated_fallbacks}))
await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, updated_fallbacks)
verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type)

36
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -13,6 +13,7 @@ These are members of a Team on LiteLLM
"""
import asyncio
import contextlib
import json
import traceback
from collections.abc import Awaitable, Mapping, Sequence
@ -443,30 +444,25 @@ async def add_new_user_to_default_team(
teams: Sequence[str | NewUserRequestTeam],
prisma_client: "PrismaClient",
):
tasks: Final[list[Awaitable[object]]] = []
for team in teams:
user_role: Literal["user", "admin"] = "user"
max_budget_in_team: float | None = None
if isinstance(team, str):
team_id = team
elif isinstance(team, NewUserRequestTeam):
team_id = team.team_id
user_role = team.user_role
max_budget_in_team = team.max_budget_in_team
else:
raise ValueError(f"Invalid team type: {type(team)}")
tasks.append(
_add_user_to_team(
memberships: Final = tuple(_default_team_membership(team) for team in teams)
for membership in memberships:
with contextlib.suppress(Exception):
await _add_user_to_team(
user_id=user_id,
team_id=team_id,
team_id=membership.team_id,
user_email=user_email,
user_api_key_dict=user_api_key_dict,
max_budget_in_team=max_budget_in_team,
user_role=user_role,
max_budget_in_team=membership.max_budget_in_team,
user_role=membership.user_role,
)
)
await asyncio.gather(*tasks, return_exceptions=True)
def _default_team_membership(team: str | NewUserRequestTeam) -> NewUserRequestTeam:
if isinstance(team, str):
return NewUserRequestTeam(team_id=team)
if isinstance(team, NewUserRequestTeam):
return team
raise ValueError(f"Invalid team type: {type(team)}")
async def _fetch_user_team_ids(user_id: str, prisma_client: "PrismaClient") -> tuple[str, ...]:

315
litellm/proxy/management_endpoints/model_access_group_management_endpoints.py
View file

@ -12,23 +12,30 @@ Endpoints here:
- DELETE /access_group/{access_group}/budget - Clear an access group's shared budget
"""
import asyncio
import contextlib
import itertools
import json
from collections.abc import Mapping, Sequence
from datetime import datetime
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from datetime import datetime, timedelta
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Any, Final, Protocol
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.config_sync_pubsub import publish_model_rows_written
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
model_access_group_cache_key,
model_access_group_registry_cache_key,
)
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper
from litellm.proxy.management_endpoints.common_utils import validate_budget_duration
# Clear cache and reload models to pick up the access group changes
@ -116,6 +123,28 @@ class _ModelTableClient(Protocol):
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
class _ModelTableBatchActions(Protocol):
def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ...
class _ModelTableBatch(Protocol):
@property
def litellm_proxymodeltable(self) -> _ModelTableBatchActions: ...
class _ModelTableTxActions(Protocol):
async def find_many(self) -> Sequence[_DeploymentRow]: ...
class _ModelTableTx(Protocol):
@property
def litellm_proxymodeltable(self) -> _ModelTableTxActions: ...
async def query_raw(self, query: str, *args: object) -> object: ...
def batch_(self) -> contextlib.AbstractAsyncContextManager[_ModelTableBatch]: ...
class _BudgetRow(Protocol):
@property
def budget_id(self) -> str: ...
@ -168,7 +197,42 @@ class _ModelAccessGroupBudgetTableClient(Protocol):
def _model_table(prisma_client: PrismaClient) -> _ModelTableClient:
return ModelRepository(prisma_client).table
return ModelRepository(WriterPinnedClient(prisma_client.db)).table
_TRY_LOCK_ACCESS_GROUP: Final = (
"SELECT pg_try_advisory_xact_lock(hashtext('model_access_group'), hashtext($1)) AS locked"
)
_LOCK_MODEL_ROWS: Final = (
'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) '
"ORDER BY model_id FOR UPDATE SKIP LOCKED"
)
_MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30)
_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds()
_LOCK_RETRY_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8)
_LOCK_RETRY_LONGEST_DELAY_SECONDS: Final = 1.0
class _LockedModelRow(BaseModel):
model_config = ConfigDict(frozen=True)
model_id: str
_LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...])
class _AccessGroupLock(BaseModel):
model_config = ConfigDict(frozen=True)
locked: bool
_ACCESS_GROUP_LOCK: Final = TypeAdapter(tuple[_AccessGroupLock])
def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]:
return writer_wrapper(prisma_client.db).tx(timeout=_MEMBERSHIP_TX_TIMEOUT)
def _model_access_group_budget_table(prisma_client: PrismaClient) -> _ModelAccessGroupBudgetTableClient:
@ -379,11 +443,10 @@ async def _tag_deployment_with_access_group(
"""Write `access_group` into one deployment's model_info; returns the
(model_id, updated model_info) pair when a write happened, None when the
deployment already carried the group."""
updated_model_info, was_modified = add_access_group_to_deployment(
model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
access_group=access_group,
updated_model_info: Final = _model_info_with_access_group(
model_id=model_id, model_info=model_info, access_group=access_group
)
if not was_modified:
if updated_model_info is None:
return None
await _model_table(prisma_client).update(
where={"model_id": model_id},
@ -393,6 +456,141 @@ async def _tag_deployment_with_access_group(
return (model_id, updated_model_info)
def _model_info_with_access_group(model_id: str, model_info: object, access_group: str) -> Mapping[str, object] | None:
updated_model_info, was_modified = add_access_group_to_deployment(
model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
access_group=access_group,
)
return updated_model_info if was_modified else None
def _model_info_without_access_group(
model_id: str, model_info: object, access_group: str
) -> Mapping[str, object] | None:
updated_model_info, was_modified = remove_access_group_from_deployment(
model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
access_group=access_group,
)
return updated_model_info if was_modified else None
def _membership_rewrite(deployment: _DeploymentRow, selected: bool, access_group: str) -> Mapping[str, object] | None:
if selected:
return _model_info_with_access_group(
model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group
)
return _model_info_without_access_group(
model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group
)
def _membership_rewrites(
deployments: Sequence[_DeploymentRow], selected_model_ids: frozenset[str], access_group: str
) -> tuple[tuple[str, Mapping[str, object]], ...]:
rewrites: Final = (
(
deployment.model_id,
_membership_rewrite(
deployment=deployment,
selected=deployment.model_id in selected_model_ids,
access_group=access_group,
),
)
for deployment in deployments
)
return tuple((model_id, model_info) for model_id, model_info in rewrites if model_info is not None)
def _carries_access_group(model_info: object, access_group: str) -> bool:
access_groups: Final = (model_info_as_mapping(model_info) or {}).get("access_groups")
return isinstance(access_groups, list) and access_group in access_groups
def _model_ids_to_lock(
deployments: Sequence[_DeploymentRow], access_group: str, data: UpdateModelGroupRequest
) -> frozenset[str]:
selected_model_ids: Final = frozenset(data.model_ids or ())
selected_model_names: Final = frozenset(() if data.model_ids else data.model_names or ())
return frozenset(
deployment.model_id
for deployment in deployments
if deployment.model_id in selected_model_ids
or deployment.model_name in selected_model_names
or _carries_access_group(deployment.model_info, access_group)
)
async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client: PrismaClient) -> None:
if access_group not in _access_groups_of_deployments(await _model_table(prisma_client).find_many()):
raise _http_error(404, f"Access group '{access_group}' not found")
async def _rewrite_membership_unless_locked(
prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None:
async with _model_table_tx(prisma_client) as tx:
(group_lock,) = _ACCESS_GROUP_LOCK.validate_python(await tx.query_raw(_TRY_LOCK_ACCESS_GROUP, access_group))
if not group_lock.locked:
return None
model_ids_to_lock: Final = _model_ids_to_lock(
deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data
)
locked_rows: Final = _LOCKED_MODEL_ROWS.validate_python(
await tx.query_raw(_LOCK_MODEL_ROWS, sorted(model_ids_to_lock))
)
if frozenset(row.model_id for row in locked_rows) != model_ids_to_lock:
return None
deployments: Final = tuple(
deployment
for deployment in await tx.litellm_proxymodeltable.find_many()
if deployment.model_id in model_ids_to_lock
)
if access_group not in _access_groups_of_deployments(deployments):
raise HTTPException(
status_code=404,
detail={"error": f"Access group '{access_group}' not found"},
)
selected_deployments: Final = _selected_deployments_or_400(deployments=deployments, data=data)
rewrites: Final = _membership_rewrites(
deployments=deployments,
selected_model_ids=frozenset(deployment.model_id for deployment in selected_deployments),
access_group=access_group,
)
if rewrites:
async with tx.batch_() as batch:
for model_id, model_info in rewrites:
batch.litellm_proxymodeltable.update(
where={"model_id": model_id}, data={"model_info": json.dumps(model_info)}
)
return selected_deployments, rewrites
def _lock_retry_delays() -> Iterator[float]:
return itertools.chain(_LOCK_RETRY_BACKOFF_SECONDS, itertools.repeat(_LOCK_RETRY_LONGEST_DELAY_SECONDS))
async def _rewrite_membership_once_unlocked(
prisma_client: PrismaClient,
access_group: str,
data: UpdateModelGroupRequest,
retry_budget_seconds: float = _LOCK_RETRY_BUDGET_SECONDS,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
clock: Callable[[], float] = time.monotonic,
) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]:
deadline: Final = clock() + retry_budget_seconds
for delay in _lock_retry_delays():
if (
rewritten := await _rewrite_membership_unless_locked(
prisma_client=prisma_client, access_group=access_group, data=data
)
) is not None:
return rewritten
if clock() + delay > deadline:
break
await sleep(delay)
raise _http_error(409, f"Another edit holds access group '{access_group}' or one of its models, try again")
def _readable_model_info_or_raise(model_id: str, model_info: object) -> Mapping[str, object]:
"""These helpers rewrite the model_info column wholesale, so a present-but-unreadable
value must refuse loudly rather than be silently replaced with a fresh object; an
@ -412,11 +610,10 @@ async def _strip_access_group_from_deployment(
"""Remove `access_group` from one deployment's model_info; returns the
(model_id, updated model_info) pair when a write happened, None when the
deployment did not carry the group."""
updated_model_info, was_modified = remove_access_group_from_deployment(
model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
access_group=access_group,
updated_model_info: Final = _model_info_without_access_group(
model_id=model_id, model_info=model_info, access_group=access_group
)
if not was_modified:
if updated_model_info is None:
return None
await _model_table(prisma_client).update(
where={"model_id": model_id},
@ -493,6 +690,35 @@ async def update_specific_deployments_with_access_group(
return tuple(pair for pair in tagged if pair is not None)
def _selected_deployments_or_400(
deployments: Sequence[_DeploymentRow], data: UpdateModelGroupRequest
) -> tuple[_DeploymentRow, ...]:
if data.model_ids:
known_model_ids: Final = frozenset(deployment.model_id for deployment in deployments)
unknown_model_id: Final = next(
(model_id for model_id in data.model_ids if model_id not in known_model_ids), None
)
if unknown_model_id is not None:
raise HTTPException(
status_code=400,
detail={"error": f"Deployment with model_id '{unknown_model_id}' not found in Database."},
)
selected_model_ids: Final = frozenset(data.model_ids)
return tuple(deployment for deployment in deployments if deployment.model_id in selected_model_ids)
model_names: Final = data.model_names or ()
known_model_names: Final = frozenset(deployment.model_name for deployment in deployments)
unknown_model_name: Final = next((name for name in model_names if name not in known_model_names), None)
if unknown_model_name is not None:
raise HTTPException(
status_code=400,
detail={
"error": f"Can't find model '{unknown_model_name}' in Database. Access group management is only supported for database models."
},
)
selected_model_names: Final = frozenset(model_names)
return tuple(deployment for deployment in deployments if deployment.model_name in selected_model_names)
async def _find_deployment_or_400(model_id: str, prisma_client: PrismaClient) -> object:
deployment: Final = await _model_table(prisma_client).find_unique(where={"model_id": model_id})
if deployment is None:
@ -536,10 +762,10 @@ async def get_all_access_groups_from_db(
Returns:
Dict[str, AccessGroupInfo]: Dictionary mapping access_group name to info
"""
# Get all deployments
deployments: Final = await ModelRepository(prisma_client).table.find_many()
return dict(_access_groups_of_deployments(await ModelRepository(prisma_client).table.find_many()))
# Build access group map
def _access_groups_of_deployments(deployments: Sequence[_DeploymentRow]) -> Mapping[str, AccessGroupInfo]:
model_names_by_group: Final[dict[str, list[str]]] = {}
for deployment in deployments:
@ -835,8 +1061,8 @@ async def update_access_group(
Update an access group's model names.
This will:
1. Remove the access group from all current deployments
2. Add the access group to all deployments for the new model_names list
1. Add the access group to all deployments for the new model_names list
2. Remove the access group from every other deployment
Example:
```bash
@ -858,6 +1084,7 @@ async def update_access_group(
Raises:
- HTTPException 400: If any model names don't exist
- HTTPException 404: If access group not found
- HTTPException 409: If another edit holds the group or one of its models for 30 s
"""
from litellm.proxy.proxy_server import llm_router, prisma_client
@ -879,26 +1106,14 @@ async def update_access_group(
detail={"error": "Either model_names or model_ids must be provided and non-empty"},
)
use_model_ids: Final = has_model_ids
# Validation: Check if access group exists
try:
access_groups_map: Final = await get_all_access_groups_from_db(prisma_client=prisma_client)
if access_group not in access_groups_map:
raise HTTPException(
status_code=404,
detail={"error": f"Access group '{access_group}' not found"},
)
await _raise_404_unless_the_primary_carries(access_group=access_group, prisma_client=prisma_client)
except HTTPException:
raise
except Exception as e:
raise HTTPException(
status_code=500,
detail={"error": f"Failed to check access group existence: {e}"},
)
raise _http_error(500, f"Failed to check access group existence: {e}")
# Validation: Check if all new models exist (only if using model_names path)
if not use_model_ids and has_model_names:
if not has_model_ids and has_model_names:
assert data.model_names is not None
missing_models: Final = await _missing_models_after_read_through(
model_names=data.model_names,
@ -912,43 +1127,19 @@ async def update_access_group(
)
try:
# Step 1: Remove access group from ALL DB deployments (skip config models)
all_deployments: Final = await _model_table(prisma_client).find_many()
stripped: Final = [
await _strip_access_group_from_deployment(
model_id=deployment.model_id,
model_info=deployment.model_info,
access_group=access_group,
prisma_client=prisma_client,
)
for deployment in all_deployments
]
stripped_pairs: Final = tuple(pair for pair in stripped if pair is not None)
# Step 2: Add access group using the appropriate method
if use_model_ids:
assert data.model_ids is not None
updated_pairs = await update_specific_deployments_with_access_group(
model_ids=data.model_ids,
access_group=access_group,
prisma_client=prisma_client,
)
else:
assert data.model_names is not None
updated_pairs = await update_deployments_with_access_group(
model_names=data.model_names,
access_group=access_group,
prisma_client=prisma_client,
)
models_updated: Final = len(updated_pairs)
selected_deployments, rewrites = await _rewrite_membership_once_unlocked(
prisma_client=prisma_client, access_group=access_group, data=data
)
if rewrites:
await publish_model_rows_written(tuple(model_id for model_id, _ in rewrites))
models_updated: Final = len(selected_deployments)
# Clear cache and reload models to pick up the access group changes
live_before_reload: Final = live_model_ids_snapshot()
reload_outcome: Final = await clear_cache()
_raise_http_if_reload_degraded_serving(
before=live_before_reload,
written_models=list({**dict(stripped_pairs), **dict(updated_pairs)}.items()),
written_models=rewrites,
access_group=access_group,
still_desired=reload_outcome.still_desired,
live_after=reload_outcome.live_after,

6
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -779,7 +779,11 @@ async def _auto_router_capability_slot(
limit=limit,
)
yield tables.litellm_proxymodeltable
await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable")
await publish_config_change(
redis_cache=coordination_redis_cache(),
object_type="litellm_proxymodeltable",
model_ids=() if model_id is None else (model_id,),
)
ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add"

83
litellm/proxy/management_endpoints/scim/scim_v2.py
View file

@ -229,7 +229,11 @@ class UserProvisionerHelpers:
return None
requested_teams: Final = list(dict.fromkeys(new_user_request.teams or []))
new_teams: Final = requested_teams if requested_teams else list(existing_user.teams or [])
new_teams: Final = (
list(_keep_record_team_order(existing_user.teams or [], requested_teams))
if requested_teams
else list(existing_user.teams or [])
)
if new_user_request.user_id != existing_user.user_id:
verbose_proxy_logger.info(
@ -1165,7 +1169,7 @@ async def _handle_team_membership_changes(
existing_teams_set: Final = set(existing_teams)
new_teams_set: Final = set(new_teams)
teams_to_add: Final = new_teams_set - existing_teams_set
teams_to_add: Final = tuple(team for team in dict.fromkeys(new_teams) if team not in existing_teams_set)
teams_to_remove: Final = existing_teams_set - new_teams_set
if teams_to_add or teams_to_remove:
@ -1948,7 +1952,11 @@ async def update_user(
# SCIM User.groups is readOnly (RFC 7643 4.1.2): IdPs sync membership via /Groups and send
# no groups or `groups: []` on profile PUTs, so empty means unspecified, not "remove from every team"
target_teams: Final = user_data["teams"] or existing_user.teams
target_teams: Final = (
list(_keep_record_team_order(existing_user.teams, user_data["teams"]))
if user_data["teams"]
else existing_user.teams
)
await _handle_team_membership_changes(
user_id=user_id,
existing_teams=existing_user.teams,
@ -2241,19 +2249,29 @@ def _handle_name_update(path: str, op_type: str, value: object, scim_metadata: d
scim_metadata["familyName"] = str(value)
def _handle_group_operations(op_type: str, value: object, teams_set: set[str], path: str | None) -> set[str] | None:
"""Handle group/team membership operations."""
group_values = _extract_group_values(value)
if not group_values and value is None:
group_values = _extract_ids_from_path_filter(path, "groups")
if op_type == "replace":
return set(group_values)
elif op_type == "add":
teams_set.update(group_values)
elif op_type == "remove":
for gid in group_values:
teams_set.discard(gid)
return None
def _is_group_path(path: str | None) -> bool:
return isinstance(path, str) and path.lower().startswith("groups")
def _apply_group_operation(teams: tuple[str, ...], op: SCIMPatchOperation) -> tuple[str, ...]:
listed: Final = _extract_group_values(op.value)
group_values: Final = listed if listed or op.value is not None else _extract_ids_from_path_filter(op.path, "groups")
if op.op == "replace":
return tuple(dict.fromkeys(group_values))
if op.op == "add":
return tuple(dict.fromkeys((*teams, *group_values)))
removed: Final = frozenset(group_values)
return tuple(team for team in teams if team not in removed)
def _keep_record_team_order(record_teams: Sequence[str], target_teams: Sequence[str]) -> tuple[str, ...]:
target: Final = frozenset(target_teams)
record: Final = tuple(dict.fromkeys(record_teams))
known: Final = frozenset(record)
return (
*(team for team in record if team in target),
*(team for team in dict.fromkeys(target_teams) if team not in known),
)
def _multi_valued_attribute_base(path: str) -> str:
@ -2520,8 +2538,8 @@ def _handle_generic_metadata(path: str, op_type: str, value: object, metadata: d
def _apply_patch_ops(
existing_user: LiteLLM_UserTable,
patch_ops: SCIMPatchOp,
) -> tuple[dict[str, object], set[str]]:
"""Apply patch operations and return update data and final team set."""
) -> tuple[dict[str, object], tuple[str, ...]]:
"""Apply patch operations and return update data and the final teams in order."""
update_data: Final[dict[str, object]] = {}
stored_metadata: Final[object] = TypeAdapter(object).validate_python(existing_user.model_dump().get("metadata"))
metadata: Final = _JSON_OBJECT_ADAPTER.validate_python(stored_metadata) if isinstance(stored_metadata, dict) else {}
@ -2541,9 +2559,6 @@ def _apply_patch_ops(
if isinstance(enterprise, _InvalidEnterpriseValue):
_raise_invalid_enterprise_value(enterprise)
teams_set: Final[set[str]] = set(existing_user.teams or [])
replace_team_set: set[str] | None = None
for op in patch_ops.Operations:
path = op.path.lower() if isinstance(op.path, str) else ""
value = op.value
@ -2585,16 +2600,20 @@ def _apply_patch_ops(
_handle_name_update(path, op_type, value, scim_metadata)
elif _multi_valued_attribute_base(path) in SCIM_MULTI_VALUED_ATTRIBUTE_METADATA_KEYS:
_handle_multi_valued_attribute_update(path, op_type, value, metadata)
elif path.startswith("groups"):
new_replace_set = _handle_group_operations(op_type, value, teams_set, op.path)
if new_replace_set is not None:
replace_team_set = new_replace_set
elif _enterprise_extension_sub_path(path) is not None:
elif _is_group_path(path) or _enterprise_extension_sub_path(path) is not None:
continue
else:
_handle_generic_metadata(path, op_type, value, metadata)
final_team_set: Final = replace_team_set if replace_team_set is not None else teams_set
record_teams: Final = tuple(existing_user.teams or [])
group_ops: Final = tuple(op for op in patch_ops.Operations if _is_group_path(op.path))
last_replace: Final = next((op for op in reversed(group_ops) if op.op == "replace"), None)
final_teams: Final = _keep_record_team_order(
record_teams,
_apply_group_operation(record_teams, last_replace)
if last_replace is not None
else reduce(_apply_group_operation, group_ops, record_teams),
)
final_metadata: Final = (
{
**{key: metadata_value for key, metadata_value in metadata.items() if key != SCIM_ENTERPRISE_METADATA_KEY},
@ -2605,7 +2624,7 @@ def _apply_patch_ops(
else {**metadata, "scim_metadata": scim_metadata}
)
update_data["metadata"] = final_metadata
return update_data, final_team_set
return update_data, final_teams
def _is_user_not_in_team_error(exc: HTTPException) -> bool:
@ -2758,7 +2777,7 @@ async def patch_user(
prev_active: Final = _user_scim_active(existing_user)
update_data, final_team_set = _apply_patch_ops(
update_data, final_teams = _apply_patch_ops(
existing_user=existing_user,
patch_ops=patch_ops,
)
@ -2770,15 +2789,15 @@ async def patch_user(
await _handle_team_membership_changes(
user_id=user_id,
existing_teams=existing_user.teams or [],
new_teams=list(final_team_set),
new_teams=list(final_teams),
)
update_data["teams"] = list(final_team_set)
update_data["teams"] = list(final_teams)
admin_group: Final = await _get_scim_admin_group()
if admin_group is not None:
update_data["user_role"] = _resolve_scim_user_role(
await _scim_groups_from_team_ids(prisma_client, list(final_team_set)),
await _scim_groups_from_team_ids(prisma_client, list(final_teams)),
admin_group,
_default_scim_user_role(),
)

335
litellm/proxy/management_endpoints/subscription_accounts.py Normal file
View file

@ -0,0 +1,335 @@
"""
Subscription accounts: the operator-entered recurring fee of a provider account whose
requests are covered by a flat subscription, shown beside metered spend on the Usage page.
"""
import re
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass
from datetime import date
from itertools import chain
from typing import TYPE_CHECKING, Annotated, Final, TypeAlias
from fastapi import APIRouter, Depends, HTTPException, Query, status
from pydantic import TypeAdapter
from litellm.constants import LITELLM_PROXY_ADMIN_NAME, SUBSCRIPTION_BACKED_PROVIDERS
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.llms.subscription_accounts import resolve_subscription_account_id
from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth, user_api_key_has_admin_view
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.spend_tracking.subscription_billing_periods import billing_periods_starting_within
from litellm.proxy.utils import get_prisma_client_or_throw
from litellm.repositories.subscription_account_repository import (
DuplicateSubscriptionAccount,
KeepLabel,
SetLabel,
SubscriptionAccountFeeChanges,
SubscriptionAccountFeeFields,
SubscriptionAccountRepository,
SubscriptionAccountStore,
)
from litellm.types.proxy.management_endpoints.subscription_accounts import (
BillingPeriod,
DeleteSubscriptionAccountRequest,
SubscriptionAccountFeeRequest,
SubscriptionAccountUsage,
SubscriptionUsageResponse,
UpdateSubscriptionAccountFeeRequest,
)
from litellm.types.router import DeploymentTypedDict
if TYPE_CHECKING:
from litellm.router import Router
router: Final = APIRouter(prefix="/subscription_accounts", tags=["subscription accounts"])
Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]
@dataclass(frozen=True, slots=True)
class SubscriptionDeployment:
model_name: str
custom_llm_provider: str
account_id: str | None
AccountIdResolver: TypeAlias = Callable[[str, Mapping[str, object]], str | None]
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _declared_provider(litellm_params: Mapping[str, object]) -> str | None:
model: Final = litellm_params.get("model")
configured_provider: Final = litellm_params.get("custom_llm_provider")
return declared_authenticating_provider(
model if isinstance(model, str) else None,
configured_provider if isinstance(configured_provider, str) else None,
)
def _model_info(deployment: Mapping[str, object]) -> Mapping[str, object]:
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
def _public_model_name(entry: DeploymentTypedDict) -> str:
team_public_model_name: Final = _model_info(entry).get("team_public_model_name")
return team_public_model_name if isinstance(team_public_model_name, str) else entry["model_name"]
def _subscription_deployment(
entry: DeploymentTypedDict, account_id_resolver: AccountIdResolver
) -> SubscriptionDeployment | None:
litellm_params: Final = entry["litellm_params"]
provider: Final = _declared_provider(litellm_params)
if provider is None or provider not in SUBSCRIPTION_BACKED_PROVIDERS:
return None
return SubscriptionDeployment(
model_name=_public_model_name(entry),
custom_llm_provider=provider,
account_id=account_id_resolver(provider, litellm_params),
)
def subscription_deployments_from_model_list(
model_list: Iterable[DeploymentTypedDict], account_id_resolver: AccountIdResolver
) -> tuple[SubscriptionDeployment, ...]:
candidates: Final = (_subscription_deployment(entry, account_id_resolver) for entry in model_list)
return tuple(candidate for candidate in candidates if candidate is not None)
def subscription_deployments_of_router(
llm_router: "Router | None", account_id_resolver: AccountIdResolver
) -> tuple[SubscriptionDeployment, ...]:
if llm_router is None:
return ()
registry: Final = llm_router.registry()
indexed_names: Final = tuple(registry.index("model_name"))
deployments: Final = chain.from_iterable(registry.deployments(name, source="named") or () for name in indexed_names)
return subscription_deployments_from_model_list(deployments, account_id_resolver)
def get_subscription_deployments() -> tuple[SubscriptionDeployment, ...]:
from litellm.proxy.proxy_server import llm_router
return subscription_deployments_of_router(llm_router, resolve_subscription_account_id)
def get_subscription_account_repository() -> SubscriptionAccountStore:
return SubscriptionAccountRepository(get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value))
Store: TypeAlias = Annotated[SubscriptionAccountStore, Depends(get_subscription_account_repository)]
Deployments: TypeAlias = Annotated[tuple[SubscriptionDeployment, ...], Depends(get_subscription_deployments)]
def _forbidden(user_api_key_dict: UserAPIKeyAuth) -> HTTPException:
return HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail={"error": f"{CommonProxyErrors.not_allowed_access.value}, your role={user_api_key_dict.user_role}"},
)
def _account_usage(
custom_llm_provider: str,
account_id: str | None,
deployments: Iterable[SubscriptionDeployment],
fee_row: LiteLLM_SubscriptionAccountTable | None,
start_date: date,
end_date: date,
) -> SubscriptionAccountUsage:
model_names: Final = tuple(sorted({deployment.model_name for deployment in deployments}))
if fee_row is None:
return SubscriptionAccountUsage(
custom_llm_provider=custom_llm_provider,
account_id=account_id,
deployments=model_names,
fee=None,
billing_periods=(),
fixed_cost=None,
)
periods: Final = billing_periods_starting_within(
date.fromisoformat(fee_row.billing_period_start), start_date, end_date
)
return SubscriptionAccountUsage(
custom_llm_provider=custom_llm_provider,
account_id=account_id,
deployments=model_names,
fee=fee_row,
billing_periods=tuple(BillingPeriod(start=period.start, end=period.end) for period in periods),
fixed_cost=len(periods) * fee_row.monthly_fee,
)
def build_subscription_usage(
deployments: Iterable[SubscriptionDeployment],
fee_rows: Iterable[LiteLLM_SubscriptionAccountTable],
start_date: date,
end_date: date,
) -> SubscriptionUsageResponse:
deployment_tuple: Final = tuple(deployments)
fees_by_account: Final = {(row.custom_llm_provider, row.account_id): row for row in fee_rows}
deployment_keys: Final = {(d.custom_llm_provider, d.account_id) for d in deployment_tuple}
account_keys: Final = sorted(deployment_keys | set(fees_by_account), key=lambda key: (key[0], key[1] or "~"))
accounts: Final = tuple(
_account_usage(
provider,
account_id,
(d for d in deployment_tuple if (d.custom_llm_provider, d.account_id) == (provider, account_id)),
fees_by_account.get((provider, account_id)) if account_id is not None else None,
start_date,
end_date,
)
for provider, account_id in account_keys
)
return SubscriptionUsageResponse(
start_date=start_date,
end_date=end_date,
subscription_providers=tuple(sorted(SUBSCRIPTION_BACKED_PROVIDERS)),
accounts=accounts,
)
_DAY_PATTERN: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
def _parse_day(value: str) -> date | None:
if _DAY_PATTERN.fullmatch(value) is None:
return None
try:
return date.fromisoformat(value)
except ValueError:
return None
def _parse_date(value: str, name: str) -> date:
parsed: Final = _parse_day(value)
if parsed is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail={"error": f"{name} must be YYYY-MM-DD, got {value!r}"}
)
return parsed
@router.get("/usage", response_model=SubscriptionUsageResponse, dependencies=[Depends(user_api_key_auth)])
async def get_subscription_usage(
start_date: Annotated[str, Query(description="First day of the Usage page filter, YYYY-MM-DD")],
end_date: Annotated[str, Query(description="Last day of the Usage page filter, YYYY-MM-DD")],
user_api_key_dict: Auth,
repository: Store,
deployments: Deployments,
) -> SubscriptionUsageResponse:
"""
Subscription-backed accounts behind the proxy's deployments or carrying a stored fee, each with
its configured monthly fee and the fixed cost attributed to the date range: the fee once per billing
period whose start date falls inside the range, however many deployments share the account.
"""
if not user_api_key_has_admin_view(user_api_key_dict):
raise _forbidden(user_api_key_dict)
start: Final = _parse_date(start_date, "start_date")
end: Final = _parse_date(end_date, "end_date")
if end < start:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail={"error": "end_date must not be before start_date"}
)
return build_subscription_usage(deployments, await repository.list_all(), start, end)
def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None:
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
raise _forbidden(user_api_key_dict)
def _actor(user_api_key_dict: UserAPIKeyAuth) -> str:
return user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME
@router.post("/new", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)])
async def new_subscription_account(
request: SubscriptionAccountFeeRequest,
user_api_key_dict: Auth,
repository: Store,
) -> LiteLLM_SubscriptionAccountTable:
"""Record the monthly fee of one subscription-backed provider account."""
_require_proxy_admin(user_api_key_dict)
if request.custom_llm_provider not in SUBSCRIPTION_BACKED_PROVIDERS:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={
"error": f"{request.custom_llm_provider!r} is not a subscription-backed provider; "
f"expected one of {sorted(SUBSCRIPTION_BACKED_PROVIDERS)}"
},
)
created: Final = await repository.create_account(
request.custom_llm_provider,
request.account_id,
SubscriptionAccountFeeFields(
monthly_fee=request.monthly_fee,
currency=request.currency,
billing_period_start=request.billing_period_start.isoformat(),
label=request.label,
),
_actor(user_api_key_dict),
)
if isinstance(created, DuplicateSubscriptionAccount):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={
"error": f"A fee is already configured for {created.custom_llm_provider} account "
f"{created.account_id}; use /subscription_accounts/update to change it"
},
)
return created
def _fee_changes(request: UpdateSubscriptionAccountFeeRequest) -> SubscriptionAccountFeeChanges:
return SubscriptionAccountFeeChanges(
monthly_fee=request.monthly_fee,
currency=request.currency,
billing_period_start=(
request.billing_period_start.isoformat() if request.billing_period_start is not None else None
),
label=SetLabel(request.label) if "label" in request.model_fields_set else KeepLabel(),
)
@router.post("/update", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)])
async def update_subscription_account(
request: UpdateSubscriptionAccountFeeRequest,
user_api_key_dict: Auth,
repository: Store,
) -> LiteLLM_SubscriptionAccountTable:
"""
Change the fee, currency, billing period start, or label of a subscription account.
Only the fields named in the request are written, so two admins changing different
fields at the same time both land.
"""
_require_proxy_admin(user_api_key_dict)
updated: Final = await repository.update_fee(
request.subscription_account_id, _fee_changes(request), _actor(user_api_key_dict)
)
if updated is None:
raise _not_found(request.subscription_account_id)
return updated
@router.post("/delete", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)])
async def delete_subscription_account(
request: DeleteSubscriptionAccountRequest,
user_api_key_dict: Auth,
repository: Store,
) -> LiteLLM_SubscriptionAccountTable:
"""Stop tracking a subscription account's fee; its requests stay marked as subscription-covered."""
_require_proxy_admin(user_api_key_dict)
if await repository.find_by_id(request.subscription_account_id) is None:
raise _not_found(request.subscription_account_id)
deleted: Final = await repository.delete_account(request.subscription_account_id)
if deleted is None:
raise _not_found(request.subscription_account_id)
return deleted
def _not_found(subscription_account_id: str) -> HTTPException:
return HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={"error": f"No subscription account with id {subscription_account_id}"},
)

55
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -8,7 +8,6 @@ Has all /sso/* routes
/sso/debug/callback - returns the OpenID object returned by the SSO provider
"""
import asyncio
import base64
import hashlib
import inspect
@ -151,6 +150,7 @@ from litellm.proxy.utils import (
get_custom_url,
get_server_root_path,
)
from litellm.repositories.chunked_in import find_many_in
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import SSOConfigRepository
from litellm.repositories.team_repository import TeamRepository
@ -1855,7 +1855,7 @@ async def warn_if_id_jag_capture_gap(*, retention_enabled: RetentionCheck | None
verbose_proxy_logger.warning("SSO debug callback ran with an oauth2_id_jag capture gap: %s", gap)
async def create_team_member_add_task(team_id, user_info):
async def create_team_member_add_task(team_id: str, user_info: NewUserResponse | LiteLLM_UserTable):
"""Create a task for adding a member to a team."""
try:
member: Final = Member(user_id=user_info.user_id, role="user")
@ -1871,22 +1871,43 @@ async def create_team_member_add_task(team_id, user_info):
verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e)
async def add_missing_team_member(user_info: NewUserResponse | LiteLLM_UserTable, sso_teams: list[str]):
async def find_existing_team_ids(team_ids: Sequence[str]) -> frozenset[str] | None:
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
return None
try:
rows: Final = await find_many_in(TeamRepository(prisma_client, use_writer=True).table, "team_id", team_ids)
except Exception as e:
verbose_proxy_logger.debug("[Non-Blocking] Error looking up SSO teams: %s", e)
return None
return frozenset(row.team_id for row in rows)
async def add_missing_team_member(
user_info: NewUserResponse | LiteLLM_UserTable,
sso_teams: Sequence[str],
*,
add_team_member: Callable[
[str, NewUserResponse | LiteLLM_UserTable], Awaitable[object]
] = create_team_member_add_task,
existing_team_ids: Callable[[Sequence[str]], Awaitable[frozenset[str] | None]] = find_existing_team_ids,
) -> None:
"""
- Get missing teams (diff b/w user_info.team_ids and sso_teams)
- Add missing user to missing teams
- Drop the IdP groups that are not teams with one lookup, since group claims often carry many of them
- Add missing user to missing teams, one at a time in sso_teams order, so user.teams keeps the claim's order
"""
# Handle None as empty list for new users
user_teams: Final = user_info.teams if user_info.teams is not None else []
missing_teams: Final = set(sso_teams) - set(user_teams)
missing_teams_list: Final = list(missing_teams)
tasks = []
tasks = [create_team_member_add_task(team_id, user_info) for team_id in missing_teams_list]
try:
await asyncio.gather(*tasks)
except Exception as e:
verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e)
held_teams: Final = frozenset(user_info.teams or ())
missing_teams: Final = tuple(team_id for team_id in dict.fromkeys(sso_teams) if team_id not in held_teams)
if not missing_teams:
return
existing: Final = await existing_team_ids(missing_teams)
teams_to_add: Final = (
missing_teams if existing is None else tuple(team_id for team_id in missing_teams if team_id in existing)
)
for team_id in teams_to_add:
await add_team_member(team_id, user_info)
def get_disabled_non_admin_personal_key_creation():
@ -2451,7 +2472,9 @@ async def fetch_cli_sso_team_details(
except Exception as e:
verbose_proxy_logger.error("Error fetching team details for CLI SSO session: %s", e)
return None
return tuple(_cli_sso_team_detail(team_row.model_dump()) for team_row in prisma_teams)
fetched_details: Final = tuple(_cli_sso_team_detail(team_row.model_dump()) for team_row in prisma_teams)
details_by_team_id: Final = MappingProxyType({detail.team_id: detail for detail in fetched_details})
return tuple(details_by_team_id[team_id] for team_id in dict.fromkeys(teams) if team_id in details_by_team_id)
def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str]:

36
litellm/proxy/policy_engine/pipeline_executor.py
View file

@ -98,15 +98,11 @@ def _count_change(sent: tuple[object, ...] | None, returned: tuple[object, ...]
return (len(sent), len(returned))
def _tool_call_mismatch_reason(
sent: tuple[tuple[object, object], ...] | None, returned: tuple[tuple[object, object], ...] | None
) -> str | None:
if sent == returned:
return None
def _tool_call_count_mismatch_reason(sent: Sequence[object] | None, returned: Sequence[object] | None) -> str | None:
sent_count: Final = len(sent or ())
returned_count: Final = len(returned or ())
if sent_count == returned_count:
return "the legacy hook changed a tool call's name or arguments, which this path cannot write back"
return None
return f"the legacy hook returned {returned_count} tool calls for a stream that carried {sent_count}"
@ -205,11 +201,10 @@ class _LegacyHookStreamAdapter(CustomGuardrail):
non-streaming hooks, an exception it raises ends the stream through the executor's
fail/error classification, and the response it hands back, or the one it changed in place
and returned ``None`` for, is re-scanned by the same translation so its texts reach the
client through the translation's ended-stream write-back. A
replacement whose scanned texts do not line up with the originals, or whose tool calls
differ from them, is undeliverable, so the executor releases the original chunks. A stream
that carried no text to scan, such as a tool-only Anthropic message, stays deliverable as
long as the hook left the tool calls alone."""
client through the translation's ended-stream write-back. Tool calls the hook rewrote go to the translation's
tool-call write-back like a unified guardrail's. A replacement whose scanned texts or tool
calls do not line up with the originals by count is undeliverable, so the executor releases
the original chunks."""
def __init__(
self,
@ -258,14 +253,19 @@ class _LegacyHookStreamAdapter(CustomGuardrail):
guardrail_name,
f"the legacy hook returned {returned_text_count} texts for a stream that carried {sent_text_count}",
)
tool_call_mismatch: Final = _tool_call_mismatch_reason(
_tool_call_shapes(inputs.get("tool_calls")), _tool_call_shapes(rescanned.get("tool_calls"))
sent_tool_calls: Final = inputs.get("tool_calls")
rewritten_tool_calls: Final = rescanned.get("tool_calls")
tool_call_count_mismatch: Final = _tool_call_count_mismatch_reason(sent_tool_calls, rewritten_tool_calls)
if tool_call_count_mismatch is not None:
raise UndeliverableStreamRewrite(guardrail_name, tool_call_count_mismatch)
texts_update: Final[GenericGuardrailAPIInputs] = {"texts": rewritten} if rewritten else {}
tool_calls_update: Final[GenericGuardrailAPIInputs] = (
{"tool_calls": rewritten_tool_calls}
if rewritten_tool_calls is not None
and _rewrote(_tool_call_shapes(sent_tool_calls), _tool_call_shapes(rewritten_tool_calls))
else {}
)
if tool_call_mismatch is not None:
raise UndeliverableStreamRewrite(guardrail_name, tool_call_mismatch)
if not rewritten:
return inputs
rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten}
rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, **texts_update, **tool_calls_update}
return rewritten_inputs
async def _rescan(

39
litellm/proxy/proxy_server.py
View file

@ -550,6 +550,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import (
write_reload_interval,
)
from litellm.proxy.common_utils.proxy_state import ProxyState
from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState
from litellm.proxy.common_utils.scheduled_job_stagger import (
@ -718,6 +719,9 @@ from litellm.proxy.management_endpoints.customer_endpoints import (
router as customer_router,
)
from litellm.proxy.management_endpoints.daily_activity_routes import router as daily_activity_router
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
apply_content_policy_fallbacks,
)
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
router as fallback_management_router,
)
@ -777,6 +781,9 @@ from litellm.proxy.management_endpoints.session_endpoints import (
router as session_management_router,
)
from litellm.proxy.management_endpoints.sso.saml_sso import SAML_AUTHN_REQUEST_TTL_SECONDS
from litellm.proxy.management_endpoints.subscription_accounts import (
router as subscription_accounts_router,
)
from litellm.proxy.management_endpoints.tag_management_endpoints import (
router as tag_management_router,
)
@ -8160,14 +8167,38 @@ class ProxyConfig:
if db_router_settings is not None and db_router_settings.param_value is not None
else _EMPTY_SETTINGS_MAPPING
)
previous_content_policy_rules: Final = self.router_settings.resolved().get("content_policy_fallbacks")
self.router_settings.apply_db_row("router_settings", db_values)
combined_router_settings: Final = self.router_settings.resolved()
if combined_router_settings:
self._apply_router_settings(llm_router, combined_router_settings)
self._apply_router_settings(
llm_router,
combined_router_settings,
content_policy_owned_by_config=self.router_settings.owned_by_config("content_policy_fallbacks"),
content_policy_rules_changed=(
combined_router_settings.get("content_policy_fallbacks") != previous_content_policy_rules
),
)
@staticmethod
def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None:
llm_router.apply(ChangeSet(settings={k: v for k, v in router_settings.items() if k != "routing_groups"}))
def _apply_router_settings(
llm_router: Router,
router_settings: Mapping[str, object],
*,
content_policy_owned_by_config: bool,
content_policy_rules_changed: bool,
) -> None:
llm_router.apply(
ChangeSet(
settings={
k: v for k, v in router_settings.items() if k not in ("routing_groups", "content_policy_fallbacks")
}
)
)
if "content_policy_fallbacks" in router_settings and not content_policy_owned_by_config:
apply_content_policy_fallbacks(
llm_router, router_settings["content_policy_fallbacks"], log_skipped_rules=content_policy_rules_changed
)
if "routing_groups" not in router_settings:
return
try:
@ -8751,6 +8782,7 @@ class ProxyConfig:
subscriber: Final = ConfigSyncSubscriber(
redis_cache=redis_cache,
resync_callbacks=(_resync_config_from_db,),
apply_model_ids=resync_model_deployments_by_ids,
)
self.config_sync_subscriber = subscriber
subscriber.start()
@ -21803,6 +21835,7 @@ app.include_router(memory_router)
app.include_router(lens_router)
app.include_router(plugin_router)
app.include_router(cost_tracking_settings_router)
app.include_router(subscription_accounts_router)
app.include_router(prompt_caching_requests_router)
app.include_router(router_settings_router)
app.include_router(fallback_management_router)

12
litellm/proxy/response_api_endpoints/endpoints.py
View file

@ -79,6 +79,7 @@ from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decry
from litellm.proxy.litellm_pre_call_utils import (
_CLIENT_PRICING_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list
_UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list
is_chat_completions_body,
)
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
request_tags_from_request_data,
@ -213,13 +214,6 @@ def _normalize_tool_dialect(
return {**data, **{key: value for key, value in replaceable if key in data}}
def _is_chat_completions_body(data: Mapping[str, object]) -> bool:
messages: Final = data.get("messages")
if isinstance(messages, list) and messages:
return True
return "messages" in data and "input" not in data
_CURSOR_THINKING_SEPARATOR: Final = "-thinking-"
_CURSOR_FAST_SUFFIX: Final = "-fast"
_CURSOR_THINKING_LEVELS: Final[frozenset[str]] = frozenset(get_args(REASONING_EFFORT))
@ -261,7 +255,7 @@ def _resolve_cursor_model_variant(
resolved: Final = {**data, "model": variant.base_model}
if variant.reasoning_effort is None:
return resolved
if _is_chat_completions_body(data):
if is_chat_completions_body(data):
if "reasoning_effort" in data:
return resolved
return {**resolved, "reasoning_effort": variant.reasoning_effort}
@ -646,7 +640,7 @@ async def cursor_chat_completions(
raw_body: Final = await read_request_body(request=request)
if _is_chat_completions_body(raw_body):
if is_chat_completions_body(raw_body):
# Genuine chat completions body (Cursor sends these for models whose BYOK it
# already fixed); delegate so behavior matches /chat/completions exactly.
# Keyed on messages CONTENT, not key presence: Cursor can send a null or

16
litellm/proxy/schema.prisma
View file

@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
@@id([event_id, target, version])
@@index([target, state, next_attempt_at])
}
model LiteLLM_SubscriptionAccountTable {
subscription_account_id String @id @default(uuid())
custom_llm_provider String
account_id String
label String?
monthly_fee Float
currency String
billing_period_start String
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
@@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
}

54
litellm/proxy/spend_tracking/subscription_billing_periods.py Normal file
View file

@ -0,0 +1,54 @@
from calendar import monthrange
from dataclasses import dataclass
from datetime import date, timedelta
from typing import Final
@dataclass(frozen=True, slots=True)
class BillingPeriodRange:
start: date
end: date
def _month_index(anchor: date, months_after_anchor: int) -> int:
return anchor.year * 12 + (anchor.month - 1) + months_after_anchor
_LAST_MONTH_INDEX: Final = _month_index(date.max, 0)
def _month_start(anchor: date, months_after_anchor: int) -> date:
month_index: Final = _month_index(anchor, months_after_anchor)
year: Final = month_index // 12
month: Final = month_index % 12 + 1
return date(year, month, min(anchor.day, monthrange(year, month)[1]))
def _period_end(anchor: date, months_after_anchor: int) -> date:
if _month_index(anchor, months_after_anchor + 1) > _LAST_MONTH_INDEX:
return date.max
return _month_start(anchor, months_after_anchor + 1) - timedelta(days=1)
def _first_candidate_index(anchor: date, start_date: date) -> int:
months_apart: Final = (start_date.year - anchor.year) * 12 + (start_date.month - anchor.month)
return max(0, months_apart - 1)
def billing_periods_starting_within(anchor: date, start_date: date, end_date: date) -> tuple[BillingPeriodRange, ...]:
"""Monthly periods anchored on `anchor`'s day of month whose start lies inside [start_date, end_date]."""
if end_date < start_date or end_date < anchor:
return ()
first_index: Final = _first_candidate_index(anchor, start_date)
last_index: Final = min(
first_index + _month_span(start_date, end_date) + 1, _LAST_MONTH_INDEX - _month_index(anchor, 0)
)
candidates: Final = (
BillingPeriodRange(start=_month_start(anchor, index), end=_period_end(anchor, index))
for index in range(first_index, last_index + 1)
)
return tuple(period for period in candidates if start_date <= period.start <= end_date)
def _month_span(start_date: date, end_date: date) -> int:
return (end_date.year - start_date.year) * 12 + (end_date.month - start_date.month)

10
litellm/proxy/utils.py
View file

@ -253,7 +253,11 @@ from litellm.proxy.hooks.sensitive_data_routing import ( # noqa: F401, RUF100
PROXY_SensitiveDataRoutingHandler,
_PROXY_SensitiveDataRoutingHandler, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
)
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_guardrails_from_auth_metadata
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
add_guardrails_from_auth_metadata,
normalized_metadata_slot,
)
from litellm.proxy.management_helpers.key_settings_audit import with_settings_updated_at
from litellm.proxy.policy_engine.pipeline_executor import (
PipelineExecutor,
@ -3981,9 +3985,7 @@ class ProxyLogging:
)
request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller
request_data.update({REJECTED_BEFORE_REQUEST_SETUP_KEY: REJECTED_BEFORE_REQUEST_SETUP})
if "metadata" not in request_data:
request_data["metadata"] = {}
request_data["metadata"].update(user_api_key_logged_metadata)
normalized_metadata_slot(request_data, "metadata").update(user_api_key_logged_metadata)
if litellm_logging_obj is not None:
## UPDATE LOGGING INPUT

134
litellm/repositories/subscription_account_repository.py Normal file
View file

@ -0,0 +1,134 @@
"""
Subscription account repository for database operations on LiteLLM_SubscriptionAccountTable.
"""
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
from litellm.repositories.base_repository import BaseRepository, is_unique_violation
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import PrismaTableRepository
if TYPE_CHECKING:
from prisma import models as prisma_models
@dataclass(frozen=True, slots=True)
class DuplicateSubscriptionAccount:
custom_llm_provider: str
account_id: str
@dataclass(frozen=True, slots=True)
class SubscriptionAccountFeeFields:
monthly_fee: float
currency: str
billing_period_start: str
label: str | None
@dataclass(frozen=True, slots=True)
class KeepLabel:
pass
@dataclass(frozen=True, slots=True)
class SetLabel:
value: str | None
LabelChange: TypeAlias = KeepLabel | SetLabel
@dataclass(frozen=True, slots=True)
class SubscriptionAccountFeeChanges:
monthly_fee: float | None = None
currency: str | None = None
billing_period_start: str | None = None
label: LabelChange = KeepLabel()
def columns(self) -> Mapping[str, float | str | None]:
named: Final = {
"monthly_fee": self.monthly_fee,
"currency": self.currency,
"billing_period_start": self.billing_period_start,
}
present: Final = {column: value for column, value in named.items() if value is not None}
label_column: Final = {"label": self.label.value} if isinstance(self.label, SetLabel) else {}
return MappingProxyType({**present, **label_column})
class SubscriptionAccountStore(Protocol):
async def list_all(self) -> tuple[LiteLLM_SubscriptionAccountTable, ...]: ...
async def find_by_id(self, id_value: str) -> LiteLLM_SubscriptionAccountTable | None: ...
async def create_account(
self, custom_llm_provider: str, account_id: str, fee: SubscriptionAccountFeeFields, created_by: str
) -> LiteLLM_SubscriptionAccountTable | DuplicateSubscriptionAccount: ...
async def update_fee(
self, subscription_account_id: str, changes: SubscriptionAccountFeeChanges, updated_by: str
) -> LiteLLM_SubscriptionAccountTable | None: ...
async def delete_account(self, subscription_account_id: str) -> LiteLLM_SubscriptionAccountTable | None: ...
class _SubscriptionAccountTable(PrismaTableRepository["prisma_models.LiteLLM_SubscriptionAccountTable"]):
table_name = "litellm_subscriptionaccounttable"
class SubscriptionAccountRepository(BaseRepository[LiteLLM_SubscriptionAccountTable]):
def __init__(self, prisma_client: object) -> None:
super().__init__(prisma_client)
self._table: Final = _SubscriptionAccountTable(prisma_client)
@property
def table(self) -> TableActions["prisma_models.LiteLLM_SubscriptionAccountTable"]:
return self._table.table
@property
def model_class(self) -> type[LiteLLM_SubscriptionAccountTable]:
return LiteLLM_SubscriptionAccountTable
async def list_all(self) -> tuple[LiteLLM_SubscriptionAccountTable, ...]:
return tuple(await self.find_many(order={"created_at": "asc"}))
async def find_by_id(
self, id_value: str, id_field: str = "subscription_account_id"
) -> LiteLLM_SubscriptionAccountTable | None:
return await super().find_by_id(id_value, id_field)
async def create_account(
self, custom_llm_provider: str, account_id: str, fee: SubscriptionAccountFeeFields, created_by: str
) -> LiteLLM_SubscriptionAccountTable | DuplicateSubscriptionAccount:
try:
return await self.create(
{
"custom_llm_provider": custom_llm_provider,
"account_id": account_id,
"monthly_fee": fee.monthly_fee,
"currency": fee.currency,
"billing_period_start": fee.billing_period_start,
"label": fee.label,
"created_by": created_by,
"updated_by": created_by,
}
)
except Exception as exc:
if is_unique_violation(exc):
return DuplicateSubscriptionAccount(custom_llm_provider=custom_llm_provider, account_id=account_id)
raise
async def update_fee(
self, subscription_account_id: str, changes: SubscriptionAccountFeeChanges, updated_by: str
) -> LiteLLM_SubscriptionAccountTable | None:
return await self.update(
subscription_account_id, {**changes.columns(), "updated_by": updated_by}, id_field="subscription_account_id"
)
async def delete_account(self, subscription_account_id: str) -> LiteLLM_SubscriptionAccountTable | None:
return await self.delete(subscription_account_id, id_field="subscription_account_id")

14
litellm/responses/main.py
View file

@ -31,7 +31,10 @@ from litellm.integrations.anthropic_cache_control_hook import (
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
get_provider_affinity_header,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -759,7 +762,7 @@ def _dispatched_model_name(
from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
provider_model, _ = resolve_model_and_provider_for_metadata(model=model, custom_llm_provider=custom_llm_provider)
return _strip_responses_routing_prefix(provider_model)
return strip_responses_routing_prefix(provider_model)
def _will_bridge_to_chat_completions(
@ -805,6 +808,7 @@ def provider_stores_the_conversation(model: str, custom_llm_provider: str | None
return not _will_bridge_to_chat_completions(
model,
custom_llm_provider
or declared_authenticating_provider(model)
or litellm.get_llm_provider(model=model, api_base=api_base, litellm_params=litellm_params)[1],
bool(kwargs.get("use_chat_completions_api")),
kwargs.get("model_info"),
@ -1245,7 +1249,7 @@ def _pop_use_chat_completions_api_kw(kwargs: dict[str, object]) -> bool:
_RESPONSES_ROUTING_PREFIX: Final = "responses/"
def _strip_responses_routing_prefix(model: str) -> str:
def strip_responses_routing_prefix(model: str) -> str:
if not model.startswith(_RESPONSES_ROUTING_PREFIX):
return model
return model[len(_RESPONSES_ROUTING_PREFIX) :]
@ -1273,7 +1277,7 @@ def _resolve_model_provider_for_responses(
litellm_params.api_key = dynamic_api_key
if dynamic_api_base is not None:
litellm_params.api_base = dynamic_api_base
return _strip_responses_routing_prefix(provider_model), resolved_provider
return strip_responses_routing_prefix(provider_model), resolved_provider
def _apply_managed_file_id_mapping(
@ -2893,7 +2897,7 @@ async def _aresponses_websocket(
api_key=api_key,
litellm_params=litellm_params,
)
resolved_model: Final = _strip_responses_routing_prefix(provider_model)
resolved_model: Final = strip_responses_routing_prefix(provider_model)
litellm_params_dict["data_residency"] = infer_openai_data_residency(
_custom_llm_provider,

3
litellm/responses/streaming_iterator.py
View file

@ -43,6 +43,7 @@ from litellm.litellm_core_utils.core_helpers import (
process_response_headers,
provider_reported_usage_cost,
set_response_cost_in_hidden_params,
stamp_litellm_bill_on_usage,
)
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
from litellm.litellm_core_utils.internal_call_metadata import (
@ -1869,7 +1870,7 @@ def stamp_responses_usage_cost(
if isinstance(bill, bool) or not isinstance(bill, (int, float)):
return
if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)):
setattr(usage_obj, "cost", float(bill))
stamp_litellm_bill_on_usage(response_obj, usage_obj, float(bill))
def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]:

186
litellm/responses/utils.py
View file

@ -1,10 +1,11 @@
import base64
import os
import re
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import reduce
from types import MappingProxyType
from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload
from typing import Any, Final, NoReturn, Optional, TypeVar, Union, cast, get_type_hints, overload
from pydantic import BaseModel, TypeAdapter
from typing_extensions import TypedDict as ExtraItemsTypedDict
@ -15,7 +16,13 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
from litellm.llms.base_llm.responses.transformation import (
ANY_REASONING_FIELD_VALUES,
REASONING_SUMMARY_VALUES,
BaseResponsesAPIConfig,
ReasoningFieldValues,
ReasoningSummaryLevel,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.llms.openai import (
AllMessageValues,
@ -1411,9 +1418,6 @@ def reasoning_with_summary_default(
return {**reasoning} if summary is None else {**reasoning, "summary": summary}
_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
def _openai_model_info(model: str) -> ModelInfo | None:
from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
@ -1425,24 +1429,115 @@ def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | No
return None if values is None else frozenset(values)
def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]:
_REFUSABLE_REASONING_FIELDS: Final = frozenset({"mode", "context"})
def _openai_reasoning_field_values(model: str) -> ReasoningFieldValues:
model_info: Final = _openai_model_info(model)
if model_info is None:
return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None})
return ANY_REASONING_FIELD_VALUES
if not model_info.get("supports_reasoning"):
return MappingProxyType({})
return MappingProxyType(
{
"summary": _SUMMARY_VALUES,
"summary": REASONING_SUMMARY_VALUES,
"mode": _declared_values(model_info.get("reasoning_modes")),
"context": _declared_values(model_info.get("reasoning_contexts")),
}
)
def _openai_accepts_reasoning_field(
key: str, value: object, values_by_field: Mapping[str, frozenset[str | None] | None]
) -> bool:
_CATALOG_REASONING_FIELDS: Final = (("mode", "reasoning_modes"), ("context", "reasoning_contexts"))
def _catalog_values(values: object) -> frozenset[str | None]:
return frozenset(value for value in iter_items(values) if isinstance(value, str))
def _declared_reasoning_values(row: Mapping[str, object] | None) -> Mapping[str, frozenset[str | None]]:
if row is None:
return {}
return {
field: _catalog_values(row[key]) for field, key in _CATALOG_REASONING_FIELDS if isinstance(row.get(key), list)
}
def _with_declared_reasoning_values(
defaults: ReasoningFieldValues | None,
model: str,
custom_llm_provider: str,
deployment_model_info: Mapping[str, object] | None,
) -> ReasoningFieldValues | None:
catalog: Final = cast(Mapping[str, object], litellm.model_cost) # cast-ok: read-only view of the legacy registry
catalog_row: Final = as_str_mapping(catalog.get(f"{custom_llm_provider}/{model.removeprefix('responses/')}"))
overrides: Final = {
**_declared_reasoning_values(catalog_row),
**_declared_reasoning_values(deployment_model_info),
}
if not overrides:
return defaults
return MappingProxyType({**(ANY_REASONING_FIELD_VALUES if defaults is None else defaults), **overrides})
def _is_version_number(part: str) -> bool:
return part.isdigit() and len(part) <= 2
def _splits_a_version(parts: Sequence[str], end: int) -> bool:
return end < len(parts) and _is_version_number(parts[end - 1]) and _is_version_number(parts[end])
def _hyphen_runs_longest_first(name: str) -> Iterator[str]:
parts: Final = name.rsplit("/", 1)[-1].split("-")
for length in range(len(parts), 0, -1):
yield from (
"-".join(parts[start : start + length])
for start in range(len(parts) - length + 1)
if not _splits_a_version(parts, start + length)
)
def _openai_catalog_family(name: str) -> str | None:
runs: Final = _hyphen_runs_longest_first(name)
return next((run for run in runs if OpenAIGPTConfig.is_openai_catalog_model(run)), None)
def _openai_catalog_name(name: str) -> str | None:
last_segment: Final = name.rsplit("/", 1)[-1]
return last_segment if OpenAIGPTConfig.is_openai_catalog_model(last_segment) else None
def _openai_catalog_model(model: str, base_model: str | None, names_wrap_models: bool) -> str | None:
find: Final = _openai_catalog_family if names_wrap_models else _openai_catalog_name
families: Final = (find(name) for name in (base_model, model) if name)
return next((family for family in families if family is not None), None)
def _default_reasoning_field_values(
model: str, base_model: str | None, responses_config: BaseResponsesAPIConfig | None
) -> ReasoningFieldValues | None:
declared: Final = None if responses_config is None else responses_config.reasoning_field_values()
if declared is not None:
return declared
names_wrap_models: Final = responses_config is not None and responses_config.deployment_names_wrap_openai_models()
catalog_model: Final = _openai_catalog_model(model, base_model, names_wrap_models)
if catalog_model is not None:
return _openai_reasoning_field_values(catalog_model)
return None if responses_config is None else responses_config.unresolved_model_reasoning_field_values(model)
def _reasoning_field_values(
model: str,
base_model: str | None,
custom_llm_provider: str,
responses_config: BaseResponsesAPIConfig | None,
deployment_model_info: Mapping[str, object] | None,
) -> ReasoningFieldValues | None:
defaults: Final = _default_reasoning_field_values(model, base_model, responses_config)
return _with_declared_reasoning_values(defaults, model, custom_llm_provider, deployment_model_info)
def _accepts_reasoning_field(key: str, value: object, values_by_field: ReasoningFieldValues) -> bool:
if key == "effort":
return True
if key not in values_by_field:
@ -1451,11 +1546,8 @@ def _openai_accepts_reasoning_field(
return allowed is None or (isinstance(value, str | None) and value in allowed)
def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]:
values_by_field: Final = _openai_reasoning_field_values(model)
return {
key: value for key, value in reasoning.items() if _openai_accepts_reasoning_field(key, value, values_by_field)
}
def _accepted_reasoning(reasoning: Mapping[str, object], values_by_field: ReasoningFieldValues) -> Mapping[str, object]:
return {key: value for key, value in reasoning.items() if _accepts_reasoning_field(key, value, values_by_field)}
def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]:
@ -1464,26 +1556,66 @@ def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]:
return tuple(path for path in iter_items(additional_drop_params) if isinstance(path, str))
def openai_bridge_reasoning(
@dataclass(frozen=True, slots=True)
class RefusedReasoningFields:
refused: tuple[str, ...]
def raise_refused_reasoning_fields(refusal: RefusedReasoningFields, model: str, custom_llm_provider: str) -> NoReturn:
raise litellm.UnsupportedParamsError(
message=(
f"{custom_llm_provider} does not support parameters: "
f"{[f'reasoning_effort.{key}' for key in refusal.refused]}, for model={model}. "
"To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=custom_llm_provider,
model=model,
)
def bridge_reasoning(
*,
requested: Mapping[str, object],
effort_kept: bool,
summary_alias: object,
model: str,
base_model: str | None,
custom_llm_provider: str,
api_base: str | None,
deployment_model_info: Mapping[str, object] | None,
drop_params: bool,
additional_drop_params: object,
) -> Mapping[str, object] | None:
) -> Mapping[str, object] | RefusedReasoningFields | None:
from litellm.utils import ProviderConfigManager
drop_paths: Final = _additional_drop_paths(additional_drop_params)
if "reasoning_effort" in drop_paths:
return None
kept: Final = requested if effort_kept else {key: value for key, value in requested.items() if key != "effort"}
with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, "openai")
accepted: Final = (
_openai_accepted_reasoning(with_summary, model)
if drop_params and OpenAIGPTConfig.is_openai_catalog_model(model)
else {**with_summary}
with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, custom_llm_provider)
sent: Final = (
as_str_mapping(
_apply_nested_drop_params({"reasoning_effort": {**with_summary}}, list(drop_paths)).get("reasoning_effort")
)
or {}
)
remaining: Final = as_str_mapping(
_apply_nested_drop_params({"reasoning_effort": accepted}, list(drop_paths)).get("reasoning_effort")
responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(
custom_llm_provider, model, api_base
)
return remaining or None
values_by_field: Final = _reasoning_field_values(
model, base_model, custom_llm_provider, responses_config, deployment_model_info
)
accepted: Final = sent if values_by_field is None else _accepted_reasoning(sent, values_by_field)
refused: Final = tuple(
key
for key, value in sent.items()
if key in _REFUSABLE_REASONING_FIELDS and key not in accepted and value is not None
)
if (
refused
and not drop_params
and responses_config is not None
and responses_config.ignores_unsupported_reasoning_fields()
):
return RefusedReasoningFields(refused)
return (accepted if drop_params else sent) or None

Some files were not shown because too many files have changed in this diff Show more