Merge remote-tracking branch 'origin/main' into litellm_vertex_identity_labels_wt

# Conflicts:
#	litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py
This commit is contained in:
mateo-berri 2026-11-10 16:18:34 -08:00
commit ffe74b23cf
437 changed files with 47948 additions and 2019 deletions

View file

@ -397,6 +397,7 @@ jobs:
tests/unit/llms/voyage
tests/unit/llms/wandb
tests/unit/llms/watsonx
tests/unit/llms/webiq
tests/unit/llms/xai
tests/unit/llms/xinference
tests/unit/llms/you_com

View file

@ -20,6 +20,7 @@ from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs
from litellm.constants import (
BATCH_LINE_ITEMS_MAX_PART_BYTES,
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
@ -765,7 +766,7 @@ class CheckBatchCost:
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
_file_content = await afile_content(
file_id=cast(str, raw_output_file_id),
**cast(FileContentCallOptions, credentials),
**cast(FileContentCallOptions, {**credentials, **stored_s3_bucket_kwargs(cast(str, raw_output_file_id))}),
)
content_bytes: Final = await _file_content_bytes(_file_content)

View file

@ -49,6 +49,7 @@ from litellm.constants import (
)
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.internal_call_metadata import requests_background
@ -78,6 +79,7 @@ from litellm.proxy.litellm_pre_call_utils import (
from litellm.proxy.openai_files_endpoints.common_utils import (
BATCH_CREATE_HIDDEN_PARAM,
FILE_LIST_CONTINUATION_CHUNK_SIZE,
TeamProviderDeployment,
_is_base64_encoded_unified_file_id,
apply_unified_file_ids,
decode_model_from_file_id,
@ -395,6 +397,7 @@ class _CursorPageArgs(TypedDict, total=False):
class _RouterFileCallKwargs(TypedDict, total=False):
client: ReadOnly[AsyncOpenAI | None]
custom_llm_provider: ReadOnly[str | None]
stored_s3_bucket_name: ReadOnly[str]
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
@ -462,7 +465,7 @@ async def _delete_provider_file(
await llm_router.afile_delete(
model=model_id,
file_id=provider_file_id,
**cast(_RouterFileCallKwargs, delete_data),
**cast(_RouterFileCallKwargs, {**delete_data, **stored_s3_bucket_kwargs(provider_file_id)}),
)
@ -1156,7 +1159,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve)
retrieve_options: Final = cast(
FileRetrieveCallOptions,
{**call_options, "max_retries": 0},
{**call_options, **stored_s3_bucket_kwargs(provider_file_id), "max_retries": 0},
)
for attempt in range(len(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS) + 1):
if attempt > 0:
@ -1426,6 +1429,43 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
user_api_key_dict=owner,
)
async def register_provider_file(
self,
*,
provider_file_id: str,
content_type: str,
deployment: TeamProviderDeployment,
owner: UserAPIKeyAuth,
llm_router: Router | None,
) -> str | None:
unified_file_id: Final = self.get_unified_output_file_id(
provider_file_id, deployment.deployment_id, deployment.public_model_name, file_type=content_type
)
provider_file_object, _ = await self._fetch_provider_file_object(
unified_file_id=unified_file_id,
provider_file_id=provider_file_id,
model_id=deployment.deployment_id,
model_name=None,
llm_router=llm_router,
)
if provider_file_object is None:
return None
try:
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=provider_file_object,
litellm_parent_otel_span=None,
model_mappings={deployment.deployment_id: provider_file_id},
user_api_key_dict=owner,
)
except Exception as error: # noqa: BLE001 # the upload succeeded, so the caller keeps the provider id
verbose_logger.warning(
f"Failed to register provider_file_id={sanitize_for_log(provider_file_id)} as a managed file: "
f"{type(error).__name__} {sanitize_for_log(error)}"
)
return None
return unified_file_id
async def list_user_batches(
self,
user_api_key_dict: UserAPIKeyAuth,
@ -1757,8 +1797,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if potential_file_id and "llm_output_file_id," in potential_file_id:
model_id = self.get_model_id_from_unified_file_id(potential_file_id)
if model_id:
provider_file_id: Final = self.get_output_file_id_from_unified_file_id(potential_file_id)
data["model"] = model_id
data["file_id"] = self.get_output_file_id_from_unified_file_id(potential_file_id)
data["file_id"] = provider_file_id
data.update(stored_s3_bucket_kwargs(provider_file_id))
elif call_type == CallTypes.acreate_batch.value:
input_file_id = cast(Optional[str], data.get("input_file_id"))
if input_file_id:
@ -2253,10 +2295,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
unified_batch_id = SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format(model_id, batch_id)
return base64.urlsafe_b64encode(unified_batch_id.encode()).decode().rstrip("=")
def get_unified_output_file_id(self, output_file_id: str, model_id: str, model_name: Optional[str]) -> str:
def get_unified_output_file_id(
self, output_file_id: str, model_id: str, model_name: Optional[str], file_type: str = "application/json"
) -> str:
deterministic_uuid: Final = uuid5(uuid5(NAMESPACE_URL, model_id), output_file_id)
unified_output_file_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format(
"application/json",
file_type,
str(deterministic_uuid),
model_name or "",
output_file_id,
@ -2854,7 +2898,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
await llm_router.afile_content(
model=model_id,
file_id=provider_file_id,
**cast(_RouterFileCallKwargs, data),
**cast(
_RouterFileCallKwargs,
{
**data,
**(
stored_s3_bucket_kwargs(provider_file_id)
if isinstance(provider_file_id, str)
else {}
),
},
),
),
)
except Exception as e:

View file

@ -0,0 +1,9 @@
SET LOCAL lock_timeout = '10s';
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_BackgroundResponseSettlement" (
"response_id" TEXT NOT NULL,
"settled_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_BackgroundResponseSettlement_pkey" PRIMARY KEY ("response_id")
);

View file

@ -0,0 +1,7 @@
SET LOCAL lock_timeout = '10s';
-- AlterTable
ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "model_tpm_limit" JSONB;
-- AlterTable
ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "model_rpm_limit" JSONB;

View file

@ -0,0 +1,22 @@
SET LOCAL lock_timeout = '10s';
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyGlobalSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

View file

@ -19,6 +19,8 @@ model LiteLLM_BudgetTable {
rpm_limit BigInt?
tpd_limit BigInt?
model_max_budget Json?
model_tpm_limit Json?
model_rpm_limit Json?
budget_duration String?
budget_reset_at DateTime?
allowed_models String[] @default([]) // per-member model scope; empty = inherit team models
@ -881,6 +883,7 @@ model LiteLLM_DailyUserSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
guardrail_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -917,6 +920,7 @@ model LiteLLM_DailyGlobalSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
guardrail_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -950,6 +954,7 @@ model LiteLLM_DailyOrganizationSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
guardrail_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -988,6 +993,7 @@ model LiteLLM_DailyEndUserSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
guardrail_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -1025,6 +1031,7 @@ model LiteLLM_DailyAgentSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
guardrail_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -1062,6 +1069,7 @@ model LiteLLM_DailyTeamSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
guardrail_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -1102,6 +1110,7 @@ model LiteLLM_DailyTagSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
guardrail_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -2053,6 +2062,11 @@ model LiteLLM_BackgroundInteractionSettlement {
@@index([settled_at], map: "idx_background_interaction_settlement_settled_at")
}
model LiteLLM_BackgroundResponseSettlement {
response_id String @id
settled_at DateTime @default(now())
}
model LiteLLM_Lens {
reviews LiteLLM_LensReview[]
id String @id

View file

@ -28,6 +28,7 @@ pub struct ModelInfoFields {
pub cache_read_input_audio_token_cost_flex: Option<f64>,
pub cache_read_input_audio_token_cost_priority: Option<f64>,
pub cache_read_input_audio_token_cost_batches: Option<f64>,
pub cache_read_input_image_token_cost_batches: Option<f64>,
pub explicit_cache_read_input_token_cost: Option<f64>,
pub prompt_cache_min_tokens: Option<i64>,
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,

View file

@ -133,6 +133,7 @@ def _daily_counters(row: Mapping[str, object], metadata: Mapping[str, object]) -
("prompt_caching_savings_spend", _number(savings.get("prompt_caching"))),
("gateway_injected_caching_savings_spend", _number(savings.get("gateway_injected_caching"))),
("autorouter_savings_spend", 0.0 if internal else _number(savings.get("autorouter"))),
("guardrail_spend", _number(metadata.get("accounting_guardrail_spend"))),
("total_response_time_ms", _number(duration) if timed else 0),
("timed_requests", int(timed)),
)

View file

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_share_of_spend
from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure
from litellm.proxy._types import SpendLogsPayload
from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
@ -73,6 +74,7 @@ def build_accounting_payload(
**{key: _accounting_metadata_field(key, value) for key, value in metadata.items() if key in _METADATA_FIELDS},
"accounting_entities": entities,
"accounting_savings": savings._asdict(),
"accounting_guardrail_spend": guardrail_share_of_spend(metadata, payload["spend"]),
"accounting_model_access_groups": model_access_groups,
"accounting_windows": windows,
}

View file

@ -46,9 +46,6 @@ def _progressed(result: DeliveryBatchResult) -> bool:
async def flush_accounting_deliveries(
writer: "DBSpendUpdateWriter", prisma_client: "PrismaClient", proxy_logging_obj: "ProxyLogging"
) -> None:
acquired: Final = await writer.pod_lock_manager.acquire_lock(cronjob_id=_JOB, allow_reentrant=False)
if acquired is False:
return
applied: tuple[Contribution, ...] = () # rebind-ok: keep committed contributions for cache invalidation
async def apply(transaction: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None:
@ -57,36 +54,39 @@ async def flush_accounting_deliveries(
transaction, deliveries, prisma_client=prisma_client, writer=writer, proxy_logging_obj=proxy_logging_obj
)
database: Final = cast( # cast-ok: PrismaClient.tx owns writer routing and yields the generated transaction
LedgerDatabase, prisma_client
)
repository: Final = AccountingDeliveryRepository(database)
started: Final = time.monotonic()
batch_size: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", "500"))
async def run_batch(*, retry_only: bool) -> DeliveryBatchResult:
result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only)
if result.applied:
await writer.invalidate_accounting_entities(applied, proxy_logging_obj)
if result.parked:
verbose_proxy_logger.error(
"Accounting delivery parked %d requests after isolated failures", len(result.parked)
)
await _alert(
proxy_logging_obj,
message=f"Accounting delivery parked {len(result.parked)} requests. Included spend totals require repair.",
)
return result
fresh_deadline: Final = time.monotonic() + 5
for _ in range(batch_size):
if not _filled(await run_batch(retry_only=False), batch_size) or time.monotonic() >= fresh_deadline:
break
retry_deadline: Final = time.monotonic() + 5
for _ in range(batch_size):
if time.monotonic() >= retry_deadline or not _progressed(await run_batch(retry_only=True)):
break
acquired: Final = await writer.pod_lock_manager.acquire_lock(cronjob_id=_JOB, allow_reentrant=False)
if acquired is False:
return
try:
database: Final = cast( # cast-ok: PrismaClient.tx owns writer routing and yields the generated transaction
LedgerDatabase, prisma_client
)
repository: Final = AccountingDeliveryRepository(database)
started: Final = time.monotonic()
batch_size: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", "500"))
async def run_batch(*, retry_only: bool) -> DeliveryBatchResult:
result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only)
if result.applied:
await writer.invalidate_accounting_entities(applied, proxy_logging_obj)
if result.parked:
verbose_proxy_logger.error(
"Accounting delivery parked %d requests after isolated failures", len(result.parked)
)
await _alert(
proxy_logging_obj,
message=f"Accounting delivery parked {len(result.parked)} requests. Included spend totals require repair.",
)
return result
fresh_deadline: Final = time.monotonic() + 5
for _ in range(batch_size):
if not _filled(await run_batch(retry_only=False), batch_size) or time.monotonic() >= fresh_deadline:
break
retry_deadline: Final = time.monotonic() + 5
for _ in range(batch_size):
if time.monotonic() >= retry_deadline or not _progressed(await run_batch(retry_only=True)):
break
retention_days: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_RETENTION_DAYS", "1"))
cleanup_deadline: Final = time.monotonic() + 5
for _ in range(batch_size):

View file

@ -11,6 +11,7 @@ from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils import batch_utils as core_batch_utils
from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
@ -551,7 +552,7 @@ async def fetch_batch_managed_file_content(
"file_id": provider_output_file_id,
"custom_llm_provider": custom_llm_provider,
**cast( # cast-ok: preserve dynamic provider credentials without validation
FileContentCallOptions, credentials
FileContentCallOptions, {**credentials, **stored_s3_bucket_kwargs(provider_output_file_id)}
),
}

View file

@ -16,7 +16,7 @@ import logging
import time
import weakref
from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence
from contextvars import ContextVar, Token
from contextvars import ContextVar, Token, copy_context
from dataclasses import dataclass, field
from datetime import timedelta
from enum import Enum, auto
@ -528,6 +528,7 @@ class RequestRedisBatches:
"_deadline",
"_deadline_flush",
"_native",
"_native_context",
"_native_factory",
"_native_flush_hooks",
"_native_misses",
@ -551,6 +552,7 @@ class RequestRedisBatches:
self._native_factory: Final = native_factory
self._native_send_factory: Final = native_send_factory
self._native: NativeRequestRedis | None = None
self._native_context: Final = copy_context()
self._native_flush_hooks: Final[list[Callable[[], None]]] = [] # mutable-ok: append-only registry
self._native_misses: frozenset[str] = frozenset()
self._deadline: asyncio.TimerHandle | None = None
@ -580,9 +582,12 @@ class RequestRedisBatches:
return batch
def native(self) -> NativeRequestRedis:
"""The request's native batches, built on first use so requests that touch no native store pay nothing."""
"""The request's native batches, built on first use so requests that touch no native store pay nothing.
They keep a copy of the context they are built in, and the cycle collector cannot see through native
objects, so they are built in the context this scope was opened from, which cannot reach the scope."""
if self._native is None:
self._native = self._native_factory(self.post_call_deadline)
self._native = self._native_context.run(self._native_factory, self.post_call_deadline)
return self._native
def native_post_call(self) -> NativeRequestRedis:

View file

@ -36,7 +36,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
)
from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
parse_chat_allowed_tools_tool_choice,
parse_allowed_tools_tool_choice,
responses_allowed_tools_tool_choice,
)
from litellm.litellm_core_utils.core_helpers import (
@ -900,7 +900,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
return choice_type
if choice_type == "allowed_tools":
parsed: Final = parse_chat_allowed_tools_tool_choice(
parsed: Final = parse_allowed_tools_tool_choice(
cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
object, tool_choice
)

View file

@ -1706,6 +1706,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_usage_based_routing_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
@ -1968,6 +1969,7 @@ BATCH_LINE_ITEMS_MAX_PART_BYTES: Final = get_env_int_in_range(
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, get_env_int("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7))
PENDING_RESPONSE_STATUSES: Final = ("queued", "in_progress")
TERMINAL_RESPONSE_STATUSES: Final = frozenset({"completed", "failed", "cancelled", "incomplete"})
BACKGROUND_RESPONSE_SETTLEMENT_TIMEOUT_SECONDS: Final = 2.0
BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS: Final = 10.0
BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS: Final = 60
STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, get_env_int("STALE_OBJECT_CLEANUP_BATCH_SIZE", 1000))

View file

@ -49,7 +49,6 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
BilledTokenRates,
CostCalculatorUtils,
calculate_cache_writing_cost,
calculate_cost_component,
generic_cost_per_character,
generic_cost_per_token,
get_batch_cost_rates,
@ -3177,6 +3176,21 @@ def default_video_cost_calculator(
return 0.0
def _optional_batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"cache_read_input_audio_token_cost_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
prompt_tokens: int,
) -> float | None:
context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
context_rate: Final = get_cost_per_unit(model_info, context_key, None)
return model_info.get(key) if context_rate is None else context_rate
def _batch_rate(
model_info: ModelInfo,
key: Literal[
@ -3189,9 +3203,7 @@ def _batch_rate(
*,
prompt_tokens: int = 0,
) -> float:
context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
context_rate: Final = get_cost_per_unit(model_info, context_key, None)
rate: Final = model_info.get(key) if context_rate is None else context_rate
rate: Final = _optional_batch_rate(model_info, key, prompt_tokens)
return fallback if rate is None else rate
@ -3275,30 +3287,14 @@ def batch_cost_calculator(
model_info,
batch_rates.input,
batch_rates.cache_read,
batch_rates.cache_read_image,
batch_rates.cache_creation,
batch_rates.cache_creation_rates_by_ttl,
)
elif input_cost_per_token is not None:
details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens: Final = details["cache_hit_tokens"]
cache_creation_tokens: Final = details["cache_creation_tokens"]
# Subtract cached tokens from prompt_tokens before calculating cost
# Fixes issue where cached tokens are being charged again
base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens
total_prompt_cost = (
base_input_tokens * (input_cost_per_token) / 2
) # batch cost is usually half of the regular token cost
# Add cache read cost if applicable
cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None)
total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2
configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost")
cache_creation_cost: Final = (
input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost
total_prompt_cost = _half_price_prompt_cost(
usage, model_info, input_cost_per_token, batch_rates.cache_read_image
)
total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token)
image_rate: Final = _batch_or_half(
model_info.get("output_cost_per_image_token_batches"),
@ -3324,15 +3320,54 @@ def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float
return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None)
def _cached_audio_batch_fallback(
model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int
) -> float:
def _cached_modality_rate(
model_info: ModelInfo,
standard_key: Literal["cache_read_input_audio_token_cost", "cache_read_input_image_token_cost"],
batch_modality_rate: float | None,
batch_cache_read_rate: float | None,
cache_read_rate: float | None,
prompt_tokens: int,
) -> float | None:
if batch_modality_rate is not None:
return batch_modality_rate
if batch_cache_read_rate is not None:
return batch_cache_read_rate
standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens)
if standard_audio_rate is not None:
return standard_audio_rate
return 0.0 if cache_read_rate is None else cache_read_rate
standard_modality_rate: Final = _standard_rate(model_info, standard_key, prompt_tokens)
return cache_read_rate if standard_modality_rate is None else standard_modality_rate
def _half_price_prompt_cost(
usage: Usage, model_info: ModelInfo, input_cost_per_token: float, batch_cache_read_image_rate: float | None
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cache_creation_tokens: Final = details["cache_creation_tokens"]
base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens
cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None)
half_cache_read_rate: Final = (get_cost_per_unit(model_info, cache_read_cost_key, 0.0) or 0.0) / 2
cached_audio_rate: Final = _batch_or_half(
_optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens),
_standard_rate(model_info, "cache_read_input_audio_token_cost", usage.prompt_tokens),
default=half_cache_read_rate,
)
cached_image_rate: Final = _batch_or_half(
batch_cache_read_image_rate,
_standard_rate(model_info, "cache_read_input_image_token_cost", usage.prompt_tokens),
default=half_cache_read_rate,
)
cached_audio_tokens: Final = details["cache_hit_audio_tokens"]
cached_image_tokens: Final = details["cached_image_tokens"]
cached_text_tokens: Final = max(details["cache_hit_tokens"] - cached_audio_tokens - cached_image_tokens, 0)
configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost")
cache_creation_cost: Final = (
input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost
)
return (
base_input_tokens * input_cost_per_token / 2
+ cached_text_tokens * half_cache_read_rate
+ cached_audio_tokens * cached_audio_rate
+ cached_image_tokens * cached_image_rate
+ cache_creation_tokens * cache_creation_cost / 2
)
def _batch_prompt_cost(
@ -3340,6 +3375,7 @@ def _batch_prompt_cost(
model_info: ModelInfo,
input_rate: float,
batch_cache_read_rate: float | None,
batch_cache_read_image_rate: float | None,
cache_creation_rate: float | None,
cache_creation_rates_by_ttl: Mapping[str, float],
) -> float:
@ -3349,8 +3385,30 @@ def _batch_prompt_cost(
if batch_cache_read_rate is None
else batch_cache_read_rate
)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
cached_audio_rate: Final = _cached_modality_rate(
model_info,
"cache_read_input_audio_token_cost",
_optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens),
batch_cache_read_rate,
cache_read_rate,
usage.prompt_tokens,
)
cached_image_rate: Final = _cached_modality_rate(
model_info,
"cache_read_input_image_token_cost",
batch_cache_read_image_rate,
batch_cache_read_rate,
cache_read_rate,
usage.prompt_tokens,
)
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cached_audio_rate is not None else 0
cached_image_tokens: Final = details["cached_image_tokens"] if cached_image_rate is not None else 0
cached_text_tokens: Final = (
max(details["cache_hit_tokens"] - details["cache_hit_audio_tokens"] - details["cached_image_tokens"], 0)
if cache_read_rate is not None
else 0
)
cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
audio_tokens, image_tokens, video_tokens = (
@ -3367,13 +3425,9 @@ def _batch_prompt_cost(
* _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+ cached_audio_tokens
* _batch_rate(
model_info,
"cache_read_input_audio_token_cost_batches",
_cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens),
)
+ cached_text_tokens * (cache_read_rate or 0.0)
+ cached_audio_tokens * (cached_audio_rate or 0.0)
+ cached_image_tokens * (cached_image_rate or 0.0)
+ (
calculate_cache_writing_cost(
written_tokens,

View file

@ -16,6 +16,7 @@ from pydantic import JsonValue, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY
from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
from litellm.integrations.clickhouse.context import is_lens_analysis
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
@ -37,7 +38,14 @@ TRACE_INGEST_ROUTE: Final = "/v1/traces"
_METADATA_MAPPING: Final = TypeAdapter(Mapping[str, object])
_METADATA_VALUE: Final = TypeAdapter(JsonValue)
_INTERNAL_METADATA_KEYS: Final = frozenset(
("user_api_key", "user_api_key_auth", "user_api_key_budget_reservation", "proxy_server_request")
(
"user_api_key",
"user_api_key_auth",
"user_api_key_budget_reservation",
"proxy_server_request",
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
)
)

View file

@ -18,6 +18,10 @@ from pydantic import ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
allowed_tools_tool_choice_in_auto_mode,
allowed_tools_tool_choice_with_builtin_as_function,
)
from litellm.llms.base_llm.sandbox.transformation import (
CodeExecutionResult,
ContainerHandle,
@ -175,6 +179,10 @@ def _sandbox_unavailable(step: str, error: Exception, output: str = _SANDBOX_UNA
return _SandboxUnavailable(output=output)
def _is_code_interpreter_tool(tool: Mapping[str, object]) -> bool:
return tool.get("type") == "code_interpreter"
def _output_item_type(item: object) -> object:
if isinstance(item, dict):
item_mapping: Final[dict[str, object]] = item
@ -343,8 +351,8 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
(function_tool if isinstance(tool, dict) and tool.get("type") == "code_interpreter" else tool)
for tool in tools
]
if self._tool_choice_targets_code_interpreter(kwargs.get("tool_choice")):
kwargs["tool_choice"] = self._get_function_tool_choice(call_type=call_type)
if "tool_choice" in kwargs:
kwargs["tool_choice"] = self._intercepted_tool_choice(kwargs["tool_choice"], call_type)
return kwargs
@staticmethod
@ -428,6 +436,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
or (isinstance(function, dict) and function.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME)
)
@classmethod
def _intercepted_tool_choice(cls, tool_choice: object, call_type: CallTypes | None) -> object:
if cls._tool_choice_targets_code_interpreter(tool_choice):
return cls._get_function_tool_choice(call_type=call_type)
return allowed_tools_tool_choice_with_builtin_as_function(
tool_choice, _is_code_interpreter_tool, LITELLM_CODE_EXECUTION_TOOL_NAME
)
def _resolve_provider(self, kwargs: dict[str, object]) -> str | None:
provider: Final = kwargs.get("custom_llm_provider")
if isinstance(provider, str) and provider:
@ -694,7 +710,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
def _get_followup_optional_params(self, optional_params: dict[str, object]) -> dict[str, object]:
drop_tool_choice: Final = self._tool_choice_targets_code_interpreter(optional_params.get("tool_choice"))
return {
k: v for k, v in optional_params.items() if k != "tools" and not (k == "tool_choice" and drop_tool_choice)
k: allowed_tools_tool_choice_in_auto_mode(v) if k == "tool_choice" else v
for k, v in optional_params.items()
if k != "tools" and not (k == "tool_choice" and drop_tool_choice)
}
async def async_post_agentic_loop_response_hook(

View file

@ -15,6 +15,7 @@ from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.compression import compress
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.allowed_tools_tool_choice import allowed_tools_tool_choice_with_function
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.integrations.compression_interception import (
CompressionInterceptionConfig,
@ -180,6 +181,10 @@ class CompressionInterceptionLogger(CustomLogger):
existing_tools=cast(list[dict[str, object]] | None, kwargs.get("tools")),
compressed_tools=compressed_tools,
)
if "tool_choice" in kwargs:
kwargs["tool_choice"] = allowed_tools_tool_choice_with_function(
kwargs["tool_choice"], LITELLM_CONTENT_RETRIEVE_TOOL_NAME
)
call_id = cast(str | None, kwargs.get("litellm_call_id"))
if not call_id:
call_id = str(uuid.uuid4())

View file

@ -17,6 +17,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
safe_set_attribute,
)
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.safe_json_dumps import jsonable_or_str
from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse
from litellm.types.utils import (
EmbeddingResponse,
@ -103,7 +104,7 @@ class LangfuseLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
prompt["tools"] = tools
input: Final = prompt
safe_set_attribute(span, "langfuse.observation.input", json.dumps(input))
safe_set_attribute(span, "langfuse.observation.input", json.dumps(input, default=jsonable_or_str))
@staticmethod
@override

View file

@ -1,6 +1,7 @@
import json
from typing import TYPE_CHECKING, Any, Final
from litellm.litellm_core_utils.safe_json_dumps import jsonable_or_str
from litellm.types.integrations.otel_span_attributes import SpanAttributes
if TYPE_CHECKING:
@ -60,7 +61,7 @@ class LangtraceAttributes:
if prompts:
span.add_event(
name="gen_ai.content.prompt",
attributes={SpanAttributes.LLM_PROMPTS.value: json.dumps(prompts)},
attributes={SpanAttributes.LLM_PROMPTS.value: json.dumps(prompts, default=jsonable_or_str)},
)
self.set_span_attributes(span, span_attributes)

View file

@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast
from urllib.parse import urlsplit
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
from pydantic import TypeAdapter, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.otel.model.metadata import (
@ -519,7 +519,7 @@ class LLMCallSpanData:
promoted_metadata=promoted_metadata,
is_streaming=as_bool(payload.get("stream")),
tools=_extract_tools(params),
messages_in=_dicts(payload.get("messages")) if capture_content else (),
messages_in=_message_dicts(payload.get("messages")) if capture_content else (),
choices_out=choices_out if capture_content else (),
system_fingerprint=as_str(response.get("system_fingerprint")),
time_to_first_chunk_seconds=time_to_first_chunk_seconds,
@ -783,6 +783,27 @@ def _dicts(value: object) -> tuple[Mapping[str, object], ...]:
return tuple(item for item in value if isinstance(item, dict))
_OBJECTS: Final = TypeAdapter(tuple[object, ...])
_ANY_KEY_MAPPING: Final = TypeAdapter(Mapping[object, object])
def _message_dict(message: object) -> Mapping[str, object] | None:
if isinstance(message, dict):
return {str(key): value for key, value in _ANY_KEY_MAPPING.validate_python(message).items()}
if not isinstance(message, BaseModel):
return None
try:
return message.model_dump(mode="json", fallback=str)
except ValueError:
return None
def _message_dicts(value: object) -> tuple[Mapping[str, object], ...]:
if not isinstance(value, list):
return ()
return tuple(message for item in _OBJECTS.validate_python(value) if (message := _message_dict(item)) is not None)
def _finish_reasons(choices: tuple[Mapping[str, object], ...]) -> tuple[str, ...]:
"""Non-empty ``finish_reason`` of each response choice."""
return tuple(r for c in choices if (r := as_str(c.get("finish_reason"))))

View file

@ -292,6 +292,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_SpendLogs",
"LiteLLM_BudgetWindowSpend",
"LiteLLM_BackgroundInteractionSettlement",
"LiteLLM_BackgroundResponseSettlement",
"LiteLLM_ErrorLogs",
"LiteLLM_UserNotifications",
"LiteLLM_UserProviderCredentials",

View file

@ -21,7 +21,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
safe_set_attribute,
)
from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_dumps import jsonable_or_str, safe_dumps
from litellm.types.integrations.weave_otel import WeaveOtelConfig, WeaveSpanAttributes
from litellm.types.utils import StandardCallbackDynamicParams
@ -74,7 +74,11 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
prompt["functions"] = functions
if tools is not None:
prompt["tools"] = tools
safe_set_attribute(span, OpenInferenceSpanAttributes.INPUT_VALUE, json.dumps(prompt))
safe_set_attribute(
span,
OpenInferenceSpanAttributes.INPUT_VALUE,
json.dumps(prompt, default=jsonable_or_str),
)
def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object):

View file

@ -9,11 +9,12 @@ server-side using litellm router's search tools.
import asyncio
import math
import uuid
from collections.abc import AsyncIterator, Mapping, Sequence
from collections.abc import AsyncIterator, Callable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast
from pydantic import TypeAdapter
from typing_extensions import Never, ReadOnly
import litellm
@ -38,6 +39,10 @@ from litellm.integrations.websearch_interception.transformation import (
from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
allowed_tools_tool_choice_kept_on_follow_up,
allowed_tools_tool_choice_with_builtin_as_function,
)
from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.integrations.custom_logger import (
@ -100,6 +105,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks"
_RESPONSE_CONTENT_FIELD: Final = "content"
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:"
_ResponseT: Final = TypeVar("_ResponseT")
@ -263,6 +270,22 @@ class _ParentRequestCorrelation:
_NO_ACOMPLETION_NAMED: Final[_AcompletionNamedParams] = {}
def _tool_choice_naming_litellm_web_search(
kwargs: Mapping[str, object], is_search_tool: Callable[[Mapping[str, object]], bool]
) -> Mapping[str, object]:
if "tool_choice" not in kwargs:
return MappingProxyType({})
return {
"tool_choice": allowed_tools_tool_choice_with_builtin_as_function(
kwargs["tool_choice"], is_search_tool, LITELLM_WEB_SEARCH_TOOL_NAME
)
}
def _kept_allowed_tools_tool_choice(optional_params: Mapping[str, object]) -> Mapping[str, object]:
return allowed_tools_tool_choice_kept_on_follow_up(optional_params.get("tool_choice"))
class WebSearchInterceptionLogger(CustomLogger):
"""
CustomLogger that intercepts WebSearch tool calls for models that don't
@ -514,13 +537,17 @@ class WebSearchInterceptionLogger(CustomLogger):
# Keep other tools as-is
converted_tools.append(tool)
kwargs["tools"] = converted_tools
converted_kwargs: Final[dict[str, object]] = {
**kwargs,
"tools": converted_tools,
**_tool_choice_naming_litellm_web_search(kwargs, lambda tool: is_search_tool(dict(tool))),
}
if kwargs.get("stream"):
verbose_logger.debug("WebSearchInterception: deployment hook converting stream=True to stream=False")
return as_converted_stream(kwargs, WEBSEARCH_INTERCEPTION_PREFIX)
return as_converted_stream(converted_kwargs, WEBSEARCH_INTERCEPTION_PREFIX)
return kwargs
return converted_kwargs
def _convert_responses_tools(
self, kwargs: Mapping[str, object], tools: Sequence[dict[str, object]]
@ -535,7 +562,11 @@ class WebSearchInterceptionLogger(CustomLogger):
get_litellm_web_search_tool_responses() if is_web_search_tool_responses(tool) else tool for tool in tools
]
converted_kwargs: Final = {**kwargs, "tools": converted_tools}
converted_kwargs: Final = {
**kwargs,
"tools": converted_tools,
**_tool_choice_naming_litellm_web_search(kwargs, is_web_search_tool_responses),
}
if kwargs.get("stream"):
verbose_logger.debug("WebSearchInterception: deployment hook converting stream=True to stream=False")
@ -1256,9 +1287,13 @@ class WebSearchInterceptionLogger(CustomLogger):
tools_param: Final = optional_params.get("tools")
optional_params_clean: Final = {
k: v
for k, v in optional_params.items()
if k not in {"tools", "tool_choice", "stream", "model_alias_map", "stream_response", "custom_prompt_dict"}
**{
k: v
for k, v in optional_params.items()
if k
not in {"tools", "tool_choice", "stream", "model_alias_map", "stream_response", "custom_prompt_dict"}
},
**_kept_allowed_tools_tool_choice(_OPTIONAL_PARAMS.validate_python(optional_params)),
}
kwargs_for_followup: Final = {
@ -1843,9 +1878,11 @@ class WebSearchInterceptionLogger(CustomLogger):
)
if request_patch.messages is None:
raise ValueError("WebSearchInterception: missing follow-up messages")
params: Final = dict(optional_params)
params.update(request_patch.optional_params)
params.pop("tool_choice", None)
merged_params: Final = {**optional_params, **request_patch.optional_params}
params: Final = {
**{key: value for key, value in merged_params.items() if key != "tool_choice"},
**_kept_allowed_tools_tool_choice(merged_params),
}
patch_kwargs: Final = dict[str, object](request_patch.kwargs)
return await litellm.acompletion(
model=request_patch.model or model,
@ -1958,17 +1995,20 @@ class WebSearchInterceptionLogger(CustomLogger):
tools_param: Final = optional_params.get("tools")
optional_params_clean: Final = {
k: v
for k, v in optional_params.items()
if k
not in {
"tools",
"tool_choice",
"extra_body",
"model_alias_map",
"stream_response",
"custom_prompt_dict",
}
**{
k: v
for k, v in optional_params.items()
if k
not in {
"tools",
"tool_choice",
"extra_body",
"model_alias_map",
"stream_response",
"custom_prompt_dict",
}
},
**_kept_allowed_tools_tool_choice(_OPTIONAL_PARAMS.validate_python(optional_params)),
}
if tools_param is not None:
optional_params_clean["tools"] = tools_param

View file

@ -40,6 +40,16 @@ async def every_claimed_tool_call(
return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))})
_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "tool_use"})
def called_tool_names(response: object) -> frozenset[str]:
items: Final = (*parts_of(message_field(response, "output")), *parts_of(message_field(response, "content")))
return _strings(
message_field(item, "name") for item in items if message_field(item, "type") in _TOOL_CALL_ITEM_TYPES
)
def _call_ids(call: object) -> frozenset[str]:
return _strings((message_field(call, "id"), message_field(call, "call_id")))

View file

@ -1,12 +1,17 @@
from collections.abc import Mapping
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Literal
from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam
from openai.types.chat.chat_completion_allowed_tools_param import ChatCompletionAllowedToolsParam
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from litellm.exceptions import BadRequestError
ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"})
_FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE: Final = "2023-12-01"
HOSTED_TOOL_CHOICE_TYPES: Final = frozenset(
{
@ -29,6 +34,9 @@ HOSTED_TOOL_CHOICE_TYPES: Final = frozenset(
CHAT_ALLOWED_TOOLS_LAYOUT: Final = (
'{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": [<at least one tool>]}}'
)
RESPONSES_ALLOWED_TOOLS_LAYOUT: Final = (
'{"type": "allowed_tools", "mode": "auto" | "required", "tools": [<at least one tool>]}'
)
class ChatAllowedTools(BaseModel):
@ -45,7 +53,17 @@ class ChatAllowedToolsToolChoice(BaseModel):
allowed_tools: ChatAllowedTools
_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice)
class ResponsesAllowedToolsToolChoice(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
type: Literal["allowed_tools"]
mode: Literal["auto", "required"]
tools: tuple[Mapping[str, object], ...] = Field(min_length=1)
_ALLOWED_TOOLS_TOOL_CHOICE: Final[TypeAdapter[ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice]] = (
TypeAdapter(ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice)
)
_SELECTOR: Final = TypeAdapter(dict[str, object])
@ -67,21 +85,22 @@ def is_allowed_tools_tool_choice(tool_choice: object) -> bool:
return False
def raise_if_allowed_tools_tool_choice_unsupported(
def drops_unsupported_allowed_tools_tool_choice(
tool_choice: object,
custom_llm_provider: str,
model: str,
) -> None:
"""Providers outside the set would silently send the request with no tool restriction at all, so the call stops
here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call."""
drop_params: bool,
) -> bool:
if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS:
return
return False
if drop_params:
return True
raise BadRequestError(
message=(
"tool_choice type 'allowed_tools' is only supported for "
f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. "
"drop_params does not drop it, since that would let the model call tools the request excluded. "
"Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead."
"Send only the allowed tools in `tools` with tool_choice='auto' or 'required', or set drop_params=True "
"to drop the tool_choice, which lets the model call any tool in `tools`."
),
model=model,
llm_provider=custom_llm_provider,
@ -119,18 +138,295 @@ def raise_if_hosted_tool_choice_unsupported(
)
def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
try:
return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
except ValidationError:
return None
def _has_name(selector: Mapping[str, object]) -> bool:
name: Final = selector.get("name")
return isinstance(name, str) and name != ""
def _names_a_tool_or_nothing(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type")
if not isinstance(tool_type, str) or tool_type not in ("function", "custom"):
return tool.get("name") is None or _has_name(tool)
match tool.get(tool_type):
case {"name": str()}:
return True
case {"name": object()}:
return False
case _:
return True
def _names_are_strings(tools: Iterable[Mapping[str, object]]) -> bool:
return all(map(_names_a_tool_or_nothing, tools))
def _chat_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
tool_type: Final = tool.get("type")
if not isinstance(tool_type, str) or tool_type not in ("function", "custom"):
return tool
if tool_type in tool:
return None if "name" in tool else tool
if not _has_name(tool):
return None
return {**{key: value for key, value in tool.items() if key != "name"}, tool_type: {"name": tool["name"]}}
def _chat_layout(choice: ResponsesAllowedToolsToolChoice) -> ChatAllowedToolsToolChoice | None:
nested: Final = tuple(_chat_allowed_tool(tool) for tool in choice.tools)
if any(tool is None for tool in nested):
return None
return ChatAllowedToolsToolChoice(
type="allowed_tools",
allowed_tools=ChatAllowedTools(mode=choice.mode, tools=tuple(tool for tool in nested if tool is not None)),
)
def parse_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
try:
parsed: Final = _ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
except ValidationError:
return None
match parsed:
case ChatAllowedToolsToolChoice():
return parsed if _names_are_strings(parsed.allowed_tools.tools) else None
case ResponsesAllowedToolsToolChoice():
return _chat_layout(parsed) if _names_are_strings(parsed.tools) else None
def azure_api_version_takes_tool_choice(api_version: str) -> bool:
return api_version.count("-") < 2 or api_version >= _FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE
def _azure_request_takes_tool_choice(api_version: str | None) -> bool:
import litellm
from litellm.secret_managers.main import get_secret_str
return azure_api_version_takes_tool_choice(
api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION
)
def _enforces_allowed_tools(custom_llm_provider: str | None, api_version: str | None) -> bool:
if custom_llm_provider == "azure":
return _azure_request_takes_tool_choice(api_version)
return custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS
def unenforced_allowed_tools_tool_choice(
tool_choice: object, custom_llm_provider: str | None, api_version: str | None = None
) -> bool:
return is_allowed_tools_tool_choice(tool_choice) and (
not _enforces_allowed_tools(custom_llm_provider, api_version)
or parse_allowed_tools_tool_choice(tool_choice) is None
)
def _allowed_tool_names(tool_choice: object) -> frozenset[object]:
parsed: Final = parse_allowed_tools_tool_choice(tool_choice)
if parsed is None:
return frozenset()
flattened: Final = tuple(_responses_allowed_tool(tool) for tool in parsed.allowed_tools.tools)
return frozenset(tool.get("name") for tool in flattened if tool is not None and _server_entry(tool) is None)
def parse_allowed_tools_tool_choice_or_refuse(
tool_choice: object, model: str, drop_params: bool
) -> ChatAllowedToolsToolChoice | None:
parsed: Final = parse_allowed_tools_tool_choice(tool_choice)
if parsed is not None or drop_params:
return parsed
raise BadRequestError(
message=(
f"Invalid tool choice, tool_choice={tool_choice}. An allowed_tools tool_choice must be "
f"{CHAT_ALLOWED_TOOLS_LAYOUT} or {RESPONSES_ALLOWED_TOOLS_LAYOUT}"
),
model=model,
llm_provider="",
)
def allowed_tools_tool_choice_in_effect(tool_choice: object, model: str, drop_params: bool) -> object:
if not is_allowed_tools_tool_choice(tool_choice):
return tool_choice
return None if parse_allowed_tools_tool_choice_or_refuse(tool_choice, model, drop_params) is None else tool_choice
def _server_entry(tool: Mapping[str, object]) -> tuple[str, str | None] | None:
label: Final = tool.get("server_label")
name: Final = tool.get("name")
if tool.get("type") != "mcp" or not isinstance(label, str):
return None
return label, name if isinstance(name, str) else None
def _server_entries(tool_choice: object) -> frozenset[tuple[str, str | None]]:
parsed: Final = parse_allowed_tools_tool_choice(tool_choice)
entries: Final = () if parsed is None else tuple(map(_server_entry, parsed.allowed_tools.tools))
return frozenset(entry for entry in entries if entry is not None)
@dataclass(frozen=True, slots=True)
class GatewayTool:
names: frozenset[str]
server_labels: frozenset[str]
@dataclass(frozen=True, slots=True)
class GatewayTools:
server_labels: frozenset[str]
tools: Mapping[str, GatewayTool]
NO_GATEWAY_TOOLS: Final = GatewayTools(server_labels=frozenset(), tools=MappingProxyType({}))
def _server_entry_allows(entry: tuple[str, str | None], tool: GatewayTool | None) -> bool:
label, name = entry
return tool is not None and label in tool.server_labels and (name is None or name in tool.names)
def _a_server_entry_allows(entries: Iterable[tuple[str, str | None]], tool: GatewayTool | None) -> bool:
return any(_server_entry_allows(entry, tool) for entry in entries)
def tool_calls_the_choice_excludes(
tool_choice: object, tool_names: Iterable[str], gateway: GatewayTools = NO_GATEWAY_TOOLS
) -> tuple[str, ...]:
if not is_allowed_tools_tool_choice(tool_choice):
return ()
allowed: Final = _allowed_tool_names(tool_choice)
server_entries: Final = _server_entries(tool_choice)
return tuple(
name
for name in tool_names
if name not in allowed and not _a_server_entry_allows(server_entries, gateway.tools.get(name))
)
def refuse_tool_calls_the_choice_excludes(tool_choice: object, tool_names: Iterable[str], model: str) -> None:
excluded: Final = tool_calls_the_choice_excludes(tool_choice, tool_names)
if excluded:
raise BadRequestError(
message=(
f"The model called {', '.join(excluded)}, which the allowed_tools tool_choice excludes, because this "
"request did not reach a model that enforces the choice. LiteLLM did not run the call. Send only the "
"allowed tools in `tools` with tool_choice='auto' or 'required'."
),
model=model,
llm_provider="",
)
def _with_builtin_as(
tool: Mapping[str, object], is_builtin: Callable[[Mapping[str, object]], bool], function_entry: Mapping[str, object]
) -> Mapping[str, object]:
return function_entry if tool.get("type") not in ("function", "custom", "mcp") and is_builtin(tool) else tool
def _function_entry(
layout: ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice, name: str
) -> Mapping[str, object]:
match layout:
case ChatAllowedToolsToolChoice():
return {"type": "function", "function": {"name": name}}
case ResponsesAllowedToolsToolChoice():
return {"type": "function", "name": name}
def _entries_of(
layout: ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice,
) -> tuple[Mapping[str, object], ...]:
match layout:
case ChatAllowedToolsToolChoice():
return layout.allowed_tools.tools
case ResponsesAllowedToolsToolChoice():
return layout.tools
def _with_entries(
layout: ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice, tools: Iterable[Mapping[str, object]]
) -> Mapping[str, object]:
entries: Final = [dict(tool) for tool in tools]
match layout:
case ChatAllowedToolsToolChoice():
return {"type": "allowed_tools", "allowed_tools": {"mode": layout.allowed_tools.mode, "tools": entries}}
case ResponsesAllowedToolsToolChoice():
return {"type": "allowed_tools", "mode": layout.mode, "tools": entries}
def _callers_layout(tool_choice: object) -> ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice | None:
if parse_allowed_tools_tool_choice(tool_choice) is None:
return None
return _ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
def allowed_tools_tool_choice_with_builtin_as_function(
tool_choice: object, is_builtin: Callable[[Mapping[str, object]], bool], function_name: str
) -> object:
layout: Final = _callers_layout(tool_choice)
if layout is None:
return tool_choice
entry: Final = _function_entry(layout, function_name)
return _with_entries(layout, (_with_builtin_as(tool, is_builtin, entry) for tool in _entries_of(layout)))
def allowed_tools_tool_choice_with_function(tool_choice: object, function_name: str) -> object:
layout: Final = _callers_layout(tool_choice)
if layout is None or function_name in _allowed_tool_names(tool_choice):
return tool_choice
return _with_entries(layout, (*_entries_of(layout), _function_entry(layout, function_name)))
def allowed_tools_tool_choice_in_auto_mode(tool_choice: object) -> object:
layout: Final = _callers_layout(tool_choice)
match layout:
case None:
return tool_choice
case ChatAllowedToolsToolChoice():
return layout.model_copy(
update={"allowed_tools": layout.allowed_tools.model_copy(update={"mode": "auto"})}
).model_dump(mode="json")
case ResponsesAllowedToolsToolChoice():
return layout.model_copy(update={"mode": "auto"}).model_dump(mode="json")
def allowed_tools_tool_choice_kept_on_follow_up(tool_choice: object) -> Mapping[str, object]:
if not is_allowed_tools_tool_choice(tool_choice):
return MappingProxyType({})
return MappingProxyType({"tool_choice": allowed_tools_tool_choice_in_auto_mode(tool_choice)})
def _gateway_entry(tool: Mapping[str, object], gateway_labels: frozenset[str]) -> tuple[str, str | None] | None:
entry: Final = _server_entry(tool)
return entry if entry is not None and entry[0] in gateway_labels else None
def allowed_tools_tool_choice_with_gateway_tools_as_functions(tool_choice: object, gateway: GatewayTools) -> object:
layout: Final = _callers_layout(tool_choice)
entries: Final = () if layout is None else _entries_of(layout)
gateway_entries: Final = tuple(
entry for entry in (_gateway_entry(tool, gateway.server_labels) for tool in entries) if entry is not None
)
if layout is None or not gateway_entries:
return tool_choice
kept: Final = tuple(tool for tool in entries if _gateway_entry(tool, gateway.server_labels) is None)
allowed: Final = tuple(
name for name, tool in gateway.tools.items() if _a_server_entry_allows(gateway_entries, tool)
)
if not kept and not allowed:
return "none"
return _with_entries(layout, (*kept, *(_function_entry(layout, name) for name in allowed)))
def chat_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ChatCompletionAllowedToolChoiceParam:
return ChatCompletionAllowedToolChoiceParam(
type="allowed_tools",
allowed_tools=ChatCompletionAllowedToolsParam(
mode=choice.allowed_tools.mode, tools=[dict(tool) for tool in choice.allowed_tools.tools]
),
)
def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
tool_type: Final = tool.get("type")
if tool_type not in ("function", "custom"):

View file

@ -31,6 +31,9 @@ def normalized_audio_duration_seconds(value: object) -> float | None:
return seconds if math.isfinite(seconds) and seconds >= 0 else None
LIBSNDFILE_UNKNOWN_FRAME_COUNT: Final = 2**63 - 1
@dataclass
class ProcessedAudioFile:
"""
@ -340,7 +343,10 @@ def calculate_request_duration(file: FileTypes) -> float | None:
# Extract duration using soundfile
file_object: Final = io.BytesIO(file_content)
with sf.SoundFile(file_object) as audio:
duration: Final = len(audio) / audio.samplerate
frames: Final = len(audio)
if frames >= LIBSNDFILE_UNKNOWN_FRAME_COUNT:
return None
duration: Final = frames / audio.samplerate
return duration
except Exception:

View file

@ -1,7 +1,7 @@
# this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api
import json
from collections.abc import Awaitable, Iterator, Mapping, Sequence
from collections.abc import Awaitable, Iterable, Iterator, Mapping, Sequence
from itertools import chain
from types import MappingProxyType
from typing import Final, TypeVar
@ -18,7 +18,12 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
parse_allowed_tools_tool_choice,
tool_calls_the_choice_excludes,
)
from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model
from litellm.litellm_core_utils.get_litellm_params import AGENTIC_LOOP_TOOL_CHOICE_KEY
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict, object_items
@ -34,7 +39,11 @@ from litellm.types.integrations.custom_logger import (
is_interception_internal_key,
stashed_stream_options,
)
from litellm.types.utils import ModelResponse
from litellm.types.utils import (
ChatCompletionMessageCustomToolCall,
ChatCompletionMessageToolCall,
ModelResponse,
)
from litellm.utils import CustomStreamWrapper
_DispatchResult: Final = TypeVar("_DispatchResult")
@ -181,6 +190,40 @@ def _end_refused_turn(
return finalized if replay is None else replay
def _tool_call_name(call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall) -> str:
match call:
case ChatCompletionMessageCustomToolCall():
return call.custom.name
case ChatCompletionMessageToolCall():
return call.function.name or ""
def _called_tool_names(response: ModelResponse) -> tuple[str, ...]:
calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in response.choices)
return tuple(map(_tool_call_name, calls))
def _server_fulfilled_tool_names(callbacks: Iterable[CustomLogger]) -> frozenset[str]:
return frozenset(chain.from_iterable(callback.server_fulfilled_tool_names for callback in callbacks))
def holds_a_call_the_choice_excludes(
tool_names: Iterable[str], kwargs: Mapping[str, object], callbacks: Iterable[CustomLogger]
) -> bool:
server_fulfilled: Final = _server_fulfilled_tool_names(callbacks)
client_tool_names: Final = (name for name in tool_names if name not in server_fulfilled)
return bool(tool_calls_the_choice_excludes(kwargs.get(AGENTIC_LOOP_TOOL_CHOICE_KEY), client_tool_names))
def with_caller_tool_choice(
kwargs: Mapping[str, object], optional_params: Mapping[str, object]
) -> Mapping[str, object]:
tool_choice: Final = optional_params.get("tool_choice")
if AGENTIC_LOOP_TOOL_CHOICE_KEY in kwargs or parse_allowed_tools_tool_choice(tool_choice) is None:
return kwargs
return {**kwargs, AGENTIC_LOOP_TOOL_CHOICE_KEY: tool_choice}
def _request_tools(optional_params: Mapping[str, object]) -> list[dict[str, object]]:
return [tool for tool in object_items(optional_params.get("tools")) if is_str_keyed_dict(tool)]
@ -238,6 +281,35 @@ async def _refused_chat_turn(
return finalize_refused_chat_turn(response, claimed)
def _fingerprint_of_a_turn_to_run(
*,
response: ModelResponse,
request_kwargs: Mapping[str, object],
callbacks: Sequence[CustomLogger],
tool_calls: object,
fingerprints: Sequence[str],
depth: int,
max_loops: int,
model: str,
stream: bool,
) -> str | None:
if holds_a_call_the_choice_excludes(_called_tool_names(response), request_kwargs, callbacks):
return None
try:
return _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if stream:
raise
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
return None
def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]:
metadata: Final = kwargs_for_followup.get("litellm_metadata")
return MappingProxyType(
@ -371,7 +443,8 @@ async def maybe_run_chat_completion_agentic_loop(
stream: bool,
) -> ModelResponse | CustomStreamWrapper | None:
callbacks: Final = _custom_loggers(logging_obj)
depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
request_kwargs: Final = with_caller_tool_choice(kwargs, optional_params)
depth, max_loops, fingerprints = _agentic_loop_settings(request_kwargs)
tools: Final = _request_tools(optional_params)
for callback in callbacks:
@ -379,7 +452,7 @@ async def maybe_run_chat_completion_agentic_loop(
continue
hook_kwargs = {
**kwargs,
**request_kwargs,
"_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
"custom_llm_provider": custom_llm_provider,
}
@ -403,18 +476,18 @@ async def maybe_run_chat_completion_agentic_loop(
if not should_run:
continue
try:
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if stream:
raise
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
fingerprint = _fingerprint_of_a_turn_to_run(
response=response,
request_kwargs=request_kwargs,
callbacks=callbacks,
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
stream=stream,
)
if fingerprint is None:
return _end_refused_turn(
await _refused_chat_turn(
callbacks,
@ -425,9 +498,9 @@ async def maybe_run_chat_completion_agentic_loop(
messages=messages,
tools=tools,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
kwargs=request_kwargs,
),
kwargs=kwargs,
kwargs=request_kwargs,
depth=depth,
model=model,
custom_llm_provider=custom_llm_provider,
@ -471,7 +544,7 @@ async def maybe_run_chat_completion_agentic_loop(
callback=callback,
model=model,
optional_params=optional_params,
kwargs=kwargs,
kwargs=dict(request_kwargs),
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
depth=depth,
@ -482,7 +555,7 @@ async def maybe_run_chat_completion_agentic_loop(
return _converted_stream_replay(
response,
kwargs=kwargs,
kwargs=request_kwargs,
depth=depth,
model=model,
custom_llm_provider=custom_llm_provider,
@ -526,18 +599,18 @@ async def run_chat_completion_agentic_hooks(
if not should_run:
continue
try:
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if stream:
raise
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
fingerprint = _fingerprint_of_a_turn_to_run(
response=response,
request_kwargs=kwargs,
callbacks=callbacks,
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
stream=stream,
)
if fingerprint is None:
return await _refused_chat_turn(
callbacks,
callback,

View file

@ -17,6 +17,28 @@ BEDROCK_MANAGED_S3_PREFIXES: Final = (
BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
)
MANAGED_CLOUD_STORAGE_SCHEMES: Final = ("s3://", "gs://")
STORED_S3_BUCKET_PARAM: Final = "stored_s3_bucket_name"
REQUEST_S3_BUCKET_PARAMS: Final = frozenset({"s3_bucket_name", "s3_output_bucket_name"})
def stored_s3_bucket_kwargs(stored_file_id: str) -> Mapping[str, str]:
"""The bucket (and any prefix before LiteLLM's own path) of an ``s3://`` file id LiteLLM recorded itself,
as the param that lets Bedrock read it back.
Only pass ids from LiteLLM's own records (managed file rows, provider batch responses), never one a caller sent.
"""
if not stored_file_id.startswith("s3://"):
return {}
bucket, _, object_key = stored_file_id.removeprefix("s3://").partition("/")
if not bucket:
return {}
segments: Final = object_key.split("/")
prefix_depth: Final = next(
(depth for depth in range(len(segments)) if "/".join(segments[depth:]).startswith(BEDROCK_MANAGED_S3_PREFIXES)),
0,
)
prefix: Final = "/".join(segments[:prefix_depth])
return {STORED_S3_BUCKET_PARAM: f"{bucket}/{prefix}" if prefix else bucket}
MAX_FILE_ID_DECODE_PASSES: Final = 8

View file

@ -123,7 +123,7 @@ def proxy_request_headers(container: Mapping[str, object] | None) -> Mapping[str
def stream_cost_injection_requested(request_headers: Mapping[str, object] | None) -> bool:
"""The global include_cost_in_streaming_usage flag or the per-request header asks for usage.cost on streams."""
if bool(getattr(litellm, "include_cost_in_streaming_usage", False)):
if litellm.include_cost_in_streaming_usage:
return True
if not request_headers:
return False

View file

@ -1,6 +1,7 @@
"""Utils for accessing credentials."""
from collections.abc import Mapping
from copy import deepcopy
from typing import Final
import litellm
@ -27,7 +28,9 @@ class CredentialAccessor:
"""Safe accessor for credentials."""
credential: Final = CredentialAccessor.find_credential(credential_name)
return {} if credential is None else dict(credential_api_key_secret_reference(credential.credential_values))
if credential is None:
return {}
return deepcopy(dict(credential_api_key_secret_reference(credential.credential_values)))
@staticmethod
def upsert_credentials(credentials: list[CredentialItem]):

View file

@ -32,8 +32,9 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
)
PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header"
AGENTIC_LOOP_TOOL_CHOICE_KEY: Final = "_agentic_loop_tool_choice"
AGENTIC_LOOP_CONTROL_KWARGS_KEYS: Final = frozenset(
{"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops"}
{"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops", AGENTIC_LOOP_TOOL_CHOICE_KEY}
)
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS: Final = frozenset(
{
@ -62,6 +63,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"bucket_name",
"s3_bucket_name",
"s3_output_bucket_name",
"stored_s3_bucket_name",
"allow_legacy_cloud_file_ids",
"s3_endpoint_url",
"s3_region_name",

View file

@ -906,6 +906,7 @@ class Logging(LiteLLMLoggingBaseClass):
# Timing for results that cannot carry ``_hidden_params`` (plain-dict /v1/messages
# responses and the bridge stream wrappers); see ``update_response_metadata``.
self.response_timing_metrics: Mapping[str, float] = {}
self.first_chunk_overhead_ms: float | None = None
# Passthrough endpoint guardrails config for field targeting
self.passthrough_guardrails_config: dict[str, object] | None = None
@ -983,6 +984,11 @@ class Logging(LiteLLMLoggingBaseClass):
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
self.response_timing_metrics = dict(timing_metrics)
def set_first_chunk_overhead(self, seconds: float) -> None:
"""The proxy's own work on the first chunk of a stream, the delay every later chunk inherits."""
if self.first_chunk_overhead_ms is None:
self.first_chunk_overhead_ms = seconds * 1000
def add_dynamic_callback(self, callback: CustomLogger) -> None:
self.dynamic_input_callbacks = self._with_dynamic_callback(self.dynamic_input_callbacks, callback)
self.dynamic_success_callbacks = self._with_dynamic_callback(self.dynamic_success_callbacks, callback)
@ -7984,8 +7990,15 @@ def get_standard_logging_object_payload(
if clean_hidden_params["litellm_overhead_time_ms"] is None and status == "success":
# /v1/messages dict results and the bridge stream wrappers keep it on the logging object;
# failure payloads stay None like every response type that carries its own _hidden_params
timing_metrics: Final = getattr(logging_obj, "response_timing_metrics", None) or {}
timing_metrics: Final[Mapping[str, float]] = getattr(logging_obj, "response_timing_metrics", None) or {}
clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms")
# the overhead above was stamped when the stream object came back, before any chunk was
# read; the proxy's work on the first chunk (hook, cost injection, serialization) is the
# delay the client sees on every chunk, so only that one is added, never a sum over chunks
first_chunk_overhead_ms: Final = getattr(logging_obj, "first_chunk_overhead_ms", None)
stamped_overhead_ms: Final = clean_hidden_params["litellm_overhead_time_ms"]
if status == "success" and stamped_overhead_ms is not None and isinstance(first_chunk_overhead_ms, float):
clean_hidden_params["litellm_overhead_time_ms"] = round(stamped_overhead_ms + first_chunk_overhead_ms, 4)
model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information(
base_model=base_model,

View file

@ -311,6 +311,12 @@ def guardrail_information_cost(guardrail_information: object) -> float:
return _validated_entry_cost(guardrail_information)
def guardrail_share_of_spend(spend_log_metadata: Mapping[str, object], spend: float) -> float:
"""The part of a spend row's ``spend`` billed for its guardrails, priced the way success and failure rows add
it to spend and capped at the row's spend, since an allowance discount or a repeat job poll can charge less."""
return min(guardrail_information_cost(spend_log_metadata.get("guardrail_information")), max(spend, 0.0))
def cost_breakdown_with_guardrail(cost_breakdown: CostBreakdown | None, guardrail_cost: float) -> CostBreakdown | None:
if guardrail_cost <= 0.0:
return cost_breakdown

View file

@ -112,6 +112,7 @@ _BATCH_RATE_PREFIXES: Final = (
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"cache_read_input_image_token_cost",
"cache_creation_input_token_cost",
)
_BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values())
@ -326,6 +327,7 @@ class BatchCostRates:
cache_read: float | None
cache_creation: float | None
cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
cache_read_image: float | None = None
def _batch_rate(model_info: ModelInfo, key: str) -> float | None:
@ -388,6 +390,7 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide
if (rate := _batch_rate_for_prefix(model_info, key, usage, inclusive)) is not None
}
),
cache_read_image=_batch_rate_for_prefix(model_info, "cache_read_input_image_token_cost", usage, inclusive),
)

View file

@ -299,7 +299,7 @@ def update_response_metadata(
# cannot hold ``_hidden_params``: keep only the timing on the logging object (no cost
# recompute) so the proxy headers and the standard logging payload can still read it.
logging_obj.set_response_timing_metrics(
response_timing_metrics(start_time, end_time, logging_obj, include_overhead)
_logging_obj_timing_metrics(start_time, end_time, logging_obj, include_overhead)
)
return
@ -307,3 +307,24 @@ def update_response_metadata(
metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead)
metadata.set_hidden_params(logging_obj, model, kwargs, cache_hit=cache_hit)
metadata.apply()
if logging_obj.stream is True:
# the response assembled from the chunks never inherits the stream wrapper's
# ``_hidden_params``, so the logging payload reads the timing from the logging object
logging_obj.set_response_timing_metrics(
_logging_obj_timing_metrics(start_time, end_time, logging_obj, include_overhead)
)
def _logging_obj_timing_metrics(
start_time: datetime.datetime,
end_time: datetime.datetime,
logging_obj: LiteLLMLoggingObject,
include_overhead: bool,
) -> Mapping[str, float]:
"""A stream read to completion keeps the overhead stamped when the stream was handed back."""
timing_metrics: Final = response_timing_metrics(start_time, end_time, logging_obj, include_overhead)
stamped: Final[Mapping[str, float]] = getattr(logging_obj, "response_timing_metrics", None) or {}
stamped_overhead_ms: Final = stamped.get("litellm_overhead_time_ms")
if include_overhead or stamped_overhead_ms is None:
return timing_metrics
return {**timing_metrics, "litellm_overhead_time_ms": stamped_overhead_ms}

View file

@ -5,7 +5,7 @@ Helper functions to handle images passed in messages
import asyncio
import base64
import time
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from functools import partial
@ -531,6 +531,29 @@ async def async_convert_urls_to_base64(
)
async def _download_outcome(url: str, timeout: float | httpx.Timeout | None) -> "str | litellm.ImageFetchError":
try:
return await async_convert_url_to_base64(url, timeout=timeout)
except litellm.ImageFetchError as error:
return error
async def _downloads_through_first_failure(
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None
) -> "AsyncIterator[str | litellm.ImageFetchError]":
for url in remote_urls:
yield (outcome := await _download_outcome(url, timeout))
if isinstance(outcome, litellm.ImageFetchError):
return
async def async_fetch_urls_to_base64(
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
) -> "Mapping[str, str | litellm.ImageFetchError]":
outcomes: Final = tuple([outcome async for outcome in _downloads_through_first_failure(remote_urls, timeout)])
return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True)))
def _remote_urls_to_inline(
messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool]
) -> tuple[str, ...]:
@ -539,6 +562,41 @@ def _remote_urls_to_inline(
return tuple(dict.fromkeys(remote.url for remote in remotes if should_inline(_remote_media(remote))))
def _message_parts(messages: Iterable[AllMessageValues]) -> Iterator[object]:
return chain.from_iterable(_content_parts(message) for message in messages)
def _file_of_file_part(part: object) -> Mapping[str, object] | None:
fields: Final = _as_mapping(part)
return _as_mapping(fields.get("file")) if fields is not None and fields.get("type") == "file" else None
def _file_id_url(part: object, url_starts: tuple[str, ...]) -> str | None:
file: Final = _file_of_file_part(part)
file_id: Final = file.get("file_id") if file is not None else None
return file_id if isinstance(file_id, str) and file_id.startswith(url_starts) else None
def _is_readable_file_part_or_other_part(part: object) -> bool:
fields: Final = _as_mapping(part)
if fields is None or fields.get("type") != "file":
return fields is not None
file: Final = _file_of_file_part(part)
file_id: Final = file.get("file_id") if file is not None else None
return file is not None and (not file_id or isinstance(file_id, str))
def has_file_id_urls(messages: Iterable[AllMessageValues], url_starts: tuple[str, ...]) -> bool:
return any(_file_id_url(part, url_starts) is not None for part in _message_parts(messages))
def file_id_urls_before_an_unreadable_part(
messages: Iterable[AllMessageValues], url_starts: tuple[str, ...]
) -> tuple[str, ...]:
readable_parts: Final = takewhile(_is_readable_file_part_or_other_part, _message_parts(messages))
return tuple(dict.fromkeys(url for part in readable_parts if (url := _file_id_url(part, url_starts)) is not None))
def inline_remote_media(
messages: list[AllMessageValues], # mutable-ok: every transform_request takes list[AllMessageValues]
should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url,

View file

@ -35,11 +35,14 @@ from operator import itemgetter
from types import MappingProxyType
from typing import Final, Literal, TypeAlias, TypeVar
from pydantic import ValidationError
from litellm.types.llms.anthropic import (
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
ANTHROPIC_TOOL_SEARCH_TOOL_NAMES,
AnthropicMessagesSystemMessageParam,
AnthropicSystemMessageContent,
AnthropicToolSearchToolResultBlock,
)
from litellm.types.llms.openai import (
AllMessageValues,
@ -299,6 +302,15 @@ def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool:
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES
def anthropic_tool_search_result(block: object) -> AnthropicToolSearchToolResultBlock | None:
if message_field(block, "type") != "tool_search_tool_result":
return None
try:
return AnthropicToolSearchToolResultBlock.model_validate(block)
except ValidationError:
return None
def find_anthropic_server_tool_result(
tool_id: str,
web_search_results: Sequence[object] | None,

View file

@ -282,14 +282,6 @@ def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]:
)
def _content_carries_encrypted_reasoning(content: object) -> bool:
return any(is_encrypted_reasoning_block(block) for block in _object_sequence(content))
def messages_carry_encrypted_reasoning(messages: Sequence[object]) -> bool:
return any(_content_carries_encrypted_reasoning(content) for content in anthropic_content_lists(messages))
def _strip_encrypted_reasoning_from_blocks(
content: object,
*,

View file

@ -3,7 +3,8 @@ import math
from collections.abc import Callable
from typing import Any, Final
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
from pydantic_core import to_jsonable_python
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
@ -113,3 +114,13 @@ def safe_dumps(
return json.dumps(
safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str
)
_JSONABLE: Final = TypeAdapter(object)
def jsonable_or_str(value: object) -> object:
try:
return _JSONABLE.validate_python(to_jsonable_python(value, fallback=str))
except ValueError:
return str(value)

View file

@ -82,6 +82,8 @@ FUNCTION_CALL_ATTRIBUTE: Final = "function_call"
_SYNC_ITER_EXHAUSTED: Final = object()
_STREAMING_CHOICE_KEYS: Final = frozenset({"finish_reason", "index", "delta", "logprobs", "message"})
_GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__)
_CompleteStreamingResponse = ModelResponse | TextCompletionResponse | ModelResponseStream | None
_USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset(
@ -139,11 +141,18 @@ def _entry_len(entries: Mapping[str, object], key: str) -> int:
def _prepend_upstream_choice_extras(
choice: StreamingChoices, upstream_choice: ChunkChoice, extra_keys: frozenset[str]
choice: StreamingChoices, upstream_choice: ChunkChoice | StreamingChoices, extra_keys: frozenset[str]
) -> None:
if not extra_keys:
return
upstream_extras: Final[Mapping[str, object]] = upstream_choice.model_dump(include=set(extra_keys))
stored_extras: Final = {
key: value for key, value in (upstream_choice.__pydantic_extra__ or {}).items() if key in extra_keys
}
upstream_extras: Final[Mapping[str, object]] = (
stored_extras
if _holds_only_scalars(stored_extras.items())
else upstream_choice.model_dump(include=set(extra_keys))
)
choice.__pydantic_extra__ = {**upstream_extras, **(choice.__pydantic_extra__ or {})} # rebind-ok: reuse choice
choice.__pydantic_fields_set__.update(upstream_extras)
@ -439,7 +448,7 @@ class CustomStreamWrapper:
self.completion_stream = completion_stream
self.sent_first_chunk = False
self.sent_last_chunk = False
self._openai_handler_receives_chat_chunks = False
self._openai_handler_receives_chunk_objects = False
self._failure_logging: asyncio.Task[None] | None = None
self._closed = False
self._stream_created_time: float = time.time()
@ -1162,13 +1171,13 @@ class CustomStreamWrapper:
def _prebuilt_plain_delta(self, chunk: object) -> Delta | None:
if (
not self._openai_handler_receives_chat_chunks
or not isinstance(chunk, ChatCompletionChunk)
not self._openai_handler_receives_chunk_objects
or not isinstance(chunk, (ChatCompletionChunk, ModelResponseStream))
or not chunk.choices
):
return None
upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent
ChoiceDelta | None,
ChoiceDelta | Delta | None,
chunk.choices[0].delta,
)
if (
@ -1296,22 +1305,23 @@ class CustomStreamWrapper:
upstream_choice: Final = (
original_chunk_for_reuse.choices[0]
if plain_delta_created
and isinstance(original_chunk_for_reuse, ChatCompletionChunk)
and isinstance(original_chunk_for_reuse, (ChatCompletionChunk, ModelResponseStream))
and len(original_chunk_for_reuse.choices) == 1
else None
)
upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent
ChoiceDelta | None,
ChoiceDelta | Delta | None,
getattr(upstream_choice, "delta", None),
)
upstream_choice_extra_keys: Final = frozenset(
(upstream_choice.__pydantic_extra__ or {}) if upstream_choice is not None else ()
)
upstream_logprobs: Final[object] = getattr(upstream_choice, "logprobs", None)
can_reuse_choice: Final = (
upstream_choice is not None
and upstream_delta is not None
and "enhancements" not in upstream_choice_extra_keys
and upstream_choice.logprobs is None
and upstream_logprobs is None
and _holds_only_scalars(upstream_delta)
)
if can_reuse_choice and upstream_choice is not None:
@ -1320,7 +1330,7 @@ class CustomStreamWrapper:
_prepend_upstream_choice_extras(
model_response.choices[0],
upstream_choice,
upstream_choice_extra_keys - {"message"},
upstream_choice_extra_keys - _STREAMING_CHOICE_KEYS,
)
else:
choices: Final = []
@ -1737,14 +1747,16 @@ class CustomStreamWrapper:
self._record_finish_choice_fields(cached_chunk)
else: # openai / azure chat model
self.uses_openai_chat_stream = True
if isinstance(chunk, ChatCompletionChunk):
self._openai_handler_receives_chat_chunks = True
if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
raise litellm.InternalServerError(
message="Upstream generation aborted (finish_reason=abort)",
model=chunk.model,
llm_provider=self.custom_llm_provider,
)
if isinstance(chunk, (ChatCompletionChunk, ModelResponseStream)):
self._openai_handler_receives_chunk_objects = True
if isinstance(chunk, ChatCompletionChunk) and any(
str(choice.finish_reason) == "abort" for choice in chunk.choices or ()
):
raise litellm.InternalServerError(
message="Upstream generation aborted (finish_reason=abort)",
model=chunk.model,
llm_provider=self.custom_llm_provider,
)
if self.custom_llm_provider in [
LlmProviders.AZURE.value,
LlmProviders.AZURE_AI.value,

View file

@ -812,8 +812,9 @@ class ModelResponseIterator:
reasoning_content=reasoning_content,
speed=self.speed,
)
if usage.speed is not None:
self.speed = usage.speed
reported_speed: Final = self._cumulative_usage.get("speed")
if isinstance(reported_speed, str):
self.speed = reported_speed
return usage
def _content_block_delta_helper(

View file

@ -31,6 +31,7 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.asyncify import is_event_loop_running
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
coerce_token_limit,
drops_unsupported_params,
map_finish_reason,
@ -151,8 +152,10 @@ from ..common_utils import (
requires_native_compaction_beta,
stop_reason_provider_specific_fields,
strip_advisor_blocks_from_messages,
tool_search_result_renamed,
tools_marked_for_a_tool_config_point,
validated_object_mapping,
without_stale_tool_search_results,
)
if TYPE_CHECKING:
@ -411,6 +414,17 @@ class _AnthropicToolResultBlock(TypedDict, total=False):
content: ReadOnly[object]
def _provider_fields_with_tool_names(message: object, forward_map: Mapping[str, str]) -> Mapping[str, object] | None:
provider_fields: Final = as_str_mapping(message_field(message, "provider_specific_fields"))
replayed_results: Final = parts_of(provider_fields.get("tool_results")) if provider_fields is not None else ()
if provider_fields is None or not replayed_results:
return None
return {
**provider_fields,
"tool_results": [tool_search_result_renamed(result, forward_map) for result in replayed_results],
}
_ENUM_TYPE_CHECKS: Final[Mapping[object, Callable[[object], bool]]] = MappingProxyType(
{
"null": lambda v: v is None,
@ -457,7 +471,7 @@ def _basic_sanitize_anthropic_tool_name(name: str) -> str:
return _ANTHROPIC_TOOL_NAME_INVALID_CHARS.sub("_", name)[:_ANTHROPIC_TOOL_NAME_MAX_LEN]
def _build_anthropic_tool_name_maps(
def build_anthropic_tool_name_maps(
original_names: list[str],
) -> tuple[dict[str, str], dict[str, str]]:
"""Build (forward, reverse) tool-name maps for a single request.
@ -1458,8 +1472,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
messages: list[AllMessageValues],
name_forward_map: dict[str, str],
) -> list[AllMessageValues]:
"""Return a copy of `messages` with tool_call/function_call names
rewritten using the per-request forward map.
"""Return a copy of `messages` with tool_call/function_call names, and
the tools a replayed tool search found, rewritten using the per-request
forward map.
Only mutates messages whose tool_call/function_call name is *in* the
forward map. Names absent from the map (already valid, no collision)
@ -1475,7 +1490,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
continue
tool_calls = msg.get("tool_calls")
function_call = msg.get("function_call")
if not tool_calls and not function_call:
renamed_provider_fields = _provider_fields_with_tool_names(msg, name_forward_map)
if not tool_calls and not function_call and renamed_provider_fields is None:
new_messages.append(msg)
continue
new_msg = dict(msg)
@ -1501,6 +1517,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
new_fc = dict(function_call)
new_fc["name"] = name_forward_map[fc_name]
new_msg["function_call"] = new_fc
if renamed_provider_fields is not None:
new_msg["provider_specific_fields"] = renamed_provider_fields
new_messages.append(cast(AllMessageValues, new_msg))
return new_messages
@ -1517,7 +1535,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
callers that need to compute the maps from the raw OpenAI shape --
e.g. test setup or future pre-mapping consumers.
See _build_anthropic_tool_name_maps for the collision rules. Pulls
See build_anthropic_tool_name_maps for the collision rules. Pulls
the original name out of either ``{"function": {"name": ...}}``
(legacy OpenAI shape) or ``{"name": ...}`` (rare top-level shape).
"""
@ -1530,7 +1548,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
original = tool.get("name")
if isinstance(original, str) and original:
original_names.append(original)
return _build_anthropic_tool_name_maps(original_names)
return build_anthropic_tool_name_maps(original_names)
@staticmethod
def _sanitize_tool_names_in_request(
@ -1558,7 +1576,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# 1. Collect originals from the Anthropic-shaped custom-tool entries.
# Order matters: the first occurrence wins the canonical slot;
# later collisions get numeric suffixes (see
# ``_build_anthropic_tool_name_maps``).
# ``build_anthropic_tool_name_maps``).
original_names: Final[list[str]] = []
for t in tools:
if not isinstance(t, dict):
@ -1572,7 +1590,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if not original_names:
return {}, {}
forward, reverse = _build_anthropic_tool_name_maps(original_names)
forward, reverse = build_anthropic_tool_name_maps(original_names)
if not forward:
# Every name was already valid -- nothing to do.
return forward, reverse
@ -2544,7 +2562,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# The reverse map only contains entries for names that were actually
# rewritten -- so a tool legitimately named ``foo_bar`` is never
# incorrectly retyped to ``foo/bar`` on the response side.
# See _build_anthropic_tool_name_maps for the collision-handling
# See build_anthropic_tool_name_maps for the collision-handling
# rules and rationale.
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
@ -2600,8 +2618,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
_all_tools: Final = optional_params.get("tools") or []
_has_advisor = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _all_tools)
if not _has_advisor:
anthropic_messages = strip_advisor_blocks_from_messages(anthropic_messages)
anthropic_messages = list(
without_stale_tool_search_results(
anthropic_messages if _has_advisor else strip_advisor_blocks_from_messages(anthropic_messages),
_TOOLS_ADAPTER.validate_python(_all_tools),
)
)
## Add code_execution tool if container_upload is in messages
_tools: Final = (
@ -3239,7 +3261,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Reverse-map rewritten tool names back to caller's originals so a
# downstream OpenAI-style dispatcher can match on the registered name.
# See _build_anthropic_tool_name_maps for why this is keyed on the
# See build_anthropic_tool_name_maps for why this is keyed on the
# per-request reverse map (so a tool legitimately named `foo_bar` is
# never incorrectly retyped to `foo/bar`). No-op when the map is
# empty (the common case).

View file

@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
get_file_ids_from_messages,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_tool_search_result,
find_anthropic_server_tool_result,
message_field,
parts_of,
@ -64,10 +65,12 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS,
AllAnthropicPassThroughMessageValues,
AllAnthropicToolsValues,
AnthropicMcpServerTool,
AnthropicMessagesToolChoice,
AnthropicThinkingParam,
AnthropicToolSearchToolResultBlock,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
@ -1918,6 +1921,113 @@ def strip_advisor_blocks_from_messages(messages: list[_MessageT], replace_with_t
return messages
def _assistant_blocks(message: AllAnthropicPassThroughMessageValues) -> tuple[object, ...]:
return parts_of(message["content"]) if message["role"] == "assistant" else ()
def _results_with_unloaded_tools(
messages: Sequence[AllAnthropicPassThroughMessageValues], tool_names: frozenset[str]
) -> tuple[AnthropicToolSearchToolResultBlock, ...]:
blocks: Final = chain.from_iterable(map(_assistant_blocks, messages))
results: Final = (result for result in map(anthropic_tool_search_result, blocks) if result is not None)
return tuple(result for result in results if not tool_names.issuperset(result.tool_names))
def _is_stale_tool_search_block(block: object, stale_ids: frozenset[str]) -> bool:
match message_field(block, "type"):
case "server_tool_use":
return message_field(block, "id") in stale_ids
case "tool_search_tool_result":
return message_field(block, "tool_use_id") in stale_ids
case _:
return False
def _with_loaded_tool_references_only(block: object, tool_names: frozenset[str]) -> object:
result: Final = anthropic_tool_search_result(block)
fields: Final = as_str_mapping(block)
if result is None or fields is None or tool_names.issuperset(result.tool_names):
return block
content: Final = as_str_mapping(fields.get("content")) or MappingProxyType({})
references: Final = parts_of(content.get("tool_references"))
loaded: Final = [reference for reference in references if message_field(reference, "tool_name") in tool_names]
return {**fields, "content": {**content, "tool_references": loaded}}
def _with_loaded_tool_searches_only(
message: AllAnthropicPassThroughMessageValues, stale_ids: frozenset[str], tool_names: frozenset[str]
) -> AllAnthropicPassThroughMessageValues | None:
if message["role"] != "assistant":
return message
content: Final = message["content"]
if isinstance(content, str):
return message
blocks: Final = list(content)
kept: Final = [
_with_loaded_tool_references_only(block, tool_names)
for block in blocks
if not _is_stale_tool_search_block(block, stale_ids)
]
if kept == blocks:
return message
return {**message, "content": kept} if kept else None
def _tool_reference_renamed(reference: object, name_map: Mapping[str, str]) -> object:
fields: Final = as_str_mapping(reference)
name: Final = fields.get("tool_name") if fields is not None else None
if fields is None or not isinstance(name, str) or name not in name_map:
return reference
return {**fields, "tool_name": name_map[name]}
def tool_search_content_renamed(content: object, name_map: Mapping[str, str]) -> object:
fields: Final = as_str_mapping(content)
if fields is None or "tool_references" not in fields:
return content
references: Final = parts_of(fields["tool_references"])
return {**fields, "tool_references": [_tool_reference_renamed(reference, name_map) for reference in references]}
def tool_search_result_renamed(block: object, name_map: Mapping[str, str]) -> object:
fields: Final = as_str_mapping(block)
if fields is None or fields.get("type") != "tool_search_tool_result":
return block
return {**fields, "content": tool_search_content_renamed(fields.get("content"), name_map)}
def replays_tool_search(messages: object) -> bool:
contents: Final = (message_field(message, "content") for message in parts_of(messages))
blocks: Final = chain.from_iterable(map(parts_of, contents))
return any(anthropic_tool_search_result(block) is not None for block in blocks)
def without_stale_tool_search_results(
messages: Sequence[AllAnthropicPassThroughMessageValues], tools: Sequence[object]
) -> tuple[AllAnthropicPassThroughMessageValues, ...]:
"""
Leave out the tools a replayed tool search found that are no longer in ``tools``,
and the whole search, its ``server_tool_use`` and its ``tool_search_tool_result``,
when none of them are left. Anthropic rejects such a history with "Tool reference
'<name>' not found in available tools", for example after a follow-up turn removes
a tool or sends none.
"""
tool_names: Final = frozenset(name for tool in tools if isinstance(name := message_field(tool, "name"), str))
unloaded: Final = _results_with_unloaded_tools(messages, tool_names)
if not unloaded:
return tuple(messages)
litellm.verbose_logger.debug(
"Leaving out the tools a replayed tool search found that this request no longer sends: %s",
sorted(frozenset(chain.from_iterable(result.tool_names for result in unloaded)) - tool_names),
)
stale_ids: Final = frozenset(result.tool_use_id for result in unloaded if tool_names.isdisjoint(result.tool_names))
return tuple(
kept
for message in messages
if (kept := _with_loaded_tool_searches_only(message, stale_ids, tool_names)) is not None
)
def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool:
"""
Detect Anthropic 400 errors caused by invalid thinking blocks in replayed

View file

@ -125,6 +125,8 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
status_code=e.response.status_code,
message=e.response.text,
)
except litellm.Timeout as e:
raise AnthropicError(status_code=e.status_code, message=str(e)) from e
except Exception as e:
verbose_logger.error("Error in CountTokens handler: %s", e)
raise AnthropicError(

View file

@ -18,7 +18,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
ChatCountTranslationError,
anthropic_count_fields,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.base_llm.base_utils import BaseTokenCounter, deployment_timeout
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
# Global handler instance - reuse across all token counting requests
@ -134,6 +134,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
optional_params=count_options,
context_management=context_management.spec if context_management is not None else None,
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
timeout=deployment_timeout(litellm_params),
)
if result is not None:

View file

@ -1,17 +1,12 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Any,
Final,
cast, # noqa: TID251 # non_default_params reaches the mapper as an untyped dict
)
from typing import TYPE_CHECKING, Any, Final
from httpx._models import Headers, Response
import litellm
from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_cache_breakpoint_support
from litellm.litellm_core_utils.allowed_tools_tool_choice import is_allowed_tools_tool_choice
from litellm.litellm_core_utils.allowed_tools_tool_choice import azure_api_version_takes_tool_choice
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_tool_reference_parts_from_tool_messages,
@ -250,17 +245,11 @@ class AzureOpenAIConfig(BaseConfig):
if api_version_year is None or api_version_month is None or api_version_day is None:
optional_params["tool_choice"] = value
else:
if (
api_version_year < "2023"
or (api_version_year == "2023" and api_version_month < "12")
or (api_version_year == "2023" and api_version_month == "12" and api_version_day < "01")
):
if is_allowed_tools_tool_choice(
cast(object, value) # cast-ok: widening an untyped dict value to object is always sound
) or not (litellm.drop_params is True or (drop_params is not None and drop_params is True)):
if not azure_api_version_takes_tool_choice(api_version):
if not (litellm.drop_params is True or drop_params is True):
raise UnsupportedParamsError(
status_code=400,
message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. drop_params never drops an allowed_tools tool_choice, since that would let the model call tools the request excluded. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""",
message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""",
)
elif value == "required" and (
api_version_year == "2024" and api_version_month <= "05"

View file

@ -147,6 +147,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
status_code=e.response.status_code,
message=e.response.text,
)
except litellm.Timeout as e:
raise AnthropicError(status_code=e.status_code, message=str(e)) from e
except Exception as e:
verbose_logger.error("Error in CountTokens handler: %s", e)
raise AnthropicError(

View file

@ -20,7 +20,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
AzureAIAnthropicCountTokensHandler,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.base_llm.base_utils import BaseTokenCounter, deployment_timeout
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
# Global handler instance - reuse across all token counting requests
@ -137,6 +137,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
system=system,
optional_params=count_options,
context_management=None if context_management is None else context_management.spec,
timeout=deployment_timeout(litellm_params),
)
if result is not None:

View file

@ -15,8 +15,9 @@ from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final, TypeAlias
import httpx
from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
from pydantic import BaseModel, JsonValue, PositiveFloat, TypeAdapter, ValidationError
from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal
from litellm._logging import verbose_logger
@ -38,6 +39,18 @@ from litellm.types.utils import (
TokenCountResponse,
)
_SECONDS: Final[TypeAdapter[float]] = TypeAdapter(PositiveFloat)
def deployment_timeout(litellm_params: Mapping[str, object]) -> float | httpx.Timeout | None:
timeout: Final = litellm_params.get("timeout")
if timeout is None or isinstance(timeout, httpx.Timeout):
return timeout
try:
return _SECONDS.validate_python(timeout)
except ValidationError:
return None
class BaseTokenCounter(ABC):
@abstractmethod

View file

@ -4,6 +4,7 @@ import re
from abc import abstractmethod
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
from pydantic import ConfigDict, TypeAdapter, ValidationError
@ -29,6 +30,7 @@ if TYPE_CHECKING:
RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_DECODED_JSON: Final = TypeAdapter(object)
_NO_RESPONSE_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
class PassthroughMetadata(LiteLLMBaseModel):
@ -236,7 +238,12 @@ class BasePassthroughConfig(BaseLLMModelInfo):
return None
def create_stream_collector(
self, model: str, custom_llm_provider: str, endpoint: str
self,
model: str,
custom_llm_provider: str,
endpoint: str,
*,
response_headers: Mapping[str, str] = _NO_RESPONSE_HEADERS,
) -> PassthroughStreamCollector:
return RawBytesStreamCollector(
provider_config=self, model=model, custom_llm_provider=custom_llm_provider, endpoint=endpoint

View file

@ -23,6 +23,7 @@ from litellm.llms.anthropic.common_utils import (
OBJECT_MAPPING_ADAPTER,
AnthropicModelInfo,
carried_thinking_keys,
replays_tool_search,
)
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
@ -265,6 +266,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
optional_params=optional_params,
headers=headers,
thinking=_anthropic_request.get("thinking"),
history_holds_tool_search=replays_tool_search(
OBJECT_MAPPING_ADAPTER.validate_python(_anthropic_request).get("messages")
),
)
if beta_list:
_anthropic_request["anthropic_beta"] = beta_list
@ -397,9 +401,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
optional_params: dict,
headers: dict,
thinking: AnthropicThinkingParam | None,
history_holds_tool_search: bool,
) -> list[str]:
tools: Final = optional_params.get("tools")
tool_search_used: Final = self.is_tool_search_used(tools)
tool_search_used: Final = self.is_tool_search_used(tools) or history_holds_tool_search
programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools)
input_examples_used: Final = self.is_input_examples_used(tools)
is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages)

View file

@ -32,6 +32,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
BEDROCK_MANAGED_S3_PREFIXES,
BEDROCK_MANAGED_S3_UPLOAD_PREFIX,
STORED_S3_BUCKET_PARAM,
build_managed_cloud_object_name,
encode_s3_object_key_for_url,
sanitize_cloud_object_component,
@ -313,11 +314,19 @@ def get_configured_s3_bucket_names(litellm_params: Mapping[str, object]) -> tupl
Resolve the S3 buckets a Bedrock file id may live in.
Bedrock batch outputs land in ``s3_output_bucket_name`` when it differs from
the input bucket, so retrieval validates against both.
the input bucket, so retrieval validates against both, plus the bucket LiteLLM
recorded for a managed file uploaded or written to a request-chosen bucket.
"""
input_bucket: Final = _input_bucket_name(litellm_params)
output_bucket: Final = _output_bucket_name(litellm_params)
buckets: Final = tuple(dict.fromkeys(bucket for bucket in (input_bucket, output_bucket) if bucket))
stored_bucket: Final = litellm_params.get(STORED_S3_BUCKET_PARAM)
buckets: Final = tuple(
dict.fromkeys(
bucket
for bucket in (input_bucket, output_bucket, stored_bucket if isinstance(stored_bucket, str) else None)
if bucket
)
)
if not buckets:
raise ValueError(_S3_BUCKET_REQUIRED_ERROR)
return buckets

View file

@ -3,16 +3,21 @@ from collections.abc import Callable, Iterator, Mapping, Sequence
from functools import partial
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, Final, Optional, cast
from typing import TYPE_CHECKING, ClassVar, Final, Optional, TypeVar, cast
import httpx
from httpx import Response
from pydantic import TypeAdapter
from pydantic import BaseModel, TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import AWS_ERROR_TYPE_HEADER
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
from litellm.llms.base_llm.passthrough.transformation import (
BasePassthroughConfig,
PassthroughStreamCollector,
relayed_json_object,
)
from litellm.llms.bedrock.chat.converse_to_openai import as_json_array, as_json_object
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
bedrock_invoke_guardrail_headers,
@ -21,7 +26,12 @@ from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletio
from litellm.types.utils import ModelResponseStream
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError, BedrockEventStreamDecoderBase, BedrockModelInfo
from ..common_utils import (
BEDROCK_INVOKE_TRACE_FIELD,
BedrockError,
BedrockEventStreamDecoderBase,
BedrockModelInfo,
)
from ..request_metadata import (
BEDROCK_REQUEST_METADATA_HEADER,
BedrockRequestMetadataViolation,
@ -49,6 +59,9 @@ _BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stre
_BEDROCK_INVOKE_GUARDRAIL_HEADERS: Final = frozenset(
{"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"}
)
_BEDROCK_SERVICE_TIER_HEADER: Final = "x-amzn-bedrock-service-tier"
_NO_RESPONSE_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
_TieredResponse: Final = TypeVar("_TieredResponse", bound=BaseModel)
def _bedrock_endpoint_action(endpoint: str) -> str:
@ -111,7 +124,9 @@ def _with_request_metadata(
litellm_params: Mapping[str, object],
) -> tuple[Mapping[str, str], Mapping[str, object]]:
if action in _BEDROCK_CONVERSE_ACTIONS:
return headers, _with_body_request_metadata(action, body, litellm_params)
return _without_owned_request_metadata_header(headers), _with_body_request_metadata(
action, body, litellm_params
)
if action in _BEDROCK_INVOKE_ACTIONS and "requestMetadata" in body:
return _without_owned_request_metadata_header(headers), _with_body_request_metadata(
action, body, litellm_params
@ -137,6 +152,10 @@ def _converse_guardrail_config(action: str, guardrail_config: object) -> object:
return {key: value for key, value in fields.items() if key != "streamProcessingMode"}
def _without_invoke_guardrail_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
return {name: value for name, value in headers.items() if name.lower() not in _BEDROCK_INVOKE_GUARDRAIL_HEADERS}
def _with_deployment_guardrail(
action: str,
headers: Mapping[str, str],
@ -146,7 +165,10 @@ def _with_deployment_guardrail(
if guardrail_config is None:
return headers, body
if action in _BEDROCK_CONVERSE_ACTIONS:
return headers, {**body, "guardrailConfig": _converse_guardrail_config(action, guardrail_config)}
return _without_invoke_guardrail_headers(headers), {
**body,
"guardrailConfig": _converse_guardrail_config(action, guardrail_config),
}
if action in _BEDROCK_INVOKE_ACTIONS:
guardrail_headers: Final = tuple(bedrock_invoke_guardrail_headers(guardrail_config).items())
return merge_bedrock_invoke_headers(
@ -266,6 +288,19 @@ class _CoalescedChunks:
return self._chunks
_BEDROCK_EVENT: Final = TypeAdapter(dict[str, object])
def _with_invoke_trace(chunk: ModelResponseStream, trace: Mapping[str, object] | None) -> ModelResponseStream:
if trace is None:
return chunk
return chunk.model_copy(
update={
"provider_specific_fields": {**(chunk.provider_specific_fields or {}), BEDROCK_INVOKE_TRACE_FIELD: trace}
}
)
def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelResponseStream | None:
from litellm.litellm_core_utils.streaming_handler import (
convert_generic_chunk_to_model_response_stream,
@ -273,11 +308,15 @@ def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelR
)
from litellm.types.utils import GenericStreamingChunk
translated_chunk: Final = decoder.chunk_parser(chunk_data=json.loads(message))
event: Final = _BEDROCK_EVENT.validate_json(message)
trace: Final = object_mapping(event.get(BEDROCK_INVOKE_TRACE_FIELD))
translated_chunk: Final = decoder.chunk_parser(chunk_data=event)
if isinstance(translated_chunk, ModelResponseStream):
return translated_chunk
return _with_invoke_trace(translated_chunk, trace)
if generic_chunk_has_all_required_fields(cast(dict, translated_chunk)):
return convert_generic_chunk_to_model_response_stream(cast(GenericStreamingChunk, translated_chunk))
return _with_invoke_trace(
convert_generic_chunk_to_model_response_stream(cast(GenericStreamingChunk, translated_chunk)), trace
)
return None
@ -332,15 +371,33 @@ def _count_prompt_tokens(model: str, request_body: object) -> int:
return token_counter(model=model, messages=messages)
def _invoke_guardrail_trace(httpx_response: Response) -> Mapping[str, object]:
reply: Final = relayed_json_object(httpx_response) or {}
return {BEDROCK_INVOKE_TRACE_FIELD: reply.get(BEDROCK_INVOKE_TRACE_FIELD)}
def _served_service_tier(response_headers: Mapping[str, str]) -> str | None:
headers: Final = httpx.Headers(response_headers)
if _BEDROCK_SERVICE_TIER_HEADER not in headers:
return None
return headers[_BEDROCK_SERVICE_TIER_HEADER]
def _with_served_service_tier(response: _TieredResponse, service_tier: str | None) -> _TieredResponse:
if service_tier is None:
return response
return response.model_copy(update={"service_tier": service_tier})
def _build_logged_response(
chunks: Sequence[ModelResponseStream], litellm_logging_obj: "LiteLLMLoggingObj"
chunks: Sequence[ModelResponseStream], litellm_logging_obj: "LiteLLMLoggingObj", service_tier: str | None
) -> Optional["CostResponseTypes"]:
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
if len(chunks) == 0:
return None
return stream_chunk_builder(
chunks=list(chunks),
chunks=[*chunks[:-1], _with_served_service_tier(chunks[-1], service_tier)],
logging_obj=litellm_logging_obj,
count_prompt_tokens=partial(
_count_prompt_tokens, litellm_logging_obj.model, litellm_logging_obj.model_call_details.get("input")
@ -355,11 +412,13 @@ class BedrockEventStreamCollector:
self,
parse_event: Callable[["EventStreamMessage"], str | None],
decoder: Optional["AWSEventStreamDecoder"],
service_tier: str | None = None,
) -> None:
from botocore.eventstream import EventStreamBuffer
self._parse_event = parse_event
self._decoder = decoder
self._service_tier: Final = service_tier
self._event_stream_buffer: Final[EventStreamBuffer] = EventStreamBuffer()
self._chunks: Final = _CoalescedChunks()
@ -377,7 +436,7 @@ class BedrockEventStreamCollector:
self._chunks.add(translated)
def build_logged_response(self, litellm_logging_obj: "LiteLLMLoggingObj") -> Optional["CostResponseTypes"]:
return _build_logged_response(self._chunks.chunks(), litellm_logging_obj)
return _build_logged_response(self._chunks.chunks(), litellm_logging_obj, self._service_tier)
class BedrockAgentRuntimeAWS(BaseAWSLLM):
@ -542,15 +601,23 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
request_data=request_data,
encoding=encoding,
)
if "invoke" in endpoint:
logging_obj.record_native_bedrock_guardrail(_invoke_guardrail_trace(httpx_response))
return litellm_model_response
return _with_served_service_tier(litellm_model_response, _served_service_tier(httpx_response.headers))
def create_stream_collector(
self, model: str, custom_llm_provider: str, endpoint: str
self,
model: str,
custom_llm_provider: str,
endpoint: str,
*,
response_headers: Mapping[str, str] = _NO_RESPONSE_HEADERS,
) -> PassthroughStreamCollector:
return BedrockEventStreamCollector(
parse_event=self._parse_message_from_event,
decoder=self._get_event_stream_decoder(model=model, endpoint=endpoint),
service_tier=_served_service_tier(response_headers),
)
def _get_event_stream_decoder(self, model: str, endpoint: str) -> Optional["AWSEventStreamDecoder"]:

View file

@ -663,7 +663,9 @@ class HTTPResponseLimitError(ValueError):
class MaskedHTTPStatusError(httpx.HTTPStatusError):
def __init__(self, original_error: httpx.HTTPStatusError, message: str | None = None, text: str | None = None):
def __init__(
self, original_error: httpx.HTTPStatusError, message: str | None = None, text: str | None = None
) -> None:
# Create a new error with the masked URL
masked_url: Final = mask_sensitive_info(str(original_error.request.url))
# Mask the original exception message too (it contains the full URL)

View file

@ -48,6 +48,7 @@ from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAG
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
from litellm.litellm_core_utils.agentic_loop_refusal import (
called_tool_names,
every_claimed_tool_call,
finalize_refused_responses_turn,
is_refused_item,
@ -64,7 +65,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
)
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
bill_first_round_on_failure,
holds_a_call_the_choice_excludes,
run_chat_completion_agentic_hooks,
with_caller_tool_choice,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
@ -160,6 +163,7 @@ from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
AgenticLoopToolChoiceError,
converted_stream_requested,
is_interception_internal_key,
)
@ -268,7 +272,11 @@ if TYPE_CHECKING:
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission
from litellm.responses.streaming_iterator import (
ResponsesWsFrameGuard,
ResponsesWsTurnAdmission,
ResponsesWsTurnUsageRecorder,
)
from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -6402,7 +6410,13 @@ class BaseLLMHTTPHandler:
tools: Final = anthropic_messages_optional_request_params.get("tools", [])
depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs)
hook_kwargs: Final = {**kwargs, "_agentic_loop_api_surface": api_surface}
hook_kwargs: Final = {
**with_caller_tool_choice(
_OPTIONAL_PARAMS.validate_python(kwargs),
_OPTIONAL_PARAMS.validate_python(anthropic_messages_optional_request_params),
),
"_agentic_loop_api_surface": api_surface,
}
for callback in callbacks:
if not isinstance(callback, CustomLogger):
@ -6441,6 +6455,10 @@ class BaseLLMHTTPHandler:
# Safety guards must run OUTSIDE the callback try/except — they are
# bounded-loop / cycle-break rails, not callback bugs.
try:
if holds_a_call_the_choice_excludes(
called_tool_names(response), hook_kwargs, (cb for cb in callbacks if isinstance(cb, CustomLogger))
):
raise AgenticLoopToolChoiceError(model)
fingerprint = self._check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
@ -7241,6 +7259,7 @@ class BaseLLMHTTPHandler:
first_message: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: "ResponsesWsTurnAdmission | None" = None,
turn_usage_recorder: "ResponsesWsTurnUsageRecorder | None" = None,
frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs: object,
) -> Exception | None:
@ -7407,6 +7426,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
request_defaults=request_defaults,
turn_admission=_ws_turn_admission,
turn_usage_recorder=turn_usage_recorder,
frame_guard=frame_guard,
)
return await streaming.bidirectional_forward()

View file

@ -34,6 +34,9 @@ def _warn_on_dropped_budget(thinking: object, model: str) -> None:
class DeepSeekChatConfig(OpenAIGPTConfig):
def _flattens_content_lists(self) -> bool:
return True
def get_supported_openai_params(self, model: str) -> list[str]:
"""
DeepSeek reasoner models support thinking parameter.

View file

@ -25,6 +25,9 @@ class DockerModelRunnerChatConfig(OpenAIGPTConfig):
The engine name (e.g., "llama.cpp") is part of the API endpoint path.
"""
def _flattens_content_lists(self) -> bool:
return True
@overload
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: Literal[True]

View file

@ -170,7 +170,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
return request_body_obj.model_dump(exclude_none=True)
def _transform_image_usage(self, usage_metadata: dict):
return transform_gemini_image_usage(usage_metadata)
return transform_gemini_image_usage(_JSON_DICT.validate_python(usage_metadata))
def transform_image_generation_response(
self,

View file

@ -1,5 +1,7 @@
from typing import Final
from pydantic import TypeAdapter
from litellm.types.utils import ImageUsage, ImageUsageInputTokensDetails
@ -34,12 +36,12 @@ def _sum_modality_token_details(usage_metadata: dict, *details_keys: str) -> Ima
return tokens_details
def transform_gemini_image_usage(usage_metadata: dict) -> ImageUsage:
def transform_gemini_image_usage(usage_metadata: dict[str, object]) -> ImageUsage:
"""
Transform Gemini usageMetadata to ImageUsage format.
"""
input_tokens_details = _sum_modality_token_details(usage_metadata, "promptTokensDetails", "prompt_tokens_details")
output_tokens: Final = usage_metadata.get("candidatesTokenCount", 0)
output_tokens: Final = TypeAdapter(int).validate_python(usage_metadata.get("candidatesTokenCount", 0))
output_tokens_details: Final = _sum_modality_token_details(
usage_metadata, "candidatesTokensDetails", "candidates_tokens_details"
)

View file

@ -22,6 +22,9 @@ class HerokuError(Exception):
class HerokuChatConfig(OpenAIGPTConfig):
def _flattens_content_lists(self) -> bool:
return True
@overload
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: Literal[True]

View file

@ -87,25 +87,31 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
model: str,
drop_params: bool,
) -> dict:
_tools = non_default_params.pop("tools", None)
params: Final = dict[str, object](non_default_params)
mapped_params: Final = {
key: value
for key, value in params.items()
if params.get("tools") != [] or key not in ("tools", "tool_choice", "parallel_tool_calls")
}
_tools = mapped_params.pop("tools", None)
if _tools is not None:
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportArgumentType] # tools arrive untyped through the bare-dict params
_tools = remove_strict_from_schema(_tools)
if isinstance(_tools, list):
_tools = self._convert_custom_tools_to_function_tools(_tools)
if _tools is not None:
non_default_params["tools"] = _tools
mapped_params["tools"] = _tools
thinking: Final = non_default_params.pop("thinking", None)
thinking: Final = mapped_params.pop("thinking", None)
thinking_effort: Final = (
None
if "reasoning_effort" in non_default_params
if "reasoning_effort" in mapped_params
else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
)
if thinking_effort is not None:
non_default_params["reasoning_effort"] = thinking_effort
mapped_params["reasoning_effort"] = thinking_effort
return super().map_openai_params(non_default_params, optional_params, model, drop_params)
return super().map_openai_params(mapped_params, optional_params, model, drop_params)
def _get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None

View file

@ -142,8 +142,6 @@ class MistralConfig(OpenAIGPTConfig):
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
"""
_is_base_class = False
temperature: int | None = None
top_p: int | None = None
max_tokens: int | None = None

View file

@ -2,9 +2,11 @@
Support for gpt model family
"""
import asyncio
import json
import os
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from contextvars import ContextVar
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload
from urllib.parse import urlparse
@ -28,7 +30,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
async_fetch_urls_to_base64,
convert_url_to_base64,
file_id_urls_before_an_unreadable_part,
has_file_id_urls,
)
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
drop_non_python_regex_patterns,
@ -85,6 +90,20 @@ else:
_NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = ("user", "tool")
_PDF_URL_STARTS: Final = ("https://", "http://", "www.")
_NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({})
_prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar(
"prefetched_pdf_urls"
)
def _pdf_data_url(url: str) -> str:
prefetched: Final = _prefetched_pdf_urls.get(_NOTHING_PREFETCHED).get(url)
if isinstance(prefetched, litellm.ImageFetchError):
raise prefetched
return prefetched if prefetched is not None else convert_url_to_base64(url)
def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool:
if custom_llm_provider != "openai":
@ -191,9 +210,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
- `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling.
"""
# Add a class variable to track if this is the base class
_is_base_class = True
frequency_penalty: int | None = None
function_call: str | dict | None = None
functions: list | None = None
@ -225,8 +241,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
if key != "self" and value is not None:
setattr(self.__class__, key, value)
self.__class__._is_base_class = False
@classmethod
def get_config(cls):
return super().get_config()
@ -322,9 +336,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
)
def contains_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> bool:
potential_pdf_url_starts: Final = ["https://", "http://", "www."]
file_id: Final = content_item.get("file_id")
if file_id and any(file_id.startswith(start) for start in potential_pdf_url_starts):
if file_id and any(file_id.startswith(start) for start in _PDF_URL_STARTS):
return True
return False
@ -332,7 +345,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
content_copy: Final = content_item.copy()
file_id: Final = content_copy.get("file_id")
if file_id is not None:
base64_data: Final = convert_url_to_base64(file_id)
base64_data: Final = _pdf_data_url(file_id)
content_copy["file_data"] = base64_data
content_copy["filename"] = "my_file.pdf"
content_copy.pop("file_id")
@ -688,6 +701,12 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
**self._sanitized_tools_update_for_openai(optional_params, litellm_params),
}
def _flattens_content_lists(self) -> bool:
return False
def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]:
return messages
async def async_transform_request(
self,
model: str,
@ -696,30 +715,25 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
transformed_messages, tools = self._messages_and_tools_in_cache_mark_dialect(
model=model,
messages=await self._transform_messages(
messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True
),
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
if self.__class__._is_base_class:
if not optional_params.get("tools") and not optional_params.get("functions"):
optional_params.pop("tool_choice", None)
return {
"model": model,
"messages": transformed_messages,
**optional_params,
**self._sanitized_tools_update_for_openai(optional_params, litellm_params),
}
else:
## allow for any object specific behaviour to be handled
if not has_file_id_urls(messages, _PDF_URL_STARTS):
return self.transform_request(model, messages, optional_params, litellm_params, headers)
sent_messages: Final = self._with_provider_roles(
self._prompt_cache_ordered_messages(messages, litellm_params), model
)
pdf_urls: Final = file_id_urls_before_an_unreadable_part(
(message for message in sent_messages if message.get("role") in _ROLES_WHOSE_FILES_ARE_DOWNLOADED),
_PDF_URL_STARTS,
)
prefetched: Final = (
_NOTHING_PREFETCHED if self._flattens_content_lists() else await async_fetch_urls_to_base64(pdf_urls)
)
reset_token: Final = _prefetched_pdf_urls.set(prefetched)
try:
return await asyncio.to_thread(
self.transform_request, model, messages, optional_params, litellm_params, headers
)
finally:
_prefetched_pdf_urls.reset(reset_token)
def _passed_in_tools(self, optional_params: dict) -> bool:
return optional_params.get("tools", None) is not None

View file

@ -122,6 +122,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
return super()._map_openai_params(non_default_params, optional_params, model, drop_params)
def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]:
keeps_system_messages: Final = supports_system_messages(model, "openai")
return tuple(
message
if keeps_system_messages or message["role"] != "system"
else ChatCompletionUserMessage(content=message["content"], role="user")
for message in messages
)
def is_model_o_series_model(self, model: str) -> bool:
model = model.split("/")[-1] # could be "openai/o3" or "o3"
return (
@ -152,13 +161,7 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
- modalities: image => drop param (if user opts in to dropping param)
- role: system ==> translate to role 'user'
"""
_supports_system_messages: Final = supports_system_messages(model, "openai")
transformed_messages: Final = list(messages)
for i, message in enumerate(transformed_messages):
if message["role"] == "system" and not _supports_system_messages:
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
transformed_messages[i] = new_message # Replace the old message with the new one
transformed_messages: Final = list(self._with_provider_roles(messages, model))
if is_async:
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
else:

View file

@ -1166,7 +1166,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
shared_session: Optional["ClientSession"] = None,
):
response = None
data = provider_config.transform_request(
data = await provider_config.async_transform_request(
model=model,
messages=messages,
optional_params=optional_params,

View file

@ -111,6 +111,8 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig):
status_code=e.response.status_code,
message=e.response.text,
)
except litellm.Timeout as e:
raise OpenAIError(status_code=e.status_code, message=str(e)) from e
except (httpx.RequestError, json.JSONDecodeError, ValueError) as e:
verbose_logger.error("Error in CountTokens handler: %s", e)
raise OpenAIError(

View file

@ -7,6 +7,7 @@ from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final
import httpx
import openai
from pydantic import JsonValue
from typing_extensions import assert_never
@ -21,7 +22,7 @@ from litellm.llms.anthropic.messages.responses_adapters.handler import count_vie
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.base_llm.base_utils import BaseTokenCounter, deployment_timeout
from litellm.llms.openai.common_utils import OpenAIError, decoded_json_object, get_openai_credentials
from litellm.llms.openai.responses.count_tokens.handler import (
OpenAICountTokensHandler,
@ -61,6 +62,7 @@ def _workload_identity_config(
class _CountCredential:
api_base: str | None
credential: str | OpenAIWorkloadIdentityConfig
timeout: float | httpx.Timeout | None
def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError:
@ -82,7 +84,9 @@ def _count_credential(deployment: Mapping[str, object] | None, model_label: str)
"workload identity on the deployment, or OPENAI_API_KEY in the proxy environment."
),
)
return _CountCredential(api_base=credentials.api_base, credential=credential)
return _CountCredential(
api_base=credentials.api_base, credential=credential, timeout=deployment_timeout(litellm_params)
)
def _count_error(model_to_use: str, request_model: str, message: str, status_code: int) -> TokenCountResponse:
@ -245,6 +249,7 @@ class OpenAITokenCounter(BaseTokenCounter):
tools=counted_tools,
instructions=instructions,
text=text,
timeout=resolved.timeout,
),
model_to_use,
request_model,
@ -264,7 +269,9 @@ class OpenAITokenCounter(BaseTokenCounter):
body: Final = native_count_request(request, model_to_use, policy)
return await self._counted(
resolved,
lambda api_key, api_base: openai_count_tokens_handler.post_count_request(body, api_key, api_base),
lambda api_key, api_base: openai_count_tokens_handler.post_count_request(
body, api_key, api_base, resolved.timeout
),
model_to_use,
request_model,
)

View file

@ -111,7 +111,6 @@ from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseCompactionItem,
ResponseCompletedEvent,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
ResponsesAPIStreamingResponse,
@ -119,6 +118,7 @@ from litellm.types.llms.openai import (
from litellm.types.responses.main import (
GenericResponseOutputItem,
OutputText,
OutputToolSearchOutput,
)
from litellm.types.utils import GenericGuardrailAPIInputs
@ -276,6 +276,7 @@ _ITEM_ID_KEYED_TOOL_CALL_TYPES: Final = frozenset({_TOOL_SEARCH_CALL_TYPE, _WEB_
_OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE}
_SCANNED_TOOL_SEARCH_NAME: Final = "tool_search"
_TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object])
_RAW_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...])
_STREAM_EVENT_TYPE_ADAPTER: Final = TypeAdapter(str)
_TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call": "arguments", "custom_tool_call": "input"}
@ -297,8 +298,6 @@ _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
)
_EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {}
def _item_rewrite_field(item: Mapping[str, object]) -> str | None:
item_type: Final = item.get("type")
@ -343,6 +342,18 @@ def _is_tool_call_item(item: object) -> bool:
return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES
def _is_tool_search_item(item: object) -> bool:
match item:
case {"type": "tool_search_call" | "tool_search_output"}:
return True
case _:
return False
def _joins_assistant_turn(item: object) -> bool:
return _is_tool_call_item(item) or _is_tool_search_item(item)
def _claude_web_search_tool_call(item: object) -> ChatCompletionAssistantToolCall | None:
search: Final = web_search_call(item)
return None if search is None else claude_web_search_call_tool_call(search)
@ -490,14 +501,17 @@ def _provenance_unit_bounds(
) -> tuple[tuple[int, int], ...]:
trailing_roles: Final = tuple(
accumulate(
(_last_message_role(messages) for messages in solo_conversions),
(
"assistant" if _is_tool_search_item(item) else _last_message_role(messages)
for item, messages in zip(raw_input, solo_conversions)
),
lambda previous, current: current if current is not None else previous,
)
)
start_indexes: Final = tuple(
index
for index in range(len(raw_input))
if index == 0 or not (_is_tool_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant")
if index == 0 or not (_joins_assistant_turn(raw_input[index]) and trailing_roles[index - 1] == "assistant")
)
return tuple(zip(start_indexes, (*start_indexes[1:], len(raw_input))))
@ -505,20 +519,24 @@ def _provenance_unit_bounds(
def _input_item_provenance(
raw_input: Sequence[object],
expected_messages: Sequence[object],
request_tools: object = None,
) -> tuple[Mapping[int, int], frozenset[int]] | None:
if not all(isinstance(item, Mapping) for item in raw_input):
return None
request_params: Final = {"tools": request_tools}
solo_conversions: Final = tuple(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=cast("ResponseInputParam", [item]), # cast-ok: items checked as Mappings above
responses_api_request=_EMPTY_RESPONSES_REQUEST,
responses_api_request=request_params,
replay_tool_search=True,
)
for item in raw_input
)
full_conversion: Final = tuple(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=cast("ResponseInputParam", list(raw_input)), # cast-ok: items checked as Mappings above
responses_api_request=_EMPTY_RESPONSES_REQUEST,
responses_api_request=request_params,
replay_tool_search=True,
)
)
if full_conversion != tuple(expected_messages):
@ -530,7 +548,8 @@ def _input_item_provenance(
else tuple(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=cast("ResponseInputParam", list(raw_input[start:end])), # cast-ok: checked as Mappings above
responses_api_request=_EMPTY_RESPONSES_REQUEST,
responses_api_request=request_params,
replay_tool_search=True,
)
)
for start, end in units
@ -688,11 +707,12 @@ def _patched_request_fields(
instructions: object,
original_messages: Sequence[object],
structured_messages: Sequence[object],
request_tools: object = None,
) -> _RequestFields | None:
if not isinstance(raw_input, list) or len(original_messages) != len(structured_messages):
return None
offset: Final = 1 if instructions else 0
provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:])
provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:], request_tools)
if provenance is None:
return None
item_for_message, tainted = provenance
@ -735,16 +755,54 @@ def _patch_or_convert_request_fields(
instructions: object,
original_messages: Sequence[object],
structured_messages: Sequence[AllMessageValues],
request_tools: object = None,
) -> _RequestFields | None:
if not isinstance(structured_messages, list):
return None
patched: Final = _patched_request_fields(raw_input, instructions, original_messages, structured_messages)
patched: Final = _patched_request_fields(
raw_input, instructions, original_messages, structured_messages, request_tools
)
if patched is not None:
return patched
input_items, converted_instructions = (
LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages)
)
return _RequestFields(input=tuple(input_items), instructions=converted_instructions)
return _RequestFields(
input=_with_tool_searches_restored(input_items, raw_input), instructions=converted_instructions
)
def _tool_search_items_by_call_id(raw_input: Sequence[object], item_type: str) -> Mapping[str, object]:
return MappingProxyType(
{
call_id: item
for item in raw_input
if stream_item_field(item, "type") == item_type
and isinstance(call_id := stream_item_field(item, "call_id"), str)
}
)
def _restored_tool_search(
item: object, calls: Mapping[str, object], outputs: Mapping[str, object]
) -> tuple[object, ...]:
call_id: Final = stream_item_field(item, "call_id")
if stream_item_field(item, "type") != "function_call" or not isinstance(call_id, str) or call_id not in calls:
return (item,)
arguments: Final = stream_item_field(item, "arguments")
restored_call: Final = _rewritten_tool_search_call(
calls[call_id], arguments if isinstance(arguments, str) else None
)
return (restored_call, outputs[call_id]) if call_id in outputs else (restored_call,)
def _with_tool_searches_restored(input_items: Sequence[object], raw_input: object) -> tuple[object, ...]:
raw_items: Final = _RAW_ITEMS_ADAPTER.validate_python(raw_input) if isinstance(raw_input, list) else ()
if not any(map(_is_tool_search_item, raw_items)):
return tuple(input_items)
calls: Final = _tool_search_items_by_call_id(raw_items, _TOOL_SEARCH_CALL_TYPE)
outputs: Final = _tool_search_items_by_call_id(raw_items, "tool_search_output")
return tuple(chain.from_iterable(_restored_tool_search(item, calls, outputs) for item in input_items))
def _next_stream_sequence_number(responses_so_far: Sequence[object] | None) -> int:
@ -791,6 +849,7 @@ class OpenAIResponsesHandler(BaseTranslation):
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=input_data,
responses_api_request=data,
replay_tool_search=True,
)
return cast(list[AllMessageValues], messages) if messages else None
@ -986,7 +1045,7 @@ class OpenAIResponsesHandler(BaseTranslation):
)
)
return _patch_or_convert_request_fields(
data.get("input"), data.get("instructions"), structured_messages, merged
data.get("input"), data.get("instructions"), structured_messages, merged, data.get("tools")
)
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
@ -1953,7 +2012,7 @@ class OpenAIResponsesHandler(BaseTranslation):
if isinstance(output_item, GenericResponseOutputItem):
if output_item.content and content_idx < len(output_item.content):
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
elif isinstance(output_item, BaseModel):
elif isinstance(output_item, BaseModel) and not isinstance(output_item, OutputToolSearchOutput):
# Handle other Pydantic models by converting to GenericResponseOutputItem
try:
generic_item = GenericResponseOutputItem.model_validate(output_item.model_dump())

View file

@ -1136,7 +1136,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
return None
@staticmethod
def parse_terminal_response_from_stream_chunks(all_chunks: list[str]) -> ResponsesAPIResponse | None:
def parse_terminal_response_from_stream_chunks(all_chunks: Sequence[str]) -> ResponsesAPIResponse | None:
terminal_event: Final = OpenAIResponsesAPIConfig.parse_terminal_event_from_stream_chunks(all_chunks)
return None if terminal_event is None else terminal_event.response

View file

@ -50,6 +50,9 @@ def create_config_class(provider: SimpleProviderConfig):
base_class: Final[type] = OpenAIGPTConfig if provider.base_class == "openai_gpt" else OpenAILikeChatConfig
class JSONProviderConfig(base_class):
def _flattens_content_lists(self) -> bool:
return bool(provider.special_handling.get("convert_content_list_to_string"))
@overload
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: Literal[True]
@ -69,7 +72,7 @@ def create_config_class(provider: SimpleProviderConfig):
"""Transform messages based on special_handling config"""
# Handle content list to string conversion if configured
if provider.special_handling.get("convert_content_list_to_string"):
if self._flattens_content_lists():
messages = handle_messages_with_content_list_to_str_conversion(messages)
if is_async:

View file

@ -32,6 +32,9 @@ class SambanovaConfig(OpenAIGPTConfig):
response_format: dict | None = None
tools: list | None = None
def _flattens_content_lists(self) -> bool:
return True
def __init__(
self,
max_tokens: int | None = None,

View file

@ -6,10 +6,12 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, cast
import httpx
from httpx._types import RequestFiles
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.images.utils import ImageEditRequestUtils
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.gemini.image_usage_transformation import transform_gemini_image_usage
from litellm.llms.vertex_ai.common_utils import (
get_vertex_base_url,
map_vertex_image_size,
@ -25,7 +27,9 @@ from litellm.types.llms.vertex_ai import (
HttpxPartType,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage
from litellm.types.utils import FileTypes, ImageObject, ImageResponse
_JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -218,7 +222,6 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ImageResponse:
model_response: Final = ImageResponse()
try:
response_json: Final = _generate_content_payload(raw_response)
except Exception as exc:
@ -239,8 +242,15 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
if (inline_data := part.get("inlineData")) and (b64_json := inline_data.get("data"))
]
model_response.data = cast(list[OpenAIImage], data_list)
return model_response
usage_metadata: Final = _JSON_DICT.validate_python(response_json.get("usageMetadata") or {})
traffic_type: Final = usage_metadata.get("trafficType") if usage_metadata else None
return ImageResponse(
data=data_list,
usage=transform_gemini_image_usage(usage_metadata) if usage_metadata else None,
hidden_params={"provider_specific_fields": {"traffic_type": traffic_type}}
if isinstance(traffic_type, str) and traffic_type
else {},
)
def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[HttpxPartType]:
images: Final[list[FileTypes]] = image if isinstance(image, list) else [image]

View file

View file

View file

@ -0,0 +1,156 @@
from __future__ import annotations
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
from urllib.parse import urlsplit
import httpx
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.search.transformation import BaseSearchConfig, SearchResponse, SearchResult
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
class _WebIQResult(BaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
title: str
url: str
content: str
lastUpdatedAt: str | None = None
class _WebIQResponse(BaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
webResults: tuple[_WebIQResult, ...]
_DOMAINS: Final = TypeAdapter(tuple[str, ...])
def _extra_fields(model: BaseModel, reserved: Mapping[str, object]) -> MappingProxyType[str, object]:
extra: Final[Mapping[str, object]] = model.model_extra or {}
return MappingProxyType({key: value for key, value in extra.items() if key not in reserved})
def _query_with_domains(query: str, domains: object) -> str:
if domains is None:
return query
parsed: Final = _DOMAINS.validate_python(domains)
included: Final = tuple(f"site:{domain}" for domain in parsed if domain and not domain.startswith("-"))
excluded: Final = tuple(f"-site:{domain[1:]}" for domain in parsed if domain.startswith("-") and len(domain) > 1)
include_clause: Final = f"({' OR '.join(included)})" if included else ""
if not included and not excluded:
return query
return " ".join(part for part in (f"({query})", include_clause, *excluded) if part)
class WebIQSearchConfig(BaseSearchConfig):
DEFAULT_API_BASE: Final = "https://api.microsoft.ai/v3"
@staticmethod
def ui_friendly_name() -> str:
return "Microsoft Web IQ"
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: BaseSearchConfig interface
api_key: str | None = None,
api_base: str | None = None,
**kwargs: object, # kwargs-ok: BaseSearchConfig interface
) -> dict[str, str]: # mutable-ok: HTTP handler requires a dict
if urlsplit(api_base or self.DEFAULT_API_BASE).scheme != "https":
raise ValueError("Microsoft Web IQ search requires an HTTPS api_base")
resolved_key: Final = self.resolve_server_api_key(
caller_api_key=api_key,
caller_api_base=api_base,
key_env_vars=("WEBIQ_API_KEY",),
base_env_var=None,
default_api_base=self.DEFAULT_API_BASE,
)
if not resolved_key:
raise ValueError("Set WEBIQ_API_KEY or pass api_key for Microsoft Web IQ search")
return {
**headers,
"x-apikey": resolved_key,
"Content-Type": "application/json",
}
def get_complete_url(
self,
api_base: str | None,
optional_params: dict[str, object], # mutable-ok: BaseSearchConfig interface
data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: BaseSearchConfig interface
**kwargs: object, # kwargs-ok: BaseSearchConfig interface
) -> str:
base: Final = (api_base or self.DEFAULT_API_BASE).rstrip("/")
return base if base.endswith("/search/web") else f"{base}/search/web"
def transform_search_request(
self,
query: str | list[str], # mutable-ok: BaseSearchConfig interface
optional_params: dict[str, object], # mutable-ok: BaseSearchConfig interface
**kwargs: object, # kwargs-ok: BaseSearchConfig interface
) -> dict[str, object]: # mutable-ok: HTTP handler requires a JSON dict
country: Final = optional_params.get("country")
mapped: Final = MappingProxyType(
{
"maxResults": optional_params.get("max_results"),
"region": country.upper() if isinstance(country, str) else None,
}
)
native: Final = MappingProxyType(
{
key: value
for key, value in optional_params.items()
if key not in self.get_supported_perplexity_optional_params()
}
)
return {
"contentFormat": "passage",
"maxLength": 5000,
**MappingProxyType({key: value for key, value in mapped.items() if value is not None}),
**native,
"query": _query_with_domains(
" ".join(query) if isinstance(query, list) else query,
optional_params.get("search_domain_filter"),
),
}
def transform_search_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs: object, # kwargs-ok: BaseSearchConfig interface
) -> SearchResponse:
try:
parsed: Final = _WebIQResponse.model_validate_json(raw_response.content)
except ValidationError as exc:
raise BaseLLMException(
status_code=502,
message=f"Microsoft Web IQ returned an invalid search response: {exc}",
headers=dict(raw_response.headers),
) from exc
return SearchResponse.model_validate(
{
**_extra_fields(parsed, SearchResponse.model_fields),
"results": [
SearchResult.model_validate(
{
**_extra_fields(result, SearchResult.model_fields),
"title": result.title,
"url": result.url,
"snippet": result.content,
"date": result.lastUpdatedAt or None,
"last_updated": result.lastUpdatedAt or None,
}
)
for result in parsed.webResults
],
}
)

View file

@ -75,7 +75,8 @@ from litellm.constants import (
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
raise_if_allowed_tools_tool_choice_unsupported,
drops_unsupported_allowed_tools_tool_choice,
parse_allowed_tools_tool_choice,
raise_if_hosted_tool_choice_unsupported,
)
from litellm.litellm_core_utils.asyncify import asyncify, run_async_function
@ -91,6 +92,7 @@ from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, Str
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_litellm_params import (
AGENTIC_LOOP_CONTROL_KWARGS_KEYS,
AGENTIC_LOOP_TOOL_CHOICE_KEY,
AWS_CREDENTIAL_KWARGS_KEYS,
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
@ -5612,7 +5614,16 @@ def completion(
messages = validate_and_fix_openai_messages(messages=messages)
tools = validate_and_fix_openai_tools(tools=tools)
# validate tool_choice
tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice, model=model)
requested_tool_choice: Final = cast( # cast-ok: widening an untyped dict to object is always sound
object, tool_choice
)
tool_choice = validate_chat_completion_tool_choice(
tool_choice=tool_choice,
model=model,
drop_params=drops_unsupported_params(
cast("Mapping[str, object]", kwargs).get("drop_params") # cast-ok: kwargs is the untyped request dict
),
)
# validate optional params
stop = validate_openai_optional_params(stop=stop)
thinking = validate_and_fix_thinking_param(thinking=thinking)
@ -5909,12 +5920,13 @@ def completion(
if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model):
tools = _drop_anthropic_only_tool_keys(tools=tools)
raise_if_allowed_tools_tool_choice_unsupported(
drops_tool_choice: Final = drops_unsupported_allowed_tools_tool_choice(
tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
object, tool_choice
),
custom_llm_provider=custom_llm_provider,
model=model,
drop_params=drop_unsupported_reasoning,
)
raise_if_hosted_tool_choice_unsupported(
tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
@ -6039,7 +6051,7 @@ def completion(
"response_format": response_format,
"seed": seed,
"tools": tools,
"tool_choice": tool_choice,
"tool_choice": None if drops_tool_choice else tool_choice,
"max_retries": max_retries,
"logprobs": logprobs,
"top_logprobs": top_logprobs,
@ -6228,7 +6240,18 @@ def completion(
if key in kwargs
},
)
litellm_params: Final = with_control_options(requested_litellm_params, control_options)
litellm_params: Final = with_control_options(
{
**requested_litellm_params,
**(
{AGENTIC_LOOP_TOOL_CHOICE_KEY: requested_tool_choice}
if AGENTIC_LOOP_TOOL_CHOICE_KEY not in kwargs
and parse_allowed_tools_tool_choice(requested_tool_choice) is not None
else {}
),
},
control_options,
)
affinity_api_base: Final = (
str(client.base_url)
if isinstance(client, openai.OpenAI | openai.AsyncOpenAI)

File diff suppressed because it is too large Load diff

View file

@ -5,6 +5,7 @@ Domain models for LiteLLM backend.
from litellm.models.access_group import LiteLLM_AccessGroupTable
from litellm.models.autorouter_session import LiteLLM_AutoRouterSession
from litellm.models.budget import (
CUSTOMER_ONLY_BUDGET_FIELDS,
LiteLLM_BudgetTable,
LiteLLM_BudgetTableFull,
LiteLLM_TeamMemberTable,
@ -38,6 +39,7 @@ from litellm.models.user import LiteLLM_UserTable
from litellm.models.verification_token import LiteLLM_VerificationToken
__all__ = [
"CUSTOMER_ONLY_BUDGET_FIELDS",
"CreateCredentialItem",
"CredentialBase",
"CredentialItem",

View file

@ -5,6 +5,7 @@ Canonical definition for ``litellm_budgettable``. Re-exported from
``litellm.proxy._types`` for backwards compatibility.
"""
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Final
@ -29,6 +30,8 @@ class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase):
rpm_limit: int | None = None
tpd_limit: int | None = None
model_max_budget: dict | None = None
model_tpm_limit: Mapping[str, int] | None = None
model_rpm_limit: Mapping[str, int] | None = None
budget_duration: str | None = None
allowed_models: list[str] | None = None # per-member model scope; empty = inherit team models
temp_budget_increase: float | None = None
@ -70,3 +73,6 @@ class LiteLLM_TeamMemberTable(LiteLLM_BudgetTable):
budget_id: str | None = None
model_config = ConfigDict(protected_namespaces=())
CUSTOMER_ONLY_BUDGET_FIELDS: Final = frozenset({"model_tpm_limit", "model_rpm_limit"})

View file

@ -57,11 +57,17 @@ def _as_generator(iterable: Iterator[bytes]) -> Generator[bytes, bytes, None]:
class _SpendCollection:
"""Feeds relayed chunks to the provider's stream collector without letting spend tracking break the relay."""
def __init__(self, provider_config: BasePassthroughConfig, litellm_logging_obj: LiteLLMLoggingObj) -> None:
def __init__(
self,
provider_config: BasePassthroughConfig,
litellm_logging_obj: LiteLLMLoggingObj,
response_headers: httpx.Headers,
) -> None:
self.collector: Final[PassthroughStreamCollector] = provider_config.create_stream_collector(
model=litellm_logging_obj.model,
custom_llm_provider=litellm_logging_obj.model_call_details.get("custom_llm_provider", ""),
endpoint=litellm_logging_obj.model_call_details.get("endpoint", ""),
response_headers=response_headers,
)
self.chunk_count = 0
self._failed = False
@ -97,7 +103,8 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]):
self._response: httpx.Response
self._iterator: AsyncGenerator[bytes, bytes]
self._litellm_logging_obj = litellm_logging_obj
self._spend = _SpendCollection(provider_config, litellm_logging_obj)
self._provider_config: Final = provider_config
self._spend: _SpendCollection
self._flush_scheduled = False
self._background_tasks: set[asyncio.Task[bool]] = set() # mutable-ok: instance set for background task tracking
self._flush: asyncio.Task[bool] | None = None
@ -129,6 +136,7 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]):
self._response = await self._response_coro
self.headers = self._response.headers
self.status_code = self._response.status_code
self._spend = _SpendCollection(self._provider_config, self._litellm_logging_obj, self._response.headers)
self._initialized = True
try:
raise_for_provider_status(self._response)
@ -148,7 +156,7 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]):
return _init().__await__()
def _start_flush(self) -> None:
if self._flush_scheduled or not self._spend.should_flush:
if self._flush_scheduled or not self._initialized or not self._spend.should_flush:
return
self._flush_scheduled = True
@ -237,7 +245,7 @@ class PassthroughStreamingResponse(Generator[bytes, bytes, None]):
self.status_code = response.status_code
self._litellm_logging_obj = litellm_logging_obj
self._iterator: Generator[bytes, bytes, None] = _as_generator(response.iter_bytes())
self._spend = _SpendCollection(provider_config, litellm_logging_obj)
self._spend = _SpendCollection(provider_config, litellm_logging_obj, response.headers)
self._flush_scheduled = False
def _start_flush(self) -> None:

View file

@ -25,7 +25,16 @@ _PASS_THROUGH_PROTECTED_HEADERS: Final[frozenset] = frozenset(
)
# Header name prefix used to block AWS SigV4 signing headers from being overridden.
_PASS_THROUGH_PROTECTED_HEADER_PREFIXES: Final[tuple] = ("x-amz-",)
_PASS_THROUGH_PROTECTED_HEADER_PREFIXES: Final[tuple[str, ...]] = ("x-amz-",)
def _is_forwarded_pass_through_header(header_name: str) -> bool:
if not header_name.lower().startswith(PASS_THROUGH_HEADER_PREFIX):
return False
forwarded_name: Final = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower()
return forwarded_name not in _PASS_THROUGH_PROTECTED_HEADERS and not any(
forwarded_name.startswith(prefix) for prefix in _PASS_THROUGH_PROTECTED_HEADER_PREFIXES
)
class BasePassthroughUtils:
@ -42,18 +51,11 @@ class BasePassthroughUtils:
# parse_qs returns a dict where each value is a list, so let's flatten it
updated_existing_query_params: Final = {k: v[0] if len(v) == 1 else v for k, v in existing_query_params.items()}
# Start with default query params (lowest priority)
merged_params: Final = {}
if default_query_params:
merged_params.update(default_query_params)
return {**(default_query_params or {}), **updated_existing_query_params, **request_query_params}
# Override with existing URL query params (medium priority)
merged_params.update(updated_existing_query_params)
# Override with request query params (highest priority - client can override anything)
merged_params.update(request_query_params)
return merged_params
@staticmethod
def pass_through_prefixed_headers(request_headers: Mapping[str, str]) -> Mapping[str, str]:
return {name: value for name, value in request_headers.items() if _is_forwarded_pass_through_header(name)}
@staticmethod
def forward_headers_from_request(
@ -88,16 +90,14 @@ class BasePassthroughUtils:
# Credential and protocol-level headers are excluded from this mechanism.
for header_name, header_value in request_headers.items():
if header_name.lower().startswith(PASS_THROUGH_HEADER_PREFIX):
# Strip the 'x-pass-' prefix and normalize to lowercase
actual_header_name = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower()
if actual_header_name in _PASS_THROUGH_PROTECTED_HEADERS or any(
actual_header_name.startswith(p) for p in _PASS_THROUGH_PROTECTED_HEADER_PREFIXES
):
if not _is_forwarded_pass_through_header(header_name):
verbose_logger.debug(
"x-pass- header %s maps to a protected header name; skipping",
header_name,
)
continue
# Strip the 'x-pass-' prefix and normalize to lowercase
actual_header_name = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower()
headers[actual_header_name] = header_value
return headers

View file

@ -4098,6 +4098,32 @@
"generateContent": false
}
},
"webiq": {
"display_name": "Microsoft Web IQ (`webiq`)",
"url": "https://learn.microsoft.com/en-us/web-iq/reference/web-search",
"endpoints": {
"chat_completions": false,
"responses": false,
"text_completion": false,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": true,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": false,
"interactions": false,
"systemone": false,
"a2a": false,
"generateContent": false
}
},
"tavily": {
"display_name": "Tavily (`tavily`)",
"url": "https://docs.litellm.ai/docs/search/tavily",

View file

@ -3799,6 +3799,21 @@
},
"title": "Providers",
"type": "object"
},
"teams": {
"anyOf": [
{
"additionalProperties": {
"$ref": "#/components/schemas/MetricWithMetadata"
},
"type": "object"
},
{
"type": "null"
}
],
"description": "Customer spend by the key's current team, or last team for deleted keys. Keys without a known team are grouped under Unassigned",
"title": "Teams"
}
},
"title": "BreakdownMetrics",
@ -4867,6 +4882,12 @@
"title": "Gateway Injected Caching Savings Spend",
"type": "number"
},
"guardrail_spend": {
"default": 0.0,
"description": "Guardrail spend in USD, already included in spend, so model spend is spend - guardrail_spend. Rows recorded before this field existed report 0 even when their spend includes guardrail cost",
"title": "Guardrail Spend",
"type": "number"
},
"prompt_caching_savings_spend": {
"default": 0.0,
"title": "Prompt Caching Savings Spend",
@ -27293,6 +27314,65 @@
"title": "OutputToolSearchCall",
"type": "object"
},
"OutputToolSearchOutput": {
"additionalProperties": true,
"properties": {
"call_id": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Call Id"
},
"execution": {
"enum": [
"server",
"client"
],
"title": "Execution",
"type": "string"
},
"id": {
"title": "Id",
"type": "string"
},
"status": {
"enum": [
"in_progress",
"completed",
"incomplete"
],
"title": "Status",
"type": "string"
},
"tools": {
"default": [],
"items": {
"additionalProperties": true,
"type": "object"
},
"title": "Tools",
"type": "array"
},
"type": {
"const": "tool_search_output",
"title": "Type",
"type": "string"
}
},
"required": [
"id",
"execution",
"status",
"type"
],
"title": "OutputToolSearchOutput",
"type": "object"
},
"PendingSafetyCheck": {
"additionalProperties": true,
"description": "A pending safety check for the computer call.",
@ -29663,6 +29743,9 @@
{
"$ref": "#/components/schemas/OutputToolSearchCall"
},
{
"$ref": "#/components/schemas/OutputToolSearchOutput"
},
{
"$ref": "#/components/schemas/ResponseCompactionItem"
}

View file

@ -15,6 +15,7 @@ from pydantic import (
Field,
Json,
JsonValue,
NonNegativeInt,
PositiveInt,
TypeAdapter,
field_validator,
@ -1826,6 +1827,14 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase):
default=None,
description="Max budget for each model (e.g. {'gpt-4o': {'max_budget': '0.0000001', 'budget_duration': '1d', 'tpm_limit': 1000, 'rpm_limit': 1000}})",
)
model_tpm_limit: Mapping[str, NonNegativeInt] | None = Field(
default=None,
description="Max tokens per minute per model for this budget id, shared by every key, team and customer on it (e.g. {'gpt-4o': 400000})",
)
model_rpm_limit: Mapping[str, NonNegativeInt] | None = Field(
default=None,
description="Max requests per minute per model for this budget id, shared by every key, team and customer on it (e.g. {'gpt-4o': 200})",
)
budget_reset_at: datetime | None = Field(
default=None,
description="Datetime when the budget is reset",
@ -1851,6 +1860,21 @@ class CustomerBase(LiteLLMPydanticObjectBase):
blocked: bool = False
def reject_unknown_customer_fields(request: type[BaseModel], values: object) -> object:
if not isinstance(values, dict):
return values
unknown: Final = tuple(sorted(str(key) for key in values if key not in request.model_fields)) # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # raw request body
if not unknown:
return values
on_budget: Final = tuple(field for field in unknown if field in BudgetNewRequest.model_fields)
hint: Final = (
f" {', '.join(on_budget)}: these live on the customer's budget object, update it with /budget/update."
if on_budget
else ""
)
raise ValueError(f"Unknown fields: {', '.join(unknown)}.{hint}")
class NewCustomerRequest(BudgetNewRequest):
"""
Create a new customer, allocate a budget to them
@ -1868,6 +1892,11 @@ class NewCustomerRequest(BudgetNewRequest):
models: list[str] | None = None
object_permission: LiteLLM_ObjectPermissionBase | None = None
@model_validator(mode="before")
@classmethod
def reject_unknown_fields(cls, values: object) -> object:
return reject_unknown_customer_fields(cls, values)
@model_validator(mode="before")
@classmethod
def check_user_info(cls, values):
@ -1895,6 +1924,11 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase):
models: list[str] | None = None
object_permission: LiteLLM_ObjectPermissionBase | None = None
@model_validator(mode="before")
@classmethod
def reject_unknown_fields(cls, values: object) -> object:
return reject_unknown_customer_fields(cls, values)
class DeleteCustomerRequest(LiteLLMPydanticObjectBase):
"""
@ -5301,6 +5335,7 @@ class BaseDailySpendTransaction(TypedDict):
# missing value to zero, so requiring it here would describe a shape the aggregation
# is explicitly tested against.
autorouter_savings_spend: NotRequired[float]
guardrail_spend: NotRequired[float] # writable-ok: the rollup queue accumulates into this key in place
# request level metrics
spend: float

View file

@ -2,8 +2,9 @@ import importlib.util
import os
import re
import sys
from collections.abc import Collection, Iterator, Mapping, Sequence
from functools import lru_cache
from collections.abc import Callable, Collection, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import cache, lru_cache
from itertools import accumulate, chain, repeat
from logging import Logger
from types import MappingProxyType
@ -22,7 +23,9 @@ from litellm.constants import (
MINIMUM_CUSTOM_KEY_LENGTH,
STANDARD_CUSTOMER_ID_HEADERS,
)
from litellm.litellm_core_utils.cloud_storage_security import REQUEST_S3_BUCKET_PARAMS, STORED_S3_BUCKET_PARAM
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.url_utils import (
SSRFError,
@ -41,11 +44,13 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
from litellm.types.router import (
CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS,
Deployment,
DeploymentTypedDict,
LiteLLMParamsTypedDict,
reject_server_owned_wif_fields,
server_owned_wif_fields_present,
)
from litellm.types.router import reject_server_owned_wif_params as _reject_server_owned_wif_params
from litellm.types.utils import CustomPricingLiteLLMParams, oauth_token_exchange_litellm_params
from litellm.types.utils import CustomPricingLiteLLMParams, LlmProviders, oauth_token_exchange_litellm_params
from litellm.types.videos.main import DecodedVideoId
@ -211,35 +216,74 @@ def _is_param_allowed(
return False
def deployment_clientside_auth_params(litellm_params: LiteLLMParamsTypedDict) -> CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS:
if "configurable_clientside_auth_params" not in litellm_params:
return None
return litellm_params["configurable_clientside_auth_params"]
_DEPLOYMENT_LIST: Final = TypeAdapter(tuple[DeploymentTypedDict, ...], config=ConfigDict(arbitrary_types_allowed=True))
def _model_group_deployments(
model: str, llm_router: Router, team_id: str | None = None
) -> Sequence[DeploymentTypedDict]:
registry: Final = llm_router.registry()
provider_group: Final = model.split("/", 1)[0]
if team_id is not None and registry.routing_group(model) is None:
team_deployments: Final = registry.deployments(
model, source="all", team_id=team_id
) or _DEPLOYMENT_LIST.validate_python(registry.deployments(model, source="wildcard", team_id=team_id))
if team_deployments:
return team_deployments
return (
registry.deployments(model, source="routed_group", team_id=team_id)
or (registry.deployments(provider_group, team_id=team_id) if provider_group in provider_list else None)
or ()
)
_VERTEX_TARGET_READING_PROVIDERS: Final[frozenset[str]] = frozenset(
(LlmProviders.VERTEX_AI.value, LlmProviders.VERTEX_AI_BETA.value, LlmProviders.GEMINI.value, LlmProviders.GDC.value)
)
def _deployment_provider(litellm_params: LiteLLMParamsTypedDict) -> str | None:
match litellm_params:
case {"custom_llm_provider": str(declared)} if declared:
return declared
case {"model": str(model)}:
return inferred_provider(model)
case _:
return None
def _model_group_reads_vertex_target(model: str, llm_router: Router | None, team_id: str | None = None) -> bool:
if llm_router is None:
return True
deployments: Final = _model_group_deployments(model, llm_router, team_id)
providers: Final = frozenset(_deployment_provider(deployment["litellm_params"]) for deployment in deployments)
return not deployments or None in providers or bool(providers & _VERTEX_TARGET_READING_PROVIDERS)
def _allow_model_level_clientside_configurable_parameters(
model: str, param: str, request_body_value: object, llm_router: Router | None
model: str, param: str, request_body_value: object, llm_router: Router | None, team_id: str | None = None
) -> bool:
"""
Check if model is allowed to use configurable client-side params
- get matching model
- check if 'clientside_configurable_parameters' is set for model
-
The router, not the caller, picks which deployment serves the request, so an
opt-in counts only when every deployment of the model group, already alias
resolved, carries it
"""
if llm_router is None:
return False
# check if model is set
registry: Final = llm_router.registry()
model_info = registry.group(model)
if model_info is None:
# check if wildcard model is set
if model.split("/", 1)[0] in provider_list:
model_info = registry.group(model.split("/", 1)[0])
if model_info is None:
return False
if model_info is None or model_info.configurable_clientside_auth_params is None:
return False
return _is_param_allowed(
param=param,
request_body_value=request_body_value,
configurable_clientside_auth_params=model_info.configurable_clientside_auth_params,
deployments: Final = _model_group_deployments(model, llm_router, team_id)
return bool(deployments) and all(
_is_param_allowed(
param=param,
request_body_value=request_body_value,
configurable_clientside_auth_params=deployment_clientside_auth_params(deployment["litellm_params"]),
)
for deployment in deployments
)
@ -318,6 +362,20 @@ def route_manages_mcp_servers(route: str | None) -> bool:
return route is not None and _MCP_SERVER_MANAGEMENT_ROUTE.fullmatch(route) is not None
VERTEX_TARGET_PARAMS: Final[frozenset[str]] = frozenset(
("vertex_project", "vertex_location", "vertex_ai_project", "vertex_ai_location")
)
_ROUTES_THAT_NAME_THEIR_VERTEX_TARGET: Final[frozenset[str]] = frozenset(
("/vertex_ai/live", "/rag/ingest", "/v1/rag/ingest")
)
def _without_route_named_vertex_target(request_body: Mapping[str, object], route: str | None) -> Mapping[str, object]:
if route not in _ROUTES_THAT_NAME_THEIR_VERTEX_TARGET:
return request_body
return MappingProxyType({key: value for key, value in request_body.items() if key not in VERTEX_TARGET_PARAMS})
_NESTED_CONFIG_KEYS: Final[tuple[str, ...]] = ("litellm_embedding_config", "extra_body")
# Metadata containers that carry per-request configuration consumed by the
@ -410,6 +468,10 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
"aws_external_id",
"aws_session_tags",
"vertex_credentials",
"vertex_location",
"vertex_project",
"vertex_ai_location",
"vertex_ai_project",
# Azure managed-identity / federated-auth token. The Azure provider
# transformer reads ``azure_ad_token`` (top-level or via
# ``extra_body``) and resolves it through ``get_secret`` before
@ -444,6 +506,7 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
"s3_endpoint_url",
"s3_bucket_name",
"s3_output_bucket_name",
STORED_S3_BUCKET_PARAM,
"allow_legacy_cloud_file_ids",
"sagemaker_base_url",
"deployment_url",
@ -481,6 +544,105 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
*sorted(CustomPricingLiteLLMParams.model_fields.keys()),
)
_BANNED_METADATA_PARAMS: Final[tuple[str, ...]] = tuple(
param for param in _BANNED_REQUEST_BODY_PARAMS if param not in VERTEX_TARGET_PARAMS
)
def _routed_opt_in_hop(
frontier_and_reached: tuple[frozenset[str], frozenset[str]], router_and_team: tuple[Router, str | None]
) -> tuple[frozenset[str], frozenset[str]]:
frontier, reached = frontier_and_reached
llm_router, team_id = router_and_team
deployments: Final = chain.from_iterable(_model_group_deployments(model, llm_router, team_id) for model in frontier)
routed_groups: Final = frozenset(deployment["model_name"] for deployment in deployments) - reached
fallbacks: Final = frozenset[str]().union(
*(router_fallback_model_names(name, llm_router, _EVERY_FALLBACK_TYPE) for name in routed_groups)
)
registry: Final = llm_router.registry()
next_hop: Final = (
routed_groups | frozenset(registry.resolve(name, kind="alias") or name for name in fallbacks)
) - reached
return next_hop, reached | next_hop
def _opt_in_model_groups(
model: str, also_requested_models: tuple[str, ...], llm_router: Router | None, team_id: str | None = None
) -> frozenset[str]:
requested: Final = frozenset((model, *also_requested_models))
if llm_router is None:
return requested
registry: Final = llm_router.registry()
routed: Final = frozenset(registry.resolve(name, kind="alias") or name for name in requested)
fallbacks: Final = frozenset[str]().union(
*(router_fallback_model_names(name, llm_router, _EVERY_FALLBACK_TYPE) for name in requested | routed)
)
groups: Final = routed | frozenset(registry.resolve(name, kind="alias") or name for name in fallbacks)
hops: Final = accumulate(repeat((llm_router, team_id)), _routed_opt_in_hop, initial=(groups, groups))
_, reached = next(hop for hop in hops if not hop[0])
return reached
_MAX_OPT_IN_FALLBACK_MODELS: Final = 100
@dataclass(frozen=True, slots=True)
class _OptInGroups:
every: tuple[str, ...]
vertex_target: tuple[str, ...]
team_id: str | None = None
_OptInKey = tuple[str | tuple[str, str | None], ...]
def _opt_in_key(litellm_params: LiteLLMParamsTypedDict) -> _OptInKey:
return tuple(
item if isinstance(item, str) else ("api_base", item.get("api_base"))
for item in deployment_clientside_auth_params(litellm_params) or ()
)
def _same_answer_key(
model_group: str, llm_router: Router, team_id: str | None
) -> frozenset[tuple[str | None, _OptInKey]] | str:
deployments: Final = _model_group_deployments(model_group, llm_router, team_id)
key: Final = frozenset(
(_deployment_provider(deployment["litellm_params"]), _opt_in_key(deployment["litellm_params"]))
for deployment in deployments
)
return key or model_group
def _distinct_opt_in_groups(
model_groups: frozenset[str], llm_router: Router | None, team_id: str | None
) -> tuple[str, ...]:
if llm_router is None:
return tuple(sorted(model_groups))
return tuple(
{
_same_answer_key(model_group, llm_router, team_id): model_group for model_group in sorted(model_groups)
}.values()
)
def _resolve_opt_in_groups(
model: str, also_requested_models: tuple[str, ...], llm_router: Router | None, team_id: str | None = None
) -> _OptInGroups:
if len(frozenset(also_requested_models)) > _MAX_OPT_IN_FALLBACK_MODELS:
raise ValueError(
f"Rejected Request: the request names more than {_MAX_OPT_IN_FALLBACK_MODELS} fallback models, "
"too many to check against configurable_clientside_auth_params."
)
every: Final = _distinct_opt_in_groups(
_opt_in_model_groups(model, also_requested_models, llm_router, team_id), llm_router, team_id
)
return _OptInGroups(
every=every,
vertex_target=tuple(group for group in every if _model_group_reads_vertex_target(group, llm_router, team_id)),
team_id=team_id,
)
_ClientSideAllowlistSetting: TypeAlias = Literal[
"allowed_client_side_s3_buckets", "allowed_client_side_s3_endpoint_urls"
@ -494,6 +656,9 @@ _PROXY_ALLOWLIST_SETTING_BY_PARAM: Final[Mapping[str, _ClientSideAllowlistSettin
}
)
TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY: Final = "allowed_s3_buckets"
_BUCKET_NAMES: Final = TypeAdapter(tuple[str, ...])
class _ClientSideStorageAllowlist(BaseModel):
model_config = ConfigDict(frozen=True)
@ -511,8 +676,15 @@ class _ClientSideStorageAllowlist(BaseModel):
def _banned_param_error(param: str) -> ValueError:
setting: Final = _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param)
team_hint: Final = (
f" or in the team's `metadata.{TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY}`"
if param in REQUEST_S3_BUCKET_PARAMS
else ""
)
allowlist_hint: Final = (
"" if setting is None else f" To allow specific values only, list them in `general_settings.{setting}`."
""
if setting is None
else f" To allow specific values only, list them in `general_settings.{setting}`{team_hint}."
)
return ValueError(
f"Rejected Request: {param} is not allowed in request body. "
@ -526,17 +698,21 @@ def _banned_param_error(param: str) -> ValueError:
def _check_banned_params(
body: Mapping[str, object],
general_settings: dict,
general_settings: Mapping[str, object],
llm_router: Router | None,
model: str,
*,
manages_deployments: bool = False,
mcp_server_fields_allowed: bool = False,
banned_params: tuple[str, ...] = _BANNED_REQUEST_BODY_PARAMS,
opt_in_groups: Callable[[], _OptInGroups],
defer_model_checks: bool = False,
defer_s3_buckets: bool = False,
) -> None:
"""Raise ``ValueError`` if ``body`` carries a banned param without admin opt-in.
Shared between the root-level check and the nested-config check so a
new banned param only needs to be added in one place.
new banned param only needs to be added in one place. ``defer_s3_buckets`` lets
bucket params through for handlers that check them against the caller team's allowlist.
"""
wif_body: Final[Mapping[str, object]] = (
MappingProxyType({key: value for key, value in body.items() if key not in oauth_token_exchange_litellm_params})
@ -546,7 +722,7 @@ def _check_banned_params(
reject_server_owned_wif_params(wif_body)
if not manages_deployments:
reject_federated_credential_reference(body)
for param in _BANNED_REQUEST_BODY_PARAMS:
for param in banned_params:
if param not in body:
continue
if general_settings.get("allow_client_side_credentials") is True:
@ -557,14 +733,22 @@ def _check_banned_params(
setting := _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param)
) is not None and _ClientSideStorageAllowlist.model_validate(general_settings).admits(setting, body[param]):
continue
if (
if defer_s3_buckets and param in REQUEST_S3_BUCKET_PARAMS:
continue
if defer_model_checks:
continue
if all(
_allow_model_level_clientside_configurable_parameters(
model=model,
model=model_group,
param=param,
request_body_value=body[param],
llm_router=llm_router,
team_id=opt_in_groups().team_id,
)
is True
for model_group in (
opt_in_groups().vertex_target if param in VERTEX_TARGET_PARAMS else opt_in_groups().every
)
):
# Per-param opt-in: only THIS param is permitted by the
# deployment's ``configurable_clientside_auth_params``. Skip
@ -648,27 +832,39 @@ def fallback_target_model_name(target: object) -> str | None:
return None
def _direct_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]:
_EVERY_FALLBACK_TYPE: Final[tuple[str, ...]] = ("general", "context_window", "content_policy")
def _direct_fallback_model_names(model: str, llm_router: Router, fallback_types: tuple[str, ...]) -> frozenset[str]:
from litellm.proxy.auth.model_checks import get_all_fallbacks
return frozenset(
name for target in get_all_fallbacks(model, llm_router) if (name := fallback_target_model_name(target))
targets: Final = chain.from_iterable(
get_all_fallbacks(model, llm_router, fallback_type) for fallback_type in fallback_types
)
return frozenset(name for target in targets if (name := fallback_target_model_name(target)))
def _fallback_hop(
frontier_and_reached: tuple[frozenset[str], frozenset[str]], llm_router: Router
frontier_and_reached: tuple[frozenset[str], frozenset[str]], router_and_types: tuple[Router, tuple[str, ...]]
) -> tuple[frozenset[str], frozenset[str]]:
frontier, reached = frontier_and_reached
llm_router, fallback_types = router_and_types
next_hop: Final = (
frozenset(chain.from_iterable(_direct_fallback_model_names(model, llm_router) for model in frontier)) - reached
frozenset(
chain.from_iterable(_direct_fallback_model_names(model, llm_router, fallback_types) for model in frontier)
)
- reached
)
return next_hop, reached | next_hop
def router_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]:
def router_fallback_model_names(
model: str, llm_router: Router, fallback_types: tuple[str, ...] = ("general",)
) -> frozenset[str]:
"""The groups a failure on `model` can reach through the router's fallbacks, chained and generic (`*`) ones."""
hops: Final = accumulate(repeat(llm_router), _fallback_hop, initial=(frozenset({model}), frozenset({model})))
hops: Final = accumulate(
repeat((llm_router, fallback_types)), _fallback_hop, initial=(frozenset({model}), frozenset({model}))
)
_, reached = next(hop for hop in hops if not hop[0])
return reached - {model}
@ -698,12 +894,15 @@ def _reject_url_valued_fallback_target(value: str) -> None:
def is_request_body_safe(
request_body: dict,
general_settings: dict,
request_body: Mapping[str, object],
general_settings: Mapping[str, object],
llm_router: Router | None,
model: str,
*,
route: str | None = None,
team_id: str | None = None,
defer_model_checks: bool = False,
defer_s3_buckets: bool = False,
) -> bool:
"""
Check if the request body is safe.
@ -733,33 +932,72 @@ def is_request_body_safe(
if "model_list" in request_body:
raise ValueError("Rejected Request: model_list is not allowed in the request body.")
manages_deployments: Final = route_manages_deployments(route)
root_body: Final = _without_route_named_vertex_target(request_body, route)
route_model: Final = _model_from_route_path(route) if route is not None else None
also_requested_models: Final = (
*request_fallback_model_names(root_body),
*((route_model,) if route_model else ()),
)
opt_in_groups: Final = cache(
lambda: _resolve_opt_in_groups(model or route_model or "", also_requested_models, llm_router, team_id)
)
_check_banned_params(
request_body,
root_body,
general_settings,
llm_router,
model,
manages_deployments=manages_deployments,
mcp_server_fields_allowed=route_manages_mcp_servers(route),
opt_in_groups=opt_in_groups,
defer_model_checks=defer_model_checks,
defer_s3_buckets=defer_s3_buckets,
)
for nested_key in _NESTED_CONFIG_KEYS:
nested = _coerce_metadata_to_dict(request_body.get(nested_key))
if nested is not None:
_check_banned_params(nested, general_settings, llm_router, model, manages_deployments=manages_deployments)
_check_banned_params(
nested,
general_settings,
llm_router,
manages_deployments=manages_deployments,
opt_in_groups=opt_in_groups,
defer_model_checks=defer_model_checks,
defer_s3_buckets=defer_s3_buckets,
)
for metadata_key in _NESTED_METADATA_KEYS:
metadata = _coerce_metadata_to_dict(request_body.get(metadata_key))
if metadata is not None:
_check_banned_params(metadata, general_settings, llm_router, model, manages_deployments=manages_deployments)
if any(isinstance(key, str) and key.startswith(f"{metadata_key}[") for key in request_body):
_check_banned_params(
metadata,
general_settings,
llm_router,
manages_deployments=manages_deployments,
banned_params=_BANNED_METADATA_PARAMS,
opt_in_groups=opt_in_groups,
defer_model_checks=defer_model_checks,
defer_s3_buckets=defer_s3_buckets,
)
if any(key.startswith(f"{metadata_key}[") for key in request_body):
_check_banned_params(
extract_nested_form_metadata(form_data=request_body, prefix=f"{metadata_key}["),
general_settings,
llm_router,
model,
manages_deployments=manages_deployments,
banned_params=_BANNED_METADATA_PARAMS,
opt_in_groups=opt_in_groups,
defer_model_checks=defer_model_checks,
defer_s3_buckets=defer_s3_buckets,
)
for target in iter_request_fallback_targets(request_body):
if isinstance(target, dict):
_check_banned_params(target, general_settings, llm_router, model, manages_deployments=manages_deployments)
_check_banned_params(
target,
general_settings,
llm_router,
manages_deployments=manages_deployments,
opt_in_groups=opt_in_groups,
defer_model_checks=defer_model_checks,
defer_s3_buckets=defer_s3_buckets,
)
target_model = target.get("model")
if isinstance(target_model, str):
_reject_url_valued_fallback_target(target_model)
@ -773,9 +1011,16 @@ def is_request_body_safe(
litellm_params_metadata: Final = _coerce_metadata_to_dict(litellm_params.get("metadata"))
if litellm_params_metadata is not None:
_check_banned_params(
litellm_params_metadata, general_settings, llm_router, model, manages_deployments=manages_deployments
litellm_params_metadata,
general_settings,
llm_router,
manages_deployments=manages_deployments,
banned_params=_BANNED_METADATA_PARAMS,
opt_in_groups=opt_in_groups,
defer_model_checks=defer_model_checks,
defer_s3_buckets=defer_s3_buckets,
)
for user_config_params in _user_config_litellm_params(request_body): # pyright: ignore[reportUnknownArgumentType] # request_body is an untyped dict
for user_config_params in _user_config_litellm_params(request_body):
reject_api_key_secret_reference(user_config_params)
return True
@ -846,10 +1091,57 @@ def _coerce_metadata_to_dict(value: object) -> dict[str, object] | None:
return None
def _team_s3_buckets(team_metadata: Mapping[str, object] | None) -> tuple[str, ...]:
if team_metadata is None:
return ()
try:
return _BUCKET_NAMES.validate_python(team_metadata.get(TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY) or ())
except ValidationError:
return ()
_TEAM_S3_BUCKET_ROUTE_TEMPLATES: Final = frozenset(
{"/v1/files", "/files", "/{provider}/v1/files", "/v1/batches", "/batches", "/{provider}/v1/batches"}
)
def checks_team_s3_buckets_in_handler(request: Request) -> bool:
"""File upload and batch create check request buckets against the caller team's allowlist themselves."""
return get_request_route_template(request) in _TEAM_S3_BUCKET_ROUTE_TEMPLATES and request.method == "POST"
def enforce_team_s3_bucket_allowlist(
request_body: Mapping[str, object],
general_settings: Mapping[str, object],
llm_router: Router | None,
team_metadata: Mapping[str, object] | None,
*,
team_id: str | None = None,
) -> None:
"""Re-run the request body check that auth relaxed for bucket params, now admitting the caller team's buckets."""
proxy_buckets: Final = _ClientSideStorageAllowlist.model_validate(general_settings).allowed_client_side_s3_buckets
model: Final = request_body.get("model")
try:
is_request_body_safe(
request_body=dict(request_body),
general_settings={
**general_settings,
"allowed_client_side_s3_buckets": (*(proxy_buckets or ()), *_team_s3_buckets(team_metadata)),
},
llm_router=llm_router,
model=model if isinstance(model, str) else "",
team_id=team_id,
)
except ValueError as rejection:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=f"Authentication Error, {rejection}")
async def pre_db_read_auth_checks(
request: Request,
request_data: dict,
route: str,
*,
defer_team_checks: bool = False,
):
"""
1. Checks if request size is under max_request_size_mb (if set)
@ -869,12 +1161,18 @@ async def pre_db_read_auth_checks(
await check_if_request_size_is_safe(request=request)
# Check 2. Request body is safe
defer_model_checks: Final = (
defer_team_checks and llm_router is not None and bool(llm_router.registry().index("team_model"))
)
request.state.deferred_model_body_check = defer_model_checks # rebind-ok: hand state to auth
is_request_body_safe(
request_body=request_data,
general_settings=general_settings,
llm_router=llm_router,
model=request_data.get("model", ""), # [TODO] use model passed in url as well (azure openai routes)
model=request_data.get("model") or _model_from_route_path(route) or "",
route=route,
defer_model_checks=defer_model_checks,
defer_s3_buckets=checks_team_s3_buckets_in_handler(request),
)
# Check 3. Check if IP address is allowed
@ -1522,6 +1820,9 @@ class BatchLimitRequest(Protocol):
def model_fields_set(self) -> Collection[str]: ...
_ADMIN_ONLY_METADATA_KEYS: Final = (*ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS, TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY)
def enforce_batch_limits_are_admin_only(
data: BatchLimitRequest,
existing_metadata: Mapping[str, object] | None,
@ -1532,7 +1833,8 @@ def enforce_batch_limits_are_admin_only(
Every key in ``ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS`` caps what the holder
may do with batches, so a holder-writable copy would let a caller lift their
own quota. Gated on the resulting value rather than on presence, so a form
own quota, and the S3 bucket list grants access to buckets with the proxy's
own credentials. Gated on the resulting value rather than on presence, so a form
resending the stored value stays a no-op.
"""
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value:
@ -1542,17 +1844,19 @@ def enforce_batch_limits_are_admin_only(
(data.metadata or EMPTY_MAPPING) if "metadata" in data.model_fields_set else stored
)
changed: Final = next(
(key for key in ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS if requested.get(key) != stored.get(key)),
(key for key in _ADMIN_ONLY_METADATA_KEYS if requested.get(key) != stored.get(key)),
None,
)
if changed is None:
return
reason: Final = (
"It lets the holder read and write those buckets with the proxy's AWS credentials."
if changed == TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY
else "It limits what the holder can do with batches, so the holder cannot raise it."
)
raise HTTPException(
status_code=403,
detail={
"error": f"Only proxy admins can set {changed} on a {entity}. "
"It limits what the holder can do with batches, so the holder cannot raise it."
},
detail={"error": f"Only proxy admins can set {changed} on a {entity}. {reason}"},
)
@ -2395,37 +2699,8 @@ def get_model_from_request(
team_id=team_id,
)
model = _format_model_candidates(candidates)
# If no explicit model was found, try to extract from route
if model is None:
# Parse model from route that follows the pattern /openai/deployments/{model}/*
match: Final = re.match(r"/openai/deployments/([^/]+)", route)
if match:
model = match.group(1)
# If still not found, extract model from Google generateContent-style routes.
# These routes put the model in the path and allow "/" inside the model id.
# Examples:
# - /v1beta/models/gemini-2.0-flash:generateContent
# - /v1beta/models/bedrock/claude-sonnet-3.7:generateContent
# - /models/custom/ns/model:streamGenerateContent
if model is None and not route.lower().startswith("/vertex"):
google_match = re.search(r"/(?:v1beta|beta)/models/([^:]+):", route)
if google_match:
model = google_match.group(1)
if model is None and not route.lower().startswith("/vertex"):
google_match = re.search(r"^/models/([^:]+):", route)
if google_match:
model = google_match.group(1)
# If still not found, extract from Vertex AI passthrough route
# Pattern: /vertex_ai/.../models/{model_id}:*
# Example: /vertex_ai/v1/.../models/gemini-1.5-pro:generateContent
if model is None and route.lower().startswith("/vertex"):
vertex_match: Final = re.search(r"/models/([^:]+)", route)
if vertex_match:
model = vertex_match.group(1)
model = _model_from_route_path(route)
if route.lower().startswith("/bedrock"):
bedrock_model: Final = _model_from_bedrock_route(route)
@ -2444,6 +2719,23 @@ def get_model_from_request(
return model
def _model_from_route_path(route: str) -> str | None:
"""
Google generateContent-style routes allow "/" inside the model id, e.g.
/v1beta/models/bedrock/claude-sonnet-3.7:generateContent
"""
azure_deployment: Final = re.match(r"/openai/deployments/([^/]+)", route)
if azure_deployment:
return azure_deployment.group(1)
if route.lower().startswith("/vertex"):
vertex_model: Final = re.search(r"/models/([^:]+)", route)
return vertex_model.group(1) if vertex_model else None
google_model: Final = re.search(r"/(?:v1beta|beta)/models/([^:]+):", route) or re.search(
r"^/models/([^:]+):", route
)
return google_model.group(1) if google_model else None
def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str | None:
if llm_router is None:
return None

View file

@ -0,0 +1,198 @@
import asyncio
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, Protocol
from litellm._logging import verbose_proxy_logger
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from litellm.proxy._types import LiteLLMRoutes, SpecialModelNames
from litellm.repositories.prisma_protocols import CountTable
DEPRECATION_DISCUSSION_URL: Final = "https://github.com/BerriAI/litellm/discussions/32090"
class CountTableRepository(Protocol):
@property
def table(self) -> CountTable: ...
@dataclass(frozen=True, slots=True)
class LegacyEmptyModelsCounts:
teamless_keys_with_empty_models: int
teamless_keys_with_all_team_models: int
team_keys_with_empty_models: int
teams_with_empty_models: int
keys_with_access_groups_and_empty_models: int
teams_with_access_groups_and_empty_models: int
def _model_calling_keys(where: Mapping[str, object]) -> Mapping[str, object]:
return {
"AND": [
where,
{
"NOT": [
{"allowed_routes": {"equals": [LiteLLMRoutes.management_routes.name]}},
{"allowed_routes": {"equals": [LiteLLMRoutes.info_routes.name]}},
]
},
]
}
async def count_legacy_empty_models_rows(keys: CountTable, teams: CountTable) -> LegacyEmptyModelsCounts:
(
teamless_empty,
teamless_all_team,
team_keys_empty,
teams_empty,
keys_with_groups_empty,
teams_with_groups_empty,
) = await asyncio.gather(
keys.count(
where=_model_calling_keys(
{"team_id": None, "models": {"isEmpty": True}, "access_group_ids": {"isEmpty": True}}
)
),
keys.count(
where=_model_calling_keys(
{
"team_id": None,
"models": {"has": SpecialModelNames.all_team_models.value},
"access_group_ids": {"isEmpty": True},
}
)
),
keys.count(
where=_model_calling_keys(
{
"AND": [{"team_id": {"not": None}}, {"team_id": {"not": UI_SESSION_TOKEN_TEAM_ID}}],
"models": {"isEmpty": True},
"access_group_ids": {"isEmpty": True},
}
)
),
teams.count(where={"models": {"isEmpty": True}, "access_group_ids": {"isEmpty": True}}),
keys.count(
where=_model_calling_keys(
{
"OR": [{"team_id": None}, {"team_id": {"not": UI_SESSION_TOKEN_TEAM_ID}}],
"models": {"isEmpty": True},
"NOT": {"access_group_ids": {"isEmpty": True}},
}
)
),
teams.count(where={"models": {"isEmpty": True}, "NOT": {"access_group_ids": {"isEmpty": True}}}),
)
return LegacyEmptyModelsCounts(
teamless_keys_with_empty_models=teamless_empty,
teamless_keys_with_all_team_models=teamless_all_team,
team_keys_with_empty_models=team_keys_empty,
teams_with_empty_models=teams_empty,
keys_with_access_groups_and_empty_models=keys_with_groups_empty,
teams_with_access_groups_and_empty_models=teams_with_groups_empty,
)
def _counted(count: int, singular: str, plural: str) -> str:
return f"{count} {singular if count == 1 else plural}"
def legacy_empty_models_warning(counts: LegacyEmptyModelsCounts) -> str | None:
findings: Final = tuple(
f"{_counted(count, singular, plural)}, which means {meaning}."
for count, singular, plural, meaning in (
(
counts.teamless_keys_with_empty_models,
"key without a team has an empty models list",
"keys without a team have an empty models list",
"every proxy model",
),
(
counts.teamless_keys_with_all_team_models,
"key without a team lists all-team-models",
"keys without a team list all-team-models",
"every proxy model",
),
(
counts.team_keys_with_empty_models,
"team key has an empty models list",
"team keys have an empty models list",
"all of the team's models",
),
(
counts.teams_with_empty_models,
"team has an empty models list",
"teams have an empty models list",
"every proxy model",
),
)
if count > 0
)
access_group_findings: Final = tuple(
f"{_counted(count, singular, plural)}."
for count, singular, plural in (
(
counts.keys_with_access_groups_and_empty_models,
"key with access groups has an empty models list",
"keys with access groups have an empty models list",
),
(
counts.teams_with_access_groups_and_empty_models,
"team with access groups has an empty models list",
"teams with access groups have an empty models list",
),
)
if count > 0
)
if not findings and not access_group_findings:
return None
all_proxy_models: Final = SpecialModelNames.all_proxy_models.value
all_team_models: Final = SpecialModelNames.all_team_models.value
keep_access_advice: Final = (
(
f"To keep today's access, set models to {all_proxy_models} or a list of models on each one, "
f"or to {all_team_models} on keys that belong to a team, before you upgrade.",
)
if findings
else ()
)
teamless_all_team_models_advice: Final = (
(
f"Keys without a team that list {all_team_models} have no team to inherit from, "
f"so set each one to {all_proxy_models} or attach it to a team.",
)
if counts.teamless_keys_with_all_team_models > 0
else ()
)
access_group_advice: Final = (
("Check which models each key and team with access groups should reach before you upgrade.",)
if access_group_findings
else ()
)
return " ".join(
(
"Deprecated model access defaults are still in use.",
*findings,
*access_group_findings,
"These counts include blocked and expired keys and blocked teams.",
"LiteLLM 2.0.0 removes these defaults.",
*keep_access_advice,
*teamless_all_team_models_advice,
*access_group_advice,
"Keys and teams created without a models list still get an empty one until then, "
"so these counts can grow back.",
f"See {DEPRECATION_DISCUSSION_URL}",
)
)
async def warn_about_legacy_empty_models(keys: CountTableRepository, teams: CountTableRepository) -> None:
try:
counts: Final = await count_legacy_empty_models_rows(keys=keys.table, teams=teams.table)
except Exception as e: # noqa: BLE001 # a startup check must never take the proxy down on a DB error
verbose_proxy_logger.warning("Skipped the deprecated model access defaults check: %s", e)
return
message: Final = legacy_empty_models_warning(counts)
if message is not None:
verbose_proxy_logger.warning(message)

View file

@ -28,6 +28,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.auth.auth_checks import (
TeamNotFoundError,
forget_missing_user,
get_team_membership,
get_team_object,
get_user_object,
@ -246,6 +247,8 @@ class GrantResolver:
return ResolvedGrants(user_object, team_object, membership, effective_user_id)
async def _user(self, lookup: UserLookup) -> LiteLLM_UserTable | None:
if lookup.user_id is not None:
forget_missing_user(lookup.user_id)
return await self._load_user(
user_id=lookup.user_id,
prisma_client=self._prisma,

View file

@ -109,12 +109,14 @@ from litellm.proxy.auth.auth_object_prefetch import (
)
from litellm.proxy.auth.auth_utils import (
abbreviate_api_key,
checks_team_s3_buckets_in_handler,
fallback_target_model_name,
get_end_user_id_from_request_body,
get_model_from_request,
get_request_route,
get_request_route_template,
is_invalid_virtual_key_error,
is_request_body_safe,
iter_request_fallback_targets,
log_model_access_denial,
normalize_request_route,
@ -1313,6 +1315,12 @@ def _apply_budget_limits_to_end_user_params(
if budget_info.model_max_budget is not None:
end_user_params["end_user_model_max_budget"] = budget_info.model_max_budget
if budget_info.model_tpm_limit is not None:
end_user_params["end_user_model_tpm_limit"] = budget_info.model_tpm_limit # rebind-ok: in-place like siblings
if budget_info.model_rpm_limit is not None:
end_user_params["end_user_model_rpm_limit"] = budget_info.model_rpm_limit # rebind-ok: in-place like siblings
verbose_proxy_logger.debug("Applied budget limits to end user %s", end_user_id)
@ -1444,6 +1452,10 @@ def update_valid_token_with_end_user_params(valid_token: UserAPIKeyAuth, end_use
valid_token.allowed_model_region = end_user_params["allowed_model_region"]
if end_user_params.get("end_user_model_max_budget") is not None:
valid_token.end_user_model_max_budget = end_user_params["end_user_model_max_budget"]
if "end_user_model_tpm_limit" in end_user_params:
valid_token.end_user_model_tpm_limit = end_user_params["end_user_model_tpm_limit"] # rebind-ok: in-place
if "end_user_model_rpm_limit" in end_user_params:
valid_token.end_user_model_rpm_limit = end_user_params["end_user_model_rpm_limit"] # rebind-ok: in-place
return valid_token
@ -2266,6 +2278,7 @@ async def user_api_key_auth_builder(
azure_apim_header: str | None,
request_data: dict[str, object],
custom_litellm_key_header: str | None = None,
defer_team_checks: bool = False,
) -> UserAPIKeyAuth:
from litellm.proxy.proxy_server import (
general_settings,
@ -2300,6 +2313,7 @@ async def user_api_key_auth_builder(
request_data=request_data,
request=request,
route=route,
defer_team_checks=defer_team_checks,
)
pass_through_endpoints: Final[list[dict] | None] = general_settings.get("pass_through_endpoints", None)
## CHECK IF X-LITELM-API-KEY IS PASSED IN - supercedes Authorization header
@ -2985,6 +2999,10 @@ async def user_api_key_auth_builder(
valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit")
valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit")
valid_token.end_user_tpd_limit = end_user_params.get("end_user_tpd_limit")
if "end_user_model_tpm_limit" in end_user_params:
valid_token.end_user_model_tpm_limit = end_user_params["end_user_model_tpm_limit"]
if "end_user_model_rpm_limit" in end_user_params:
valid_token.end_user_model_rpm_limit = end_user_params["end_user_model_rpm_limit"]
valid_token.allowed_model_region = end_user_params.get("allowed_model_region")
return await validate_resolved_virtual_key(
@ -3670,6 +3688,18 @@ async def run_centralized_common_checks(
user_custom_auth,
)
if not sub_call and getattr(request.state, "deferred_model_body_check", False) is True:
body_model: Final = request_data.get("model")
is_request_body_safe(
request_body=request_data,
general_settings=_GENERAL_SETTINGS_ADAPTER.validate_python(general_settings),
llm_router=llm_router,
model=body_model if isinstance(body_model, str) else "",
route=route,
team_id=user_api_key_auth_obj.team_id,
defer_s3_buckets=checks_team_s3_buckets_in_handler(request),
)
# Public routes (e.g. /health/liveness) are exempt from
# auth in the builder — the wrapper must not retroactively apply
# authz on top, or k8s readiness probes and other unauthenticated
@ -4134,6 +4164,10 @@ async def _apply_key_end_user_default_budget_to_token(
valid_token.end_user_tpd_limit = default_budget.tpd_limit
if not keep_token_limits or valid_token.end_user_model_max_budget is None:
valid_token.end_user_model_max_budget = default_budget.model_max_budget
if not keep_token_limits or valid_token.end_user_model_tpm_limit is None:
valid_token.end_user_model_tpm_limit = default_budget.model_tpm_limit # rebind-ok: in-place like siblings
if not keep_token_limits or valid_token.end_user_model_rpm_limit is None:
valid_token.end_user_model_rpm_limit = default_budget.model_rpm_limit # rebind-ok: in-place like siblings
async def _reserve_budget_after_common_checks(
@ -4502,6 +4536,7 @@ async def user_api_key_auth(
azure_apim_header=azure_apim_header,
request_data=request_data,
custom_litellm_key_header=custom_litellm_key_header,
defer_team_checks=True,
)
except Exception:
# The body was read first, so a caller who sent both a malformed body and
@ -4981,7 +5016,7 @@ async def authorize_internal_virtual_key(
)
)
route: Final = get_request_route(request=request)
await pre_db_read_auth_checks(request_data=request_data, request=request, route=route)
await pre_db_read_auth_checks(request_data=request_data, request=request, route=route, defer_team_checks=True)
auth: Final = await validate_resolved_virtual_key(
request=request,
request_data=request_data,

View file

@ -12,6 +12,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, cast
from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -20,6 +21,7 @@ from litellm.litellm_core_utils.core_helpers import attribute_or_none, has_attri
from litellm.litellm_core_utils.hidden_params import get_hidden_params, set_hidden_param
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
from litellm.proxy._types import *
from litellm.proxy.auth.auth_utils import enforce_team_s3_bucket_allowlist
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
from litellm.proxy.batches_endpoints.litellm_executed_batches import (
@ -81,6 +83,7 @@ if TYPE_CHECKING:
from prisma.models import LiteLLM_ManagedObjectTable
router: Final = APIRouter()
_REQUEST_BODY_ADAPTER: Final = TypeAdapter(Mapping[str, object])
class _UserBatchLister(Protocol):
@ -285,6 +288,7 @@ async def create_batch(
"""
from litellm.proxy.proxy_server import (
general_settings,
general_settings_view,
llm_router,
proxy_config,
proxy_logging_obj,
@ -294,6 +298,13 @@ async def create_batch(
data: dict = {}
try:
data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line
enforce_team_s3_bucket_allowlist(
request_body=_REQUEST_BODY_ADAPTER.validate_python(data),
general_settings=general_settings_view(),
llm_router=llm_router,
team_metadata=user_api_key_dict.team_metadata,
team_id=user_api_key_dict.team_id,
)
_raise_when_routing_headers_disagree(request, data.get("model"))
verbose_proxy_logger.debug(
"Request received by LiteLLM:\n%s",
@ -384,6 +395,7 @@ async def create_batch(
prepare_data_with_credentials(
data=_create_batch_data,
credentials=credentials,
keep_request_buckets=True,
)
# Create batch using model credentials
@ -488,6 +500,7 @@ async def create_batch(
prepare_data_with_credentials(
data=_create_batch_data,
credentials=credentials,
keep_request_buckets=True,
)
# Create batch using model credentials
@ -506,6 +519,7 @@ async def create_batch(
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
custom_llm_provider=custom_llm_provider,
keep_request_buckets=True,
)
_raise_not_found_when_openai_fallback_unservable(
requested_provider=requested_provider,

View file

@ -452,7 +452,7 @@ class LiteLLMExecutedBatchRunner:
) -> LiteLLMBatch:
endpoint: Final = _validate_endpoint(create_request.get("endpoint"))
content: Final = await self._download_input(unified_input_file_id, user_api_key_dict)
parsed: Final = parse_batch_input(content, endpoint, self._body_rejection(model))
parsed: Final = parse_batch_input(content, endpoint, self._body_rejection(model, user_api_key_dict.team_id))
if isinstance(parsed, InvalidBatchInput):
raise batch_error(400, f"Invalid batch input file: {parsed.describe()}")
llm_batch_id: Final = f"{LITELLM_EXECUTED_BATCH_ID_PREFIX}{uuid_module.uuid4().hex}"
@ -533,7 +533,7 @@ class LiteLLMExecutedBatchRunner:
return failed
return await self.batches.load_batch(batch.id) or batch
def _body_rejection(self, model: str) -> BodyRejection:
def _body_rejection(self, model: str, team_id: str | None) -> BodyRejection:
def reject(body: Mapping[str, object]) -> str | None:
try:
is_request_body_safe(
@ -541,6 +541,7 @@ class LiteLLMExecutedBatchRunner:
general_settings=dict(self.general_settings),
llm_router=self.llm_router,
model=model,
team_id=team_id,
)
except ValueError as e:
return str(e)

View file

@ -260,6 +260,16 @@ Delete a credential:
lite credentials delete <credential-name>
```
#### Connect a GitHub Copilot Credential
Connect your own GitHub Copilot account to a per-user GitHub Copilot credential, the same as the Connect button in the UI. Run it with your own virtual key, since the connection belongs to the user that key resolves to:
```bash
lite --api-key sk-your-key credentials connect <credential-name>
```
It prints a code and a GitHub link, then waits. Open the link, enter the code, and approve. The command exits 0 once GitHub confirms the login, and exits 1 if the code expires, the login is denied, or the GitHub account has no Copilot seat.
### Keys Management
The CLI provides commands for managing API keys on your LiteLLM proxy server:

View file

@ -1,5 +1,8 @@
import json
from collections.abc import Sequence
import time
from collections.abc import Callable, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Literal
import click
@ -8,9 +11,30 @@ import rich
from rich.table import Table
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.models.credentials import UserConnectionPollResponse, UserConnectionPollStatus
from ...credentials import CredentialsManagementClient
from ...exceptions import UnauthorizedError
from ._cli_context import cli_context_values
_SLOW_DOWN_STEP_SECONDS: Final = 5
_CONNECT_FAILURES: Final[MappingProxyType[UserConnectionPollStatus, str]] = MappingProxyType(
{
"expired": "The code expired before it was approved. Run the command again for a new code",
"denied": "The GitHub login was denied",
"no_copilot_seat": "That GitHub account has no GitHub Copilot seat",
}
)
@dataclass(frozen=True, slots=True)
class PollClock:
now: Callable[[], float]
sleep: Callable[[float], None]
SYSTEM_CLOCK: Final = PollClock(now=time.monotonic, sleep=time.sleep)
class _CredentialInfo(TypedDict):
custom_llm_provider: ReadOnly[NotRequired[str]]
@ -179,3 +203,80 @@ def get(ctx: click.Context, credential_name: str):
client: Final = CredentialsManagementClient(context["base_url"], context["api_key"])
response: Final = client.get(credential_name)
_print_json(response)
def _next_poll_interval(poll: UserConnectionPollResponse, current: int) -> int:
if poll.interval is not None:
return poll.interval
return current + _SLOW_DOWN_STEP_SECONDS if poll.status == "slow_down" else current
def _poll_until_settled(
client: CredentialsManagementClient,
credential_name: str,
flow_handle: str,
first_interval: int,
expires_at: float,
clock: PollClock,
) -> UserConnectionPollResponse:
interval = first_interval # rebind-ok: GitHub can ask for a longer wait on any poll
while clock.now() + interval < expires_at:
clock.sleep(interval)
if (poll := client.poll_user_connection(credential_name, flow_handle)).status not in ("pending", "slow_down"):
return poll
interval = _next_poll_interval(poll, interval)
return UserConnectionPollResponse(status="expired")
def connect_user_credential(
client: CredentialsManagementClient,
credential_name: str,
clock: PollClock,
) -> UserConnectionPollResponse:
"""Run the GitHub device login for the calling key's user and return the final poll result.
Polling stops once the next poll would land after the code expires, since the proxy rejects polls on an expired
code. The countdown starts before the code is requested, so it never runs past the proxy's own."""
requested_at: Final = clock.now()
started: Final = client.start_user_connection(credential_name)
click.echo(f"Go to {started.verification_uri} and enter code {started.user_code}")
click.echo("Waiting for approval...")
expires_at: Final = requested_at + started.expires_in
return _poll_until_settled(client, credential_name, started.flow_handle, started.interval, expires_at, clock)
def _report_http_error(response: requests.Response) -> None:
click.echo(f"Error: HTTP {response.status_code}", err=True)
try:
error_body: Final[_JsonBodyView] = {"body": response.json()}
_print_json(error_body["body"])
except json.JSONDecodeError:
click.echo(response.text, err=True)
@credentials.command()
@click.argument("credential_name")
@click.pass_context
def connect(ctx: click.Context, credential_name: str) -> None:
"""Connect your own GitHub Copilot account to a per-user credential, without the UI"""
context: Final = cli_context_values(ctx)
client: Final = CredentialsManagementClient(context["base_url"], context["api_key"])
try:
result: Final = connect_user_credential(client, credential_name, SYSTEM_CLOCK)
except UnauthorizedError as e:
if isinstance(e.orig_exception, requests.exceptions.HTTPError) and e.orig_exception.response is not None:
_report_http_error(e.orig_exception.response)
raise click.Abort()
except requests.exceptions.HTTPError as e:
_report_http_error(e.response)
raise click.Abort()
except requests.exceptions.RequestException as e:
click.echo(
f"Error: could not reach the proxy at {context['base_url']} ({type(e).__name__}). Run the command again",
err=True,
)
raise click.Abort()
if result.status != "connected":
click.echo(f"Error: {_CONNECT_FAILURES.get(result.status, result.status)}", err=True)
raise click.exceptions.Exit(1)
click.echo(f"Connected as {result.github_login} for credential '{credential_name}'")

View file

@ -5,6 +5,8 @@ from urllib.parse import quote
import requests
from pydantic import TypeAdapter
from litellm.models.credentials import UserConnectionPollResponse, UserConnectionStartResponse
from .exceptions import UnauthorizedError
_DECODED_JSON: Final = TypeAdapter(object)
@ -217,3 +219,33 @@ class CredentialsManagementClient:
if e.response.status_code == 401:
raise UnauthorizedError(e)
raise
def _send_json(self, request: requests.Request) -> object:
try:
response: Final = requests.Session().send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return _DECODED_JSON.validate_python(response.json())
except requests.exceptions.HTTPError as e:
if e.response.status_code == 401:
raise UnauthorizedError(e)
raise
def _user_connection_url(self, credential_name: str, action: str) -> str:
return f"{self._base_url}/credentials/{quote(credential_name, safe='')}/user_connection/{action}"
def start_user_connection(self, credential_name: str) -> UserConnectionStartResponse:
"""Start the GitHub device login that connects the calling key's user to a per-user credential."""
request: Final = requests.Request(
"POST", self._user_connection_url(credential_name, "start"), headers=self._get_headers()
)
return UserConnectionStartResponse.model_validate(self._send_json(request))
def poll_user_connection(self, credential_name: str, flow_handle: str) -> UserConnectionPollResponse:
"""Check once whether the user approved the device login that ``start_user_connection`` began."""
request: Final = requests.Request(
"POST",
self._user_connection_url(credential_name, "poll"),
headers=self._get_headers(),
json={"flow_handle": flow_handle},
)
return UserConnectionPollResponse.model_validate(self._send_json(request))

View file

@ -3,6 +3,7 @@ import contextlib
import json
import logging
import math
import time
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
@ -380,7 +381,7 @@ def resolve_litellm_call_id(client_call_id: str | None) -> str:
return str(uuid.uuid4())
def should_return_raw_model_name(request_data: dict[str, object]) -> bool:
def should_return_raw_model_name(request_data: Mapping[str, object]) -> bool:
return any(
isinstance(metadata, dict) and metadata.get(RETURN_RAW_MODEL_NAME_METADATA_KEY) is True
for metadata in (request_data.get("metadata"), request_data.get("litellm_metadata"))
@ -4865,6 +4866,15 @@ class ProxyBaseLLMRequestProcessing:
_finalize_streaming_generator_cleanup = finalize_streaming_generator_cleanup
@staticmethod
def record_first_chunk_overhead(
logging_obj: LiteLLMLoggingObj | None, chunk_received_at: float, is_provider_output: bool = True
) -> None:
"""The proxy's own work on a chunk; the logging object keeps the first one for the stamped overhead.
A keepalive ping carries no provider output, so it is never that first chunk."""
if logging_obj is not None and is_provider_output:
logging_obj.set_first_chunk_overhead(time.perf_counter() - chunk_received_at)
@staticmethod
async def async_streaming_data_generator(
response: object,
@ -4922,12 +4932,12 @@ class ProxyBaseLLMRequestProcessing:
response=response,
request_data=request_data,
)
release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(
stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None
)
timing_sink: Final = stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None
release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(timing_sink)
try:
str_so_far = ""
async for chunk in guarded_stream:
chunk_received_at = time.perf_counter()
# ``.format(chunk)`` was previously evaluated for every chunk
# regardless of log level; gate it behind the level check.
if debug_enabled:
@ -4968,6 +4978,9 @@ class ProxyBaseLLMRequestProcessing:
delivered_chunk = delivered_chunk or chunk != STREAM_SSE_KEEPALIVE_PING_BYTES
serialized = serialize_chunk(chunk)
recent_tail = advance_sse_tail(recent_tail, serialized)
ProxyBaseLLMRequestProcessing.record_first_chunk_overhead(
timing_sink, chunk_received_at, is_provider_output=chunk != STREAM_SSE_KEEPALIVE_PING_BYTES
)
yield serialized
held_tail: Final = flush_tail() if flush_tail is not None else b""
if held_tail:

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -6,7 +6,8 @@ feature on) and decodes the body as its own ``ModelsResponse``, ``{"models": [Mo
never the OpenAI list shape. Codex then ignores its own bundled catalog, so this body is all it knows
about the provider's models. It is built from the rows ``/v1/models`` already lists: a model Codex
knows, by its id or by the upstream model its deployments call once the provider's own prefixes are
stripped, keeps Codex's own entry for it, minus any service tier some deployment behind it prices tokens
stripped, keeps Codex's own entry for it, out of the catalog Codex itself uses for the provider those
deployments call (its Bedrock catalogs for ``bedrock`` and ``bedrock_mantle``, its OpenAI one otherwise), minus any service tier some deployment behind it prices tokens
but not that tier for; any other model gets an entry translated from the capabilities its deployments' model info
declares (see ``codex_model_translation``). ``model_info.service_tiers`` becomes the entry's
``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier``
@ -20,6 +21,15 @@ https://github.com/openai/codex/blob/rust-v0.166.0/codex-rs/models-manager/model
as ``codex_bundled_models_0.166.0.json``, and ``codex_base_instructions.md`` is
``codex-rs/models-manager/prompt.md`` at the same tag. Refresh both by downloading those two paths at
a newer tag.
Codex builds its Bedrock catalogs in code rather than shipping them as files, so the ones for Bedrock's
runtime and Mantle endpoints are vendored from Codex 0.166.0's own output, with the ``base_instructions``
copy of each entry's ``model_messages.instructions_template`` that it adds for older clients dropped:
``codex -c 'model_provider="amazon-bedrock-runtime"' debug models | jq 'del(.models[].base_instructions)'``
as ``codex_bedrock_runtime_models_0.166.0.json``, and ``model_provider="amazon-bedrock"`` the same way as
``codex_bedrock_mantle_models_0.166.0.json``. Refresh them at the same release as the bundled catalog,
which they take each entry's ``minimal_client_version`` from, and recheck
``CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION`` against the fields the new output leaves out.
"""
from __future__ import annotations
@ -34,7 +44,6 @@ from pathlib import Path
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from packaging.version import Version
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
@ -49,6 +58,7 @@ from litellm.proxy.common_utils.codex_model_translation import (
no_priced_deployment_lacks_tier,
tier_price_keys,
)
from litellm.proxy.common_utils.codex_version import codex_version_at_least
from litellm.types.llms.base_model import LiteLLMBaseModel
if TYPE_CHECKING:
@ -58,7 +68,16 @@ if TYPE_CHECKING:
CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
CODEX_BUNDLED_MODELS_PATH: Final = Path(__file__).with_name("codex_bundled_models_0.166.0.json")
CodexBedrockEndpoint: TypeAlias = Literal["bedrock", "bedrock_mantle"]
CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION: Final = "0.148.0"
CODEX_BEDROCK_MODELS_PATHS: Final[Mapping[CodexBedrockEndpoint, Path]] = MappingProxyType(
{
"bedrock": Path(__file__).with_name("codex_bedrock_runtime_models_0.166.0.json"),
"bedrock_mantle": Path(__file__).with_name("codex_bedrock_mantle_models_0.166.0.json"),
}
)
CODEX_CATALOG_BYTE_LIMIT: Final = 1024 * 1024
_FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT: Final = "0.166.0-alpha.1"
CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"})
_BODY_PREFIX: Final = '{"models":['
@ -129,8 +148,8 @@ _NATIVE_ALIAS_DEFAULTS: Final[CodexAliasDefaults] = "deployment"
@dataclass(frozen=True, slots=True)
class CodexCatalogRow:
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
input limit, the upstream model behind it and the provider its deployment declares, the reasoning
effort its deployments pin, and the
input limit, the upstream model behind it and the provider its deployment declares, every routable
deployment's `upstreams` (model and declared provider), the reasoning effort its deployments pin, and the
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
value per deployment behind the id, None where a deployment sets none, and
`configured_max_input_tokens` one `model_info.max_input_tokens` per deployment the same way.
@ -142,6 +161,7 @@ class CodexCatalogRow:
max_input_tokens: int | None = None
upstream_model: str | None = None
upstream_provider: str | None = None
upstreams: tuple[tuple[str, str | None], ...] = ()
reasoning_effort: str | None = None
display_name: str | None = None
service_tiers: tuple[object, ...] = ()
@ -158,6 +178,40 @@ def bundled_codex_models() -> Mapping[str, CodexStockModel]:
return MappingProxyType({model.slug: model for model in catalog.models})
def _with_minimal_client_version(model: CodexStockModel, built_from: str | tuple[int, ...] | None) -> CodexStockModel:
source_minimum: Final = built_from if not isinstance(built_from, tuple) else ".".join(map(str, built_from))
minimum: Final = (
source_minimum
if codex_version_at_least(source_minimum, CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION)
else CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION
)
return model.model_copy(update={"minimal_client_version": minimum})
@cache
def bundled_bedrock_codex_models(endpoint: CodexBedrockEndpoint) -> Mapping[str, CodexStockModel]:
"""Codex's catalog for a Bedrock endpoint by slug, read once from the vendored file. `codex debug models`
prints no `minimal_client_version`, so each entry takes the later of the bundled entry's Codex built it from
and `CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION`, the first release that decodes an entry without the fields
`debug models` leaves out, such as `supports_parallel_tool_calls`."""
from litellm.llms.bedrock.common_utils import bedrock_model_name_without_provider
catalog: Final = CodexStockCatalog.model_validate_json(
CODEX_BEDROCK_MODELS_PATHS[endpoint].read_text(encoding="utf-8")
)
built_from: Final = MappingProxyType(
{slug: model.minimal_client_version for slug, model in bundled_codex_models().items()}
)
return MappingProxyType(
{
model.slug: _with_minimal_client_version(
model, built_from.get(bedrock_model_name_without_provider(model.slug))
)
for model in catalog.models
}
)
@cache
def codex_base_instructions() -> str:
return CODEX_BASE_INSTRUCTIONS_PATH.read_text(encoding="utf-8")
@ -274,15 +328,67 @@ def _provider_model_name(upstream_model: str, upstream_provider: str | None) ->
return model if name_for is None else name_for(model)
def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None:
@dataclass(frozen=True, slots=True)
class _StockMatch:
"""Codex's entry for a row, and whether the row lists it under another name than Codex's (an alias)."""
model: CodexStockModel
alias: bool
def _on_endpoint(
declared: Sequence[tuple[str | None, str]], endpoint: CodexBedrockEndpoint
) -> tuple[CodexBedrockEndpoint, str] | None:
return next(((endpoint, model) for provider, model in declared if provider == endpoint), None)
def _bedrock_upstream(row: CodexCatalogRow) -> tuple[CodexBedrockEndpoint, str] | None:
"""The Bedrock endpoint a request to the row can reach whose Codex catalog promises the least, with the model
a deployment on it calls. A request can route to any deployment, and Codex's Bedrock entries leave out what
Bedrock does not support yet, so one Bedrock deployment puts the whole row on its catalog, the runtime one
(no tool search) ahead of Mantle's."""
from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider
first: Final = () if row.upstream_model is None else ((row.upstream_model, row.upstream_provider),)
declared: Final = tuple((declared_provider(model, provider), model) for model, provider in (*first, *row.upstreams))
on_bedrock: Final = (_on_endpoint(declared, endpoint) for endpoint in CODEX_BEDROCK_MODELS_PATHS)
return next((found for found in on_bedrock if found is not None), None)
def _bedrock_stock_for(row_id: str, endpoint: CodexBedrockEndpoint, upstream_model: str) -> _StockMatch | None:
"""Codex's entry for the row's id or the Bedrock model as written, else for the same model under any of
Bedrock's routing and region prefixes (`eu.openai.gpt-6.1-sol` takes `global.openai.gpt-6.1-sol`'s)."""
from litellm.llms.bedrock.common_utils import bedrock_model_name_without_provider
stock: Final = bundled_bedrock_codex_models(endpoint)
names: Final = (row_id, upstream_model.removeprefix(f"{endpoint}/"))
exact: Final = next((stock[name] for name in names if name in stock), None)
model_names: Final = tuple(bedrock_model_name_without_provider(name) for name in names)
same_model: Final = (
next((model for slug, model in stock.items() if bedrock_model_name_without_provider(slug) == name), None)
for name in model_names
)
found: Final = exact if exact is not None else next((model for model in same_model if model is not None), None)
if found is None:
return None
return _StockMatch(found, model_names[0] != bedrock_model_name_without_provider(found.slug))
def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> _StockMatch | None:
"""Codex's entry for the row's id, else for its upstream model as written, else for that model's name
once its provider's own prefixes are stripped (`bedrock/us.openai.gpt-6.1-sol` is `gpt-6.1-sol`)."""
once its provider's own prefixes are stripped (`openai/gpt-6.1-sol` is `gpt-6.1-sol`), out of Codex's
Bedrock catalog when a deployment behind the row is on Bedrock and `stock` otherwise. A Bedrock model that
catalog does not know has no entry, since `stock`'s would promise what Bedrock does not support."""
bedrock: Final = _bedrock_upstream(row)
if bedrock is not None:
return _bedrock_stock_for(row.id, *bedrock)
names: Final = (
(row.id,)
if row.upstream_model is None
else (row.id, row.upstream_model, _provider_model_name(row.upstream_model, row.upstream_provider))
)
return next((stock[name] for name in names if name in stock), None)
found: Final = next((stock[name] for name in names if name in stock), None)
return None if found is None else _StockMatch(found, row.id != found.slug)
def _readable_by(stock_model: CodexStockModel, client_version: str | None) -> bool:
@ -292,10 +398,7 @@ def _readable_by(stock_model: CodexStockModel, client_version: str | None) -> bo
minimum: Final = stock_model.minimal_client_version
if minimum is None or client_version is None:
return True
try:
return Version(client_version) >= Version(minimum if isinstance(minimum, str) else ".".join(map(str, minimum)))
except ValueError:
return False
return codex_version_at_least(client_version, minimum if isinstance(minimum, str) else ".".join(map(str, minimum)))
def _tier_fields(stock_model: CodexStockModel, tiers: tuple[CodexServiceTier, ...]) -> Mapping[str, object]:
@ -391,7 +494,7 @@ def _entry(
client_version: str | None,
) -> CodexCatalogEntry:
matched: Final = _stock_for(row, stock)
stock_model: Final = matched if matched is not None and _readable_by(matched, client_version) else None
stock_model: Final = matched.model if matched is not None and _readable_by(matched.model, client_version) else None
stock_tiers: Final = MappingProxyType(
{tier.id: tier for tier in (stock_model.service_tiers if stock_model is not None else ())}
)
@ -412,7 +515,7 @@ def _entry(
upgrade: Final = (
stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
)
is_alias: Final = row.id != stock_model.slug
is_alias: Final = matched is not None and matched.alias
deployment_decides: Final = is_alias and alias_defaults == "deployment"
stock_name: Final = row.id if is_alias else stock_model.display_name
listing: Final = {
@ -497,6 +600,16 @@ class CodexCatalogBody:
left_out: tuple[str, ...]
def codex_catalog_byte_limit(client_version: str | None) -> int | None:
"""None, no limit, for a Codex that reads a catalog of any size since https://github.com/openai/codex/pull/56525.
An older Codex, or one sending no version, drops a catalog over 1 MiB. 0.165.0 still has the limit."""
return (
None
if codex_version_at_least(client_version, _FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT)
else CODEX_CATALOG_BYTE_LIMIT
)
def codex_models_response_json(
rows: Sequence[CodexCatalogRow],
*,
@ -555,7 +668,7 @@ def merged_codex_catalog(
*,
stock: Mapping[str, CodexStockModel] | None = None,
instructions: str | None = None,
byte_limit: int = CODEX_CATALOG_BYTE_LIMIT,
byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
client_version: str | None = None,
) -> str | None:
try:
@ -569,7 +682,7 @@ def merged_codex_catalog(
entries: Final = _entries(added, stock, instructions, first_priority, _NATIVE_ALIAS_DEFAULTS, client_version)
added_models: Final = tuple(_entry_json(entry) for entry in entries)
serialized: Final = tuple(_compact_json(model) for model in added_models)
budget: Final = byte_limit - len(_compact_json({**top, "models": upstream}))
budget: Final = None if byte_limit is None else byte_limit - len(_compact_json({**top, "models": upstream}))
kept: Final = tuple(sorted(_kept_positions(entries, serialized, budget)))
if len(kept) < len(entries):
verbose_proxy_logger.warning(
@ -659,6 +772,7 @@ def _catalog_row(
upstream_provider=(
registry.resolve(lookup_id, kind="upstream_provider", team_id=team_id) if registry is not None else None
),
upstreams=registry.resolve(lookup_id, kind="upstreams", team_id=team_id) if registry is not None else (),
display_name=registry.group(label_id, field="display_name") if registry is not None else None,
reasoning_effort=(
_shared_pinned_effort(registry.group(lookup_id, field="pinned_reasoning_efforts", team_id=team_id))
@ -714,17 +828,19 @@ def codex_model_list_body(
) -> str:
"""The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out. `stock` and
`instructions` are the requesting Codex release's catalog and prompt, the bundled ones when None."""
byte_limit: Final = codex_catalog_byte_limit(client_version)
body: Final = codex_models_response_json(
codex_catalog_rows(rows, entries, llm_router, team_id, label_entries),
stock=stock,
instructions=instructions,
byte_limit=byte_limit,
client_version=client_version,
)
if body.left_out:
verbose_proxy_logger.warning(
"Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, "
"then model_list order; list the models Codex users need first in model_list",
CODEX_CATALOG_BYTE_LIMIT,
byte_limit,
", ".join(body.left_out),
)
return body.json

View file

@ -15,10 +15,10 @@ from functools import cache
from types import NoneType
from typing import Annotated, Final, Literal, TypeAlias
from packaging.version import Version
from pydantic import ConfigDict, Field, StringConstraints, field_serializer
from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort
from litellm.proxy.common_utils.codex_version import codex_version_at_least
from litellm.types.llms.base_model import LiteLLMBaseModel
TierId: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
@ -27,7 +27,7 @@ CodexInputModality: TypeAlias = Literal["text", "image"]
_TEXT_ONLY: Final[tuple[CodexInputModality, ...]] = ("text",)
_TEXT_AND_IMAGE: Final[tuple[CodexInputModality, ...]] = ("text", "image")
TOKEN_PRICE_KEY: Final = "input_cost_per_token"
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0")
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = "0.138.0"
_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
_PREFERRED_DEFAULT_EFFORT: Final = "medium"
_CODEX_UNSET_EFFORT: Final = "none"
@ -95,7 +95,7 @@ class CodexFallbackModel(LiteLLMBaseModel):
apply_patch_tool_type: Literal["freeform"] | None = None
truncation_policy: CodexTruncationPolicy = CodexTruncationPolicy()
experimental_supported_tools: tuple[()] = ()
input_modalities: tuple[CodexInputModality, ...] = _TEXT_AND_IMAGE
input_modalities: tuple[CodexInputModality, ...] = _TEXT_ONLY
context_window: int | None
base_instructions: str
supports_search_tool: bool = False
@ -148,8 +148,8 @@ def no_priced_deployment_lacks_tier(tier_id: str, deployments: Sequence[CodexDep
@dataclass(frozen=True, slots=True)
class CodexCapabilities:
"""What a request to a listed name can count on whichever deployment serves it: a capability is False
when one deployment lacks it, True when one has it and none lacks it, None when none says, and the
context window is the smallest one any deployment declares."""
when one deployment lacks it, True only when none lacks it, None otherwise, and the context window is
the smallest one any deployment declares."""
context_window: int | None = None
supports_vision: bool | None = None
@ -163,6 +163,13 @@ def _combined(flags: Iterable[bool | None]) -> bool | None:
return None if not said else False not in said
def _declared_by_every_deployment(flags: Iterable[bool | None]) -> bool | None:
said: Final = frozenset(flags)
if False in said:
return False
return None if not said or None in said else True
def combined_capabilities(
deployments: Sequence[CodexDeploymentCapabilities], listed_context_window: int | None = None
) -> CodexCapabilities:
@ -175,7 +182,7 @@ def combined_capabilities(
)
return CodexCapabilities(
context_window=min(windows) if windows else listed_context_window,
supports_vision=_combined(deployment.supports_vision for deployment in deployments),
supports_vision=_declared_by_every_deployment(deployment.supports_vision for deployment in deployments),
supports_function_calling=_combined(deployment.supports_function_calling for deployment in deployments),
supports_parallel_function_calling=_combined(
deployment.supports_parallel_function_calling for deployment in deployments
@ -184,13 +191,6 @@ def combined_capabilities(
)
def _parses_custom_efforts(client_version: str | None) -> bool:
try:
return client_version is not None and Version(client_version) >= _FIRST_CODEX_WITH_CUSTOM_EFFORTS
except ValueError:
return False
def codex_reasoning_levels(
efforts: Sequence[str], client_version: str | None
) -> tuple[tuple[CodexFallbackReasoningLevel, ...], str | None]:
@ -202,7 +202,7 @@ def codex_reasoning_levels(
group accepts it, so a user who picks no level sends what Codex sent before, else medium, else the
nearest weaker declared level, so picking no level never costs more reasoning than Codex's usual medium,
else the weakest declared level, else the first one."""
parses_any: Final = _parses_custom_efforts(client_version)
parses_any: Final = codex_version_at_least(client_version, _FIRST_CODEX_WITH_CUSTOM_EFFORTS)
offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES)
if not offered:
return (), None
@ -230,10 +230,10 @@ def codex_fallback_entry(
) -> CodexFallbackModel:
"""Codex's entry for a model it does not know, from the capabilities its deployments share.
A capability no deployment declares keeps Codex's own default for an unknown model, except the freeform
`apply_patch` tool, which goes out only to a model that calls tools: `supports_reasoning_summary_parameter`
is left out, which Codex reads as true, and the fields only Codex before 0.150.0 reads keep the false
it requires.
A capability no deployment declares keeps Codex's own default for an unknown model, except image input
and the freeform `apply_patch` tool, which go out only to a model known to read images or call tools:
`supports_reasoning_summary_parameter` is left out, which Codex reads as true, and the fields only Codex
before 0.150.0 reads keep the false it requires.
"""
levels, default_level = codex_reasoning_levels(reasoning_efforts, client_version)
return CodexFallbackModel(
@ -247,7 +247,7 @@ def codex_fallback_entry(
supports_reasoning_summaries=capabilities.supports_reasoning is True,
supports_parallel_tool_calls=capabilities.supports_parallel_function_calling is True,
apply_patch_tool_type="freeform" if capabilities.supports_function_calling is True else None,
input_modalities=_TEXT_ONLY if capabilities.supports_vision is False else _TEXT_AND_IMAGE,
input_modalities=_TEXT_AND_IMAGE if capabilities.supports_vision is True else _TEXT_ONLY,
context_window=capabilities.context_window,
base_instructions=instructions,
supports_search_tool=supports_search_tool,

Some files were not shown because too many files have changed in this diff Show more