mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_vertex_identity_labels_wt
# Conflicts: # litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py
This commit is contained in:
commit
ffe74b23cf
437 changed files with 47948 additions and 2019 deletions
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -397,6 +397,7 @@ jobs:
|
|||
tests/unit/llms/voyage
|
||||
tests/unit/llms/wandb
|
||||
tests/unit/llms/watsonx
|
||||
tests/unit/llms/webiq
|
||||
tests/unit/llms/xai
|
||||
tests/unit/llms/xinference
|
||||
tests/unit/llms/you_com
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs
|
||||
from litellm.constants import (
|
||||
BATCH_LINE_ITEMS_MAX_PART_BYTES,
|
||||
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
|
||||
|
|
@ -765,7 +766,7 @@ class CheckBatchCost:
|
|||
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
|
||||
_file_content = await afile_content(
|
||||
file_id=cast(str, raw_output_file_id),
|
||||
**cast(FileContentCallOptions, credentials),
|
||||
**cast(FileContentCallOptions, {**credentials, **stored_s3_bucket_kwargs(cast(str, raw_output_file_id))}),
|
||||
)
|
||||
|
||||
content_bytes: Final = await _file_content_bytes(_file_content)
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs
|
||||
from litellm.litellm_core_utils.core_helpers import require_attributes
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
|
||||
from litellm.litellm_core_utils.internal_call_metadata import requests_background
|
||||
|
|
@ -78,6 +79,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
BATCH_CREATE_HIDDEN_PARAM,
|
||||
FILE_LIST_CONTINUATION_CHUNK_SIZE,
|
||||
TeamProviderDeployment,
|
||||
_is_base64_encoded_unified_file_id,
|
||||
apply_unified_file_ids,
|
||||
decode_model_from_file_id,
|
||||
|
|
@ -395,6 +397,7 @@ class _CursorPageArgs(TypedDict, total=False):
|
|||
class _RouterFileCallKwargs(TypedDict, total=False):
|
||||
client: ReadOnly[AsyncOpenAI | None]
|
||||
custom_llm_provider: ReadOnly[str | None]
|
||||
stored_s3_bucket_name: ReadOnly[str]
|
||||
|
||||
|
||||
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
|
||||
|
|
@ -462,7 +465,7 @@ async def _delete_provider_file(
|
|||
await llm_router.afile_delete(
|
||||
model=model_id,
|
||||
file_id=provider_file_id,
|
||||
**cast(_RouterFileCallKwargs, delete_data),
|
||||
**cast(_RouterFileCallKwargs, {**delete_data, **stored_s3_bucket_kwargs(provider_file_id)}),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1156,7 +1159,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve)
|
||||
retrieve_options: Final = cast(
|
||||
FileRetrieveCallOptions,
|
||||
{**call_options, "max_retries": 0},
|
||||
{**call_options, **stored_s3_bucket_kwargs(provider_file_id), "max_retries": 0},
|
||||
)
|
||||
for attempt in range(len(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS) + 1):
|
||||
if attempt > 0:
|
||||
|
|
@ -1426,6 +1429,43 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
user_api_key_dict=owner,
|
||||
)
|
||||
|
||||
async def register_provider_file(
|
||||
self,
|
||||
*,
|
||||
provider_file_id: str,
|
||||
content_type: str,
|
||||
deployment: TeamProviderDeployment,
|
||||
owner: UserAPIKeyAuth,
|
||||
llm_router: Router | None,
|
||||
) -> str | None:
|
||||
unified_file_id: Final = self.get_unified_output_file_id(
|
||||
provider_file_id, deployment.deployment_id, deployment.public_model_name, file_type=content_type
|
||||
)
|
||||
provider_file_object, _ = await self._fetch_provider_file_object(
|
||||
unified_file_id=unified_file_id,
|
||||
provider_file_id=provider_file_id,
|
||||
model_id=deployment.deployment_id,
|
||||
model_name=None,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
if provider_file_object is None:
|
||||
return None
|
||||
try:
|
||||
await self.store_unified_file_id(
|
||||
file_id=unified_file_id,
|
||||
file_object=provider_file_object,
|
||||
litellm_parent_otel_span=None,
|
||||
model_mappings={deployment.deployment_id: provider_file_id},
|
||||
user_api_key_dict=owner,
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # the upload succeeded, so the caller keeps the provider id
|
||||
verbose_logger.warning(
|
||||
f"Failed to register provider_file_id={sanitize_for_log(provider_file_id)} as a managed file: "
|
||||
f"{type(error).__name__} {sanitize_for_log(error)}"
|
||||
)
|
||||
return None
|
||||
return unified_file_id
|
||||
|
||||
async def list_user_batches(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
@ -1757,8 +1797,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
if potential_file_id and "llm_output_file_id," in potential_file_id:
|
||||
model_id = self.get_model_id_from_unified_file_id(potential_file_id)
|
||||
if model_id:
|
||||
provider_file_id: Final = self.get_output_file_id_from_unified_file_id(potential_file_id)
|
||||
data["model"] = model_id
|
||||
data["file_id"] = self.get_output_file_id_from_unified_file_id(potential_file_id)
|
||||
data["file_id"] = provider_file_id
|
||||
data.update(stored_s3_bucket_kwargs(provider_file_id))
|
||||
elif call_type == CallTypes.acreate_batch.value:
|
||||
input_file_id = cast(Optional[str], data.get("input_file_id"))
|
||||
if input_file_id:
|
||||
|
|
@ -2253,10 +2295,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
unified_batch_id = SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format(model_id, batch_id)
|
||||
return base64.urlsafe_b64encode(unified_batch_id.encode()).decode().rstrip("=")
|
||||
|
||||
def get_unified_output_file_id(self, output_file_id: str, model_id: str, model_name: Optional[str]) -> str:
|
||||
def get_unified_output_file_id(
|
||||
self, output_file_id: str, model_id: str, model_name: Optional[str], file_type: str = "application/json"
|
||||
) -> str:
|
||||
deterministic_uuid: Final = uuid5(uuid5(NAMESPACE_URL, model_id), output_file_id)
|
||||
unified_output_file_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format(
|
||||
"application/json",
|
||||
file_type,
|
||||
str(deterministic_uuid),
|
||||
model_name or "",
|
||||
output_file_id,
|
||||
|
|
@ -2854,7 +2898,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
await llm_router.afile_content(
|
||||
model=model_id,
|
||||
file_id=provider_file_id,
|
||||
**cast(_RouterFileCallKwargs, data),
|
||||
**cast(
|
||||
_RouterFileCallKwargs,
|
||||
{
|
||||
**data,
|
||||
**(
|
||||
stored_s3_bucket_kwargs(provider_file_id)
|
||||
if isinstance(provider_file_id, str)
|
||||
else {}
|
||||
),
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_BackgroundResponseSettlement" (
|
||||
"response_id" TEXT NOT NULL,
|
||||
"settled_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "LiteLLM_BackgroundResponseSettlement_pkey" PRIMARY KEY ("response_id")
|
||||
);
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "model_tpm_limit" JSONB;
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "model_rpm_limit" JSONB;
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_DailyGlobalSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
|
||||
|
|
@ -19,6 +19,8 @@ model LiteLLM_BudgetTable {
|
|||
rpm_limit BigInt?
|
||||
tpd_limit BigInt?
|
||||
model_max_budget Json?
|
||||
model_tpm_limit Json?
|
||||
model_rpm_limit Json?
|
||||
budget_duration String?
|
||||
budget_reset_at DateTime?
|
||||
allowed_models String[] @default([]) // per-member model scope; empty = inherit team models
|
||||
|
|
@ -881,6 +883,7 @@ model LiteLLM_DailyUserSpend {
|
|||
prompt_caching_savings_spend Float @default(0.0)
|
||||
gateway_injected_caching_savings_spend Float @default(0.0)
|
||||
autorouter_savings_spend Float @default(0.0)
|
||||
guardrail_spend Float @default(0.0)
|
||||
spend Float @default(0.0)
|
||||
api_requests BigInt @default(0)
|
||||
successful_requests BigInt @default(0)
|
||||
|
|
@ -917,6 +920,7 @@ model LiteLLM_DailyGlobalSpend {
|
|||
prompt_caching_savings_spend Float @default(0.0)
|
||||
gateway_injected_caching_savings_spend Float @default(0.0)
|
||||
autorouter_savings_spend Float @default(0.0)
|
||||
guardrail_spend Float @default(0.0)
|
||||
spend Float @default(0.0)
|
||||
api_requests BigInt @default(0)
|
||||
successful_requests BigInt @default(0)
|
||||
|
|
@ -950,6 +954,7 @@ model LiteLLM_DailyOrganizationSpend {
|
|||
prompt_caching_savings_spend Float @default(0.0)
|
||||
gateway_injected_caching_savings_spend Float @default(0.0)
|
||||
autorouter_savings_spend Float @default(0.0)
|
||||
guardrail_spend Float @default(0.0)
|
||||
spend Float @default(0.0)
|
||||
api_requests BigInt @default(0)
|
||||
successful_requests BigInt @default(0)
|
||||
|
|
@ -988,6 +993,7 @@ model LiteLLM_DailyEndUserSpend {
|
|||
prompt_caching_savings_spend Float @default(0.0)
|
||||
gateway_injected_caching_savings_spend Float @default(0.0)
|
||||
autorouter_savings_spend Float @default(0.0)
|
||||
guardrail_spend Float @default(0.0)
|
||||
spend Float @default(0.0)
|
||||
api_requests BigInt @default(0)
|
||||
successful_requests BigInt @default(0)
|
||||
|
|
@ -1025,6 +1031,7 @@ model LiteLLM_DailyAgentSpend {
|
|||
prompt_caching_savings_spend Float @default(0.0)
|
||||
gateway_injected_caching_savings_spend Float @default(0.0)
|
||||
autorouter_savings_spend Float @default(0.0)
|
||||
guardrail_spend Float @default(0.0)
|
||||
spend Float @default(0.0)
|
||||
api_requests BigInt @default(0)
|
||||
successful_requests BigInt @default(0)
|
||||
|
|
@ -1062,6 +1069,7 @@ model LiteLLM_DailyTeamSpend {
|
|||
prompt_caching_savings_spend Float @default(0.0)
|
||||
gateway_injected_caching_savings_spend Float @default(0.0)
|
||||
autorouter_savings_spend Float @default(0.0)
|
||||
guardrail_spend Float @default(0.0)
|
||||
spend Float @default(0.0)
|
||||
api_requests BigInt @default(0)
|
||||
successful_requests BigInt @default(0)
|
||||
|
|
@ -1102,6 +1110,7 @@ model LiteLLM_DailyTagSpend {
|
|||
prompt_caching_savings_spend Float @default(0.0)
|
||||
gateway_injected_caching_savings_spend Float @default(0.0)
|
||||
autorouter_savings_spend Float @default(0.0)
|
||||
guardrail_spend Float @default(0.0)
|
||||
spend Float @default(0.0)
|
||||
api_requests BigInt @default(0)
|
||||
successful_requests BigInt @default(0)
|
||||
|
|
@ -2053,6 +2062,11 @@ model LiteLLM_BackgroundInteractionSettlement {
|
|||
@@index([settled_at], map: "idx_background_interaction_settlement_settled_at")
|
||||
}
|
||||
|
||||
model LiteLLM_BackgroundResponseSettlement {
|
||||
response_id String @id
|
||||
settled_at DateTime @default(now())
|
||||
}
|
||||
|
||||
model LiteLLM_Lens {
|
||||
reviews LiteLLM_LensReview[]
|
||||
id String @id
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ pub struct ModelInfoFields {
|
|||
pub cache_read_input_audio_token_cost_flex: Option<f64>,
|
||||
pub cache_read_input_audio_token_cost_priority: Option<f64>,
|
||||
pub cache_read_input_audio_token_cost_batches: Option<f64>,
|
||||
pub cache_read_input_image_token_cost_batches: Option<f64>,
|
||||
pub explicit_cache_read_input_token_cost: Option<f64>,
|
||||
pub prompt_cache_min_tokens: Option<i64>,
|
||||
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
|
||||
|
|
|
|||
|
|
@ -133,6 +133,7 @@ def _daily_counters(row: Mapping[str, object], metadata: Mapping[str, object]) -
|
|||
("prompt_caching_savings_spend", _number(savings.get("prompt_caching"))),
|
||||
("gateway_injected_caching_savings_spend", _number(savings.get("gateway_injected_caching"))),
|
||||
("autorouter_savings_spend", 0.0 if internal else _number(savings.get("autorouter"))),
|
||||
("guardrail_spend", _number(metadata.get("accounting_guardrail_spend"))),
|
||||
("total_response_time_ms", _number(duration) if timed else 0),
|
||||
("timed_requests", int(timed)),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_share_of_spend
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure
|
||||
from litellm.proxy._types import SpendLogsPayload
|
||||
from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
|
||||
|
|
@ -73,6 +74,7 @@ def build_accounting_payload(
|
|||
**{key: _accounting_metadata_field(key, value) for key, value in metadata.items() if key in _METADATA_FIELDS},
|
||||
"accounting_entities": entities,
|
||||
"accounting_savings": savings._asdict(),
|
||||
"accounting_guardrail_spend": guardrail_share_of_spend(metadata, payload["spend"]),
|
||||
"accounting_model_access_groups": model_access_groups,
|
||||
"accounting_windows": windows,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,9 +46,6 @@ def _progressed(result: DeliveryBatchResult) -> bool:
|
|||
async def flush_accounting_deliveries(
|
||||
writer: "DBSpendUpdateWriter", prisma_client: "PrismaClient", proxy_logging_obj: "ProxyLogging"
|
||||
) -> None:
|
||||
acquired: Final = await writer.pod_lock_manager.acquire_lock(cronjob_id=_JOB, allow_reentrant=False)
|
||||
if acquired is False:
|
||||
return
|
||||
applied: tuple[Contribution, ...] = () # rebind-ok: keep committed contributions for cache invalidation
|
||||
|
||||
async def apply(transaction: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None:
|
||||
|
|
@ -57,36 +54,39 @@ async def flush_accounting_deliveries(
|
|||
transaction, deliveries, prisma_client=prisma_client, writer=writer, proxy_logging_obj=proxy_logging_obj
|
||||
)
|
||||
|
||||
database: Final = cast( # cast-ok: PrismaClient.tx owns writer routing and yields the generated transaction
|
||||
LedgerDatabase, prisma_client
|
||||
)
|
||||
repository: Final = AccountingDeliveryRepository(database)
|
||||
started: Final = time.monotonic()
|
||||
batch_size: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", "500"))
|
||||
|
||||
async def run_batch(*, retry_only: bool) -> DeliveryBatchResult:
|
||||
result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only)
|
||||
if result.applied:
|
||||
await writer.invalidate_accounting_entities(applied, proxy_logging_obj)
|
||||
if result.parked:
|
||||
verbose_proxy_logger.error(
|
||||
"Accounting delivery parked %d requests after isolated failures", len(result.parked)
|
||||
)
|
||||
await _alert(
|
||||
proxy_logging_obj,
|
||||
message=f"Accounting delivery parked {len(result.parked)} requests. Included spend totals require repair.",
|
||||
)
|
||||
return result
|
||||
|
||||
fresh_deadline: Final = time.monotonic() + 5
|
||||
for _ in range(batch_size):
|
||||
if not _filled(await run_batch(retry_only=False), batch_size) or time.monotonic() >= fresh_deadline:
|
||||
break
|
||||
retry_deadline: Final = time.monotonic() + 5
|
||||
for _ in range(batch_size):
|
||||
if time.monotonic() >= retry_deadline or not _progressed(await run_batch(retry_only=True)):
|
||||
break
|
||||
acquired: Final = await writer.pod_lock_manager.acquire_lock(cronjob_id=_JOB, allow_reentrant=False)
|
||||
if acquired is False:
|
||||
return
|
||||
try:
|
||||
database: Final = cast( # cast-ok: PrismaClient.tx owns writer routing and yields the generated transaction
|
||||
LedgerDatabase, prisma_client
|
||||
)
|
||||
repository: Final = AccountingDeliveryRepository(database)
|
||||
started: Final = time.monotonic()
|
||||
batch_size: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", "500"))
|
||||
|
||||
async def run_batch(*, retry_only: bool) -> DeliveryBatchResult:
|
||||
result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only)
|
||||
if result.applied:
|
||||
await writer.invalidate_accounting_entities(applied, proxy_logging_obj)
|
||||
if result.parked:
|
||||
verbose_proxy_logger.error(
|
||||
"Accounting delivery parked %d requests after isolated failures", len(result.parked)
|
||||
)
|
||||
await _alert(
|
||||
proxy_logging_obj,
|
||||
message=f"Accounting delivery parked {len(result.parked)} requests. Included spend totals require repair.",
|
||||
)
|
||||
return result
|
||||
|
||||
fresh_deadline: Final = time.monotonic() + 5
|
||||
for _ in range(batch_size):
|
||||
if not _filled(await run_batch(retry_only=False), batch_size) or time.monotonic() >= fresh_deadline:
|
||||
break
|
||||
retry_deadline: Final = time.monotonic() + 5
|
||||
for _ in range(batch_size):
|
||||
if time.monotonic() >= retry_deadline or not _progressed(await run_batch(retry_only=True)):
|
||||
break
|
||||
retention_days: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_RETENTION_DAYS", "1"))
|
||||
cleanup_deadline: Final = time.monotonic() + 5
|
||||
for _ in range(batch_size):
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from pydantic import JsonValue, TypeAdapter
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils import batch_utils as core_batch_utils
|
||||
from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs
|
||||
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
|
||||
|
|
@ -551,7 +552,7 @@ async def fetch_batch_managed_file_content(
|
|||
"file_id": provider_output_file_id,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
**cast( # cast-ok: preserve dynamic provider credentials without validation
|
||||
FileContentCallOptions, credentials
|
||||
FileContentCallOptions, {**credentials, **stored_s3_bucket_kwargs(provider_output_file_id)}
|
||||
),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import logging
|
|||
import time
|
||||
import weakref
|
||||
from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence
|
||||
from contextvars import ContextVar, Token
|
||||
from contextvars import ContextVar, Token, copy_context
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import timedelta
|
||||
from enum import Enum, auto
|
||||
|
|
@ -528,6 +528,7 @@ class RequestRedisBatches:
|
|||
"_deadline",
|
||||
"_deadline_flush",
|
||||
"_native",
|
||||
"_native_context",
|
||||
"_native_factory",
|
||||
"_native_flush_hooks",
|
||||
"_native_misses",
|
||||
|
|
@ -551,6 +552,7 @@ class RequestRedisBatches:
|
|||
self._native_factory: Final = native_factory
|
||||
self._native_send_factory: Final = native_send_factory
|
||||
self._native: NativeRequestRedis | None = None
|
||||
self._native_context: Final = copy_context()
|
||||
self._native_flush_hooks: Final[list[Callable[[], None]]] = [] # mutable-ok: append-only registry
|
||||
self._native_misses: frozenset[str] = frozenset()
|
||||
self._deadline: asyncio.TimerHandle | None = None
|
||||
|
|
@ -580,9 +582,12 @@ class RequestRedisBatches:
|
|||
return batch
|
||||
|
||||
def native(self) -> NativeRequestRedis:
|
||||
"""The request's native batches, built on first use so requests that touch no native store pay nothing."""
|
||||
"""The request's native batches, built on first use so requests that touch no native store pay nothing.
|
||||
|
||||
They keep a copy of the context they are built in, and the cycle collector cannot see through native
|
||||
objects, so they are built in the context this scope was opened from, which cannot reach the scope."""
|
||||
if self._native is None:
|
||||
self._native = self._native_factory(self.post_call_deadline)
|
||||
self._native = self._native_context.run(self._native_factory, self.post_call_deadline)
|
||||
return self._native
|
||||
|
||||
def native_post_call(self) -> NativeRequestRedis:
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
|
|||
)
|
||||
from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
parse_chat_allowed_tools_tool_choice,
|
||||
parse_allowed_tools_tool_choice,
|
||||
responses_allowed_tools_tool_choice,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
|
|
@ -900,7 +900,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
|
||||
return choice_type
|
||||
if choice_type == "allowed_tools":
|
||||
parsed: Final = parse_chat_allowed_tools_tool_choice(
|
||||
parsed: Final = parse_allowed_tools_tool_choice(
|
||||
cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
|
||||
object, tool_choice
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1706,6 +1706,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
|
|||
BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
|
||||
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_usage_based_routing_counted_tokens"
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
|
||||
|
|
@ -1968,6 +1969,7 @@ BATCH_LINE_ITEMS_MAX_PART_BYTES: Final = get_env_int_in_range(
|
|||
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, get_env_int("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7))
|
||||
PENDING_RESPONSE_STATUSES: Final = ("queued", "in_progress")
|
||||
TERMINAL_RESPONSE_STATUSES: Final = frozenset({"completed", "failed", "cancelled", "incomplete"})
|
||||
BACKGROUND_RESPONSE_SETTLEMENT_TIMEOUT_SECONDS: Final = 2.0
|
||||
BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS: Final = 10.0
|
||||
BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS: Final = 60
|
||||
STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, get_env_int("STALE_OBJECT_CLEANUP_BATCH_SIZE", 1000))
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
BilledTokenRates,
|
||||
CostCalculatorUtils,
|
||||
calculate_cache_writing_cost,
|
||||
calculate_cost_component,
|
||||
generic_cost_per_character,
|
||||
generic_cost_per_token,
|
||||
get_batch_cost_rates,
|
||||
|
|
@ -3177,6 +3176,21 @@ def default_video_cost_calculator(
|
|||
return 0.0
|
||||
|
||||
|
||||
def _optional_batch_rate(
|
||||
model_info: ModelInfo,
|
||||
key: Literal[
|
||||
"input_cost_per_audio_token_batches",
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
"input_cost_per_image_token_batches",
|
||||
"input_cost_per_video_token_batches",
|
||||
],
|
||||
prompt_tokens: int,
|
||||
) -> float | None:
|
||||
context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
|
||||
context_rate: Final = get_cost_per_unit(model_info, context_key, None)
|
||||
return model_info.get(key) if context_rate is None else context_rate
|
||||
|
||||
|
||||
def _batch_rate(
|
||||
model_info: ModelInfo,
|
||||
key: Literal[
|
||||
|
|
@ -3189,9 +3203,7 @@ def _batch_rate(
|
|||
*,
|
||||
prompt_tokens: int = 0,
|
||||
) -> float:
|
||||
context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
|
||||
context_rate: Final = get_cost_per_unit(model_info, context_key, None)
|
||||
rate: Final = model_info.get(key) if context_rate is None else context_rate
|
||||
rate: Final = _optional_batch_rate(model_info, key, prompt_tokens)
|
||||
return fallback if rate is None else rate
|
||||
|
||||
|
||||
|
|
@ -3275,30 +3287,14 @@ def batch_cost_calculator(
|
|||
model_info,
|
||||
batch_rates.input,
|
||||
batch_rates.cache_read,
|
||||
batch_rates.cache_read_image,
|
||||
batch_rates.cache_creation,
|
||||
batch_rates.cache_creation_rates_by_ttl,
|
||||
)
|
||||
elif input_cost_per_token is not None:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens: Final = details["cache_hit_tokens"]
|
||||
cache_creation_tokens: Final = details["cache_creation_tokens"]
|
||||
|
||||
# Subtract cached tokens from prompt_tokens before calculating cost
|
||||
# Fixes issue where cached tokens are being charged again
|
||||
base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens
|
||||
total_prompt_cost = (
|
||||
base_input_tokens * (input_cost_per_token) / 2
|
||||
) # batch cost is usually half of the regular token cost
|
||||
|
||||
# Add cache read cost if applicable
|
||||
cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None)
|
||||
total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2
|
||||
|
||||
configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost")
|
||||
cache_creation_cost: Final = (
|
||||
input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost
|
||||
total_prompt_cost = _half_price_prompt_cost(
|
||||
usage, model_info, input_cost_per_token, batch_rates.cache_read_image
|
||||
)
|
||||
total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
|
||||
text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token)
|
||||
image_rate: Final = _batch_or_half(
|
||||
model_info.get("output_cost_per_image_token_batches"),
|
||||
|
|
@ -3324,15 +3320,54 @@ def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float
|
|||
return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None)
|
||||
|
||||
|
||||
def _cached_audio_batch_fallback(
|
||||
model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int
|
||||
) -> float:
|
||||
def _cached_modality_rate(
|
||||
model_info: ModelInfo,
|
||||
standard_key: Literal["cache_read_input_audio_token_cost", "cache_read_input_image_token_cost"],
|
||||
batch_modality_rate: float | None,
|
||||
batch_cache_read_rate: float | None,
|
||||
cache_read_rate: float | None,
|
||||
prompt_tokens: int,
|
||||
) -> float | None:
|
||||
if batch_modality_rate is not None:
|
||||
return batch_modality_rate
|
||||
if batch_cache_read_rate is not None:
|
||||
return batch_cache_read_rate
|
||||
standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens)
|
||||
if standard_audio_rate is not None:
|
||||
return standard_audio_rate
|
||||
return 0.0 if cache_read_rate is None else cache_read_rate
|
||||
standard_modality_rate: Final = _standard_rate(model_info, standard_key, prompt_tokens)
|
||||
return cache_read_rate if standard_modality_rate is None else standard_modality_rate
|
||||
|
||||
|
||||
def _half_price_prompt_cost(
|
||||
usage: Usage, model_info: ModelInfo, input_cost_per_token: float, batch_cache_read_image_rate: float | None
|
||||
) -> float:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cache_creation_tokens: Final = details["cache_creation_tokens"]
|
||||
base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens
|
||||
cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None)
|
||||
half_cache_read_rate: Final = (get_cost_per_unit(model_info, cache_read_cost_key, 0.0) or 0.0) / 2
|
||||
cached_audio_rate: Final = _batch_or_half(
|
||||
_optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens),
|
||||
_standard_rate(model_info, "cache_read_input_audio_token_cost", usage.prompt_tokens),
|
||||
default=half_cache_read_rate,
|
||||
)
|
||||
cached_image_rate: Final = _batch_or_half(
|
||||
batch_cache_read_image_rate,
|
||||
_standard_rate(model_info, "cache_read_input_image_token_cost", usage.prompt_tokens),
|
||||
default=half_cache_read_rate,
|
||||
)
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"]
|
||||
cached_image_tokens: Final = details["cached_image_tokens"]
|
||||
cached_text_tokens: Final = max(details["cache_hit_tokens"] - cached_audio_tokens - cached_image_tokens, 0)
|
||||
configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost")
|
||||
cache_creation_cost: Final = (
|
||||
input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost
|
||||
)
|
||||
return (
|
||||
base_input_tokens * input_cost_per_token / 2
|
||||
+ cached_text_tokens * half_cache_read_rate
|
||||
+ cached_audio_tokens * cached_audio_rate
|
||||
+ cached_image_tokens * cached_image_rate
|
||||
+ cache_creation_tokens * cache_creation_cost / 2
|
||||
)
|
||||
|
||||
|
||||
def _batch_prompt_cost(
|
||||
|
|
@ -3340,6 +3375,7 @@ def _batch_prompt_cost(
|
|||
model_info: ModelInfo,
|
||||
input_rate: float,
|
||||
batch_cache_read_rate: float | None,
|
||||
batch_cache_read_image_rate: float | None,
|
||||
cache_creation_rate: float | None,
|
||||
cache_creation_rates_by_ttl: Mapping[str, float],
|
||||
) -> float:
|
||||
|
|
@ -3349,8 +3385,30 @@ def _batch_prompt_cost(
|
|||
if batch_cache_read_rate is None
|
||||
else batch_cache_read_rate
|
||||
)
|
||||
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
|
||||
cached_audio_rate: Final = _cached_modality_rate(
|
||||
model_info,
|
||||
"cache_read_input_audio_token_cost",
|
||||
_optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens),
|
||||
batch_cache_read_rate,
|
||||
cache_read_rate,
|
||||
usage.prompt_tokens,
|
||||
)
|
||||
cached_image_rate: Final = _cached_modality_rate(
|
||||
model_info,
|
||||
"cache_read_input_image_token_cost",
|
||||
batch_cache_read_image_rate,
|
||||
batch_cache_read_rate,
|
||||
cache_read_rate,
|
||||
usage.prompt_tokens,
|
||||
)
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cached_audio_rate is not None else 0
|
||||
cached_image_tokens: Final = details["cached_image_tokens"] if cached_image_rate is not None else 0
|
||||
cached_text_tokens: Final = (
|
||||
max(details["cache_hit_tokens"] - details["cache_hit_audio_tokens"] - details["cached_image_tokens"], 0)
|
||||
if cache_read_rate is not None
|
||||
else 0
|
||||
)
|
||||
cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens
|
||||
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
|
||||
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
|
||||
audio_tokens, image_tokens, video_tokens = (
|
||||
|
|
@ -3367,13 +3425,9 @@ def _batch_prompt_cost(
|
|||
* _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens)
|
||||
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
|
||||
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
|
||||
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
|
||||
+ cached_audio_tokens
|
||||
* _batch_rate(
|
||||
model_info,
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
_cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens),
|
||||
)
|
||||
+ cached_text_tokens * (cache_read_rate or 0.0)
|
||||
+ cached_audio_tokens * (cached_audio_rate or 0.0)
|
||||
+ cached_image_tokens * (cached_image_rate or 0.0)
|
||||
+ (
|
||||
calculate_cache_writing_cost(
|
||||
written_tokens,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from pydantic import JsonValue, TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY
|
||||
from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
|
||||
from litellm.integrations.clickhouse.context import is_lens_analysis
|
||||
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
|
||||
|
|
@ -37,7 +38,14 @@ TRACE_INGEST_ROUTE: Final = "/v1/traces"
|
|||
_METADATA_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_METADATA_VALUE: Final = TypeAdapter(JsonValue)
|
||||
_INTERNAL_METADATA_KEYS: Final = frozenset(
|
||||
("user_api_key", "user_api_key_auth", "user_api_key_budget_reservation", "proxy_server_request")
|
||||
(
|
||||
"user_api_key",
|
||||
"user_api_key_auth",
|
||||
"user_api_key_budget_reservation",
|
||||
"proxy_server_request",
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@ from pydantic import ValidationError
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
allowed_tools_tool_choice_in_auto_mode,
|
||||
allowed_tools_tool_choice_with_builtin_as_function,
|
||||
)
|
||||
from litellm.llms.base_llm.sandbox.transformation import (
|
||||
CodeExecutionResult,
|
||||
ContainerHandle,
|
||||
|
|
@ -175,6 +179,10 @@ def _sandbox_unavailable(step: str, error: Exception, output: str = _SANDBOX_UNA
|
|||
return _SandboxUnavailable(output=output)
|
||||
|
||||
|
||||
def _is_code_interpreter_tool(tool: Mapping[str, object]) -> bool:
|
||||
return tool.get("type") == "code_interpreter"
|
||||
|
||||
|
||||
def _output_item_type(item: object) -> object:
|
||||
if isinstance(item, dict):
|
||||
item_mapping: Final[dict[str, object]] = item
|
||||
|
|
@ -343,8 +351,8 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
(function_tool if isinstance(tool, dict) and tool.get("type") == "code_interpreter" else tool)
|
||||
for tool in tools
|
||||
]
|
||||
if self._tool_choice_targets_code_interpreter(kwargs.get("tool_choice")):
|
||||
kwargs["tool_choice"] = self._get_function_tool_choice(call_type=call_type)
|
||||
if "tool_choice" in kwargs:
|
||||
kwargs["tool_choice"] = self._intercepted_tool_choice(kwargs["tool_choice"], call_type)
|
||||
return kwargs
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -428,6 +436,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
or (isinstance(function, dict) and function.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _intercepted_tool_choice(cls, tool_choice: object, call_type: CallTypes | None) -> object:
|
||||
if cls._tool_choice_targets_code_interpreter(tool_choice):
|
||||
return cls._get_function_tool_choice(call_type=call_type)
|
||||
return allowed_tools_tool_choice_with_builtin_as_function(
|
||||
tool_choice, _is_code_interpreter_tool, LITELLM_CODE_EXECUTION_TOOL_NAME
|
||||
)
|
||||
|
||||
def _resolve_provider(self, kwargs: dict[str, object]) -> str | None:
|
||||
provider: Final = kwargs.get("custom_llm_provider")
|
||||
if isinstance(provider, str) and provider:
|
||||
|
|
@ -694,7 +710,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
def _get_followup_optional_params(self, optional_params: dict[str, object]) -> dict[str, object]:
|
||||
drop_tool_choice: Final = self._tool_choice_targets_code_interpreter(optional_params.get("tool_choice"))
|
||||
return {
|
||||
k: v for k, v in optional_params.items() if k != "tools" and not (k == "tool_choice" and drop_tool_choice)
|
||||
k: allowed_tools_tool_choice_in_auto_mode(v) if k == "tool_choice" else v
|
||||
for k, v in optional_params.items()
|
||||
if k != "tools" and not (k == "tool_choice" and drop_tool_choice)
|
||||
}
|
||||
|
||||
async def async_post_agentic_loop_response_hook(
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.compression import compress
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import allowed_tools_tool_choice_with_function
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.types.integrations.compression_interception import (
|
||||
CompressionInterceptionConfig,
|
||||
|
|
@ -180,6 +181,10 @@ class CompressionInterceptionLogger(CustomLogger):
|
|||
existing_tools=cast(list[dict[str, object]] | None, kwargs.get("tools")),
|
||||
compressed_tools=compressed_tools,
|
||||
)
|
||||
if "tool_choice" in kwargs:
|
||||
kwargs["tool_choice"] = allowed_tools_tool_choice_with_function(
|
||||
kwargs["tool_choice"], LITELLM_CONTENT_RETRIEVE_TOOL_NAME
|
||||
)
|
||||
call_id = cast(str | None, kwargs.get("litellm_call_id"))
|
||||
if not call_id:
|
||||
call_id = str(uuid.uuid4())
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
|
|||
safe_set_attribute,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import require_attributes
|
||||
from litellm.litellm_core_utils.safe_json_dumps import jsonable_or_str
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse
|
||||
from litellm.types.utils import (
|
||||
EmbeddingResponse,
|
||||
|
|
@ -103,7 +104,7 @@ class LangfuseLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
|
|||
prompt["tools"] = tools
|
||||
|
||||
input: Final = prompt
|
||||
safe_set_attribute(span, "langfuse.observation.input", json.dumps(input))
|
||||
safe_set_attribute(span, "langfuse.observation.input", json.dumps(input, default=jsonable_or_str))
|
||||
|
||||
@staticmethod
|
||||
@override
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import json
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from litellm.litellm_core_utils.safe_json_dumps import jsonable_or_str
|
||||
from litellm.types.integrations.otel_span_attributes import SpanAttributes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -60,7 +61,7 @@ class LangtraceAttributes:
|
|||
if prompts:
|
||||
span.add_event(
|
||||
name="gen_ai.content.prompt",
|
||||
attributes={SpanAttributes.LLM_PROMPTS.value: json.dumps(prompts)},
|
||||
attributes={SpanAttributes.LLM_PROMPTS.value: json.dumps(prompts, default=jsonable_or_str)},
|
||||
)
|
||||
|
||||
self.set_span_attributes(span, span_attributes)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast
|
|||
from urllib.parse import urlsplit
|
||||
|
||||
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.integrations.otel.model.metadata import (
|
||||
|
|
@ -519,7 +519,7 @@ class LLMCallSpanData:
|
|||
promoted_metadata=promoted_metadata,
|
||||
is_streaming=as_bool(payload.get("stream")),
|
||||
tools=_extract_tools(params),
|
||||
messages_in=_dicts(payload.get("messages")) if capture_content else (),
|
||||
messages_in=_message_dicts(payload.get("messages")) if capture_content else (),
|
||||
choices_out=choices_out if capture_content else (),
|
||||
system_fingerprint=as_str(response.get("system_fingerprint")),
|
||||
time_to_first_chunk_seconds=time_to_first_chunk_seconds,
|
||||
|
|
@ -783,6 +783,27 @@ def _dicts(value: object) -> tuple[Mapping[str, object], ...]:
|
|||
return tuple(item for item in value if isinstance(item, dict))
|
||||
|
||||
|
||||
_OBJECTS: Final = TypeAdapter(tuple[object, ...])
|
||||
_ANY_KEY_MAPPING: Final = TypeAdapter(Mapping[object, object])
|
||||
|
||||
|
||||
def _message_dict(message: object) -> Mapping[str, object] | None:
|
||||
if isinstance(message, dict):
|
||||
return {str(key): value for key, value in _ANY_KEY_MAPPING.validate_python(message).items()}
|
||||
if not isinstance(message, BaseModel):
|
||||
return None
|
||||
try:
|
||||
return message.model_dump(mode="json", fallback=str)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _message_dicts(value: object) -> tuple[Mapping[str, object], ...]:
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
return tuple(message for item in _OBJECTS.validate_python(value) if (message := _message_dict(item)) is not None)
|
||||
|
||||
|
||||
def _finish_reasons(choices: tuple[Mapping[str, object], ...]) -> tuple[str, ...]:
|
||||
"""Non-empty ``finish_reason`` of each response choice."""
|
||||
return tuple(r for c in choices if (r := as_str(c.get("finish_reason"))))
|
||||
|
|
|
|||
|
|
@ -292,6 +292,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
|
|||
"LiteLLM_SpendLogs",
|
||||
"LiteLLM_BudgetWindowSpend",
|
||||
"LiteLLM_BackgroundInteractionSettlement",
|
||||
"LiteLLM_BackgroundResponseSettlement",
|
||||
"LiteLLM_ErrorLogs",
|
||||
"LiteLLM_UserNotifications",
|
||||
"LiteLLM_UserProviderCredentials",
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
|
|||
safe_set_attribute,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_dumps import jsonable_or_str, safe_dumps
|
||||
from litellm.types.integrations.weave_otel import WeaveOtelConfig, WeaveSpanAttributes
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
|
|
@ -74,7 +74,11 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
|
|||
prompt["functions"] = functions
|
||||
if tools is not None:
|
||||
prompt["tools"] = tools
|
||||
safe_set_attribute(span, OpenInferenceSpanAttributes.INPUT_VALUE, json.dumps(prompt))
|
||||
safe_set_attribute(
|
||||
span,
|
||||
OpenInferenceSpanAttributes.INPUT_VALUE,
|
||||
json.dumps(prompt, default=jsonable_or_str),
|
||||
)
|
||||
|
||||
|
||||
def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object):
|
||||
|
|
|
|||
|
|
@ -9,11 +9,12 @@ server-side using litellm router's search tools.
|
|||
import asyncio
|
||||
import math
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import Never, ReadOnly
|
||||
|
||||
import litellm
|
||||
|
|
@ -38,6 +39,10 @@ from litellm.integrations.websearch_interception.transformation import (
|
|||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
allowed_tools_tool_choice_kept_on_follow_up,
|
||||
allowed_tools_tool_choice_with_builtin_as_function,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model
|
||||
from litellm.llms.base_llm.search.transformation import SearchResponse
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
|
|
@ -100,6 +105,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks"
|
|||
|
||||
_RESPONSE_CONTENT_FIELD: Final = "content"
|
||||
|
||||
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:"
|
||||
|
||||
_ResponseT: Final = TypeVar("_ResponseT")
|
||||
|
|
@ -263,6 +270,22 @@ class _ParentRequestCorrelation:
|
|||
_NO_ACOMPLETION_NAMED: Final[_AcompletionNamedParams] = {}
|
||||
|
||||
|
||||
def _tool_choice_naming_litellm_web_search(
|
||||
kwargs: Mapping[str, object], is_search_tool: Callable[[Mapping[str, object]], bool]
|
||||
) -> Mapping[str, object]:
|
||||
if "tool_choice" not in kwargs:
|
||||
return MappingProxyType({})
|
||||
return {
|
||||
"tool_choice": allowed_tools_tool_choice_with_builtin_as_function(
|
||||
kwargs["tool_choice"], is_search_tool, LITELLM_WEB_SEARCH_TOOL_NAME
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _kept_allowed_tools_tool_choice(optional_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return allowed_tools_tool_choice_kept_on_follow_up(optional_params.get("tool_choice"))
|
||||
|
||||
|
||||
class WebSearchInterceptionLogger(CustomLogger):
|
||||
"""
|
||||
CustomLogger that intercepts WebSearch tool calls for models that don't
|
||||
|
|
@ -514,13 +537,17 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
# Keep other tools as-is
|
||||
converted_tools.append(tool)
|
||||
|
||||
kwargs["tools"] = converted_tools
|
||||
converted_kwargs: Final[dict[str, object]] = {
|
||||
**kwargs,
|
||||
"tools": converted_tools,
|
||||
**_tool_choice_naming_litellm_web_search(kwargs, lambda tool: is_search_tool(dict(tool))),
|
||||
}
|
||||
|
||||
if kwargs.get("stream"):
|
||||
verbose_logger.debug("WebSearchInterception: deployment hook converting stream=True to stream=False")
|
||||
return as_converted_stream(kwargs, WEBSEARCH_INTERCEPTION_PREFIX)
|
||||
return as_converted_stream(converted_kwargs, WEBSEARCH_INTERCEPTION_PREFIX)
|
||||
|
||||
return kwargs
|
||||
return converted_kwargs
|
||||
|
||||
def _convert_responses_tools(
|
||||
self, kwargs: Mapping[str, object], tools: Sequence[dict[str, object]]
|
||||
|
|
@ -535,7 +562,11 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
get_litellm_web_search_tool_responses() if is_web_search_tool_responses(tool) else tool for tool in tools
|
||||
]
|
||||
|
||||
converted_kwargs: Final = {**kwargs, "tools": converted_tools}
|
||||
converted_kwargs: Final = {
|
||||
**kwargs,
|
||||
"tools": converted_tools,
|
||||
**_tool_choice_naming_litellm_web_search(kwargs, is_web_search_tool_responses),
|
||||
}
|
||||
|
||||
if kwargs.get("stream"):
|
||||
verbose_logger.debug("WebSearchInterception: deployment hook converting stream=True to stream=False")
|
||||
|
|
@ -1256,9 +1287,13 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
|
||||
tools_param: Final = optional_params.get("tools")
|
||||
optional_params_clean: Final = {
|
||||
k: v
|
||||
for k, v in optional_params.items()
|
||||
if k not in {"tools", "tool_choice", "stream", "model_alias_map", "stream_response", "custom_prompt_dict"}
|
||||
**{
|
||||
k: v
|
||||
for k, v in optional_params.items()
|
||||
if k
|
||||
not in {"tools", "tool_choice", "stream", "model_alias_map", "stream_response", "custom_prompt_dict"}
|
||||
},
|
||||
**_kept_allowed_tools_tool_choice(_OPTIONAL_PARAMS.validate_python(optional_params)),
|
||||
}
|
||||
|
||||
kwargs_for_followup: Final = {
|
||||
|
|
@ -1843,9 +1878,11 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
)
|
||||
if request_patch.messages is None:
|
||||
raise ValueError("WebSearchInterception: missing follow-up messages")
|
||||
params: Final = dict(optional_params)
|
||||
params.update(request_patch.optional_params)
|
||||
params.pop("tool_choice", None)
|
||||
merged_params: Final = {**optional_params, **request_patch.optional_params}
|
||||
params: Final = {
|
||||
**{key: value for key, value in merged_params.items() if key != "tool_choice"},
|
||||
**_kept_allowed_tools_tool_choice(merged_params),
|
||||
}
|
||||
patch_kwargs: Final = dict[str, object](request_patch.kwargs)
|
||||
return await litellm.acompletion(
|
||||
model=request_patch.model or model,
|
||||
|
|
@ -1958,17 +1995,20 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
|
||||
tools_param: Final = optional_params.get("tools")
|
||||
optional_params_clean: Final = {
|
||||
k: v
|
||||
for k, v in optional_params.items()
|
||||
if k
|
||||
not in {
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"extra_body",
|
||||
"model_alias_map",
|
||||
"stream_response",
|
||||
"custom_prompt_dict",
|
||||
}
|
||||
**{
|
||||
k: v
|
||||
for k, v in optional_params.items()
|
||||
if k
|
||||
not in {
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"extra_body",
|
||||
"model_alias_map",
|
||||
"stream_response",
|
||||
"custom_prompt_dict",
|
||||
}
|
||||
},
|
||||
**_kept_allowed_tools_tool_choice(_OPTIONAL_PARAMS.validate_python(optional_params)),
|
||||
}
|
||||
if tools_param is not None:
|
||||
optional_params_clean["tools"] = tools_param
|
||||
|
|
|
|||
|
|
@ -40,6 +40,16 @@ async def every_claimed_tool_call(
|
|||
return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))})
|
||||
|
||||
|
||||
_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "tool_use"})
|
||||
|
||||
|
||||
def called_tool_names(response: object) -> frozenset[str]:
|
||||
items: Final = (*parts_of(message_field(response, "output")), *parts_of(message_field(response, "content")))
|
||||
return _strings(
|
||||
message_field(item, "name") for item in items if message_field(item, "type") in _TOOL_CALL_ITEM_TYPES
|
||||
)
|
||||
|
||||
|
||||
def _call_ids(call: object) -> frozenset[str]:
|
||||
return _strings((message_field(call, "id"), message_field(call, "call_id")))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,17 @@
|
|||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam
|
||||
from openai.types.chat.chat_completion_allowed_tools_param import ChatCompletionAllowedToolsParam
|
||||
from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.exceptions import BadRequestError
|
||||
|
||||
ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"})
|
||||
_FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE: Final = "2023-12-01"
|
||||
|
||||
HOSTED_TOOL_CHOICE_TYPES: Final = frozenset(
|
||||
{
|
||||
|
|
@ -29,6 +34,9 @@ HOSTED_TOOL_CHOICE_TYPES: Final = frozenset(
|
|||
CHAT_ALLOWED_TOOLS_LAYOUT: Final = (
|
||||
'{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": [<at least one tool>]}}'
|
||||
)
|
||||
RESPONSES_ALLOWED_TOOLS_LAYOUT: Final = (
|
||||
'{"type": "allowed_tools", "mode": "auto" | "required", "tools": [<at least one tool>]}'
|
||||
)
|
||||
|
||||
|
||||
class ChatAllowedTools(BaseModel):
|
||||
|
|
@ -45,7 +53,17 @@ class ChatAllowedToolsToolChoice(BaseModel):
|
|||
allowed_tools: ChatAllowedTools
|
||||
|
||||
|
||||
_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice)
|
||||
class ResponsesAllowedToolsToolChoice(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
type: Literal["allowed_tools"]
|
||||
mode: Literal["auto", "required"]
|
||||
tools: tuple[Mapping[str, object], ...] = Field(min_length=1)
|
||||
|
||||
|
||||
_ALLOWED_TOOLS_TOOL_CHOICE: Final[TypeAdapter[ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice]] = (
|
||||
TypeAdapter(ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice)
|
||||
)
|
||||
|
||||
|
||||
_SELECTOR: Final = TypeAdapter(dict[str, object])
|
||||
|
|
@ -67,21 +85,22 @@ def is_allowed_tools_tool_choice(tool_choice: object) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def raise_if_allowed_tools_tool_choice_unsupported(
|
||||
def drops_unsupported_allowed_tools_tool_choice(
|
||||
tool_choice: object,
|
||||
custom_llm_provider: str,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""Providers outside the set would silently send the request with no tool restriction at all, so the call stops
|
||||
here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call."""
|
||||
drop_params: bool,
|
||||
) -> bool:
|
||||
if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS:
|
||||
return
|
||||
return False
|
||||
if drop_params:
|
||||
return True
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
"tool_choice type 'allowed_tools' is only supported for "
|
||||
f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. "
|
||||
"drop_params does not drop it, since that would let the model call tools the request excluded. "
|
||||
"Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead."
|
||||
"Send only the allowed tools in `tools` with tool_choice='auto' or 'required', or set drop_params=True "
|
||||
"to drop the tool_choice, which lets the model call any tool in `tools`."
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
|
|
@ -119,18 +138,295 @@ def raise_if_hosted_tool_choice_unsupported(
|
|||
)
|
||||
|
||||
|
||||
def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
|
||||
try:
|
||||
return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _has_name(selector: Mapping[str, object]) -> bool:
|
||||
name: Final = selector.get("name")
|
||||
return isinstance(name, str) and name != ""
|
||||
|
||||
|
||||
def _names_a_tool_or_nothing(tool: Mapping[str, object]) -> bool:
|
||||
tool_type: Final = tool.get("type")
|
||||
if not isinstance(tool_type, str) or tool_type not in ("function", "custom"):
|
||||
return tool.get("name") is None or _has_name(tool)
|
||||
match tool.get(tool_type):
|
||||
case {"name": str()}:
|
||||
return True
|
||||
case {"name": object()}:
|
||||
return False
|
||||
case _:
|
||||
return True
|
||||
|
||||
|
||||
def _names_are_strings(tools: Iterable[Mapping[str, object]]) -> bool:
|
||||
return all(map(_names_a_tool_or_nothing, tools))
|
||||
|
||||
|
||||
def _chat_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
tool_type: Final = tool.get("type")
|
||||
if not isinstance(tool_type, str) or tool_type not in ("function", "custom"):
|
||||
return tool
|
||||
if tool_type in tool:
|
||||
return None if "name" in tool else tool
|
||||
if not _has_name(tool):
|
||||
return None
|
||||
return {**{key: value for key, value in tool.items() if key != "name"}, tool_type: {"name": tool["name"]}}
|
||||
|
||||
|
||||
def _chat_layout(choice: ResponsesAllowedToolsToolChoice) -> ChatAllowedToolsToolChoice | None:
|
||||
nested: Final = tuple(_chat_allowed_tool(tool) for tool in choice.tools)
|
||||
if any(tool is None for tool in nested):
|
||||
return None
|
||||
return ChatAllowedToolsToolChoice(
|
||||
type="allowed_tools",
|
||||
allowed_tools=ChatAllowedTools(mode=choice.mode, tools=tuple(tool for tool in nested if tool is not None)),
|
||||
)
|
||||
|
||||
|
||||
def parse_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None:
|
||||
try:
|
||||
parsed: Final = _ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
|
||||
except ValidationError:
|
||||
return None
|
||||
match parsed:
|
||||
case ChatAllowedToolsToolChoice():
|
||||
return parsed if _names_are_strings(parsed.allowed_tools.tools) else None
|
||||
case ResponsesAllowedToolsToolChoice():
|
||||
return _chat_layout(parsed) if _names_are_strings(parsed.tools) else None
|
||||
|
||||
|
||||
def azure_api_version_takes_tool_choice(api_version: str) -> bool:
|
||||
return api_version.count("-") < 2 or api_version >= _FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE
|
||||
|
||||
|
||||
def _azure_request_takes_tool_choice(api_version: str | None) -> bool:
|
||||
import litellm
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
return azure_api_version_takes_tool_choice(
|
||||
api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION
|
||||
)
|
||||
|
||||
|
||||
def _enforces_allowed_tools(custom_llm_provider: str | None, api_version: str | None) -> bool:
|
||||
if custom_llm_provider == "azure":
|
||||
return _azure_request_takes_tool_choice(api_version)
|
||||
return custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS
|
||||
|
||||
|
||||
def unenforced_allowed_tools_tool_choice(
|
||||
tool_choice: object, custom_llm_provider: str | None, api_version: str | None = None
|
||||
) -> bool:
|
||||
return is_allowed_tools_tool_choice(tool_choice) and (
|
||||
not _enforces_allowed_tools(custom_llm_provider, api_version)
|
||||
or parse_allowed_tools_tool_choice(tool_choice) is None
|
||||
)
|
||||
|
||||
|
||||
def _allowed_tool_names(tool_choice: object) -> frozenset[object]:
|
||||
parsed: Final = parse_allowed_tools_tool_choice(tool_choice)
|
||||
if parsed is None:
|
||||
return frozenset()
|
||||
flattened: Final = tuple(_responses_allowed_tool(tool) for tool in parsed.allowed_tools.tools)
|
||||
return frozenset(tool.get("name") for tool in flattened if tool is not None and _server_entry(tool) is None)
|
||||
|
||||
|
||||
def parse_allowed_tools_tool_choice_or_refuse(
|
||||
tool_choice: object, model: str, drop_params: bool
|
||||
) -> ChatAllowedToolsToolChoice | None:
|
||||
parsed: Final = parse_allowed_tools_tool_choice(tool_choice)
|
||||
if parsed is not None or drop_params:
|
||||
return parsed
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
f"Invalid tool choice, tool_choice={tool_choice}. An allowed_tools tool_choice must be "
|
||||
f"{CHAT_ALLOWED_TOOLS_LAYOUT} or {RESPONSES_ALLOWED_TOOLS_LAYOUT}"
|
||||
),
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
||||
|
||||
def allowed_tools_tool_choice_in_effect(tool_choice: object, model: str, drop_params: bool) -> object:
|
||||
if not is_allowed_tools_tool_choice(tool_choice):
|
||||
return tool_choice
|
||||
return None if parse_allowed_tools_tool_choice_or_refuse(tool_choice, model, drop_params) is None else tool_choice
|
||||
|
||||
|
||||
def _server_entry(tool: Mapping[str, object]) -> tuple[str, str | None] | None:
|
||||
label: Final = tool.get("server_label")
|
||||
name: Final = tool.get("name")
|
||||
if tool.get("type") != "mcp" or not isinstance(label, str):
|
||||
return None
|
||||
return label, name if isinstance(name, str) else None
|
||||
|
||||
|
||||
def _server_entries(tool_choice: object) -> frozenset[tuple[str, str | None]]:
|
||||
parsed: Final = parse_allowed_tools_tool_choice(tool_choice)
|
||||
entries: Final = () if parsed is None else tuple(map(_server_entry, parsed.allowed_tools.tools))
|
||||
return frozenset(entry for entry in entries if entry is not None)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GatewayTool:
|
||||
names: frozenset[str]
|
||||
server_labels: frozenset[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GatewayTools:
|
||||
server_labels: frozenset[str]
|
||||
tools: Mapping[str, GatewayTool]
|
||||
|
||||
|
||||
NO_GATEWAY_TOOLS: Final = GatewayTools(server_labels=frozenset(), tools=MappingProxyType({}))
|
||||
|
||||
|
||||
def _server_entry_allows(entry: tuple[str, str | None], tool: GatewayTool | None) -> bool:
|
||||
label, name = entry
|
||||
return tool is not None and label in tool.server_labels and (name is None or name in tool.names)
|
||||
|
||||
|
||||
def _a_server_entry_allows(entries: Iterable[tuple[str, str | None]], tool: GatewayTool | None) -> bool:
|
||||
return any(_server_entry_allows(entry, tool) for entry in entries)
|
||||
|
||||
|
||||
def tool_calls_the_choice_excludes(
|
||||
tool_choice: object, tool_names: Iterable[str], gateway: GatewayTools = NO_GATEWAY_TOOLS
|
||||
) -> tuple[str, ...]:
|
||||
if not is_allowed_tools_tool_choice(tool_choice):
|
||||
return ()
|
||||
allowed: Final = _allowed_tool_names(tool_choice)
|
||||
server_entries: Final = _server_entries(tool_choice)
|
||||
return tuple(
|
||||
name
|
||||
for name in tool_names
|
||||
if name not in allowed and not _a_server_entry_allows(server_entries, gateway.tools.get(name))
|
||||
)
|
||||
|
||||
|
||||
def refuse_tool_calls_the_choice_excludes(tool_choice: object, tool_names: Iterable[str], model: str) -> None:
|
||||
excluded: Final = tool_calls_the_choice_excludes(tool_choice, tool_names)
|
||||
if excluded:
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
f"The model called {', '.join(excluded)}, which the allowed_tools tool_choice excludes, because this "
|
||||
"request did not reach a model that enforces the choice. LiteLLM did not run the call. Send only the "
|
||||
"allowed tools in `tools` with tool_choice='auto' or 'required'."
|
||||
),
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
||||
|
||||
def _with_builtin_as(
|
||||
tool: Mapping[str, object], is_builtin: Callable[[Mapping[str, object]], bool], function_entry: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
return function_entry if tool.get("type") not in ("function", "custom", "mcp") and is_builtin(tool) else tool
|
||||
|
||||
|
||||
def _function_entry(
|
||||
layout: ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice, name: str
|
||||
) -> Mapping[str, object]:
|
||||
match layout:
|
||||
case ChatAllowedToolsToolChoice():
|
||||
return {"type": "function", "function": {"name": name}}
|
||||
case ResponsesAllowedToolsToolChoice():
|
||||
return {"type": "function", "name": name}
|
||||
|
||||
|
||||
def _entries_of(
|
||||
layout: ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice,
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
match layout:
|
||||
case ChatAllowedToolsToolChoice():
|
||||
return layout.allowed_tools.tools
|
||||
case ResponsesAllowedToolsToolChoice():
|
||||
return layout.tools
|
||||
|
||||
|
||||
def _with_entries(
|
||||
layout: ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice, tools: Iterable[Mapping[str, object]]
|
||||
) -> Mapping[str, object]:
|
||||
entries: Final = [dict(tool) for tool in tools]
|
||||
match layout:
|
||||
case ChatAllowedToolsToolChoice():
|
||||
return {"type": "allowed_tools", "allowed_tools": {"mode": layout.allowed_tools.mode, "tools": entries}}
|
||||
case ResponsesAllowedToolsToolChoice():
|
||||
return {"type": "allowed_tools", "mode": layout.mode, "tools": entries}
|
||||
|
||||
|
||||
def _callers_layout(tool_choice: object) -> ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice | None:
|
||||
if parse_allowed_tools_tool_choice(tool_choice) is None:
|
||||
return None
|
||||
return _ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice)
|
||||
|
||||
|
||||
def allowed_tools_tool_choice_with_builtin_as_function(
|
||||
tool_choice: object, is_builtin: Callable[[Mapping[str, object]], bool], function_name: str
|
||||
) -> object:
|
||||
layout: Final = _callers_layout(tool_choice)
|
||||
if layout is None:
|
||||
return tool_choice
|
||||
entry: Final = _function_entry(layout, function_name)
|
||||
return _with_entries(layout, (_with_builtin_as(tool, is_builtin, entry) for tool in _entries_of(layout)))
|
||||
|
||||
|
||||
def allowed_tools_tool_choice_with_function(tool_choice: object, function_name: str) -> object:
|
||||
layout: Final = _callers_layout(tool_choice)
|
||||
if layout is None or function_name in _allowed_tool_names(tool_choice):
|
||||
return tool_choice
|
||||
return _with_entries(layout, (*_entries_of(layout), _function_entry(layout, function_name)))
|
||||
|
||||
|
||||
def allowed_tools_tool_choice_in_auto_mode(tool_choice: object) -> object:
|
||||
layout: Final = _callers_layout(tool_choice)
|
||||
match layout:
|
||||
case None:
|
||||
return tool_choice
|
||||
case ChatAllowedToolsToolChoice():
|
||||
return layout.model_copy(
|
||||
update={"allowed_tools": layout.allowed_tools.model_copy(update={"mode": "auto"})}
|
||||
).model_dump(mode="json")
|
||||
case ResponsesAllowedToolsToolChoice():
|
||||
return layout.model_copy(update={"mode": "auto"}).model_dump(mode="json")
|
||||
|
||||
|
||||
def allowed_tools_tool_choice_kept_on_follow_up(tool_choice: object) -> Mapping[str, object]:
|
||||
if not is_allowed_tools_tool_choice(tool_choice):
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType({"tool_choice": allowed_tools_tool_choice_in_auto_mode(tool_choice)})
|
||||
|
||||
|
||||
def _gateway_entry(tool: Mapping[str, object], gateway_labels: frozenset[str]) -> tuple[str, str | None] | None:
|
||||
entry: Final = _server_entry(tool)
|
||||
return entry if entry is not None and entry[0] in gateway_labels else None
|
||||
|
||||
|
||||
def allowed_tools_tool_choice_with_gateway_tools_as_functions(tool_choice: object, gateway: GatewayTools) -> object:
|
||||
layout: Final = _callers_layout(tool_choice)
|
||||
entries: Final = () if layout is None else _entries_of(layout)
|
||||
gateway_entries: Final = tuple(
|
||||
entry for entry in (_gateway_entry(tool, gateway.server_labels) for tool in entries) if entry is not None
|
||||
)
|
||||
if layout is None or not gateway_entries:
|
||||
return tool_choice
|
||||
kept: Final = tuple(tool for tool in entries if _gateway_entry(tool, gateway.server_labels) is None)
|
||||
allowed: Final = tuple(
|
||||
name for name, tool in gateway.tools.items() if _a_server_entry_allows(gateway_entries, tool)
|
||||
)
|
||||
if not kept and not allowed:
|
||||
return "none"
|
||||
return _with_entries(layout, (*kept, *(_function_entry(layout, name) for name in allowed)))
|
||||
|
||||
|
||||
def chat_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ChatCompletionAllowedToolChoiceParam:
|
||||
return ChatCompletionAllowedToolChoiceParam(
|
||||
type="allowed_tools",
|
||||
allowed_tools=ChatCompletionAllowedToolsParam(
|
||||
mode=choice.allowed_tools.mode, tools=[dict(tool) for tool in choice.allowed_tools.tools]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
tool_type: Final = tool.get("type")
|
||||
if tool_type not in ("function", "custom"):
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ def normalized_audio_duration_seconds(value: object) -> float | None:
|
|||
return seconds if math.isfinite(seconds) and seconds >= 0 else None
|
||||
|
||||
|
||||
LIBSNDFILE_UNKNOWN_FRAME_COUNT: Final = 2**63 - 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessedAudioFile:
|
||||
"""
|
||||
|
|
@ -340,7 +343,10 @@ def calculate_request_duration(file: FileTypes) -> float | None:
|
|||
# Extract duration using soundfile
|
||||
file_object: Final = io.BytesIO(file_content)
|
||||
with sf.SoundFile(file_object) as audio:
|
||||
duration: Final = len(audio) / audio.samplerate
|
||||
frames: Final = len(audio)
|
||||
if frames >= LIBSNDFILE_UNKNOWN_FRAME_COUNT:
|
||||
return None
|
||||
duration: Final = frames / audio.samplerate
|
||||
return duration
|
||||
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api
|
||||
|
||||
import json
|
||||
from collections.abc import Awaitable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Awaitable, Iterable, Iterator, Mapping, Sequence
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
|
|
@ -18,7 +18,12 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
|
|||
DEFAULT_MAX_AGENTIC_LOOPS,
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
parse_allowed_tools_tool_choice,
|
||||
tool_calls_the_choice_excludes,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model
|
||||
from litellm.litellm_core_utils.get_litellm_params import AGENTIC_LOOP_TOOL_CHOICE_KEY
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict, object_items
|
||||
|
|
@ -34,7 +39,11 @@ from litellm.types.integrations.custom_logger import (
|
|||
is_interception_internal_key,
|
||||
stashed_stream_options,
|
||||
)
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionMessageCustomToolCall,
|
||||
ChatCompletionMessageToolCall,
|
||||
ModelResponse,
|
||||
)
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
_DispatchResult: Final = TypeVar("_DispatchResult")
|
||||
|
|
@ -181,6 +190,40 @@ def _end_refused_turn(
|
|||
return finalized if replay is None else replay
|
||||
|
||||
|
||||
def _tool_call_name(call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall) -> str:
|
||||
match call:
|
||||
case ChatCompletionMessageCustomToolCall():
|
||||
return call.custom.name
|
||||
case ChatCompletionMessageToolCall():
|
||||
return call.function.name or ""
|
||||
|
||||
|
||||
def _called_tool_names(response: ModelResponse) -> tuple[str, ...]:
|
||||
calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in response.choices)
|
||||
return tuple(map(_tool_call_name, calls))
|
||||
|
||||
|
||||
def _server_fulfilled_tool_names(callbacks: Iterable[CustomLogger]) -> frozenset[str]:
|
||||
return frozenset(chain.from_iterable(callback.server_fulfilled_tool_names for callback in callbacks))
|
||||
|
||||
|
||||
def holds_a_call_the_choice_excludes(
|
||||
tool_names: Iterable[str], kwargs: Mapping[str, object], callbacks: Iterable[CustomLogger]
|
||||
) -> bool:
|
||||
server_fulfilled: Final = _server_fulfilled_tool_names(callbacks)
|
||||
client_tool_names: Final = (name for name in tool_names if name not in server_fulfilled)
|
||||
return bool(tool_calls_the_choice_excludes(kwargs.get(AGENTIC_LOOP_TOOL_CHOICE_KEY), client_tool_names))
|
||||
|
||||
|
||||
def with_caller_tool_choice(
|
||||
kwargs: Mapping[str, object], optional_params: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
tool_choice: Final = optional_params.get("tool_choice")
|
||||
if AGENTIC_LOOP_TOOL_CHOICE_KEY in kwargs or parse_allowed_tools_tool_choice(tool_choice) is None:
|
||||
return kwargs
|
||||
return {**kwargs, AGENTIC_LOOP_TOOL_CHOICE_KEY: tool_choice}
|
||||
|
||||
|
||||
def _request_tools(optional_params: Mapping[str, object]) -> list[dict[str, object]]:
|
||||
return [tool for tool in object_items(optional_params.get("tools")) if is_str_keyed_dict(tool)]
|
||||
|
||||
|
|
@ -238,6 +281,35 @@ async def _refused_chat_turn(
|
|||
return finalize_refused_chat_turn(response, claimed)
|
||||
|
||||
|
||||
def _fingerprint_of_a_turn_to_run(
|
||||
*,
|
||||
response: ModelResponse,
|
||||
request_kwargs: Mapping[str, object],
|
||||
callbacks: Sequence[CustomLogger],
|
||||
tool_calls: object,
|
||||
fingerprints: Sequence[str],
|
||||
depth: int,
|
||||
max_loops: int,
|
||||
model: str,
|
||||
stream: bool,
|
||||
) -> str | None:
|
||||
if holds_a_call_the_choice_excludes(_called_tool_names(response), request_kwargs, callbacks):
|
||||
return None
|
||||
try:
|
||||
return _check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
except AgenticLoopSafetyError as e:
|
||||
if stream:
|
||||
raise
|
||||
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
|
||||
return None
|
||||
|
||||
|
||||
def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]:
|
||||
metadata: Final = kwargs_for_followup.get("litellm_metadata")
|
||||
return MappingProxyType(
|
||||
|
|
@ -371,7 +443,8 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
stream: bool,
|
||||
) -> ModelResponse | CustomStreamWrapper | None:
|
||||
callbacks: Final = _custom_loggers(logging_obj)
|
||||
depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
|
||||
request_kwargs: Final = with_caller_tool_choice(kwargs, optional_params)
|
||||
depth, max_loops, fingerprints = _agentic_loop_settings(request_kwargs)
|
||||
tools: Final = _request_tools(optional_params)
|
||||
|
||||
for callback in callbacks:
|
||||
|
|
@ -379,7 +452,7 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
continue
|
||||
|
||||
hook_kwargs = {
|
||||
**kwargs,
|
||||
**request_kwargs,
|
||||
"_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
}
|
||||
|
|
@ -403,18 +476,18 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
if not should_run:
|
||||
continue
|
||||
|
||||
try:
|
||||
fingerprint = _check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
except AgenticLoopSafetyError as e:
|
||||
if stream:
|
||||
raise
|
||||
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
|
||||
fingerprint = _fingerprint_of_a_turn_to_run(
|
||||
response=response,
|
||||
request_kwargs=request_kwargs,
|
||||
callbacks=callbacks,
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
stream=stream,
|
||||
)
|
||||
if fingerprint is None:
|
||||
return _end_refused_turn(
|
||||
await _refused_chat_turn(
|
||||
callbacks,
|
||||
|
|
@ -425,9 +498,9 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
messages=messages,
|
||||
tools=tools,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
kwargs=request_kwargs,
|
||||
),
|
||||
kwargs=kwargs,
|
||||
kwargs=request_kwargs,
|
||||
depth=depth,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -471,7 +544,7 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
callback=callback,
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
kwargs=kwargs,
|
||||
kwargs=dict(request_kwargs),
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
depth=depth,
|
||||
|
|
@ -482,7 +555,7 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
|
||||
return _converted_stream_replay(
|
||||
response,
|
||||
kwargs=kwargs,
|
||||
kwargs=request_kwargs,
|
||||
depth=depth,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -526,18 +599,18 @@ async def run_chat_completion_agentic_hooks(
|
|||
if not should_run:
|
||||
continue
|
||||
|
||||
try:
|
||||
fingerprint = _check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
except AgenticLoopSafetyError as e:
|
||||
if stream:
|
||||
raise
|
||||
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
|
||||
fingerprint = _fingerprint_of_a_turn_to_run(
|
||||
response=response,
|
||||
request_kwargs=kwargs,
|
||||
callbacks=callbacks,
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
stream=stream,
|
||||
)
|
||||
if fingerprint is None:
|
||||
return await _refused_chat_turn(
|
||||
callbacks,
|
||||
callback,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,28 @@ BEDROCK_MANAGED_S3_PREFIXES: Final = (
|
|||
BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
|
||||
)
|
||||
MANAGED_CLOUD_STORAGE_SCHEMES: Final = ("s3://", "gs://")
|
||||
STORED_S3_BUCKET_PARAM: Final = "stored_s3_bucket_name"
|
||||
REQUEST_S3_BUCKET_PARAMS: Final = frozenset({"s3_bucket_name", "s3_output_bucket_name"})
|
||||
|
||||
|
||||
def stored_s3_bucket_kwargs(stored_file_id: str) -> Mapping[str, str]:
|
||||
"""The bucket (and any prefix before LiteLLM's own path) of an ``s3://`` file id LiteLLM recorded itself,
|
||||
as the param that lets Bedrock read it back.
|
||||
|
||||
Only pass ids from LiteLLM's own records (managed file rows, provider batch responses), never one a caller sent.
|
||||
"""
|
||||
if not stored_file_id.startswith("s3://"):
|
||||
return {}
|
||||
bucket, _, object_key = stored_file_id.removeprefix("s3://").partition("/")
|
||||
if not bucket:
|
||||
return {}
|
||||
segments: Final = object_key.split("/")
|
||||
prefix_depth: Final = next(
|
||||
(depth for depth in range(len(segments)) if "/".join(segments[depth:]).startswith(BEDROCK_MANAGED_S3_PREFIXES)),
|
||||
0,
|
||||
)
|
||||
prefix: Final = "/".join(segments[:prefix_depth])
|
||||
return {STORED_S3_BUCKET_PARAM: f"{bucket}/{prefix}" if prefix else bucket}
|
||||
|
||||
|
||||
MAX_FILE_ID_DECODE_PASSES: Final = 8
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ def proxy_request_headers(container: Mapping[str, object] | None) -> Mapping[str
|
|||
|
||||
def stream_cost_injection_requested(request_headers: Mapping[str, object] | None) -> bool:
|
||||
"""The global include_cost_in_streaming_usage flag or the per-request header asks for usage.cost on streams."""
|
||||
if bool(getattr(litellm, "include_cost_in_streaming_usage", False)):
|
||||
if litellm.include_cost_in_streaming_usage:
|
||||
return True
|
||||
if not request_headers:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""Utils for accessing credentials."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -27,7 +28,9 @@ class CredentialAccessor:
|
|||
"""Safe accessor for credentials."""
|
||||
|
||||
credential: Final = CredentialAccessor.find_credential(credential_name)
|
||||
return {} if credential is None else dict(credential_api_key_secret_reference(credential.credential_values))
|
||||
if credential is None:
|
||||
return {}
|
||||
return deepcopy(dict(credential_api_key_secret_reference(credential.credential_values)))
|
||||
|
||||
@staticmethod
|
||||
def upsert_credentials(credentials: list[CredentialItem]):
|
||||
|
|
|
|||
|
|
@ -32,8 +32,9 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
|
|||
)
|
||||
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header"
|
||||
AGENTIC_LOOP_TOOL_CHOICE_KEY: Final = "_agentic_loop_tool_choice"
|
||||
AGENTIC_LOOP_CONTROL_KWARGS_KEYS: Final = frozenset(
|
||||
{"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops"}
|
||||
{"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops", AGENTIC_LOOP_TOOL_CHOICE_KEY}
|
||||
)
|
||||
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS: Final = frozenset(
|
||||
{
|
||||
|
|
@ -62,6 +63,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"bucket_name",
|
||||
"s3_bucket_name",
|
||||
"s3_output_bucket_name",
|
||||
"stored_s3_bucket_name",
|
||||
"allow_legacy_cloud_file_ids",
|
||||
"s3_endpoint_url",
|
||||
"s3_region_name",
|
||||
|
|
|
|||
|
|
@ -906,6 +906,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
# Timing for results that cannot carry ``_hidden_params`` (plain-dict /v1/messages
|
||||
# responses and the bridge stream wrappers); see ``update_response_metadata``.
|
||||
self.response_timing_metrics: Mapping[str, float] = {}
|
||||
self.first_chunk_overhead_ms: float | None = None
|
||||
|
||||
# Passthrough endpoint guardrails config for field targeting
|
||||
self.passthrough_guardrails_config: dict[str, object] | None = None
|
||||
|
|
@ -983,6 +984,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
|
||||
self.response_timing_metrics = dict(timing_metrics)
|
||||
|
||||
def set_first_chunk_overhead(self, seconds: float) -> None:
|
||||
"""The proxy's own work on the first chunk of a stream, the delay every later chunk inherits."""
|
||||
if self.first_chunk_overhead_ms is None:
|
||||
self.first_chunk_overhead_ms = seconds * 1000
|
||||
|
||||
def add_dynamic_callback(self, callback: CustomLogger) -> None:
|
||||
self.dynamic_input_callbacks = self._with_dynamic_callback(self.dynamic_input_callbacks, callback)
|
||||
self.dynamic_success_callbacks = self._with_dynamic_callback(self.dynamic_success_callbacks, callback)
|
||||
|
|
@ -7984,8 +7990,15 @@ def get_standard_logging_object_payload(
|
|||
if clean_hidden_params["litellm_overhead_time_ms"] is None and status == "success":
|
||||
# /v1/messages dict results and the bridge stream wrappers keep it on the logging object;
|
||||
# failure payloads stay None like every response type that carries its own _hidden_params
|
||||
timing_metrics: Final = getattr(logging_obj, "response_timing_metrics", None) or {}
|
||||
timing_metrics: Final[Mapping[str, float]] = getattr(logging_obj, "response_timing_metrics", None) or {}
|
||||
clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms")
|
||||
# the overhead above was stamped when the stream object came back, before any chunk was
|
||||
# read; the proxy's work on the first chunk (hook, cost injection, serialization) is the
|
||||
# delay the client sees on every chunk, so only that one is added, never a sum over chunks
|
||||
first_chunk_overhead_ms: Final = getattr(logging_obj, "first_chunk_overhead_ms", None)
|
||||
stamped_overhead_ms: Final = clean_hidden_params["litellm_overhead_time_ms"]
|
||||
if status == "success" and stamped_overhead_ms is not None and isinstance(first_chunk_overhead_ms, float):
|
||||
clean_hidden_params["litellm_overhead_time_ms"] = round(stamped_overhead_ms + first_chunk_overhead_ms, 4)
|
||||
|
||||
model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information(
|
||||
base_model=base_model,
|
||||
|
|
|
|||
|
|
@ -311,6 +311,12 @@ def guardrail_information_cost(guardrail_information: object) -> float:
|
|||
return _validated_entry_cost(guardrail_information)
|
||||
|
||||
|
||||
def guardrail_share_of_spend(spend_log_metadata: Mapping[str, object], spend: float) -> float:
|
||||
"""The part of a spend row's ``spend`` billed for its guardrails, priced the way success and failure rows add
|
||||
it to spend and capped at the row's spend, since an allowance discount or a repeat job poll can charge less."""
|
||||
return min(guardrail_information_cost(spend_log_metadata.get("guardrail_information")), max(spend, 0.0))
|
||||
|
||||
|
||||
def cost_breakdown_with_guardrail(cost_breakdown: CostBreakdown | None, guardrail_cost: float) -> CostBreakdown | None:
|
||||
if guardrail_cost <= 0.0:
|
||||
return cost_breakdown
|
||||
|
|
|
|||
|
|
@ -112,6 +112,7 @@ _BATCH_RATE_PREFIXES: Final = (
|
|||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_read_input_image_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
)
|
||||
_BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values())
|
||||
|
|
@ -326,6 +327,7 @@ class BatchCostRates:
|
|||
cache_read: float | None
|
||||
cache_creation: float | None
|
||||
cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
|
||||
cache_read_image: float | None = None
|
||||
|
||||
|
||||
def _batch_rate(model_info: ModelInfo, key: str) -> float | None:
|
||||
|
|
@ -388,6 +390,7 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide
|
|||
if (rate := _batch_rate_for_prefix(model_info, key, usage, inclusive)) is not None
|
||||
}
|
||||
),
|
||||
cache_read_image=_batch_rate_for_prefix(model_info, "cache_read_input_image_token_cost", usage, inclusive),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -299,7 +299,7 @@ def update_response_metadata(
|
|||
# cannot hold ``_hidden_params``: keep only the timing on the logging object (no cost
|
||||
# recompute) so the proxy headers and the standard logging payload can still read it.
|
||||
logging_obj.set_response_timing_metrics(
|
||||
response_timing_metrics(start_time, end_time, logging_obj, include_overhead)
|
||||
_logging_obj_timing_metrics(start_time, end_time, logging_obj, include_overhead)
|
||||
)
|
||||
return
|
||||
|
||||
|
|
@ -307,3 +307,24 @@ def update_response_metadata(
|
|||
metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead)
|
||||
metadata.set_hidden_params(logging_obj, model, kwargs, cache_hit=cache_hit)
|
||||
metadata.apply()
|
||||
if logging_obj.stream is True:
|
||||
# the response assembled from the chunks never inherits the stream wrapper's
|
||||
# ``_hidden_params``, so the logging payload reads the timing from the logging object
|
||||
logging_obj.set_response_timing_metrics(
|
||||
_logging_obj_timing_metrics(start_time, end_time, logging_obj, include_overhead)
|
||||
)
|
||||
|
||||
|
||||
def _logging_obj_timing_metrics(
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
logging_obj: LiteLLMLoggingObject,
|
||||
include_overhead: bool,
|
||||
) -> Mapping[str, float]:
|
||||
"""A stream read to completion keeps the overhead stamped when the stream was handed back."""
|
||||
timing_metrics: Final = response_timing_metrics(start_time, end_time, logging_obj, include_overhead)
|
||||
stamped: Final[Mapping[str, float]] = getattr(logging_obj, "response_timing_metrics", None) or {}
|
||||
stamped_overhead_ms: Final = stamped.get("litellm_overhead_time_ms")
|
||||
if include_overhead or stamped_overhead_ms is None:
|
||||
return timing_metrics
|
||||
return {**timing_metrics, "litellm_overhead_time_ms": stamped_overhead_ms}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Helper functions to handle images passed in messages
|
|||
import asyncio
|
||||
import base64
|
||||
import time
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
|
|
@ -531,6 +531,29 @@ async def async_convert_urls_to_base64(
|
|||
)
|
||||
|
||||
|
||||
async def _download_outcome(url: str, timeout: float | httpx.Timeout | None) -> "str | litellm.ImageFetchError":
|
||||
try:
|
||||
return await async_convert_url_to_base64(url, timeout=timeout)
|
||||
except litellm.ImageFetchError as error:
|
||||
return error
|
||||
|
||||
|
||||
async def _downloads_through_first_failure(
|
||||
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None
|
||||
) -> "AsyncIterator[str | litellm.ImageFetchError]":
|
||||
for url in remote_urls:
|
||||
yield (outcome := await _download_outcome(url, timeout))
|
||||
if isinstance(outcome, litellm.ImageFetchError):
|
||||
return
|
||||
|
||||
|
||||
async def async_fetch_urls_to_base64(
|
||||
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
|
||||
) -> "Mapping[str, str | litellm.ImageFetchError]":
|
||||
outcomes: Final = tuple([outcome async for outcome in _downloads_through_first_failure(remote_urls, timeout)])
|
||||
return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True)))
|
||||
|
||||
|
||||
def _remote_urls_to_inline(
|
||||
messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool]
|
||||
) -> tuple[str, ...]:
|
||||
|
|
@ -539,6 +562,41 @@ def _remote_urls_to_inline(
|
|||
return tuple(dict.fromkeys(remote.url for remote in remotes if should_inline(_remote_media(remote))))
|
||||
|
||||
|
||||
def _message_parts(messages: Iterable[AllMessageValues]) -> Iterator[object]:
|
||||
return chain.from_iterable(_content_parts(message) for message in messages)
|
||||
|
||||
|
||||
def _file_of_file_part(part: object) -> Mapping[str, object] | None:
|
||||
fields: Final = _as_mapping(part)
|
||||
return _as_mapping(fields.get("file")) if fields is not None and fields.get("type") == "file" else None
|
||||
|
||||
|
||||
def _file_id_url(part: object, url_starts: tuple[str, ...]) -> str | None:
|
||||
file: Final = _file_of_file_part(part)
|
||||
file_id: Final = file.get("file_id") if file is not None else None
|
||||
return file_id if isinstance(file_id, str) and file_id.startswith(url_starts) else None
|
||||
|
||||
|
||||
def _is_readable_file_part_or_other_part(part: object) -> bool:
|
||||
fields: Final = _as_mapping(part)
|
||||
if fields is None or fields.get("type") != "file":
|
||||
return fields is not None
|
||||
file: Final = _file_of_file_part(part)
|
||||
file_id: Final = file.get("file_id") if file is not None else None
|
||||
return file is not None and (not file_id or isinstance(file_id, str))
|
||||
|
||||
|
||||
def has_file_id_urls(messages: Iterable[AllMessageValues], url_starts: tuple[str, ...]) -> bool:
|
||||
return any(_file_id_url(part, url_starts) is not None for part in _message_parts(messages))
|
||||
|
||||
|
||||
def file_id_urls_before_an_unreadable_part(
|
||||
messages: Iterable[AllMessageValues], url_starts: tuple[str, ...]
|
||||
) -> tuple[str, ...]:
|
||||
readable_parts: Final = takewhile(_is_readable_file_part_or_other_part, _message_parts(messages))
|
||||
return tuple(dict.fromkeys(url for part in readable_parts if (url := _file_id_url(part, url_starts)) is not None))
|
||||
|
||||
|
||||
def inline_remote_media(
|
||||
messages: list[AllMessageValues], # mutable-ok: every transform_request takes list[AllMessageValues]
|
||||
should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url,
|
||||
|
|
|
|||
|
|
@ -35,11 +35,14 @@ from operator import itemgetter
|
|||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias, TypeVar
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
|
||||
ANTHROPIC_TOOL_SEARCH_TOOL_NAMES,
|
||||
AnthropicMessagesSystemMessageParam,
|
||||
AnthropicSystemMessageContent,
|
||||
AnthropicToolSearchToolResultBlock,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -299,6 +302,15 @@ def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool:
|
|||
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES
|
||||
|
||||
|
||||
def anthropic_tool_search_result(block: object) -> AnthropicToolSearchToolResultBlock | None:
|
||||
if message_field(block, "type") != "tool_search_tool_result":
|
||||
return None
|
||||
try:
|
||||
return AnthropicToolSearchToolResultBlock.model_validate(block)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def find_anthropic_server_tool_result(
|
||||
tool_id: str,
|
||||
web_search_results: Sequence[object] | None,
|
||||
|
|
|
|||
|
|
@ -282,14 +282,6 @@ def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]:
|
|||
)
|
||||
|
||||
|
||||
def _content_carries_encrypted_reasoning(content: object) -> bool:
|
||||
return any(is_encrypted_reasoning_block(block) for block in _object_sequence(content))
|
||||
|
||||
|
||||
def messages_carry_encrypted_reasoning(messages: Sequence[object]) -> bool:
|
||||
return any(_content_carries_encrypted_reasoning(content) for content in anthropic_content_lists(messages))
|
||||
|
||||
|
||||
def _strip_encrypted_reasoning_from_blocks(
|
||||
content: object,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ import math
|
|||
from collections.abc import Callable
|
||||
from typing import Any, Final
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic_core import to_jsonable_python
|
||||
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
|
||||
|
|
@ -113,3 +114,13 @@ def safe_dumps(
|
|||
return json.dumps(
|
||||
safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str
|
||||
)
|
||||
|
||||
|
||||
_JSONABLE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def jsonable_or_str(value: object) -> object:
|
||||
try:
|
||||
return _JSONABLE.validate_python(to_jsonable_python(value, fallback=str))
|
||||
except ValueError:
|
||||
return str(value)
|
||||
|
|
|
|||
|
|
@ -82,6 +82,8 @@ FUNCTION_CALL_ATTRIBUTE: Final = "function_call"
|
|||
|
||||
_SYNC_ITER_EXHAUSTED: Final = object()
|
||||
|
||||
_STREAMING_CHOICE_KEYS: Final = frozenset({"finish_reason", "index", "delta", "logprobs", "message"})
|
||||
|
||||
_GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__)
|
||||
_CompleteStreamingResponse = ModelResponse | TextCompletionResponse | ModelResponseStream | None
|
||||
_USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset(
|
||||
|
|
@ -139,11 +141,18 @@ def _entry_len(entries: Mapping[str, object], key: str) -> int:
|
|||
|
||||
|
||||
def _prepend_upstream_choice_extras(
|
||||
choice: StreamingChoices, upstream_choice: ChunkChoice, extra_keys: frozenset[str]
|
||||
choice: StreamingChoices, upstream_choice: ChunkChoice | StreamingChoices, extra_keys: frozenset[str]
|
||||
) -> None:
|
||||
if not extra_keys:
|
||||
return
|
||||
upstream_extras: Final[Mapping[str, object]] = upstream_choice.model_dump(include=set(extra_keys))
|
||||
stored_extras: Final = {
|
||||
key: value for key, value in (upstream_choice.__pydantic_extra__ or {}).items() if key in extra_keys
|
||||
}
|
||||
upstream_extras: Final[Mapping[str, object]] = (
|
||||
stored_extras
|
||||
if _holds_only_scalars(stored_extras.items())
|
||||
else upstream_choice.model_dump(include=set(extra_keys))
|
||||
)
|
||||
choice.__pydantic_extra__ = {**upstream_extras, **(choice.__pydantic_extra__ or {})} # rebind-ok: reuse choice
|
||||
choice.__pydantic_fields_set__.update(upstream_extras)
|
||||
|
||||
|
|
@ -439,7 +448,7 @@ class CustomStreamWrapper:
|
|||
self.completion_stream = completion_stream
|
||||
self.sent_first_chunk = False
|
||||
self.sent_last_chunk = False
|
||||
self._openai_handler_receives_chat_chunks = False
|
||||
self._openai_handler_receives_chunk_objects = False
|
||||
self._failure_logging: asyncio.Task[None] | None = None
|
||||
self._closed = False
|
||||
self._stream_created_time: float = time.time()
|
||||
|
|
@ -1162,13 +1171,13 @@ class CustomStreamWrapper:
|
|||
|
||||
def _prebuilt_plain_delta(self, chunk: object) -> Delta | None:
|
||||
if (
|
||||
not self._openai_handler_receives_chat_chunks
|
||||
or not isinstance(chunk, ChatCompletionChunk)
|
||||
not self._openai_handler_receives_chunk_objects
|
||||
or not isinstance(chunk, (ChatCompletionChunk, ModelResponseStream))
|
||||
or not chunk.choices
|
||||
):
|
||||
return None
|
||||
upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent
|
||||
ChoiceDelta | None,
|
||||
ChoiceDelta | Delta | None,
|
||||
chunk.choices[0].delta,
|
||||
)
|
||||
if (
|
||||
|
|
@ -1296,22 +1305,23 @@ class CustomStreamWrapper:
|
|||
upstream_choice: Final = (
|
||||
original_chunk_for_reuse.choices[0]
|
||||
if plain_delta_created
|
||||
and isinstance(original_chunk_for_reuse, ChatCompletionChunk)
|
||||
and isinstance(original_chunk_for_reuse, (ChatCompletionChunk, ModelResponseStream))
|
||||
and len(original_chunk_for_reuse.choices) == 1
|
||||
else None
|
||||
)
|
||||
upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent
|
||||
ChoiceDelta | None,
|
||||
ChoiceDelta | Delta | None,
|
||||
getattr(upstream_choice, "delta", None),
|
||||
)
|
||||
upstream_choice_extra_keys: Final = frozenset(
|
||||
(upstream_choice.__pydantic_extra__ or {}) if upstream_choice is not None else ()
|
||||
)
|
||||
upstream_logprobs: Final[object] = getattr(upstream_choice, "logprobs", None)
|
||||
can_reuse_choice: Final = (
|
||||
upstream_choice is not None
|
||||
and upstream_delta is not None
|
||||
and "enhancements" not in upstream_choice_extra_keys
|
||||
and upstream_choice.logprobs is None
|
||||
and upstream_logprobs is None
|
||||
and _holds_only_scalars(upstream_delta)
|
||||
)
|
||||
if can_reuse_choice and upstream_choice is not None:
|
||||
|
|
@ -1320,7 +1330,7 @@ class CustomStreamWrapper:
|
|||
_prepend_upstream_choice_extras(
|
||||
model_response.choices[0],
|
||||
upstream_choice,
|
||||
upstream_choice_extra_keys - {"message"},
|
||||
upstream_choice_extra_keys - _STREAMING_CHOICE_KEYS,
|
||||
)
|
||||
else:
|
||||
choices: Final = []
|
||||
|
|
@ -1737,14 +1747,16 @@ class CustomStreamWrapper:
|
|||
self._record_finish_choice_fields(cached_chunk)
|
||||
else: # openai / azure chat model
|
||||
self.uses_openai_chat_stream = True
|
||||
if isinstance(chunk, ChatCompletionChunk):
|
||||
self._openai_handler_receives_chat_chunks = True
|
||||
if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
|
||||
raise litellm.InternalServerError(
|
||||
message="Upstream generation aborted (finish_reason=abort)",
|
||||
model=chunk.model,
|
||||
llm_provider=self.custom_llm_provider,
|
||||
)
|
||||
if isinstance(chunk, (ChatCompletionChunk, ModelResponseStream)):
|
||||
self._openai_handler_receives_chunk_objects = True
|
||||
if isinstance(chunk, ChatCompletionChunk) and any(
|
||||
str(choice.finish_reason) == "abort" for choice in chunk.choices or ()
|
||||
):
|
||||
raise litellm.InternalServerError(
|
||||
message="Upstream generation aborted (finish_reason=abort)",
|
||||
model=chunk.model,
|
||||
llm_provider=self.custom_llm_provider,
|
||||
)
|
||||
if self.custom_llm_provider in [
|
||||
LlmProviders.AZURE.value,
|
||||
LlmProviders.AZURE_AI.value,
|
||||
|
|
|
|||
|
|
@ -812,8 +812,9 @@ class ModelResponseIterator:
|
|||
reasoning_content=reasoning_content,
|
||||
speed=self.speed,
|
||||
)
|
||||
if usage.speed is not None:
|
||||
self.speed = usage.speed
|
||||
reported_speed: Final = self._cumulative_usage.get("speed")
|
||||
if isinstance(reported_speed, str):
|
||||
self.speed = reported_speed
|
||||
return usage
|
||||
|
||||
def _content_block_delta_helper(
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.litellm_core_utils.asyncify import is_event_loop_running
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
coerce_token_limit,
|
||||
drops_unsupported_params,
|
||||
map_finish_reason,
|
||||
|
|
@ -151,8 +152,10 @@ from ..common_utils import (
|
|||
requires_native_compaction_beta,
|
||||
stop_reason_provider_specific_fields,
|
||||
strip_advisor_blocks_from_messages,
|
||||
tool_search_result_renamed,
|
||||
tools_marked_for_a_tool_config_point,
|
||||
validated_object_mapping,
|
||||
without_stale_tool_search_results,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -411,6 +414,17 @@ class _AnthropicToolResultBlock(TypedDict, total=False):
|
|||
content: ReadOnly[object]
|
||||
|
||||
|
||||
def _provider_fields_with_tool_names(message: object, forward_map: Mapping[str, str]) -> Mapping[str, object] | None:
|
||||
provider_fields: Final = as_str_mapping(message_field(message, "provider_specific_fields"))
|
||||
replayed_results: Final = parts_of(provider_fields.get("tool_results")) if provider_fields is not None else ()
|
||||
if provider_fields is None or not replayed_results:
|
||||
return None
|
||||
return {
|
||||
**provider_fields,
|
||||
"tool_results": [tool_search_result_renamed(result, forward_map) for result in replayed_results],
|
||||
}
|
||||
|
||||
|
||||
_ENUM_TYPE_CHECKS: Final[Mapping[object, Callable[[object], bool]]] = MappingProxyType(
|
||||
{
|
||||
"null": lambda v: v is None,
|
||||
|
|
@ -457,7 +471,7 @@ def _basic_sanitize_anthropic_tool_name(name: str) -> str:
|
|||
return _ANTHROPIC_TOOL_NAME_INVALID_CHARS.sub("_", name)[:_ANTHROPIC_TOOL_NAME_MAX_LEN]
|
||||
|
||||
|
||||
def _build_anthropic_tool_name_maps(
|
||||
def build_anthropic_tool_name_maps(
|
||||
original_names: list[str],
|
||||
) -> tuple[dict[str, str], dict[str, str]]:
|
||||
"""Build (forward, reverse) tool-name maps for a single request.
|
||||
|
|
@ -1458,8 +1472,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
name_forward_map: dict[str, str],
|
||||
) -> list[AllMessageValues]:
|
||||
"""Return a copy of `messages` with tool_call/function_call names
|
||||
rewritten using the per-request forward map.
|
||||
"""Return a copy of `messages` with tool_call/function_call names, and
|
||||
the tools a replayed tool search found, rewritten using the per-request
|
||||
forward map.
|
||||
|
||||
Only mutates messages whose tool_call/function_call name is *in* the
|
||||
forward map. Names absent from the map (already valid, no collision)
|
||||
|
|
@ -1475,7 +1490,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
continue
|
||||
tool_calls = msg.get("tool_calls")
|
||||
function_call = msg.get("function_call")
|
||||
if not tool_calls and not function_call:
|
||||
renamed_provider_fields = _provider_fields_with_tool_names(msg, name_forward_map)
|
||||
if not tool_calls and not function_call and renamed_provider_fields is None:
|
||||
new_messages.append(msg)
|
||||
continue
|
||||
new_msg = dict(msg)
|
||||
|
|
@ -1501,6 +1517,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
new_fc = dict(function_call)
|
||||
new_fc["name"] = name_forward_map[fc_name]
|
||||
new_msg["function_call"] = new_fc
|
||||
if renamed_provider_fields is not None:
|
||||
new_msg["provider_specific_fields"] = renamed_provider_fields
|
||||
new_messages.append(cast(AllMessageValues, new_msg))
|
||||
return new_messages
|
||||
|
||||
|
|
@ -1517,7 +1535,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
callers that need to compute the maps from the raw OpenAI shape --
|
||||
e.g. test setup or future pre-mapping consumers.
|
||||
|
||||
See _build_anthropic_tool_name_maps for the collision rules. Pulls
|
||||
See build_anthropic_tool_name_maps for the collision rules. Pulls
|
||||
the original name out of either ``{"function": {"name": ...}}``
|
||||
(legacy OpenAI shape) or ``{"name": ...}`` (rare top-level shape).
|
||||
"""
|
||||
|
|
@ -1530,7 +1548,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
original = tool.get("name")
|
||||
if isinstance(original, str) and original:
|
||||
original_names.append(original)
|
||||
return _build_anthropic_tool_name_maps(original_names)
|
||||
return build_anthropic_tool_name_maps(original_names)
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_tool_names_in_request(
|
||||
|
|
@ -1558,7 +1576,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# 1. Collect originals from the Anthropic-shaped custom-tool entries.
|
||||
# Order matters: the first occurrence wins the canonical slot;
|
||||
# later collisions get numeric suffixes (see
|
||||
# ``_build_anthropic_tool_name_maps``).
|
||||
# ``build_anthropic_tool_name_maps``).
|
||||
original_names: Final[list[str]] = []
|
||||
for t in tools:
|
||||
if not isinstance(t, dict):
|
||||
|
|
@ -1572,7 +1590,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if not original_names:
|
||||
return {}, {}
|
||||
|
||||
forward, reverse = _build_anthropic_tool_name_maps(original_names)
|
||||
forward, reverse = build_anthropic_tool_name_maps(original_names)
|
||||
if not forward:
|
||||
# Every name was already valid -- nothing to do.
|
||||
return forward, reverse
|
||||
|
|
@ -2544,7 +2562,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# The reverse map only contains entries for names that were actually
|
||||
# rewritten -- so a tool legitimately named ``foo_bar`` is never
|
||||
# incorrectly retyped to ``foo/bar`` on the response side.
|
||||
# See _build_anthropic_tool_name_maps for the collision-handling
|
||||
# See build_anthropic_tool_name_maps for the collision-handling
|
||||
# rules and rationale.
|
||||
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
|
||||
optional_params=optional_params,
|
||||
|
|
@ -2600,8 +2618,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
|
||||
_all_tools: Final = optional_params.get("tools") or []
|
||||
_has_advisor = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _all_tools)
|
||||
if not _has_advisor:
|
||||
anthropic_messages = strip_advisor_blocks_from_messages(anthropic_messages)
|
||||
anthropic_messages = list(
|
||||
without_stale_tool_search_results(
|
||||
anthropic_messages if _has_advisor else strip_advisor_blocks_from_messages(anthropic_messages),
|
||||
_TOOLS_ADAPTER.validate_python(_all_tools),
|
||||
)
|
||||
)
|
||||
|
||||
## Add code_execution tool if container_upload is in messages
|
||||
_tools: Final = (
|
||||
|
|
@ -3239,7 +3261,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
# Reverse-map rewritten tool names back to caller's originals so a
|
||||
# downstream OpenAI-style dispatcher can match on the registered name.
|
||||
# See _build_anthropic_tool_name_maps for why this is keyed on the
|
||||
# See build_anthropic_tool_name_maps for why this is keyed on the
|
||||
# per-request reverse map (so a tool legitimately named `foo_bar` is
|
||||
# never incorrectly retyped to `foo/bar`). No-op when the map is
|
||||
# empty (the common case).
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
|||
get_file_ids_from_messages,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
|
||||
anthropic_tool_search_result,
|
||||
find_anthropic_server_tool_result,
|
||||
message_field,
|
||||
parts_of,
|
||||
|
|
@ -64,10 +65,12 @@ from litellm.types.llms.anthropic import (
|
|||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
|
||||
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS,
|
||||
AllAnthropicPassThroughMessageValues,
|
||||
AllAnthropicToolsValues,
|
||||
AnthropicMcpServerTool,
|
||||
AnthropicMessagesToolChoice,
|
||||
AnthropicThinkingParam,
|
||||
AnthropicToolSearchToolResultBlock,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -1918,6 +1921,113 @@ def strip_advisor_blocks_from_messages(messages: list[_MessageT], replace_with_t
|
|||
return messages
|
||||
|
||||
|
||||
def _assistant_blocks(message: AllAnthropicPassThroughMessageValues) -> tuple[object, ...]:
|
||||
return parts_of(message["content"]) if message["role"] == "assistant" else ()
|
||||
|
||||
|
||||
def _results_with_unloaded_tools(
|
||||
messages: Sequence[AllAnthropicPassThroughMessageValues], tool_names: frozenset[str]
|
||||
) -> tuple[AnthropicToolSearchToolResultBlock, ...]:
|
||||
blocks: Final = chain.from_iterable(map(_assistant_blocks, messages))
|
||||
results: Final = (result for result in map(anthropic_tool_search_result, blocks) if result is not None)
|
||||
return tuple(result for result in results if not tool_names.issuperset(result.tool_names))
|
||||
|
||||
|
||||
def _is_stale_tool_search_block(block: object, stale_ids: frozenset[str]) -> bool:
|
||||
match message_field(block, "type"):
|
||||
case "server_tool_use":
|
||||
return message_field(block, "id") in stale_ids
|
||||
case "tool_search_tool_result":
|
||||
return message_field(block, "tool_use_id") in stale_ids
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _with_loaded_tool_references_only(block: object, tool_names: frozenset[str]) -> object:
|
||||
result: Final = anthropic_tool_search_result(block)
|
||||
fields: Final = as_str_mapping(block)
|
||||
if result is None or fields is None or tool_names.issuperset(result.tool_names):
|
||||
return block
|
||||
content: Final = as_str_mapping(fields.get("content")) or MappingProxyType({})
|
||||
references: Final = parts_of(content.get("tool_references"))
|
||||
loaded: Final = [reference for reference in references if message_field(reference, "tool_name") in tool_names]
|
||||
return {**fields, "content": {**content, "tool_references": loaded}}
|
||||
|
||||
|
||||
def _with_loaded_tool_searches_only(
|
||||
message: AllAnthropicPassThroughMessageValues, stale_ids: frozenset[str], tool_names: frozenset[str]
|
||||
) -> AllAnthropicPassThroughMessageValues | None:
|
||||
if message["role"] != "assistant":
|
||||
return message
|
||||
content: Final = message["content"]
|
||||
if isinstance(content, str):
|
||||
return message
|
||||
blocks: Final = list(content)
|
||||
kept: Final = [
|
||||
_with_loaded_tool_references_only(block, tool_names)
|
||||
for block in blocks
|
||||
if not _is_stale_tool_search_block(block, stale_ids)
|
||||
]
|
||||
if kept == blocks:
|
||||
return message
|
||||
return {**message, "content": kept} if kept else None
|
||||
|
||||
|
||||
def _tool_reference_renamed(reference: object, name_map: Mapping[str, str]) -> object:
|
||||
fields: Final = as_str_mapping(reference)
|
||||
name: Final = fields.get("tool_name") if fields is not None else None
|
||||
if fields is None or not isinstance(name, str) or name not in name_map:
|
||||
return reference
|
||||
return {**fields, "tool_name": name_map[name]}
|
||||
|
||||
|
||||
def tool_search_content_renamed(content: object, name_map: Mapping[str, str]) -> object:
|
||||
fields: Final = as_str_mapping(content)
|
||||
if fields is None or "tool_references" not in fields:
|
||||
return content
|
||||
references: Final = parts_of(fields["tool_references"])
|
||||
return {**fields, "tool_references": [_tool_reference_renamed(reference, name_map) for reference in references]}
|
||||
|
||||
|
||||
def tool_search_result_renamed(block: object, name_map: Mapping[str, str]) -> object:
|
||||
fields: Final = as_str_mapping(block)
|
||||
if fields is None or fields.get("type") != "tool_search_tool_result":
|
||||
return block
|
||||
return {**fields, "content": tool_search_content_renamed(fields.get("content"), name_map)}
|
||||
|
||||
|
||||
def replays_tool_search(messages: object) -> bool:
|
||||
contents: Final = (message_field(message, "content") for message in parts_of(messages))
|
||||
blocks: Final = chain.from_iterable(map(parts_of, contents))
|
||||
return any(anthropic_tool_search_result(block) is not None for block in blocks)
|
||||
|
||||
|
||||
def without_stale_tool_search_results(
|
||||
messages: Sequence[AllAnthropicPassThroughMessageValues], tools: Sequence[object]
|
||||
) -> tuple[AllAnthropicPassThroughMessageValues, ...]:
|
||||
"""
|
||||
Leave out the tools a replayed tool search found that are no longer in ``tools``,
|
||||
and the whole search, its ``server_tool_use`` and its ``tool_search_tool_result``,
|
||||
when none of them are left. Anthropic rejects such a history with "Tool reference
|
||||
'<name>' not found in available tools", for example after a follow-up turn removes
|
||||
a tool or sends none.
|
||||
"""
|
||||
tool_names: Final = frozenset(name for tool in tools if isinstance(name := message_field(tool, "name"), str))
|
||||
unloaded: Final = _results_with_unloaded_tools(messages, tool_names)
|
||||
if not unloaded:
|
||||
return tuple(messages)
|
||||
litellm.verbose_logger.debug(
|
||||
"Leaving out the tools a replayed tool search found that this request no longer sends: %s",
|
||||
sorted(frozenset(chain.from_iterable(result.tool_names for result in unloaded)) - tool_names),
|
||||
)
|
||||
stale_ids: Final = frozenset(result.tool_use_id for result in unloaded if tool_names.isdisjoint(result.tool_names))
|
||||
return tuple(
|
||||
kept
|
||||
for message in messages
|
||||
if (kept := _with_loaded_tool_searches_only(message, stale_ids, tool_names)) is not None
|
||||
)
|
||||
|
||||
|
||||
def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool:
|
||||
"""
|
||||
Detect Anthropic 400 errors caused by invalid thinking blocks in replayed
|
||||
|
|
|
|||
|
|
@ -125,6 +125,8 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
status_code=e.response.status_code,
|
||||
message=e.response.text,
|
||||
)
|
||||
except litellm.Timeout as e:
|
||||
raise AnthropicError(status_code=e.status_code, message=str(e)) from e
|
||||
except Exception as e:
|
||||
verbose_logger.error("Error in CountTokens handler: %s", e)
|
||||
raise AnthropicError(
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
|
|||
ChatCountTranslationError,
|
||||
anthropic_count_fields,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter, deployment_timeout
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
# Global handler instance - reuse across all token counting requests
|
||||
|
|
@ -134,6 +134,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
optional_params=count_options,
|
||||
context_management=context_management.spec if context_management is not None else None,
|
||||
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
|
||||
timeout=deployment_timeout(litellm_params),
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -1,17 +1,12 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Final,
|
||||
cast, # noqa: TID251 # non_default_params reaches the mapper as an untyped dict
|
||||
)
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from httpx._models import Headers, Response
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_cache_breakpoint_support
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import is_allowed_tools_tool_choice
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import azure_api_version_takes_tool_choice
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
drop_tool_reference_parts_from_tool_messages,
|
||||
|
|
@ -250,17 +245,11 @@ class AzureOpenAIConfig(BaseConfig):
|
|||
if api_version_year is None or api_version_month is None or api_version_day is None:
|
||||
optional_params["tool_choice"] = value
|
||||
else:
|
||||
if (
|
||||
api_version_year < "2023"
|
||||
or (api_version_year == "2023" and api_version_month < "12")
|
||||
or (api_version_year == "2023" and api_version_month == "12" and api_version_day < "01")
|
||||
):
|
||||
if is_allowed_tools_tool_choice(
|
||||
cast(object, value) # cast-ok: widening an untyped dict value to object is always sound
|
||||
) or not (litellm.drop_params is True or (drop_params is not None and drop_params is True)):
|
||||
if not azure_api_version_takes_tool_choice(api_version):
|
||||
if not (litellm.drop_params is True or drop_params is True):
|
||||
raise UnsupportedParamsError(
|
||||
status_code=400,
|
||||
message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. drop_params never drops an allowed_tools tool_choice, since that would let the model call tools the request excluded. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""",
|
||||
message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""",
|
||||
)
|
||||
elif value == "required" and (
|
||||
api_version_year == "2024" and api_version_month <= "05"
|
||||
|
|
|
|||
|
|
@ -147,6 +147,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
status_code=e.response.status_code,
|
||||
message=e.response.text,
|
||||
)
|
||||
except litellm.Timeout as e:
|
||||
raise AnthropicError(status_code=e.status_code, message=str(e)) from e
|
||||
except Exception as e:
|
||||
verbose_logger.error("Error in CountTokens handler: %s", e)
|
||||
raise AnthropicError(
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
|
|||
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
|
||||
AzureAIAnthropicCountTokensHandler,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter, deployment_timeout
|
||||
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
# Global handler instance - reuse across all token counting requests
|
||||
|
|
@ -137,6 +137,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
system=system,
|
||||
optional_params=count_options,
|
||||
context_management=None if context_management is None else context_management.spec,
|
||||
timeout=deployment_timeout(litellm_params),
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@ from dataclasses import dataclass
|
|||
from types import MappingProxyType
|
||||
from typing import Any, Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, JsonValue, PositiveFloat, TypeAdapter, ValidationError
|
||||
from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -38,6 +39,18 @@ from litellm.types.utils import (
|
|||
TokenCountResponse,
|
||||
)
|
||||
|
||||
_SECONDS: Final[TypeAdapter[float]] = TypeAdapter(PositiveFloat)
|
||||
|
||||
|
||||
def deployment_timeout(litellm_params: Mapping[str, object]) -> float | httpx.Timeout | None:
|
||||
timeout: Final = litellm_params.get("timeout")
|
||||
if timeout is None or isinstance(timeout, httpx.Timeout):
|
||||
return timeout
|
||||
try:
|
||||
return _SECONDS.validate_python(timeout)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
class BaseTokenCounter(ABC):
|
||||
@abstractmethod
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import re
|
|||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
|
|
@ -29,6 +30,7 @@ if TYPE_CHECKING:
|
|||
|
||||
RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_NO_RESPONSE_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
class PassthroughMetadata(LiteLLMBaseModel):
|
||||
|
|
@ -236,7 +238,12 @@ class BasePassthroughConfig(BaseLLMModelInfo):
|
|||
return None
|
||||
|
||||
def create_stream_collector(
|
||||
self, model: str, custom_llm_provider: str, endpoint: str
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
endpoint: str,
|
||||
*,
|
||||
response_headers: Mapping[str, str] = _NO_RESPONSE_HEADERS,
|
||||
) -> PassthroughStreamCollector:
|
||||
return RawBytesStreamCollector(
|
||||
provider_config=self, model=model, custom_llm_provider=custom_llm_provider, endpoint=endpoint
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.llms.anthropic.common_utils import (
|
|||
OBJECT_MAPPING_ADAPTER,
|
||||
AnthropicModelInfo,
|
||||
carried_thinking_keys,
|
||||
replays_tool_search,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
AmazonInvokeConfig,
|
||||
|
|
@ -265,6 +266,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
optional_params=optional_params,
|
||||
headers=headers,
|
||||
thinking=_anthropic_request.get("thinking"),
|
||||
history_holds_tool_search=replays_tool_search(
|
||||
OBJECT_MAPPING_ADAPTER.validate_python(_anthropic_request).get("messages")
|
||||
),
|
||||
)
|
||||
if beta_list:
|
||||
_anthropic_request["anthropic_beta"] = beta_list
|
||||
|
|
@ -397,9 +401,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
optional_params: dict,
|
||||
headers: dict,
|
||||
thinking: AnthropicThinkingParam | None,
|
||||
history_holds_tool_search: bool,
|
||||
) -> list[str]:
|
||||
tools: Final = optional_params.get("tools")
|
||||
tool_search_used: Final = self.is_tool_search_used(tools)
|
||||
tool_search_used: Final = self.is_tool_search_used(tools) or history_holds_tool_search
|
||||
programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools)
|
||||
input_examples_used: Final = self.is_input_examples_used(tools)
|
||||
is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
|
|||
BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
|
||||
BEDROCK_MANAGED_S3_PREFIXES,
|
||||
BEDROCK_MANAGED_S3_UPLOAD_PREFIX,
|
||||
STORED_S3_BUCKET_PARAM,
|
||||
build_managed_cloud_object_name,
|
||||
encode_s3_object_key_for_url,
|
||||
sanitize_cloud_object_component,
|
||||
|
|
@ -313,11 +314,19 @@ def get_configured_s3_bucket_names(litellm_params: Mapping[str, object]) -> tupl
|
|||
Resolve the S3 buckets a Bedrock file id may live in.
|
||||
|
||||
Bedrock batch outputs land in ``s3_output_bucket_name`` when it differs from
|
||||
the input bucket, so retrieval validates against both.
|
||||
the input bucket, so retrieval validates against both, plus the bucket LiteLLM
|
||||
recorded for a managed file uploaded or written to a request-chosen bucket.
|
||||
"""
|
||||
input_bucket: Final = _input_bucket_name(litellm_params)
|
||||
output_bucket: Final = _output_bucket_name(litellm_params)
|
||||
buckets: Final = tuple(dict.fromkeys(bucket for bucket in (input_bucket, output_bucket) if bucket))
|
||||
stored_bucket: Final = litellm_params.get(STORED_S3_BUCKET_PARAM)
|
||||
buckets: Final = tuple(
|
||||
dict.fromkeys(
|
||||
bucket
|
||||
for bucket in (input_bucket, output_bucket, stored_bucket if isinstance(stored_bucket, str) else None)
|
||||
if bucket
|
||||
)
|
||||
)
|
||||
if not buckets:
|
||||
raise ValueError(_S3_BUCKET_REQUIRED_ERROR)
|
||||
return buckets
|
||||
|
|
|
|||
|
|
@ -3,16 +3,21 @@ from collections.abc import Callable, Iterator, Mapping, Sequence
|
|||
from functools import partial
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, ClassVar, Final, Optional, cast
|
||||
from typing import TYPE_CHECKING, ClassVar, Final, Optional, TypeVar, cast
|
||||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import AWS_ERROR_TYPE_HEADER
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
|
||||
from litellm.llms.base_llm.passthrough.transformation import (
|
||||
BasePassthroughConfig,
|
||||
PassthroughStreamCollector,
|
||||
relayed_json_object,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.converse_to_openai import as_json_array, as_json_object
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
bedrock_invoke_guardrail_headers,
|
||||
|
|
@ -21,7 +26,12 @@ from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletio
|
|||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM
|
||||
from ..common_utils import BedrockError, BedrockEventStreamDecoderBase, BedrockModelInfo
|
||||
from ..common_utils import (
|
||||
BEDROCK_INVOKE_TRACE_FIELD,
|
||||
BedrockError,
|
||||
BedrockEventStreamDecoderBase,
|
||||
BedrockModelInfo,
|
||||
)
|
||||
from ..request_metadata import (
|
||||
BEDROCK_REQUEST_METADATA_HEADER,
|
||||
BedrockRequestMetadataViolation,
|
||||
|
|
@ -49,6 +59,9 @@ _BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stre
|
|||
_BEDROCK_INVOKE_GUARDRAIL_HEADERS: Final = frozenset(
|
||||
{"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"}
|
||||
)
|
||||
_BEDROCK_SERVICE_TIER_HEADER: Final = "x-amzn-bedrock-service-tier"
|
||||
_NO_RESPONSE_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
_TieredResponse: Final = TypeVar("_TieredResponse", bound=BaseModel)
|
||||
|
||||
|
||||
def _bedrock_endpoint_action(endpoint: str) -> str:
|
||||
|
|
@ -111,7 +124,9 @@ def _with_request_metadata(
|
|||
litellm_params: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, str], Mapping[str, object]]:
|
||||
if action in _BEDROCK_CONVERSE_ACTIONS:
|
||||
return headers, _with_body_request_metadata(action, body, litellm_params)
|
||||
return _without_owned_request_metadata_header(headers), _with_body_request_metadata(
|
||||
action, body, litellm_params
|
||||
)
|
||||
if action in _BEDROCK_INVOKE_ACTIONS and "requestMetadata" in body:
|
||||
return _without_owned_request_metadata_header(headers), _with_body_request_metadata(
|
||||
action, body, litellm_params
|
||||
|
|
@ -137,6 +152,10 @@ def _converse_guardrail_config(action: str, guardrail_config: object) -> object:
|
|||
return {key: value for key, value in fields.items() if key != "streamProcessingMode"}
|
||||
|
||||
|
||||
def _without_invoke_guardrail_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
||||
return {name: value for name, value in headers.items() if name.lower() not in _BEDROCK_INVOKE_GUARDRAIL_HEADERS}
|
||||
|
||||
|
||||
def _with_deployment_guardrail(
|
||||
action: str,
|
||||
headers: Mapping[str, str],
|
||||
|
|
@ -146,7 +165,10 @@ def _with_deployment_guardrail(
|
|||
if guardrail_config is None:
|
||||
return headers, body
|
||||
if action in _BEDROCK_CONVERSE_ACTIONS:
|
||||
return headers, {**body, "guardrailConfig": _converse_guardrail_config(action, guardrail_config)}
|
||||
return _without_invoke_guardrail_headers(headers), {
|
||||
**body,
|
||||
"guardrailConfig": _converse_guardrail_config(action, guardrail_config),
|
||||
}
|
||||
if action in _BEDROCK_INVOKE_ACTIONS:
|
||||
guardrail_headers: Final = tuple(bedrock_invoke_guardrail_headers(guardrail_config).items())
|
||||
return merge_bedrock_invoke_headers(
|
||||
|
|
@ -266,6 +288,19 @@ class _CoalescedChunks:
|
|||
return self._chunks
|
||||
|
||||
|
||||
_BEDROCK_EVENT: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _with_invoke_trace(chunk: ModelResponseStream, trace: Mapping[str, object] | None) -> ModelResponseStream:
|
||||
if trace is None:
|
||||
return chunk
|
||||
return chunk.model_copy(
|
||||
update={
|
||||
"provider_specific_fields": {**(chunk.provider_specific_fields or {}), BEDROCK_INVOKE_TRACE_FIELD: trace}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelResponseStream | None:
|
||||
from litellm.litellm_core_utils.streaming_handler import (
|
||||
convert_generic_chunk_to_model_response_stream,
|
||||
|
|
@ -273,11 +308,15 @@ def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelR
|
|||
)
|
||||
from litellm.types.utils import GenericStreamingChunk
|
||||
|
||||
translated_chunk: Final = decoder.chunk_parser(chunk_data=json.loads(message))
|
||||
event: Final = _BEDROCK_EVENT.validate_json(message)
|
||||
trace: Final = object_mapping(event.get(BEDROCK_INVOKE_TRACE_FIELD))
|
||||
translated_chunk: Final = decoder.chunk_parser(chunk_data=event)
|
||||
if isinstance(translated_chunk, ModelResponseStream):
|
||||
return translated_chunk
|
||||
return _with_invoke_trace(translated_chunk, trace)
|
||||
if generic_chunk_has_all_required_fields(cast(dict, translated_chunk)):
|
||||
return convert_generic_chunk_to_model_response_stream(cast(GenericStreamingChunk, translated_chunk))
|
||||
return _with_invoke_trace(
|
||||
convert_generic_chunk_to_model_response_stream(cast(GenericStreamingChunk, translated_chunk)), trace
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -332,15 +371,33 @@ def _count_prompt_tokens(model: str, request_body: object) -> int:
|
|||
return token_counter(model=model, messages=messages)
|
||||
|
||||
|
||||
def _invoke_guardrail_trace(httpx_response: Response) -> Mapping[str, object]:
|
||||
reply: Final = relayed_json_object(httpx_response) or {}
|
||||
return {BEDROCK_INVOKE_TRACE_FIELD: reply.get(BEDROCK_INVOKE_TRACE_FIELD)}
|
||||
|
||||
|
||||
def _served_service_tier(response_headers: Mapping[str, str]) -> str | None:
|
||||
headers: Final = httpx.Headers(response_headers)
|
||||
if _BEDROCK_SERVICE_TIER_HEADER not in headers:
|
||||
return None
|
||||
return headers[_BEDROCK_SERVICE_TIER_HEADER]
|
||||
|
||||
|
||||
def _with_served_service_tier(response: _TieredResponse, service_tier: str | None) -> _TieredResponse:
|
||||
if service_tier is None:
|
||||
return response
|
||||
return response.model_copy(update={"service_tier": service_tier})
|
||||
|
||||
|
||||
def _build_logged_response(
|
||||
chunks: Sequence[ModelResponseStream], litellm_logging_obj: "LiteLLMLoggingObj"
|
||||
chunks: Sequence[ModelResponseStream], litellm_logging_obj: "LiteLLMLoggingObj", service_tier: str | None
|
||||
) -> Optional["CostResponseTypes"]:
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
|
||||
|
||||
if len(chunks) == 0:
|
||||
return None
|
||||
return stream_chunk_builder(
|
||||
chunks=list(chunks),
|
||||
chunks=[*chunks[:-1], _with_served_service_tier(chunks[-1], service_tier)],
|
||||
logging_obj=litellm_logging_obj,
|
||||
count_prompt_tokens=partial(
|
||||
_count_prompt_tokens, litellm_logging_obj.model, litellm_logging_obj.model_call_details.get("input")
|
||||
|
|
@ -355,11 +412,13 @@ class BedrockEventStreamCollector:
|
|||
self,
|
||||
parse_event: Callable[["EventStreamMessage"], str | None],
|
||||
decoder: Optional["AWSEventStreamDecoder"],
|
||||
service_tier: str | None = None,
|
||||
) -> None:
|
||||
from botocore.eventstream import EventStreamBuffer
|
||||
|
||||
self._parse_event = parse_event
|
||||
self._decoder = decoder
|
||||
self._service_tier: Final = service_tier
|
||||
self._event_stream_buffer: Final[EventStreamBuffer] = EventStreamBuffer()
|
||||
self._chunks: Final = _CoalescedChunks()
|
||||
|
||||
|
|
@ -377,7 +436,7 @@ class BedrockEventStreamCollector:
|
|||
self._chunks.add(translated)
|
||||
|
||||
def build_logged_response(self, litellm_logging_obj: "LiteLLMLoggingObj") -> Optional["CostResponseTypes"]:
|
||||
return _build_logged_response(self._chunks.chunks(), litellm_logging_obj)
|
||||
return _build_logged_response(self._chunks.chunks(), litellm_logging_obj, self._service_tier)
|
||||
|
||||
|
||||
class BedrockAgentRuntimeAWS(BaseAWSLLM):
|
||||
|
|
@ -542,15 +601,23 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
|
|||
request_data=request_data,
|
||||
encoding=encoding,
|
||||
)
|
||||
if "invoke" in endpoint:
|
||||
logging_obj.record_native_bedrock_guardrail(_invoke_guardrail_trace(httpx_response))
|
||||
|
||||
return litellm_model_response
|
||||
return _with_served_service_tier(litellm_model_response, _served_service_tier(httpx_response.headers))
|
||||
|
||||
def create_stream_collector(
|
||||
self, model: str, custom_llm_provider: str, endpoint: str
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
endpoint: str,
|
||||
*,
|
||||
response_headers: Mapping[str, str] = _NO_RESPONSE_HEADERS,
|
||||
) -> PassthroughStreamCollector:
|
||||
return BedrockEventStreamCollector(
|
||||
parse_event=self._parse_message_from_event,
|
||||
decoder=self._get_event_stream_decoder(model=model, endpoint=endpoint),
|
||||
service_tier=_served_service_tier(response_headers),
|
||||
)
|
||||
|
||||
def _get_event_stream_decoder(self, model: str, endpoint: str) -> Optional["AWSEventStreamDecoder"]:
|
||||
|
|
|
|||
|
|
@ -663,7 +663,9 @@ class HTTPResponseLimitError(ValueError):
|
|||
|
||||
|
||||
class MaskedHTTPStatusError(httpx.HTTPStatusError):
|
||||
def __init__(self, original_error: httpx.HTTPStatusError, message: str | None = None, text: str | None = None):
|
||||
def __init__(
|
||||
self, original_error: httpx.HTTPStatusError, message: str | None = None, text: str | None = None
|
||||
) -> None:
|
||||
# Create a new error with the masked URL
|
||||
masked_url: Final = mask_sensitive_info(str(original_error.request.url))
|
||||
# Mask the original exception message too (it contains the full URL)
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAG
|
|||
from litellm.files.types import FileContentStreamingResult
|
||||
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
|
||||
from litellm.litellm_core_utils.agentic_loop_refusal import (
|
||||
called_tool_names,
|
||||
every_claimed_tool_call,
|
||||
finalize_refused_responses_turn,
|
||||
is_refused_item,
|
||||
|
|
@ -64,7 +65,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
|
|||
)
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
|
||||
bill_first_round_on_failure,
|
||||
holds_a_call_the_choice_excludes,
|
||||
run_chat_completion_agentic_hooks,
|
||||
with_caller_tool_choice,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
||||
|
|
@ -160,6 +163,7 @@ from litellm.types.integrations.custom_logger import (
|
|||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
AgenticLoopSafetyError,
|
||||
AgenticLoopToolChoiceError,
|
||||
converted_stream_requested,
|
||||
is_interception_internal_key,
|
||||
)
|
||||
|
|
@ -268,7 +272,11 @@ if TYPE_CHECKING:
|
|||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission
|
||||
from litellm.responses.streaming_iterator import (
|
||||
ResponsesWsFrameGuard,
|
||||
ResponsesWsTurnAdmission,
|
||||
ResponsesWsTurnUsageRecorder,
|
||||
)
|
||||
from litellm.types.google_genai.main import GenerateContentResponse
|
||||
from litellm.types.llms.openai_evals import (
|
||||
CancelEvalResponse,
|
||||
|
|
@ -6402,7 +6410,13 @@ class BaseLLMHTTPHandler:
|
|||
tools: Final = anthropic_messages_optional_request_params.get("tools", [])
|
||||
depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs)
|
||||
|
||||
hook_kwargs: Final = {**kwargs, "_agentic_loop_api_surface": api_surface}
|
||||
hook_kwargs: Final = {
|
||||
**with_caller_tool_choice(
|
||||
_OPTIONAL_PARAMS.validate_python(kwargs),
|
||||
_OPTIONAL_PARAMS.validate_python(anthropic_messages_optional_request_params),
|
||||
),
|
||||
"_agentic_loop_api_surface": api_surface,
|
||||
}
|
||||
|
||||
for callback in callbacks:
|
||||
if not isinstance(callback, CustomLogger):
|
||||
|
|
@ -6441,6 +6455,10 @@ class BaseLLMHTTPHandler:
|
|||
# Safety guards must run OUTSIDE the callback try/except — they are
|
||||
# bounded-loop / cycle-break rails, not callback bugs.
|
||||
try:
|
||||
if holds_a_call_the_choice_excludes(
|
||||
called_tool_names(response), hook_kwargs, (cb for cb in callbacks if isinstance(cb, CustomLogger))
|
||||
):
|
||||
raise AgenticLoopToolChoiceError(model)
|
||||
fingerprint = self._check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
|
|
@ -7241,6 +7259,7 @@ class BaseLLMHTTPHandler:
|
|||
first_message: str | None = None,
|
||||
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
|
||||
turn_admission: "ResponsesWsTurnAdmission | None" = None,
|
||||
turn_usage_recorder: "ResponsesWsTurnUsageRecorder | None" = None,
|
||||
frame_guard: "ResponsesWsFrameGuard | None" = None,
|
||||
**kwargs: object,
|
||||
) -> Exception | None:
|
||||
|
|
@ -7407,6 +7426,7 @@ class BaseLLMHTTPHandler:
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
request_defaults=request_defaults,
|
||||
turn_admission=_ws_turn_admission,
|
||||
turn_usage_recorder=turn_usage_recorder,
|
||||
frame_guard=frame_guard,
|
||||
)
|
||||
return await streaming.bidirectional_forward()
|
||||
|
|
|
|||
|
|
@ -34,6 +34,9 @@ def _warn_on_dropped_budget(thinking: object, model: str) -> None:
|
|||
|
||||
|
||||
class DeepSeekChatConfig(OpenAIGPTConfig):
|
||||
def _flattens_content_lists(self) -> bool:
|
||||
return True
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
"""
|
||||
DeepSeek reasoner models support thinking parameter.
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ class DockerModelRunnerChatConfig(OpenAIGPTConfig):
|
|||
The engine name (e.g., "llama.cpp") is part of the API endpoint path.
|
||||
"""
|
||||
|
||||
def _flattens_content_lists(self) -> bool:
|
||||
return True
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
self, messages: list[AllMessageValues], model: str, is_async: Literal[True]
|
||||
|
|
|
|||
|
|
@ -170,7 +170,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
|
|||
return request_body_obj.model_dump(exclude_none=True)
|
||||
|
||||
def _transform_image_usage(self, usage_metadata: dict):
|
||||
return transform_gemini_image_usage(usage_metadata)
|
||||
return transform_gemini_image_usage(_JSON_DICT.validate_python(usage_metadata))
|
||||
|
||||
def transform_image_generation_response(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.utils import ImageUsage, ImageUsageInputTokensDetails
|
||||
|
||||
|
||||
|
|
@ -34,12 +36,12 @@ def _sum_modality_token_details(usage_metadata: dict, *details_keys: str) -> Ima
|
|||
return tokens_details
|
||||
|
||||
|
||||
def transform_gemini_image_usage(usage_metadata: dict) -> ImageUsage:
|
||||
def transform_gemini_image_usage(usage_metadata: dict[str, object]) -> ImageUsage:
|
||||
"""
|
||||
Transform Gemini usageMetadata to ImageUsage format.
|
||||
"""
|
||||
input_tokens_details = _sum_modality_token_details(usage_metadata, "promptTokensDetails", "prompt_tokens_details")
|
||||
output_tokens: Final = usage_metadata.get("candidatesTokenCount", 0)
|
||||
output_tokens: Final = TypeAdapter(int).validate_python(usage_metadata.get("candidatesTokenCount", 0))
|
||||
output_tokens_details: Final = _sum_modality_token_details(
|
||||
usage_metadata, "candidatesTokensDetails", "candidates_tokens_details"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ class HerokuError(Exception):
|
|||
|
||||
|
||||
class HerokuChatConfig(OpenAIGPTConfig):
|
||||
def _flattens_content_lists(self) -> bool:
|
||||
return True
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
self, messages: list[AllMessageValues], model: str, is_async: Literal[True]
|
||||
|
|
|
|||
|
|
@ -87,25 +87,31 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
_tools = non_default_params.pop("tools", None)
|
||||
params: Final = dict[str, object](non_default_params)
|
||||
mapped_params: Final = {
|
||||
key: value
|
||||
for key, value in params.items()
|
||||
if params.get("tools") != [] or key not in ("tools", "tool_choice", "parallel_tool_calls")
|
||||
}
|
||||
_tools = mapped_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportArgumentType] # tools arrive untyped through the bare-dict params
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if isinstance(_tools, list):
|
||||
_tools = self._convert_custom_tools_to_function_tools(_tools)
|
||||
if _tools is not None:
|
||||
non_default_params["tools"] = _tools
|
||||
mapped_params["tools"] = _tools
|
||||
|
||||
thinking: Final = non_default_params.pop("thinking", None)
|
||||
thinking: Final = mapped_params.pop("thinking", None)
|
||||
thinking_effort: Final = (
|
||||
None
|
||||
if "reasoning_effort" in non_default_params
|
||||
if "reasoning_effort" in mapped_params
|
||||
else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
|
||||
)
|
||||
if thinking_effort is not None:
|
||||
non_default_params["reasoning_effort"] = thinking_effort
|
||||
mapped_params["reasoning_effort"] = thinking_effort
|
||||
|
||||
return super().map_openai_params(non_default_params, optional_params, model, drop_params)
|
||||
return super().map_openai_params(mapped_params, optional_params, model, drop_params)
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self, api_base: str | None, api_key: str | None
|
||||
|
|
|
|||
|
|
@ -142,8 +142,6 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
|
||||
"""
|
||||
|
||||
_is_base_class = False
|
||||
|
||||
temperature: int | None = None
|
||||
top_p: int | None = None
|
||||
max_tokens: int | None = None
|
||||
|
|
|
|||
|
|
@ -2,9 +2,11 @@
|
|||
Support for gpt model family
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
|
||||
from contextvars import ContextVar
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload
|
||||
from urllib.parse import urlparse
|
||||
|
|
@ -28,7 +30,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
async_convert_url_to_base64,
|
||||
async_fetch_urls_to_base64,
|
||||
convert_url_to_base64,
|
||||
file_id_urls_before_an_unreadable_part,
|
||||
has_file_id_urls,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
|
||||
drop_non_python_regex_patterns,
|
||||
|
|
@ -85,6 +90,20 @@ else:
|
|||
|
||||
_NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = ("user", "tool")
|
||||
_PDF_URL_STARTS: Final = ("https://", "http://", "www.")
|
||||
_NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({})
|
||||
_prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar(
|
||||
"prefetched_pdf_urls"
|
||||
)
|
||||
|
||||
|
||||
def _pdf_data_url(url: str) -> str:
|
||||
prefetched: Final = _prefetched_pdf_urls.get(_NOTHING_PREFETCHED).get(url)
|
||||
if isinstance(prefetched, litellm.ImageFetchError):
|
||||
raise prefetched
|
||||
return prefetched if prefetched is not None else convert_url_to_base64(url)
|
||||
|
||||
|
||||
def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool:
|
||||
if custom_llm_provider != "openai":
|
||||
|
|
@ -191,9 +210,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
- `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling.
|
||||
"""
|
||||
|
||||
# Add a class variable to track if this is the base class
|
||||
_is_base_class = True
|
||||
|
||||
frequency_penalty: int | None = None
|
||||
function_call: str | dict | None = None
|
||||
functions: list | None = None
|
||||
|
|
@ -225,8 +241,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
||||
self.__class__._is_base_class = False
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
return super().get_config()
|
||||
|
|
@ -322,9 +336,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
)
|
||||
|
||||
def contains_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> bool:
|
||||
potential_pdf_url_starts: Final = ["https://", "http://", "www."]
|
||||
file_id: Final = content_item.get("file_id")
|
||||
if file_id and any(file_id.startswith(start) for start in potential_pdf_url_starts):
|
||||
if file_id and any(file_id.startswith(start) for start in _PDF_URL_STARTS):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
|
@ -332,7 +345,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
content_copy: Final = content_item.copy()
|
||||
file_id: Final = content_copy.get("file_id")
|
||||
if file_id is not None:
|
||||
base64_data: Final = convert_url_to_base64(file_id)
|
||||
base64_data: Final = _pdf_data_url(file_id)
|
||||
content_copy["file_data"] = base64_data
|
||||
content_copy["filename"] = "my_file.pdf"
|
||||
content_copy.pop("file_id")
|
||||
|
|
@ -688,6 +701,12 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
**self._sanitized_tools_update_for_openai(optional_params, litellm_params),
|
||||
}
|
||||
|
||||
def _flattens_content_lists(self) -> bool:
|
||||
return False
|
||||
|
||||
def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]:
|
||||
return messages
|
||||
|
||||
async def async_transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -696,30 +715,25 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
transformed_messages, tools = self._messages_and_tools_in_cache_mark_dialect(
|
||||
model=model,
|
||||
messages=await self._transform_messages(
|
||||
messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True
|
||||
),
|
||||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
if self.__class__._is_base_class:
|
||||
if not optional_params.get("tools") and not optional_params.get("functions"):
|
||||
optional_params.pop("tool_choice", None)
|
||||
return {
|
||||
"model": model,
|
||||
"messages": transformed_messages,
|
||||
**optional_params,
|
||||
**self._sanitized_tools_update_for_openai(optional_params, litellm_params),
|
||||
}
|
||||
else:
|
||||
## allow for any object specific behaviour to be handled
|
||||
if not has_file_id_urls(messages, _PDF_URL_STARTS):
|
||||
return self.transform_request(model, messages, optional_params, litellm_params, headers)
|
||||
sent_messages: Final = self._with_provider_roles(
|
||||
self._prompt_cache_ordered_messages(messages, litellm_params), model
|
||||
)
|
||||
pdf_urls: Final = file_id_urls_before_an_unreadable_part(
|
||||
(message for message in sent_messages if message.get("role") in _ROLES_WHOSE_FILES_ARE_DOWNLOADED),
|
||||
_PDF_URL_STARTS,
|
||||
)
|
||||
prefetched: Final = (
|
||||
_NOTHING_PREFETCHED if self._flattens_content_lists() else await async_fetch_urls_to_base64(pdf_urls)
|
||||
)
|
||||
reset_token: Final = _prefetched_pdf_urls.set(prefetched)
|
||||
try:
|
||||
return await asyncio.to_thread(
|
||||
self.transform_request, model, messages, optional_params, litellm_params, headers
|
||||
)
|
||||
finally:
|
||||
_prefetched_pdf_urls.reset(reset_token)
|
||||
|
||||
def _passed_in_tools(self, optional_params: dict) -> bool:
|
||||
return optional_params.get("tools", None) is not None
|
||||
|
|
|
|||
|
|
@ -122,6 +122,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
|
|||
|
||||
return super()._map_openai_params(non_default_params, optional_params, model, drop_params)
|
||||
|
||||
def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]:
|
||||
keeps_system_messages: Final = supports_system_messages(model, "openai")
|
||||
return tuple(
|
||||
message
|
||||
if keeps_system_messages or message["role"] != "system"
|
||||
else ChatCompletionUserMessage(content=message["content"], role="user")
|
||||
for message in messages
|
||||
)
|
||||
|
||||
def is_model_o_series_model(self, model: str) -> bool:
|
||||
model = model.split("/")[-1] # could be "openai/o3" or "o3"
|
||||
return (
|
||||
|
|
@ -152,13 +161,7 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
|
|||
- modalities: image => drop param (if user opts in to dropping param)
|
||||
- role: system ==> translate to role 'user'
|
||||
"""
|
||||
_supports_system_messages: Final = supports_system_messages(model, "openai")
|
||||
transformed_messages: Final = list(messages)
|
||||
for i, message in enumerate(transformed_messages):
|
||||
if message["role"] == "system" and not _supports_system_messages:
|
||||
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
|
||||
transformed_messages[i] = new_message # Replace the old message with the new one
|
||||
|
||||
transformed_messages: Final = list(self._with_provider_roles(messages, model))
|
||||
if is_async:
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1166,7 +1166,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
shared_session: Optional["ClientSession"] = None,
|
||||
):
|
||||
response = None
|
||||
data = provider_config.transform_request(
|
||||
data = await provider_config.async_transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
|
|
|
|||
|
|
@ -111,6 +111,8 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig):
|
|||
status_code=e.response.status_code,
|
||||
message=e.response.text,
|
||||
)
|
||||
except litellm.Timeout as e:
|
||||
raise OpenAIError(status_code=e.status_code, message=str(e)) from e
|
||||
except (httpx.RequestError, json.JSONDecodeError, ValueError) as e:
|
||||
verbose_logger.error("Error in CountTokens handler: %s", e)
|
||||
raise OpenAIError(
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from dataclasses import dataclass
|
|||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
from pydantic import JsonValue
|
||||
from typing_extensions import assert_never
|
||||
|
|
@ -21,7 +22,7 @@ from litellm.llms.anthropic.messages.responses_adapters.handler import count_vie
|
|||
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
|
||||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter, deployment_timeout
|
||||
from litellm.llms.openai.common_utils import OpenAIError, decoded_json_object, get_openai_credentials
|
||||
from litellm.llms.openai.responses.count_tokens.handler import (
|
||||
OpenAICountTokensHandler,
|
||||
|
|
@ -61,6 +62,7 @@ def _workload_identity_config(
|
|||
class _CountCredential:
|
||||
api_base: str | None
|
||||
credential: str | OpenAIWorkloadIdentityConfig
|
||||
timeout: float | httpx.Timeout | None
|
||||
|
||||
|
||||
def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError:
|
||||
|
|
@ -82,7 +84,9 @@ def _count_credential(deployment: Mapping[str, object] | None, model_label: str)
|
|||
"workload identity on the deployment, or OPENAI_API_KEY in the proxy environment."
|
||||
),
|
||||
)
|
||||
return _CountCredential(api_base=credentials.api_base, credential=credential)
|
||||
return _CountCredential(
|
||||
api_base=credentials.api_base, credential=credential, timeout=deployment_timeout(litellm_params)
|
||||
)
|
||||
|
||||
|
||||
def _count_error(model_to_use: str, request_model: str, message: str, status_code: int) -> TokenCountResponse:
|
||||
|
|
@ -245,6 +249,7 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
tools=counted_tools,
|
||||
instructions=instructions,
|
||||
text=text,
|
||||
timeout=resolved.timeout,
|
||||
),
|
||||
model_to_use,
|
||||
request_model,
|
||||
|
|
@ -264,7 +269,9 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
body: Final = native_count_request(request, model_to_use, policy)
|
||||
return await self._counted(
|
||||
resolved,
|
||||
lambda api_key, api_base: openai_count_tokens_handler.post_count_request(body, api_key, api_base),
|
||||
lambda api_key, api_base: openai_count_tokens_handler.post_count_request(
|
||||
body, api_key, api_base, resolved.timeout
|
||||
),
|
||||
model_to_use,
|
||||
request_model,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -111,7 +111,6 @@ from litellm.types.llms.openai import (
|
|||
ResponseAPIUsage,
|
||||
ResponseCompactionItem,
|
||||
ResponseCompletedEvent,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
ResponsesAPIStreamingResponse,
|
||||
|
|
@ -119,6 +118,7 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.responses.main import (
|
||||
GenericResponseOutputItem,
|
||||
OutputText,
|
||||
OutputToolSearchOutput,
|
||||
)
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
|
|
@ -276,6 +276,7 @@ _ITEM_ID_KEYED_TOOL_CALL_TYPES: Final = frozenset({_TOOL_SEARCH_CALL_TYPE, _WEB_
|
|||
_OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE}
|
||||
_SCANNED_TOOL_SEARCH_NAME: Final = "tool_search"
|
||||
_TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_RAW_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...])
|
||||
_STREAM_EVENT_TYPE_ADAPTER: Final = TypeAdapter(str)
|
||||
_TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"function_call": "arguments", "custom_tool_call": "input"}
|
||||
|
|
@ -297,8 +298,6 @@ _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
|||
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
|
||||
)
|
||||
|
||||
_EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {}
|
||||
|
||||
|
||||
def _item_rewrite_field(item: Mapping[str, object]) -> str | None:
|
||||
item_type: Final = item.get("type")
|
||||
|
|
@ -343,6 +342,18 @@ def _is_tool_call_item(item: object) -> bool:
|
|||
return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES
|
||||
|
||||
|
||||
def _is_tool_search_item(item: object) -> bool:
|
||||
match item:
|
||||
case {"type": "tool_search_call" | "tool_search_output"}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _joins_assistant_turn(item: object) -> bool:
|
||||
return _is_tool_call_item(item) or _is_tool_search_item(item)
|
||||
|
||||
|
||||
def _claude_web_search_tool_call(item: object) -> ChatCompletionAssistantToolCall | None:
|
||||
search: Final = web_search_call(item)
|
||||
return None if search is None else claude_web_search_call_tool_call(search)
|
||||
|
|
@ -490,14 +501,17 @@ def _provenance_unit_bounds(
|
|||
) -> tuple[tuple[int, int], ...]:
|
||||
trailing_roles: Final = tuple(
|
||||
accumulate(
|
||||
(_last_message_role(messages) for messages in solo_conversions),
|
||||
(
|
||||
"assistant" if _is_tool_search_item(item) else _last_message_role(messages)
|
||||
for item, messages in zip(raw_input, solo_conversions)
|
||||
),
|
||||
lambda previous, current: current if current is not None else previous,
|
||||
)
|
||||
)
|
||||
start_indexes: Final = tuple(
|
||||
index
|
||||
for index in range(len(raw_input))
|
||||
if index == 0 or not (_is_tool_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant")
|
||||
if index == 0 or not (_joins_assistant_turn(raw_input[index]) and trailing_roles[index - 1] == "assistant")
|
||||
)
|
||||
return tuple(zip(start_indexes, (*start_indexes[1:], len(raw_input))))
|
||||
|
||||
|
|
@ -505,20 +519,24 @@ def _provenance_unit_bounds(
|
|||
def _input_item_provenance(
|
||||
raw_input: Sequence[object],
|
||||
expected_messages: Sequence[object],
|
||||
request_tools: object = None,
|
||||
) -> tuple[Mapping[int, int], frozenset[int]] | None:
|
||||
if not all(isinstance(item, Mapping) for item in raw_input):
|
||||
return None
|
||||
request_params: Final = {"tools": request_tools}
|
||||
solo_conversions: Final = tuple(
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=cast("ResponseInputParam", [item]), # cast-ok: items checked as Mappings above
|
||||
responses_api_request=_EMPTY_RESPONSES_REQUEST,
|
||||
responses_api_request=request_params,
|
||||
replay_tool_search=True,
|
||||
)
|
||||
for item in raw_input
|
||||
)
|
||||
full_conversion: Final = tuple(
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=cast("ResponseInputParam", list(raw_input)), # cast-ok: items checked as Mappings above
|
||||
responses_api_request=_EMPTY_RESPONSES_REQUEST,
|
||||
responses_api_request=request_params,
|
||||
replay_tool_search=True,
|
||||
)
|
||||
)
|
||||
if full_conversion != tuple(expected_messages):
|
||||
|
|
@ -530,7 +548,8 @@ def _input_item_provenance(
|
|||
else tuple(
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=cast("ResponseInputParam", list(raw_input[start:end])), # cast-ok: checked as Mappings above
|
||||
responses_api_request=_EMPTY_RESPONSES_REQUEST,
|
||||
responses_api_request=request_params,
|
||||
replay_tool_search=True,
|
||||
)
|
||||
)
|
||||
for start, end in units
|
||||
|
|
@ -688,11 +707,12 @@ def _patched_request_fields(
|
|||
instructions: object,
|
||||
original_messages: Sequence[object],
|
||||
structured_messages: Sequence[object],
|
||||
request_tools: object = None,
|
||||
) -> _RequestFields | None:
|
||||
if not isinstance(raw_input, list) or len(original_messages) != len(structured_messages):
|
||||
return None
|
||||
offset: Final = 1 if instructions else 0
|
||||
provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:])
|
||||
provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:], request_tools)
|
||||
if provenance is None:
|
||||
return None
|
||||
item_for_message, tainted = provenance
|
||||
|
|
@ -735,16 +755,54 @@ def _patch_or_convert_request_fields(
|
|||
instructions: object,
|
||||
original_messages: Sequence[object],
|
||||
structured_messages: Sequence[AllMessageValues],
|
||||
request_tools: object = None,
|
||||
) -> _RequestFields | None:
|
||||
if not isinstance(structured_messages, list):
|
||||
return None
|
||||
patched: Final = _patched_request_fields(raw_input, instructions, original_messages, structured_messages)
|
||||
patched: Final = _patched_request_fields(
|
||||
raw_input, instructions, original_messages, structured_messages, request_tools
|
||||
)
|
||||
if patched is not None:
|
||||
return patched
|
||||
input_items, converted_instructions = (
|
||||
LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages)
|
||||
)
|
||||
return _RequestFields(input=tuple(input_items), instructions=converted_instructions)
|
||||
return _RequestFields(
|
||||
input=_with_tool_searches_restored(input_items, raw_input), instructions=converted_instructions
|
||||
)
|
||||
|
||||
|
||||
def _tool_search_items_by_call_id(raw_input: Sequence[object], item_type: str) -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
call_id: item
|
||||
for item in raw_input
|
||||
if stream_item_field(item, "type") == item_type
|
||||
and isinstance(call_id := stream_item_field(item, "call_id"), str)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _restored_tool_search(
|
||||
item: object, calls: Mapping[str, object], outputs: Mapping[str, object]
|
||||
) -> tuple[object, ...]:
|
||||
call_id: Final = stream_item_field(item, "call_id")
|
||||
if stream_item_field(item, "type") != "function_call" or not isinstance(call_id, str) or call_id not in calls:
|
||||
return (item,)
|
||||
arguments: Final = stream_item_field(item, "arguments")
|
||||
restored_call: Final = _rewritten_tool_search_call(
|
||||
calls[call_id], arguments if isinstance(arguments, str) else None
|
||||
)
|
||||
return (restored_call, outputs[call_id]) if call_id in outputs else (restored_call,)
|
||||
|
||||
|
||||
def _with_tool_searches_restored(input_items: Sequence[object], raw_input: object) -> tuple[object, ...]:
|
||||
raw_items: Final = _RAW_ITEMS_ADAPTER.validate_python(raw_input) if isinstance(raw_input, list) else ()
|
||||
if not any(map(_is_tool_search_item, raw_items)):
|
||||
return tuple(input_items)
|
||||
calls: Final = _tool_search_items_by_call_id(raw_items, _TOOL_SEARCH_CALL_TYPE)
|
||||
outputs: Final = _tool_search_items_by_call_id(raw_items, "tool_search_output")
|
||||
return tuple(chain.from_iterable(_restored_tool_search(item, calls, outputs) for item in input_items))
|
||||
|
||||
|
||||
def _next_stream_sequence_number(responses_so_far: Sequence[object] | None) -> int:
|
||||
|
|
@ -791,6 +849,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=input_data,
|
||||
responses_api_request=data,
|
||||
replay_tool_search=True,
|
||||
)
|
||||
return cast(list[AllMessageValues], messages) if messages else None
|
||||
|
||||
|
|
@ -986,7 +1045,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
)
|
||||
)
|
||||
return _patch_or_convert_request_fields(
|
||||
data.get("input"), data.get("instructions"), structured_messages, merged
|
||||
data.get("input"), data.get("instructions"), structured_messages, merged, data.get("tools")
|
||||
)
|
||||
|
||||
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
|
||||
|
|
@ -1953,7 +2012,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
if isinstance(output_item, GenericResponseOutputItem):
|
||||
if output_item.content and content_idx < len(output_item.content):
|
||||
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
|
||||
elif isinstance(output_item, BaseModel):
|
||||
elif isinstance(output_item, BaseModel) and not isinstance(output_item, OutputToolSearchOutput):
|
||||
# Handle other Pydantic models by converting to GenericResponseOutputItem
|
||||
try:
|
||||
generic_item = GenericResponseOutputItem.model_validate(output_item.model_dump())
|
||||
|
|
|
|||
|
|
@ -1136,7 +1136,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
return None
|
||||
|
||||
@staticmethod
|
||||
def parse_terminal_response_from_stream_chunks(all_chunks: list[str]) -> ResponsesAPIResponse | None:
|
||||
def parse_terminal_response_from_stream_chunks(all_chunks: Sequence[str]) -> ResponsesAPIResponse | None:
|
||||
terminal_event: Final = OpenAIResponsesAPIConfig.parse_terminal_event_from_stream_chunks(all_chunks)
|
||||
return None if terminal_event is None else terminal_event.response
|
||||
|
||||
|
|
|
|||
|
|
@ -50,6 +50,9 @@ def create_config_class(provider: SimpleProviderConfig):
|
|||
base_class: Final[type] = OpenAIGPTConfig if provider.base_class == "openai_gpt" else OpenAILikeChatConfig
|
||||
|
||||
class JSONProviderConfig(base_class):
|
||||
def _flattens_content_lists(self) -> bool:
|
||||
return bool(provider.special_handling.get("convert_content_list_to_string"))
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
self, messages: list[AllMessageValues], model: str, is_async: Literal[True]
|
||||
|
|
@ -69,7 +72,7 @@ def create_config_class(provider: SimpleProviderConfig):
|
|||
"""Transform messages based on special_handling config"""
|
||||
|
||||
# Handle content list to string conversion if configured
|
||||
if provider.special_handling.get("convert_content_list_to_string"):
|
||||
if self._flattens_content_lists():
|
||||
messages = handle_messages_with_content_list_to_str_conversion(messages)
|
||||
|
||||
if is_async:
|
||||
|
|
|
|||
|
|
@ -32,6 +32,9 @@ class SambanovaConfig(OpenAIGPTConfig):
|
|||
response_format: dict | None = None
|
||||
tools: list | None = None
|
||||
|
||||
def _flattens_content_lists(self) -> bool:
|
||||
return True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_tokens: int | None = None,
|
||||
|
|
|
|||
|
|
@ -6,10 +6,12 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, cast
|
|||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.images.utils import ImageEditRequestUtils
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.gemini.image_usage_transformation import transform_gemini_image_usage
|
||||
from litellm.llms.vertex_ai.common_utils import (
|
||||
get_vertex_base_url,
|
||||
map_vertex_image_size,
|
||||
|
|
@ -25,7 +27,9 @@ from litellm.types.llms.vertex_ai import (
|
|||
HttpxPartType,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage
|
||||
from litellm.types.utils import FileTypes, ImageObject, ImageResponse
|
||||
|
||||
_JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
|
@ -218,7 +222,6 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
|
|||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ImageResponse:
|
||||
model_response: Final = ImageResponse()
|
||||
try:
|
||||
response_json: Final = _generate_content_payload(raw_response)
|
||||
except Exception as exc:
|
||||
|
|
@ -239,8 +242,15 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
|
|||
if (inline_data := part.get("inlineData")) and (b64_json := inline_data.get("data"))
|
||||
]
|
||||
|
||||
model_response.data = cast(list[OpenAIImage], data_list)
|
||||
return model_response
|
||||
usage_metadata: Final = _JSON_DICT.validate_python(response_json.get("usageMetadata") or {})
|
||||
traffic_type: Final = usage_metadata.get("trafficType") if usage_metadata else None
|
||||
return ImageResponse(
|
||||
data=data_list,
|
||||
usage=transform_gemini_image_usage(usage_metadata) if usage_metadata else None,
|
||||
hidden_params={"provider_specific_fields": {"traffic_type": traffic_type}}
|
||||
if isinstance(traffic_type, str) and traffic_type
|
||||
else {},
|
||||
)
|
||||
|
||||
def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[HttpxPartType]:
|
||||
images: Final[list[FileTypes]] = image if isinstance(image, list) else [image]
|
||||
|
|
|
|||
0
litellm/llms/webiq/__init__.py
Normal file
0
litellm/llms/webiq/__init__.py
Normal file
0
litellm/llms/webiq/search/__init__.py
Normal file
0
litellm/llms/webiq/search/__init__.py
Normal file
156
litellm/llms/webiq/search/transformation.py
Normal file
156
litellm/llms/webiq/search/transformation.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.search.transformation import BaseSearchConfig, SearchResponse, SearchResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
|
||||
class _WebIQResult(BaseModel):
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
title: str
|
||||
url: str
|
||||
content: str
|
||||
lastUpdatedAt: str | None = None
|
||||
|
||||
|
||||
class _WebIQResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
webResults: tuple[_WebIQResult, ...]
|
||||
|
||||
|
||||
_DOMAINS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
def _extra_fields(model: BaseModel, reserved: Mapping[str, object]) -> MappingProxyType[str, object]:
|
||||
extra: Final[Mapping[str, object]] = model.model_extra or {}
|
||||
return MappingProxyType({key: value for key, value in extra.items() if key not in reserved})
|
||||
|
||||
|
||||
def _query_with_domains(query: str, domains: object) -> str:
|
||||
if domains is None:
|
||||
return query
|
||||
parsed: Final = _DOMAINS.validate_python(domains)
|
||||
included: Final = tuple(f"site:{domain}" for domain in parsed if domain and not domain.startswith("-"))
|
||||
excluded: Final = tuple(f"-site:{domain[1:]}" for domain in parsed if domain.startswith("-") and len(domain) > 1)
|
||||
include_clause: Final = f"({' OR '.join(included)})" if included else ""
|
||||
if not included and not excluded:
|
||||
return query
|
||||
return " ".join(part for part in (f"({query})", include_clause, *excluded) if part)
|
||||
|
||||
|
||||
class WebIQSearchConfig(BaseSearchConfig):
|
||||
DEFAULT_API_BASE: Final = "https://api.microsoft.ai/v3"
|
||||
|
||||
@staticmethod
|
||||
def ui_friendly_name() -> str:
|
||||
return "Microsoft Web IQ"
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict[str, str], # mutable-ok: BaseSearchConfig interface
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
**kwargs: object, # kwargs-ok: BaseSearchConfig interface
|
||||
) -> dict[str, str]: # mutable-ok: HTTP handler requires a dict
|
||||
if urlsplit(api_base or self.DEFAULT_API_BASE).scheme != "https":
|
||||
raise ValueError("Microsoft Web IQ search requires an HTTPS api_base")
|
||||
resolved_key: Final = self.resolve_server_api_key(
|
||||
caller_api_key=api_key,
|
||||
caller_api_base=api_base,
|
||||
key_env_vars=("WEBIQ_API_KEY",),
|
||||
base_env_var=None,
|
||||
default_api_base=self.DEFAULT_API_BASE,
|
||||
)
|
||||
if not resolved_key:
|
||||
raise ValueError("Set WEBIQ_API_KEY or pass api_key for Microsoft Web IQ search")
|
||||
return {
|
||||
**headers,
|
||||
"x-apikey": resolved_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
optional_params: dict[str, object], # mutable-ok: BaseSearchConfig interface
|
||||
data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: BaseSearchConfig interface
|
||||
**kwargs: object, # kwargs-ok: BaseSearchConfig interface
|
||||
) -> str:
|
||||
base: Final = (api_base or self.DEFAULT_API_BASE).rstrip("/")
|
||||
return base if base.endswith("/search/web") else f"{base}/search/web"
|
||||
|
||||
def transform_search_request(
|
||||
self,
|
||||
query: str | list[str], # mutable-ok: BaseSearchConfig interface
|
||||
optional_params: dict[str, object], # mutable-ok: BaseSearchConfig interface
|
||||
**kwargs: object, # kwargs-ok: BaseSearchConfig interface
|
||||
) -> dict[str, object]: # mutable-ok: HTTP handler requires a JSON dict
|
||||
country: Final = optional_params.get("country")
|
||||
mapped: Final = MappingProxyType(
|
||||
{
|
||||
"maxResults": optional_params.get("max_results"),
|
||||
"region": country.upper() if isinstance(country, str) else None,
|
||||
}
|
||||
)
|
||||
native: Final = MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in optional_params.items()
|
||||
if key not in self.get_supported_perplexity_optional_params()
|
||||
}
|
||||
)
|
||||
return {
|
||||
"contentFormat": "passage",
|
||||
"maxLength": 5000,
|
||||
**MappingProxyType({key: value for key, value in mapped.items() if value is not None}),
|
||||
**native,
|
||||
"query": _query_with_domains(
|
||||
" ".join(query) if isinstance(query, list) else query,
|
||||
optional_params.get("search_domain_filter"),
|
||||
),
|
||||
}
|
||||
|
||||
def transform_search_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
**kwargs: object, # kwargs-ok: BaseSearchConfig interface
|
||||
) -> SearchResponse:
|
||||
try:
|
||||
parsed: Final = _WebIQResponse.model_validate_json(raw_response.content)
|
||||
except ValidationError as exc:
|
||||
raise BaseLLMException(
|
||||
status_code=502,
|
||||
message=f"Microsoft Web IQ returned an invalid search response: {exc}",
|
||||
headers=dict(raw_response.headers),
|
||||
) from exc
|
||||
return SearchResponse.model_validate(
|
||||
{
|
||||
**_extra_fields(parsed, SearchResponse.model_fields),
|
||||
"results": [
|
||||
SearchResult.model_validate(
|
||||
{
|
||||
**_extra_fields(result, SearchResult.model_fields),
|
||||
"title": result.title,
|
||||
"url": result.url,
|
||||
"snippet": result.content,
|
||||
"date": result.lastUpdatedAt or None,
|
||||
"last_updated": result.lastUpdatedAt or None,
|
||||
}
|
||||
)
|
||||
for result in parsed.webResults
|
||||
],
|
||||
}
|
||||
)
|
||||
|
|
@ -75,7 +75,8 @@ from litellm.constants import (
|
|||
from litellm.exceptions import LiteLLMUnknownProvider
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
raise_if_allowed_tools_tool_choice_unsupported,
|
||||
drops_unsupported_allowed_tools_tool_choice,
|
||||
parse_allowed_tools_tool_choice,
|
||||
raise_if_hosted_tool_choice_unsupported,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify, run_async_function
|
||||
|
|
@ -91,6 +92,7 @@ from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, Str
|
|||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.get_litellm_params import (
|
||||
AGENTIC_LOOP_CONTROL_KWARGS_KEYS,
|
||||
AGENTIC_LOOP_TOOL_CHOICE_KEY,
|
||||
AWS_CREDENTIAL_KWARGS_KEYS,
|
||||
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
|
|
@ -5612,7 +5614,16 @@ def completion(
|
|||
messages = validate_and_fix_openai_messages(messages=messages)
|
||||
tools = validate_and_fix_openai_tools(tools=tools)
|
||||
# validate tool_choice
|
||||
tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice, model=model)
|
||||
requested_tool_choice: Final = cast( # cast-ok: widening an untyped dict to object is always sound
|
||||
object, tool_choice
|
||||
)
|
||||
tool_choice = validate_chat_completion_tool_choice(
|
||||
tool_choice=tool_choice,
|
||||
model=model,
|
||||
drop_params=drops_unsupported_params(
|
||||
cast("Mapping[str, object]", kwargs).get("drop_params") # cast-ok: kwargs is the untyped request dict
|
||||
),
|
||||
)
|
||||
# validate optional params
|
||||
stop = validate_openai_optional_params(stop=stop)
|
||||
thinking = validate_and_fix_thinking_param(thinking=thinking)
|
||||
|
|
@ -5909,12 +5920,13 @@ def completion(
|
|||
if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model):
|
||||
tools = _drop_anthropic_only_tool_keys(tools=tools)
|
||||
|
||||
raise_if_allowed_tools_tool_choice_unsupported(
|
||||
drops_tool_choice: Final = drops_unsupported_allowed_tools_tool_choice(
|
||||
tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
|
||||
object, tool_choice
|
||||
),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
drop_params=drop_unsupported_reasoning,
|
||||
)
|
||||
raise_if_hosted_tool_choice_unsupported(
|
||||
tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic
|
||||
|
|
@ -6039,7 +6051,7 @@ def completion(
|
|||
"response_format": response_format,
|
||||
"seed": seed,
|
||||
"tools": tools,
|
||||
"tool_choice": tool_choice,
|
||||
"tool_choice": None if drops_tool_choice else tool_choice,
|
||||
"max_retries": max_retries,
|
||||
"logprobs": logprobs,
|
||||
"top_logprobs": top_logprobs,
|
||||
|
|
@ -6228,7 +6240,18 @@ def completion(
|
|||
if key in kwargs
|
||||
},
|
||||
)
|
||||
litellm_params: Final = with_control_options(requested_litellm_params, control_options)
|
||||
litellm_params: Final = with_control_options(
|
||||
{
|
||||
**requested_litellm_params,
|
||||
**(
|
||||
{AGENTIC_LOOP_TOOL_CHOICE_KEY: requested_tool_choice}
|
||||
if AGENTIC_LOOP_TOOL_CHOICE_KEY not in kwargs
|
||||
and parse_allowed_tools_tool_choice(requested_tool_choice) is not None
|
||||
else {}
|
||||
),
|
||||
},
|
||||
control_options,
|
||||
)
|
||||
affinity_api_base: Final = (
|
||||
str(client.base_url)
|
||||
if isinstance(client, openai.OpenAI | openai.AsyncOpenAI)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -5,6 +5,7 @@ Domain models for LiteLLM backend.
|
|||
from litellm.models.access_group import LiteLLM_AccessGroupTable
|
||||
from litellm.models.autorouter_session import LiteLLM_AutoRouterSession
|
||||
from litellm.models.budget import (
|
||||
CUSTOMER_ONLY_BUDGET_FIELDS,
|
||||
LiteLLM_BudgetTable,
|
||||
LiteLLM_BudgetTableFull,
|
||||
LiteLLM_TeamMemberTable,
|
||||
|
|
@ -38,6 +39,7 @@ from litellm.models.user import LiteLLM_UserTable
|
|||
from litellm.models.verification_token import LiteLLM_VerificationToken
|
||||
|
||||
__all__ = [
|
||||
"CUSTOMER_ONLY_BUDGET_FIELDS",
|
||||
"CreateCredentialItem",
|
||||
"CredentialBase",
|
||||
"CredentialItem",
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Canonical definition for ``litellm_budgettable``. Re-exported from
|
|||
``litellm.proxy._types`` for backwards compatibility.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -29,6 +30,8 @@ class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase):
|
|||
rpm_limit: int | None = None
|
||||
tpd_limit: int | None = None
|
||||
model_max_budget: dict | None = None
|
||||
model_tpm_limit: Mapping[str, int] | None = None
|
||||
model_rpm_limit: Mapping[str, int] | None = None
|
||||
budget_duration: str | None = None
|
||||
allowed_models: list[str] | None = None # per-member model scope; empty = inherit team models
|
||||
temp_budget_increase: float | None = None
|
||||
|
|
@ -70,3 +73,6 @@ class LiteLLM_TeamMemberTable(LiteLLM_BudgetTable):
|
|||
budget_id: str | None = None
|
||||
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
|
||||
CUSTOMER_ONLY_BUDGET_FIELDS: Final = frozenset({"model_tpm_limit", "model_rpm_limit"})
|
||||
|
|
|
|||
|
|
@ -57,11 +57,17 @@ def _as_generator(iterable: Iterator[bytes]) -> Generator[bytes, bytes, None]:
|
|||
class _SpendCollection:
|
||||
"""Feeds relayed chunks to the provider's stream collector without letting spend tracking break the relay."""
|
||||
|
||||
def __init__(self, provider_config: BasePassthroughConfig, litellm_logging_obj: LiteLLMLoggingObj) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
provider_config: BasePassthroughConfig,
|
||||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
response_headers: httpx.Headers,
|
||||
) -> None:
|
||||
self.collector: Final[PassthroughStreamCollector] = provider_config.create_stream_collector(
|
||||
model=litellm_logging_obj.model,
|
||||
custom_llm_provider=litellm_logging_obj.model_call_details.get("custom_llm_provider", ""),
|
||||
endpoint=litellm_logging_obj.model_call_details.get("endpoint", ""),
|
||||
response_headers=response_headers,
|
||||
)
|
||||
self.chunk_count = 0
|
||||
self._failed = False
|
||||
|
|
@ -97,7 +103,8 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]):
|
|||
self._response: httpx.Response
|
||||
self._iterator: AsyncGenerator[bytes, bytes]
|
||||
self._litellm_logging_obj = litellm_logging_obj
|
||||
self._spend = _SpendCollection(provider_config, litellm_logging_obj)
|
||||
self._provider_config: Final = provider_config
|
||||
self._spend: _SpendCollection
|
||||
self._flush_scheduled = False
|
||||
self._background_tasks: set[asyncio.Task[bool]] = set() # mutable-ok: instance set for background task tracking
|
||||
self._flush: asyncio.Task[bool] | None = None
|
||||
|
|
@ -129,6 +136,7 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]):
|
|||
self._response = await self._response_coro
|
||||
self.headers = self._response.headers
|
||||
self.status_code = self._response.status_code
|
||||
self._spend = _SpendCollection(self._provider_config, self._litellm_logging_obj, self._response.headers)
|
||||
self._initialized = True
|
||||
try:
|
||||
raise_for_provider_status(self._response)
|
||||
|
|
@ -148,7 +156,7 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]):
|
|||
return _init().__await__()
|
||||
|
||||
def _start_flush(self) -> None:
|
||||
if self._flush_scheduled or not self._spend.should_flush:
|
||||
if self._flush_scheduled or not self._initialized or not self._spend.should_flush:
|
||||
return
|
||||
self._flush_scheduled = True
|
||||
|
||||
|
|
@ -237,7 +245,7 @@ class PassthroughStreamingResponse(Generator[bytes, bytes, None]):
|
|||
self.status_code = response.status_code
|
||||
self._litellm_logging_obj = litellm_logging_obj
|
||||
self._iterator: Generator[bytes, bytes, None] = _as_generator(response.iter_bytes())
|
||||
self._spend = _SpendCollection(provider_config, litellm_logging_obj)
|
||||
self._spend = _SpendCollection(provider_config, litellm_logging_obj, response.headers)
|
||||
self._flush_scheduled = False
|
||||
|
||||
def _start_flush(self) -> None:
|
||||
|
|
|
|||
|
|
@ -25,7 +25,16 @@ _PASS_THROUGH_PROTECTED_HEADERS: Final[frozenset] = frozenset(
|
|||
)
|
||||
|
||||
# Header name prefix used to block AWS SigV4 signing headers from being overridden.
|
||||
_PASS_THROUGH_PROTECTED_HEADER_PREFIXES: Final[tuple] = ("x-amz-",)
|
||||
_PASS_THROUGH_PROTECTED_HEADER_PREFIXES: Final[tuple[str, ...]] = ("x-amz-",)
|
||||
|
||||
|
||||
def _is_forwarded_pass_through_header(header_name: str) -> bool:
|
||||
if not header_name.lower().startswith(PASS_THROUGH_HEADER_PREFIX):
|
||||
return False
|
||||
forwarded_name: Final = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower()
|
||||
return forwarded_name not in _PASS_THROUGH_PROTECTED_HEADERS and not any(
|
||||
forwarded_name.startswith(prefix) for prefix in _PASS_THROUGH_PROTECTED_HEADER_PREFIXES
|
||||
)
|
||||
|
||||
|
||||
class BasePassthroughUtils:
|
||||
|
|
@ -42,18 +51,11 @@ class BasePassthroughUtils:
|
|||
# parse_qs returns a dict where each value is a list, so let's flatten it
|
||||
updated_existing_query_params: Final = {k: v[0] if len(v) == 1 else v for k, v in existing_query_params.items()}
|
||||
|
||||
# Start with default query params (lowest priority)
|
||||
merged_params: Final = {}
|
||||
if default_query_params:
|
||||
merged_params.update(default_query_params)
|
||||
return {**(default_query_params or {}), **updated_existing_query_params, **request_query_params}
|
||||
|
||||
# Override with existing URL query params (medium priority)
|
||||
merged_params.update(updated_existing_query_params)
|
||||
|
||||
# Override with request query params (highest priority - client can override anything)
|
||||
merged_params.update(request_query_params)
|
||||
|
||||
return merged_params
|
||||
@staticmethod
|
||||
def pass_through_prefixed_headers(request_headers: Mapping[str, str]) -> Mapping[str, str]:
|
||||
return {name: value for name, value in request_headers.items() if _is_forwarded_pass_through_header(name)}
|
||||
|
||||
@staticmethod
|
||||
def forward_headers_from_request(
|
||||
|
|
@ -88,16 +90,14 @@ class BasePassthroughUtils:
|
|||
# Credential and protocol-level headers are excluded from this mechanism.
|
||||
for header_name, header_value in request_headers.items():
|
||||
if header_name.lower().startswith(PASS_THROUGH_HEADER_PREFIX):
|
||||
# Strip the 'x-pass-' prefix and normalize to lowercase
|
||||
actual_header_name = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower()
|
||||
if actual_header_name in _PASS_THROUGH_PROTECTED_HEADERS or any(
|
||||
actual_header_name.startswith(p) for p in _PASS_THROUGH_PROTECTED_HEADER_PREFIXES
|
||||
):
|
||||
if not _is_forwarded_pass_through_header(header_name):
|
||||
verbose_logger.debug(
|
||||
"x-pass- header %s maps to a protected header name; skipping",
|
||||
header_name,
|
||||
)
|
||||
continue
|
||||
# Strip the 'x-pass-' prefix and normalize to lowercase
|
||||
actual_header_name = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower()
|
||||
headers[actual_header_name] = header_value
|
||||
|
||||
return headers
|
||||
|
|
|
|||
|
|
@ -4098,6 +4098,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"webiq": {
|
||||
"display_name": "Microsoft Web IQ (`webiq`)",
|
||||
"url": "https://learn.microsoft.com/en-us/web-iq/reference/web-search",
|
||||
"endpoints": {
|
||||
"chat_completions": false,
|
||||
"responses": false,
|
||||
"text_completion": false,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": true,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": false,
|
||||
"interactions": false,
|
||||
"systemone": false,
|
||||
"a2a": false,
|
||||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"tavily": {
|
||||
"display_name": "Tavily (`tavily`)",
|
||||
"url": "https://docs.litellm.ai/docs/search/tavily",
|
||||
|
|
|
|||
|
|
@ -3799,6 +3799,21 @@
|
|||
},
|
||||
"title": "Providers",
|
||||
"type": "object"
|
||||
},
|
||||
"teams": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": {
|
||||
"$ref": "#/components/schemas/MetricWithMetadata"
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Customer spend by the key's current team, or last team for deleted keys. Keys without a known team are grouped under Unassigned",
|
||||
"title": "Teams"
|
||||
}
|
||||
},
|
||||
"title": "BreakdownMetrics",
|
||||
|
|
@ -4867,6 +4882,12 @@
|
|||
"title": "Gateway Injected Caching Savings Spend",
|
||||
"type": "number"
|
||||
},
|
||||
"guardrail_spend": {
|
||||
"default": 0.0,
|
||||
"description": "Guardrail spend in USD, already included in spend, so model spend is spend - guardrail_spend. Rows recorded before this field existed report 0 even when their spend includes guardrail cost",
|
||||
"title": "Guardrail Spend",
|
||||
"type": "number"
|
||||
},
|
||||
"prompt_caching_savings_spend": {
|
||||
"default": 0.0,
|
||||
"title": "Prompt Caching Savings Spend",
|
||||
|
|
@ -27293,6 +27314,65 @@
|
|||
"title": "OutputToolSearchCall",
|
||||
"type": "object"
|
||||
},
|
||||
"OutputToolSearchOutput": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"call_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Call Id"
|
||||
},
|
||||
"execution": {
|
||||
"enum": [
|
||||
"server",
|
||||
"client"
|
||||
],
|
||||
"title": "Execution",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"title": "Id",
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"enum": [
|
||||
"in_progress",
|
||||
"completed",
|
||||
"incomplete"
|
||||
],
|
||||
"title": "Status",
|
||||
"type": "string"
|
||||
},
|
||||
"tools": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"title": "Tools",
|
||||
"type": "array"
|
||||
},
|
||||
"type": {
|
||||
"const": "tool_search_output",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"execution",
|
||||
"status",
|
||||
"type"
|
||||
],
|
||||
"title": "OutputToolSearchOutput",
|
||||
"type": "object"
|
||||
},
|
||||
"PendingSafetyCheck": {
|
||||
"additionalProperties": true,
|
||||
"description": "A pending safety check for the computer call.",
|
||||
|
|
@ -29663,6 +29743,9 @@
|
|||
{
|
||||
"$ref": "#/components/schemas/OutputToolSearchCall"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/OutputToolSearchOutput"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseCompactionItem"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from pydantic import (
|
|||
Field,
|
||||
Json,
|
||||
JsonValue,
|
||||
NonNegativeInt,
|
||||
PositiveInt,
|
||||
TypeAdapter,
|
||||
field_validator,
|
||||
|
|
@ -1826,6 +1827,14 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase):
|
|||
default=None,
|
||||
description="Max budget for each model (e.g. {'gpt-4o': {'max_budget': '0.0000001', 'budget_duration': '1d', 'tpm_limit': 1000, 'rpm_limit': 1000}})",
|
||||
)
|
||||
model_tpm_limit: Mapping[str, NonNegativeInt] | None = Field(
|
||||
default=None,
|
||||
description="Max tokens per minute per model for this budget id, shared by every key, team and customer on it (e.g. {'gpt-4o': 400000})",
|
||||
)
|
||||
model_rpm_limit: Mapping[str, NonNegativeInt] | None = Field(
|
||||
default=None,
|
||||
description="Max requests per minute per model for this budget id, shared by every key, team and customer on it (e.g. {'gpt-4o': 200})",
|
||||
)
|
||||
budget_reset_at: datetime | None = Field(
|
||||
default=None,
|
||||
description="Datetime when the budget is reset",
|
||||
|
|
@ -1851,6 +1860,21 @@ class CustomerBase(LiteLLMPydanticObjectBase):
|
|||
blocked: bool = False
|
||||
|
||||
|
||||
def reject_unknown_customer_fields(request: type[BaseModel], values: object) -> object:
|
||||
if not isinstance(values, dict):
|
||||
return values
|
||||
unknown: Final = tuple(sorted(str(key) for key in values if key not in request.model_fields)) # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # raw request body
|
||||
if not unknown:
|
||||
return values
|
||||
on_budget: Final = tuple(field for field in unknown if field in BudgetNewRequest.model_fields)
|
||||
hint: Final = (
|
||||
f" {', '.join(on_budget)}: these live on the customer's budget object, update it with /budget/update."
|
||||
if on_budget
|
||||
else ""
|
||||
)
|
||||
raise ValueError(f"Unknown fields: {', '.join(unknown)}.{hint}")
|
||||
|
||||
|
||||
class NewCustomerRequest(BudgetNewRequest):
|
||||
"""
|
||||
Create a new customer, allocate a budget to them
|
||||
|
|
@ -1868,6 +1892,11 @@ class NewCustomerRequest(BudgetNewRequest):
|
|||
models: list[str] | None = None
|
||||
object_permission: LiteLLM_ObjectPermissionBase | None = None
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def reject_unknown_fields(cls, values: object) -> object:
|
||||
return reject_unknown_customer_fields(cls, values)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_user_info(cls, values):
|
||||
|
|
@ -1895,6 +1924,11 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase):
|
|||
models: list[str] | None = None
|
||||
object_permission: LiteLLM_ObjectPermissionBase | None = None
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def reject_unknown_fields(cls, values: object) -> object:
|
||||
return reject_unknown_customer_fields(cls, values)
|
||||
|
||||
|
||||
class DeleteCustomerRequest(LiteLLMPydanticObjectBase):
|
||||
"""
|
||||
|
|
@ -5301,6 +5335,7 @@ class BaseDailySpendTransaction(TypedDict):
|
|||
# missing value to zero, so requiring it here would describe a shape the aggregation
|
||||
# is explicitly tested against.
|
||||
autorouter_savings_spend: NotRequired[float]
|
||||
guardrail_spend: NotRequired[float] # writable-ok: the rollup queue accumulates into this key in place
|
||||
|
||||
# request level metrics
|
||||
spend: float
|
||||
|
|
|
|||
|
|
@ -2,8 +2,9 @@ import importlib.util
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Collection, Iterator, Mapping, Sequence
|
||||
from functools import lru_cache
|
||||
from collections.abc import Callable, Collection, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import cache, lru_cache
|
||||
from itertools import accumulate, chain, repeat
|
||||
from logging import Logger
|
||||
from types import MappingProxyType
|
||||
|
|
@ -22,7 +23,9 @@ from litellm.constants import (
|
|||
MINIMUM_CUSTOM_KEY_LENGTH,
|
||||
STANDARD_CUSTOMER_ID_HEADERS,
|
||||
)
|
||||
from litellm.litellm_core_utils.cloud_storage_security import REQUEST_S3_BUCKET_PARAMS, STORED_S3_BUCKET_PARAM
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.url_utils import (
|
||||
SSRFError,
|
||||
|
|
@ -41,11 +44,13 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
|||
from litellm.types.router import (
|
||||
CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS,
|
||||
Deployment,
|
||||
DeploymentTypedDict,
|
||||
LiteLLMParamsTypedDict,
|
||||
reject_server_owned_wif_fields,
|
||||
server_owned_wif_fields_present,
|
||||
)
|
||||
from litellm.types.router import reject_server_owned_wif_params as _reject_server_owned_wif_params
|
||||
from litellm.types.utils import CustomPricingLiteLLMParams, oauth_token_exchange_litellm_params
|
||||
from litellm.types.utils import CustomPricingLiteLLMParams, LlmProviders, oauth_token_exchange_litellm_params
|
||||
from litellm.types.videos.main import DecodedVideoId
|
||||
|
||||
|
||||
|
|
@ -211,35 +216,74 @@ def _is_param_allowed(
|
|||
return False
|
||||
|
||||
|
||||
def deployment_clientside_auth_params(litellm_params: LiteLLMParamsTypedDict) -> CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS:
|
||||
if "configurable_clientside_auth_params" not in litellm_params:
|
||||
return None
|
||||
return litellm_params["configurable_clientside_auth_params"]
|
||||
|
||||
|
||||
_DEPLOYMENT_LIST: Final = TypeAdapter(tuple[DeploymentTypedDict, ...], config=ConfigDict(arbitrary_types_allowed=True))
|
||||
|
||||
|
||||
def _model_group_deployments(
|
||||
model: str, llm_router: Router, team_id: str | None = None
|
||||
) -> Sequence[DeploymentTypedDict]:
|
||||
registry: Final = llm_router.registry()
|
||||
provider_group: Final = model.split("/", 1)[0]
|
||||
if team_id is not None and registry.routing_group(model) is None:
|
||||
team_deployments: Final = registry.deployments(
|
||||
model, source="all", team_id=team_id
|
||||
) or _DEPLOYMENT_LIST.validate_python(registry.deployments(model, source="wildcard", team_id=team_id))
|
||||
if team_deployments:
|
||||
return team_deployments
|
||||
return (
|
||||
registry.deployments(model, source="routed_group", team_id=team_id)
|
||||
or (registry.deployments(provider_group, team_id=team_id) if provider_group in provider_list else None)
|
||||
or ()
|
||||
)
|
||||
|
||||
|
||||
_VERTEX_TARGET_READING_PROVIDERS: Final[frozenset[str]] = frozenset(
|
||||
(LlmProviders.VERTEX_AI.value, LlmProviders.VERTEX_AI_BETA.value, LlmProviders.GEMINI.value, LlmProviders.GDC.value)
|
||||
)
|
||||
|
||||
|
||||
def _deployment_provider(litellm_params: LiteLLMParamsTypedDict) -> str | None:
|
||||
match litellm_params:
|
||||
case {"custom_llm_provider": str(declared)} if declared:
|
||||
return declared
|
||||
case {"model": str(model)}:
|
||||
return inferred_provider(model)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _model_group_reads_vertex_target(model: str, llm_router: Router | None, team_id: str | None = None) -> bool:
|
||||
if llm_router is None:
|
||||
return True
|
||||
deployments: Final = _model_group_deployments(model, llm_router, team_id)
|
||||
providers: Final = frozenset(_deployment_provider(deployment["litellm_params"]) for deployment in deployments)
|
||||
return not deployments or None in providers or bool(providers & _VERTEX_TARGET_READING_PROVIDERS)
|
||||
|
||||
|
||||
def _allow_model_level_clientside_configurable_parameters(
|
||||
model: str, param: str, request_body_value: object, llm_router: Router | None
|
||||
model: str, param: str, request_body_value: object, llm_router: Router | None, team_id: str | None = None
|
||||
) -> bool:
|
||||
"""
|
||||
Check if model is allowed to use configurable client-side params
|
||||
- get matching model
|
||||
- check if 'clientside_configurable_parameters' is set for model
|
||||
-
|
||||
The router, not the caller, picks which deployment serves the request, so an
|
||||
opt-in counts only when every deployment of the model group, already alias
|
||||
resolved, carries it
|
||||
"""
|
||||
if llm_router is None:
|
||||
return False
|
||||
# check if model is set
|
||||
registry: Final = llm_router.registry()
|
||||
model_info = registry.group(model)
|
||||
if model_info is None:
|
||||
# check if wildcard model is set
|
||||
if model.split("/", 1)[0] in provider_list:
|
||||
model_info = registry.group(model.split("/", 1)[0])
|
||||
|
||||
if model_info is None:
|
||||
return False
|
||||
|
||||
if model_info is None or model_info.configurable_clientside_auth_params is None:
|
||||
return False
|
||||
|
||||
return _is_param_allowed(
|
||||
param=param,
|
||||
request_body_value=request_body_value,
|
||||
configurable_clientside_auth_params=model_info.configurable_clientside_auth_params,
|
||||
deployments: Final = _model_group_deployments(model, llm_router, team_id)
|
||||
return bool(deployments) and all(
|
||||
_is_param_allowed(
|
||||
param=param,
|
||||
request_body_value=request_body_value,
|
||||
configurable_clientside_auth_params=deployment_clientside_auth_params(deployment["litellm_params"]),
|
||||
)
|
||||
for deployment in deployments
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -318,6 +362,20 @@ def route_manages_mcp_servers(route: str | None) -> bool:
|
|||
return route is not None and _MCP_SERVER_MANAGEMENT_ROUTE.fullmatch(route) is not None
|
||||
|
||||
|
||||
VERTEX_TARGET_PARAMS: Final[frozenset[str]] = frozenset(
|
||||
("vertex_project", "vertex_location", "vertex_ai_project", "vertex_ai_location")
|
||||
)
|
||||
_ROUTES_THAT_NAME_THEIR_VERTEX_TARGET: Final[frozenset[str]] = frozenset(
|
||||
("/vertex_ai/live", "/rag/ingest", "/v1/rag/ingest")
|
||||
)
|
||||
|
||||
|
||||
def _without_route_named_vertex_target(request_body: Mapping[str, object], route: str | None) -> Mapping[str, object]:
|
||||
if route not in _ROUTES_THAT_NAME_THEIR_VERTEX_TARGET:
|
||||
return request_body
|
||||
return MappingProxyType({key: value for key, value in request_body.items() if key not in VERTEX_TARGET_PARAMS})
|
||||
|
||||
|
||||
_NESTED_CONFIG_KEYS: Final[tuple[str, ...]] = ("litellm_embedding_config", "extra_body")
|
||||
|
||||
# Metadata containers that carry per-request configuration consumed by the
|
||||
|
|
@ -410,6 +468,10 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
|
|||
"aws_external_id",
|
||||
"aws_session_tags",
|
||||
"vertex_credentials",
|
||||
"vertex_location",
|
||||
"vertex_project",
|
||||
"vertex_ai_location",
|
||||
"vertex_ai_project",
|
||||
# Azure managed-identity / federated-auth token. The Azure provider
|
||||
# transformer reads ``azure_ad_token`` (top-level or via
|
||||
# ``extra_body``) and resolves it through ``get_secret`` before
|
||||
|
|
@ -444,6 +506,7 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
|
|||
"s3_endpoint_url",
|
||||
"s3_bucket_name",
|
||||
"s3_output_bucket_name",
|
||||
STORED_S3_BUCKET_PARAM,
|
||||
"allow_legacy_cloud_file_ids",
|
||||
"sagemaker_base_url",
|
||||
"deployment_url",
|
||||
|
|
@ -481,6 +544,105 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
|
|||
*sorted(CustomPricingLiteLLMParams.model_fields.keys()),
|
||||
)
|
||||
|
||||
_BANNED_METADATA_PARAMS: Final[tuple[str, ...]] = tuple(
|
||||
param for param in _BANNED_REQUEST_BODY_PARAMS if param not in VERTEX_TARGET_PARAMS
|
||||
)
|
||||
|
||||
|
||||
def _routed_opt_in_hop(
|
||||
frontier_and_reached: tuple[frozenset[str], frozenset[str]], router_and_team: tuple[Router, str | None]
|
||||
) -> tuple[frozenset[str], frozenset[str]]:
|
||||
frontier, reached = frontier_and_reached
|
||||
llm_router, team_id = router_and_team
|
||||
deployments: Final = chain.from_iterable(_model_group_deployments(model, llm_router, team_id) for model in frontier)
|
||||
routed_groups: Final = frozenset(deployment["model_name"] for deployment in deployments) - reached
|
||||
fallbacks: Final = frozenset[str]().union(
|
||||
*(router_fallback_model_names(name, llm_router, _EVERY_FALLBACK_TYPE) for name in routed_groups)
|
||||
)
|
||||
registry: Final = llm_router.registry()
|
||||
next_hop: Final = (
|
||||
routed_groups | frozenset(registry.resolve(name, kind="alias") or name for name in fallbacks)
|
||||
) - reached
|
||||
return next_hop, reached | next_hop
|
||||
|
||||
|
||||
def _opt_in_model_groups(
|
||||
model: str, also_requested_models: tuple[str, ...], llm_router: Router | None, team_id: str | None = None
|
||||
) -> frozenset[str]:
|
||||
requested: Final = frozenset((model, *also_requested_models))
|
||||
if llm_router is None:
|
||||
return requested
|
||||
registry: Final = llm_router.registry()
|
||||
routed: Final = frozenset(registry.resolve(name, kind="alias") or name for name in requested)
|
||||
fallbacks: Final = frozenset[str]().union(
|
||||
*(router_fallback_model_names(name, llm_router, _EVERY_FALLBACK_TYPE) for name in requested | routed)
|
||||
)
|
||||
groups: Final = routed | frozenset(registry.resolve(name, kind="alias") or name for name in fallbacks)
|
||||
hops: Final = accumulate(repeat((llm_router, team_id)), _routed_opt_in_hop, initial=(groups, groups))
|
||||
_, reached = next(hop for hop in hops if not hop[0])
|
||||
return reached
|
||||
|
||||
|
||||
_MAX_OPT_IN_FALLBACK_MODELS: Final = 100
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OptInGroups:
|
||||
every: tuple[str, ...]
|
||||
vertex_target: tuple[str, ...]
|
||||
team_id: str | None = None
|
||||
|
||||
|
||||
_OptInKey = tuple[str | tuple[str, str | None], ...]
|
||||
|
||||
|
||||
def _opt_in_key(litellm_params: LiteLLMParamsTypedDict) -> _OptInKey:
|
||||
return tuple(
|
||||
item if isinstance(item, str) else ("api_base", item.get("api_base"))
|
||||
for item in deployment_clientside_auth_params(litellm_params) or ()
|
||||
)
|
||||
|
||||
|
||||
def _same_answer_key(
|
||||
model_group: str, llm_router: Router, team_id: str | None
|
||||
) -> frozenset[tuple[str | None, _OptInKey]] | str:
|
||||
deployments: Final = _model_group_deployments(model_group, llm_router, team_id)
|
||||
key: Final = frozenset(
|
||||
(_deployment_provider(deployment["litellm_params"]), _opt_in_key(deployment["litellm_params"]))
|
||||
for deployment in deployments
|
||||
)
|
||||
return key or model_group
|
||||
|
||||
|
||||
def _distinct_opt_in_groups(
|
||||
model_groups: frozenset[str], llm_router: Router | None, team_id: str | None
|
||||
) -> tuple[str, ...]:
|
||||
if llm_router is None:
|
||||
return tuple(sorted(model_groups))
|
||||
return tuple(
|
||||
{
|
||||
_same_answer_key(model_group, llm_router, team_id): model_group for model_group in sorted(model_groups)
|
||||
}.values()
|
||||
)
|
||||
|
||||
|
||||
def _resolve_opt_in_groups(
|
||||
model: str, also_requested_models: tuple[str, ...], llm_router: Router | None, team_id: str | None = None
|
||||
) -> _OptInGroups:
|
||||
if len(frozenset(also_requested_models)) > _MAX_OPT_IN_FALLBACK_MODELS:
|
||||
raise ValueError(
|
||||
f"Rejected Request: the request names more than {_MAX_OPT_IN_FALLBACK_MODELS} fallback models, "
|
||||
"too many to check against configurable_clientside_auth_params."
|
||||
)
|
||||
every: Final = _distinct_opt_in_groups(
|
||||
_opt_in_model_groups(model, also_requested_models, llm_router, team_id), llm_router, team_id
|
||||
)
|
||||
return _OptInGroups(
|
||||
every=every,
|
||||
vertex_target=tuple(group for group in every if _model_group_reads_vertex_target(group, llm_router, team_id)),
|
||||
team_id=team_id,
|
||||
)
|
||||
|
||||
|
||||
_ClientSideAllowlistSetting: TypeAlias = Literal[
|
||||
"allowed_client_side_s3_buckets", "allowed_client_side_s3_endpoint_urls"
|
||||
|
|
@ -494,6 +656,9 @@ _PROXY_ALLOWLIST_SETTING_BY_PARAM: Final[Mapping[str, _ClientSideAllowlistSettin
|
|||
}
|
||||
)
|
||||
|
||||
TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY: Final = "allowed_s3_buckets"
|
||||
_BUCKET_NAMES: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
class _ClientSideStorageAllowlist(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
|
@ -511,8 +676,15 @@ class _ClientSideStorageAllowlist(BaseModel):
|
|||
|
||||
def _banned_param_error(param: str) -> ValueError:
|
||||
setting: Final = _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param)
|
||||
team_hint: Final = (
|
||||
f" or in the team's `metadata.{TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY}`"
|
||||
if param in REQUEST_S3_BUCKET_PARAMS
|
||||
else ""
|
||||
)
|
||||
allowlist_hint: Final = (
|
||||
"" if setting is None else f" To allow specific values only, list them in `general_settings.{setting}`."
|
||||
""
|
||||
if setting is None
|
||||
else f" To allow specific values only, list them in `general_settings.{setting}`{team_hint}."
|
||||
)
|
||||
return ValueError(
|
||||
f"Rejected Request: {param} is not allowed in request body. "
|
||||
|
|
@ -526,17 +698,21 @@ def _banned_param_error(param: str) -> ValueError:
|
|||
|
||||
def _check_banned_params(
|
||||
body: Mapping[str, object],
|
||||
general_settings: dict,
|
||||
general_settings: Mapping[str, object],
|
||||
llm_router: Router | None,
|
||||
model: str,
|
||||
*,
|
||||
manages_deployments: bool = False,
|
||||
mcp_server_fields_allowed: bool = False,
|
||||
banned_params: tuple[str, ...] = _BANNED_REQUEST_BODY_PARAMS,
|
||||
opt_in_groups: Callable[[], _OptInGroups],
|
||||
defer_model_checks: bool = False,
|
||||
defer_s3_buckets: bool = False,
|
||||
) -> None:
|
||||
"""Raise ``ValueError`` if ``body`` carries a banned param without admin opt-in.
|
||||
|
||||
Shared between the root-level check and the nested-config check so a
|
||||
new banned param only needs to be added in one place.
|
||||
new banned param only needs to be added in one place. ``defer_s3_buckets`` lets
|
||||
bucket params through for handlers that check them against the caller team's allowlist.
|
||||
"""
|
||||
wif_body: Final[Mapping[str, object]] = (
|
||||
MappingProxyType({key: value for key, value in body.items() if key not in oauth_token_exchange_litellm_params})
|
||||
|
|
@ -546,7 +722,7 @@ def _check_banned_params(
|
|||
reject_server_owned_wif_params(wif_body)
|
||||
if not manages_deployments:
|
||||
reject_federated_credential_reference(body)
|
||||
for param in _BANNED_REQUEST_BODY_PARAMS:
|
||||
for param in banned_params:
|
||||
if param not in body:
|
||||
continue
|
||||
if general_settings.get("allow_client_side_credentials") is True:
|
||||
|
|
@ -557,14 +733,22 @@ def _check_banned_params(
|
|||
setting := _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param)
|
||||
) is not None and _ClientSideStorageAllowlist.model_validate(general_settings).admits(setting, body[param]):
|
||||
continue
|
||||
if (
|
||||
if defer_s3_buckets and param in REQUEST_S3_BUCKET_PARAMS:
|
||||
continue
|
||||
if defer_model_checks:
|
||||
continue
|
||||
if all(
|
||||
_allow_model_level_clientside_configurable_parameters(
|
||||
model=model,
|
||||
model=model_group,
|
||||
param=param,
|
||||
request_body_value=body[param],
|
||||
llm_router=llm_router,
|
||||
team_id=opt_in_groups().team_id,
|
||||
)
|
||||
is True
|
||||
for model_group in (
|
||||
opt_in_groups().vertex_target if param in VERTEX_TARGET_PARAMS else opt_in_groups().every
|
||||
)
|
||||
):
|
||||
# Per-param opt-in: only THIS param is permitted by the
|
||||
# deployment's ``configurable_clientside_auth_params``. Skip
|
||||
|
|
@ -648,27 +832,39 @@ def fallback_target_model_name(target: object) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _direct_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]:
|
||||
_EVERY_FALLBACK_TYPE: Final[tuple[str, ...]] = ("general", "context_window", "content_policy")
|
||||
|
||||
|
||||
def _direct_fallback_model_names(model: str, llm_router: Router, fallback_types: tuple[str, ...]) -> frozenset[str]:
|
||||
from litellm.proxy.auth.model_checks import get_all_fallbacks
|
||||
|
||||
return frozenset(
|
||||
name for target in get_all_fallbacks(model, llm_router) if (name := fallback_target_model_name(target))
|
||||
targets: Final = chain.from_iterable(
|
||||
get_all_fallbacks(model, llm_router, fallback_type) for fallback_type in fallback_types
|
||||
)
|
||||
return frozenset(name for target in targets if (name := fallback_target_model_name(target)))
|
||||
|
||||
|
||||
def _fallback_hop(
|
||||
frontier_and_reached: tuple[frozenset[str], frozenset[str]], llm_router: Router
|
||||
frontier_and_reached: tuple[frozenset[str], frozenset[str]], router_and_types: tuple[Router, tuple[str, ...]]
|
||||
) -> tuple[frozenset[str], frozenset[str]]:
|
||||
frontier, reached = frontier_and_reached
|
||||
llm_router, fallback_types = router_and_types
|
||||
next_hop: Final = (
|
||||
frozenset(chain.from_iterable(_direct_fallback_model_names(model, llm_router) for model in frontier)) - reached
|
||||
frozenset(
|
||||
chain.from_iterable(_direct_fallback_model_names(model, llm_router, fallback_types) for model in frontier)
|
||||
)
|
||||
- reached
|
||||
)
|
||||
return next_hop, reached | next_hop
|
||||
|
||||
|
||||
def router_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]:
|
||||
def router_fallback_model_names(
|
||||
model: str, llm_router: Router, fallback_types: tuple[str, ...] = ("general",)
|
||||
) -> frozenset[str]:
|
||||
"""The groups a failure on `model` can reach through the router's fallbacks, chained and generic (`*`) ones."""
|
||||
hops: Final = accumulate(repeat(llm_router), _fallback_hop, initial=(frozenset({model}), frozenset({model})))
|
||||
hops: Final = accumulate(
|
||||
repeat((llm_router, fallback_types)), _fallback_hop, initial=(frozenset({model}), frozenset({model}))
|
||||
)
|
||||
_, reached = next(hop for hop in hops if not hop[0])
|
||||
return reached - {model}
|
||||
|
||||
|
|
@ -698,12 +894,15 @@ def _reject_url_valued_fallback_target(value: str) -> None:
|
|||
|
||||
|
||||
def is_request_body_safe(
|
||||
request_body: dict,
|
||||
general_settings: dict,
|
||||
request_body: Mapping[str, object],
|
||||
general_settings: Mapping[str, object],
|
||||
llm_router: Router | None,
|
||||
model: str,
|
||||
*,
|
||||
route: str | None = None,
|
||||
team_id: str | None = None,
|
||||
defer_model_checks: bool = False,
|
||||
defer_s3_buckets: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if the request body is safe.
|
||||
|
|
@ -733,33 +932,72 @@ def is_request_body_safe(
|
|||
if "model_list" in request_body:
|
||||
raise ValueError("Rejected Request: model_list is not allowed in the request body.")
|
||||
manages_deployments: Final = route_manages_deployments(route)
|
||||
root_body: Final = _without_route_named_vertex_target(request_body, route)
|
||||
route_model: Final = _model_from_route_path(route) if route is not None else None
|
||||
also_requested_models: Final = (
|
||||
*request_fallback_model_names(root_body),
|
||||
*((route_model,) if route_model else ()),
|
||||
)
|
||||
opt_in_groups: Final = cache(
|
||||
lambda: _resolve_opt_in_groups(model or route_model or "", also_requested_models, llm_router, team_id)
|
||||
)
|
||||
_check_banned_params(
|
||||
request_body,
|
||||
root_body,
|
||||
general_settings,
|
||||
llm_router,
|
||||
model,
|
||||
manages_deployments=manages_deployments,
|
||||
mcp_server_fields_allowed=route_manages_mcp_servers(route),
|
||||
opt_in_groups=opt_in_groups,
|
||||
defer_model_checks=defer_model_checks,
|
||||
defer_s3_buckets=defer_s3_buckets,
|
||||
)
|
||||
for nested_key in _NESTED_CONFIG_KEYS:
|
||||
nested = _coerce_metadata_to_dict(request_body.get(nested_key))
|
||||
if nested is not None:
|
||||
_check_banned_params(nested, general_settings, llm_router, model, manages_deployments=manages_deployments)
|
||||
_check_banned_params(
|
||||
nested,
|
||||
general_settings,
|
||||
llm_router,
|
||||
manages_deployments=manages_deployments,
|
||||
opt_in_groups=opt_in_groups,
|
||||
defer_model_checks=defer_model_checks,
|
||||
defer_s3_buckets=defer_s3_buckets,
|
||||
)
|
||||
for metadata_key in _NESTED_METADATA_KEYS:
|
||||
metadata = _coerce_metadata_to_dict(request_body.get(metadata_key))
|
||||
if metadata is not None:
|
||||
_check_banned_params(metadata, general_settings, llm_router, model, manages_deployments=manages_deployments)
|
||||
if any(isinstance(key, str) and key.startswith(f"{metadata_key}[") for key in request_body):
|
||||
_check_banned_params(
|
||||
metadata,
|
||||
general_settings,
|
||||
llm_router,
|
||||
manages_deployments=manages_deployments,
|
||||
banned_params=_BANNED_METADATA_PARAMS,
|
||||
opt_in_groups=opt_in_groups,
|
||||
defer_model_checks=defer_model_checks,
|
||||
defer_s3_buckets=defer_s3_buckets,
|
||||
)
|
||||
if any(key.startswith(f"{metadata_key}[") for key in request_body):
|
||||
_check_banned_params(
|
||||
extract_nested_form_metadata(form_data=request_body, prefix=f"{metadata_key}["),
|
||||
general_settings,
|
||||
llm_router,
|
||||
model,
|
||||
manages_deployments=manages_deployments,
|
||||
banned_params=_BANNED_METADATA_PARAMS,
|
||||
opt_in_groups=opt_in_groups,
|
||||
defer_model_checks=defer_model_checks,
|
||||
defer_s3_buckets=defer_s3_buckets,
|
||||
)
|
||||
for target in iter_request_fallback_targets(request_body):
|
||||
if isinstance(target, dict):
|
||||
_check_banned_params(target, general_settings, llm_router, model, manages_deployments=manages_deployments)
|
||||
_check_banned_params(
|
||||
target,
|
||||
general_settings,
|
||||
llm_router,
|
||||
manages_deployments=manages_deployments,
|
||||
opt_in_groups=opt_in_groups,
|
||||
defer_model_checks=defer_model_checks,
|
||||
defer_s3_buckets=defer_s3_buckets,
|
||||
)
|
||||
target_model = target.get("model")
|
||||
if isinstance(target_model, str):
|
||||
_reject_url_valued_fallback_target(target_model)
|
||||
|
|
@ -773,9 +1011,16 @@ def is_request_body_safe(
|
|||
litellm_params_metadata: Final = _coerce_metadata_to_dict(litellm_params.get("metadata"))
|
||||
if litellm_params_metadata is not None:
|
||||
_check_banned_params(
|
||||
litellm_params_metadata, general_settings, llm_router, model, manages_deployments=manages_deployments
|
||||
litellm_params_metadata,
|
||||
general_settings,
|
||||
llm_router,
|
||||
manages_deployments=manages_deployments,
|
||||
banned_params=_BANNED_METADATA_PARAMS,
|
||||
opt_in_groups=opt_in_groups,
|
||||
defer_model_checks=defer_model_checks,
|
||||
defer_s3_buckets=defer_s3_buckets,
|
||||
)
|
||||
for user_config_params in _user_config_litellm_params(request_body): # pyright: ignore[reportUnknownArgumentType] # request_body is an untyped dict
|
||||
for user_config_params in _user_config_litellm_params(request_body):
|
||||
reject_api_key_secret_reference(user_config_params)
|
||||
return True
|
||||
|
||||
|
|
@ -846,10 +1091,57 @@ def _coerce_metadata_to_dict(value: object) -> dict[str, object] | None:
|
|||
return None
|
||||
|
||||
|
||||
def _team_s3_buckets(team_metadata: Mapping[str, object] | None) -> tuple[str, ...]:
|
||||
if team_metadata is None:
|
||||
return ()
|
||||
try:
|
||||
return _BUCKET_NAMES.validate_python(team_metadata.get(TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY) or ())
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
_TEAM_S3_BUCKET_ROUTE_TEMPLATES: Final = frozenset(
|
||||
{"/v1/files", "/files", "/{provider}/v1/files", "/v1/batches", "/batches", "/{provider}/v1/batches"}
|
||||
)
|
||||
|
||||
|
||||
def checks_team_s3_buckets_in_handler(request: Request) -> bool:
|
||||
"""File upload and batch create check request buckets against the caller team's allowlist themselves."""
|
||||
return get_request_route_template(request) in _TEAM_S3_BUCKET_ROUTE_TEMPLATES and request.method == "POST"
|
||||
|
||||
|
||||
def enforce_team_s3_bucket_allowlist(
|
||||
request_body: Mapping[str, object],
|
||||
general_settings: Mapping[str, object],
|
||||
llm_router: Router | None,
|
||||
team_metadata: Mapping[str, object] | None,
|
||||
*,
|
||||
team_id: str | None = None,
|
||||
) -> None:
|
||||
"""Re-run the request body check that auth relaxed for bucket params, now admitting the caller team's buckets."""
|
||||
proxy_buckets: Final = _ClientSideStorageAllowlist.model_validate(general_settings).allowed_client_side_s3_buckets
|
||||
model: Final = request_body.get("model")
|
||||
try:
|
||||
is_request_body_safe(
|
||||
request_body=dict(request_body),
|
||||
general_settings={
|
||||
**general_settings,
|
||||
"allowed_client_side_s3_buckets": (*(proxy_buckets or ()), *_team_s3_buckets(team_metadata)),
|
||||
},
|
||||
llm_router=llm_router,
|
||||
model=model if isinstance(model, str) else "",
|
||||
team_id=team_id,
|
||||
)
|
||||
except ValueError as rejection:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=f"Authentication Error, {rejection}")
|
||||
|
||||
|
||||
async def pre_db_read_auth_checks(
|
||||
request: Request,
|
||||
request_data: dict,
|
||||
route: str,
|
||||
*,
|
||||
defer_team_checks: bool = False,
|
||||
):
|
||||
"""
|
||||
1. Checks if request size is under max_request_size_mb (if set)
|
||||
|
|
@ -869,12 +1161,18 @@ async def pre_db_read_auth_checks(
|
|||
await check_if_request_size_is_safe(request=request)
|
||||
|
||||
# Check 2. Request body is safe
|
||||
defer_model_checks: Final = (
|
||||
defer_team_checks and llm_router is not None and bool(llm_router.registry().index("team_model"))
|
||||
)
|
||||
request.state.deferred_model_body_check = defer_model_checks # rebind-ok: hand state to auth
|
||||
is_request_body_safe(
|
||||
request_body=request_data,
|
||||
general_settings=general_settings,
|
||||
llm_router=llm_router,
|
||||
model=request_data.get("model", ""), # [TODO] use model passed in url as well (azure openai routes)
|
||||
model=request_data.get("model") or _model_from_route_path(route) or "",
|
||||
route=route,
|
||||
defer_model_checks=defer_model_checks,
|
||||
defer_s3_buckets=checks_team_s3_buckets_in_handler(request),
|
||||
)
|
||||
|
||||
# Check 3. Check if IP address is allowed
|
||||
|
|
@ -1522,6 +1820,9 @@ class BatchLimitRequest(Protocol):
|
|||
def model_fields_set(self) -> Collection[str]: ...
|
||||
|
||||
|
||||
_ADMIN_ONLY_METADATA_KEYS: Final = (*ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS, TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY)
|
||||
|
||||
|
||||
def enforce_batch_limits_are_admin_only(
|
||||
data: BatchLimitRequest,
|
||||
existing_metadata: Mapping[str, object] | None,
|
||||
|
|
@ -1532,7 +1833,8 @@ def enforce_batch_limits_are_admin_only(
|
|||
|
||||
Every key in ``ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS`` caps what the holder
|
||||
may do with batches, so a holder-writable copy would let a caller lift their
|
||||
own quota. Gated on the resulting value rather than on presence, so a form
|
||||
own quota, and the S3 bucket list grants access to buckets with the proxy's
|
||||
own credentials. Gated on the resulting value rather than on presence, so a form
|
||||
resending the stored value stays a no-op.
|
||||
"""
|
||||
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value:
|
||||
|
|
@ -1542,17 +1844,19 @@ def enforce_batch_limits_are_admin_only(
|
|||
(data.metadata or EMPTY_MAPPING) if "metadata" in data.model_fields_set else stored
|
||||
)
|
||||
changed: Final = next(
|
||||
(key for key in ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS if requested.get(key) != stored.get(key)),
|
||||
(key for key in _ADMIN_ONLY_METADATA_KEYS if requested.get(key) != stored.get(key)),
|
||||
None,
|
||||
)
|
||||
if changed is None:
|
||||
return
|
||||
reason: Final = (
|
||||
"It lets the holder read and write those buckets with the proxy's AWS credentials."
|
||||
if changed == TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY
|
||||
else "It limits what the holder can do with batches, so the holder cannot raise it."
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={
|
||||
"error": f"Only proxy admins can set {changed} on a {entity}. "
|
||||
"It limits what the holder can do with batches, so the holder cannot raise it."
|
||||
},
|
||||
detail={"error": f"Only proxy admins can set {changed} on a {entity}. {reason}"},
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -2395,37 +2699,8 @@ def get_model_from_request(
|
|||
team_id=team_id,
|
||||
)
|
||||
model = _format_model_candidates(candidates)
|
||||
|
||||
# If no explicit model was found, try to extract from route
|
||||
if model is None:
|
||||
# Parse model from route that follows the pattern /openai/deployments/{model}/*
|
||||
match: Final = re.match(r"/openai/deployments/([^/]+)", route)
|
||||
if match:
|
||||
model = match.group(1)
|
||||
|
||||
# If still not found, extract model from Google generateContent-style routes.
|
||||
# These routes put the model in the path and allow "/" inside the model id.
|
||||
# Examples:
|
||||
# - /v1beta/models/gemini-2.0-flash:generateContent
|
||||
# - /v1beta/models/bedrock/claude-sonnet-3.7:generateContent
|
||||
# - /models/custom/ns/model:streamGenerateContent
|
||||
if model is None and not route.lower().startswith("/vertex"):
|
||||
google_match = re.search(r"/(?:v1beta|beta)/models/([^:]+):", route)
|
||||
if google_match:
|
||||
model = google_match.group(1)
|
||||
|
||||
if model is None and not route.lower().startswith("/vertex"):
|
||||
google_match = re.search(r"^/models/([^:]+):", route)
|
||||
if google_match:
|
||||
model = google_match.group(1)
|
||||
|
||||
# If still not found, extract from Vertex AI passthrough route
|
||||
# Pattern: /vertex_ai/.../models/{model_id}:*
|
||||
# Example: /vertex_ai/v1/.../models/gemini-1.5-pro:generateContent
|
||||
if model is None and route.lower().startswith("/vertex"):
|
||||
vertex_match: Final = re.search(r"/models/([^:]+)", route)
|
||||
if vertex_match:
|
||||
model = vertex_match.group(1)
|
||||
model = _model_from_route_path(route)
|
||||
|
||||
if route.lower().startswith("/bedrock"):
|
||||
bedrock_model: Final = _model_from_bedrock_route(route)
|
||||
|
|
@ -2444,6 +2719,23 @@ def get_model_from_request(
|
|||
return model
|
||||
|
||||
|
||||
def _model_from_route_path(route: str) -> str | None:
|
||||
"""
|
||||
Google generateContent-style routes allow "/" inside the model id, e.g.
|
||||
/v1beta/models/bedrock/claude-sonnet-3.7:generateContent
|
||||
"""
|
||||
azure_deployment: Final = re.match(r"/openai/deployments/([^/]+)", route)
|
||||
if azure_deployment:
|
||||
return azure_deployment.group(1)
|
||||
if route.lower().startswith("/vertex"):
|
||||
vertex_model: Final = re.search(r"/models/([^:]+)", route)
|
||||
return vertex_model.group(1) if vertex_model else None
|
||||
google_model: Final = re.search(r"/(?:v1beta|beta)/models/([^:]+):", route) or re.search(
|
||||
r"^/models/([^:]+):", route
|
||||
)
|
||||
return google_model.group(1) if google_model else None
|
||||
|
||||
|
||||
def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str | None:
|
||||
if llm_router is None:
|
||||
return None
|
||||
|
|
|
|||
198
litellm/proxy/auth/legacy_empty_models_check.py
Normal file
198
litellm/proxy/auth/legacy_empty_models_check.py
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Protocol
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
|
||||
from litellm.proxy._types import LiteLLMRoutes, SpecialModelNames
|
||||
from litellm.repositories.prisma_protocols import CountTable
|
||||
|
||||
DEPRECATION_DISCUSSION_URL: Final = "https://github.com/BerriAI/litellm/discussions/32090"
|
||||
|
||||
|
||||
class CountTableRepository(Protocol):
|
||||
@property
|
||||
def table(self) -> CountTable: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LegacyEmptyModelsCounts:
|
||||
teamless_keys_with_empty_models: int
|
||||
teamless_keys_with_all_team_models: int
|
||||
team_keys_with_empty_models: int
|
||||
teams_with_empty_models: int
|
||||
keys_with_access_groups_and_empty_models: int
|
||||
teams_with_access_groups_and_empty_models: int
|
||||
|
||||
|
||||
def _model_calling_keys(where: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {
|
||||
"AND": [
|
||||
where,
|
||||
{
|
||||
"NOT": [
|
||||
{"allowed_routes": {"equals": [LiteLLMRoutes.management_routes.name]}},
|
||||
{"allowed_routes": {"equals": [LiteLLMRoutes.info_routes.name]}},
|
||||
]
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
async def count_legacy_empty_models_rows(keys: CountTable, teams: CountTable) -> LegacyEmptyModelsCounts:
|
||||
(
|
||||
teamless_empty,
|
||||
teamless_all_team,
|
||||
team_keys_empty,
|
||||
teams_empty,
|
||||
keys_with_groups_empty,
|
||||
teams_with_groups_empty,
|
||||
) = await asyncio.gather(
|
||||
keys.count(
|
||||
where=_model_calling_keys(
|
||||
{"team_id": None, "models": {"isEmpty": True}, "access_group_ids": {"isEmpty": True}}
|
||||
)
|
||||
),
|
||||
keys.count(
|
||||
where=_model_calling_keys(
|
||||
{
|
||||
"team_id": None,
|
||||
"models": {"has": SpecialModelNames.all_team_models.value},
|
||||
"access_group_ids": {"isEmpty": True},
|
||||
}
|
||||
)
|
||||
),
|
||||
keys.count(
|
||||
where=_model_calling_keys(
|
||||
{
|
||||
"AND": [{"team_id": {"not": None}}, {"team_id": {"not": UI_SESSION_TOKEN_TEAM_ID}}],
|
||||
"models": {"isEmpty": True},
|
||||
"access_group_ids": {"isEmpty": True},
|
||||
}
|
||||
)
|
||||
),
|
||||
teams.count(where={"models": {"isEmpty": True}, "access_group_ids": {"isEmpty": True}}),
|
||||
keys.count(
|
||||
where=_model_calling_keys(
|
||||
{
|
||||
"OR": [{"team_id": None}, {"team_id": {"not": UI_SESSION_TOKEN_TEAM_ID}}],
|
||||
"models": {"isEmpty": True},
|
||||
"NOT": {"access_group_ids": {"isEmpty": True}},
|
||||
}
|
||||
)
|
||||
),
|
||||
teams.count(where={"models": {"isEmpty": True}, "NOT": {"access_group_ids": {"isEmpty": True}}}),
|
||||
)
|
||||
return LegacyEmptyModelsCounts(
|
||||
teamless_keys_with_empty_models=teamless_empty,
|
||||
teamless_keys_with_all_team_models=teamless_all_team,
|
||||
team_keys_with_empty_models=team_keys_empty,
|
||||
teams_with_empty_models=teams_empty,
|
||||
keys_with_access_groups_and_empty_models=keys_with_groups_empty,
|
||||
teams_with_access_groups_and_empty_models=teams_with_groups_empty,
|
||||
)
|
||||
|
||||
|
||||
def _counted(count: int, singular: str, plural: str) -> str:
|
||||
return f"{count} {singular if count == 1 else plural}"
|
||||
|
||||
|
||||
def legacy_empty_models_warning(counts: LegacyEmptyModelsCounts) -> str | None:
|
||||
findings: Final = tuple(
|
||||
f"{_counted(count, singular, plural)}, which means {meaning}."
|
||||
for count, singular, plural, meaning in (
|
||||
(
|
||||
counts.teamless_keys_with_empty_models,
|
||||
"key without a team has an empty models list",
|
||||
"keys without a team have an empty models list",
|
||||
"every proxy model",
|
||||
),
|
||||
(
|
||||
counts.teamless_keys_with_all_team_models,
|
||||
"key without a team lists all-team-models",
|
||||
"keys without a team list all-team-models",
|
||||
"every proxy model",
|
||||
),
|
||||
(
|
||||
counts.team_keys_with_empty_models,
|
||||
"team key has an empty models list",
|
||||
"team keys have an empty models list",
|
||||
"all of the team's models",
|
||||
),
|
||||
(
|
||||
counts.teams_with_empty_models,
|
||||
"team has an empty models list",
|
||||
"teams have an empty models list",
|
||||
"every proxy model",
|
||||
),
|
||||
)
|
||||
if count > 0
|
||||
)
|
||||
access_group_findings: Final = tuple(
|
||||
f"{_counted(count, singular, plural)}."
|
||||
for count, singular, plural in (
|
||||
(
|
||||
counts.keys_with_access_groups_and_empty_models,
|
||||
"key with access groups has an empty models list",
|
||||
"keys with access groups have an empty models list",
|
||||
),
|
||||
(
|
||||
counts.teams_with_access_groups_and_empty_models,
|
||||
"team with access groups has an empty models list",
|
||||
"teams with access groups have an empty models list",
|
||||
),
|
||||
)
|
||||
if count > 0
|
||||
)
|
||||
if not findings and not access_group_findings:
|
||||
return None
|
||||
all_proxy_models: Final = SpecialModelNames.all_proxy_models.value
|
||||
all_team_models: Final = SpecialModelNames.all_team_models.value
|
||||
keep_access_advice: Final = (
|
||||
(
|
||||
f"To keep today's access, set models to {all_proxy_models} or a list of models on each one, "
|
||||
f"or to {all_team_models} on keys that belong to a team, before you upgrade.",
|
||||
)
|
||||
if findings
|
||||
else ()
|
||||
)
|
||||
teamless_all_team_models_advice: Final = (
|
||||
(
|
||||
f"Keys without a team that list {all_team_models} have no team to inherit from, "
|
||||
f"so set each one to {all_proxy_models} or attach it to a team.",
|
||||
)
|
||||
if counts.teamless_keys_with_all_team_models > 0
|
||||
else ()
|
||||
)
|
||||
access_group_advice: Final = (
|
||||
("Check which models each key and team with access groups should reach before you upgrade.",)
|
||||
if access_group_findings
|
||||
else ()
|
||||
)
|
||||
return " ".join(
|
||||
(
|
||||
"Deprecated model access defaults are still in use.",
|
||||
*findings,
|
||||
*access_group_findings,
|
||||
"These counts include blocked and expired keys and blocked teams.",
|
||||
"LiteLLM 2.0.0 removes these defaults.",
|
||||
*keep_access_advice,
|
||||
*teamless_all_team_models_advice,
|
||||
*access_group_advice,
|
||||
"Keys and teams created without a models list still get an empty one until then, "
|
||||
"so these counts can grow back.",
|
||||
f"See {DEPRECATION_DISCUSSION_URL}",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def warn_about_legacy_empty_models(keys: CountTableRepository, teams: CountTableRepository) -> None:
|
||||
try:
|
||||
counts: Final = await count_legacy_empty_models_rows(keys=keys.table, teams=teams.table)
|
||||
except Exception as e: # noqa: BLE001 # a startup check must never take the proxy down on a DB error
|
||||
verbose_proxy_logger.warning("Skipped the deprecated model access defaults check: %s", e)
|
||||
return
|
||||
message: Final = legacy_empty_models_warning(counts)
|
||||
if message is not None:
|
||||
verbose_proxy_logger.warning(message)
|
||||
|
|
@ -28,6 +28,7 @@ from litellm.proxy._types import (
|
|||
)
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
TeamNotFoundError,
|
||||
forget_missing_user,
|
||||
get_team_membership,
|
||||
get_team_object,
|
||||
get_user_object,
|
||||
|
|
@ -246,6 +247,8 @@ class GrantResolver:
|
|||
return ResolvedGrants(user_object, team_object, membership, effective_user_id)
|
||||
|
||||
async def _user(self, lookup: UserLookup) -> LiteLLM_UserTable | None:
|
||||
if lookup.user_id is not None:
|
||||
forget_missing_user(lookup.user_id)
|
||||
return await self._load_user(
|
||||
user_id=lookup.user_id,
|
||||
prisma_client=self._prisma,
|
||||
|
|
|
|||
|
|
@ -109,12 +109,14 @@ from litellm.proxy.auth.auth_object_prefetch import (
|
|||
)
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
abbreviate_api_key,
|
||||
checks_team_s3_buckets_in_handler,
|
||||
fallback_target_model_name,
|
||||
get_end_user_id_from_request_body,
|
||||
get_model_from_request,
|
||||
get_request_route,
|
||||
get_request_route_template,
|
||||
is_invalid_virtual_key_error,
|
||||
is_request_body_safe,
|
||||
iter_request_fallback_targets,
|
||||
log_model_access_denial,
|
||||
normalize_request_route,
|
||||
|
|
@ -1313,6 +1315,12 @@ def _apply_budget_limits_to_end_user_params(
|
|||
if budget_info.model_max_budget is not None:
|
||||
end_user_params["end_user_model_max_budget"] = budget_info.model_max_budget
|
||||
|
||||
if budget_info.model_tpm_limit is not None:
|
||||
end_user_params["end_user_model_tpm_limit"] = budget_info.model_tpm_limit # rebind-ok: in-place like siblings
|
||||
|
||||
if budget_info.model_rpm_limit is not None:
|
||||
end_user_params["end_user_model_rpm_limit"] = budget_info.model_rpm_limit # rebind-ok: in-place like siblings
|
||||
|
||||
verbose_proxy_logger.debug("Applied budget limits to end user %s", end_user_id)
|
||||
|
||||
|
||||
|
|
@ -1444,6 +1452,10 @@ def update_valid_token_with_end_user_params(valid_token: UserAPIKeyAuth, end_use
|
|||
valid_token.allowed_model_region = end_user_params["allowed_model_region"]
|
||||
if end_user_params.get("end_user_model_max_budget") is not None:
|
||||
valid_token.end_user_model_max_budget = end_user_params["end_user_model_max_budget"]
|
||||
if "end_user_model_tpm_limit" in end_user_params:
|
||||
valid_token.end_user_model_tpm_limit = end_user_params["end_user_model_tpm_limit"] # rebind-ok: in-place
|
||||
if "end_user_model_rpm_limit" in end_user_params:
|
||||
valid_token.end_user_model_rpm_limit = end_user_params["end_user_model_rpm_limit"] # rebind-ok: in-place
|
||||
return valid_token
|
||||
|
||||
|
||||
|
|
@ -2266,6 +2278,7 @@ async def user_api_key_auth_builder(
|
|||
azure_apim_header: str | None,
|
||||
request_data: dict[str, object],
|
||||
custom_litellm_key_header: str | None = None,
|
||||
defer_team_checks: bool = False,
|
||||
) -> UserAPIKeyAuth:
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
|
|
@ -2300,6 +2313,7 @@ async def user_api_key_auth_builder(
|
|||
request_data=request_data,
|
||||
request=request,
|
||||
route=route,
|
||||
defer_team_checks=defer_team_checks,
|
||||
)
|
||||
pass_through_endpoints: Final[list[dict] | None] = general_settings.get("pass_through_endpoints", None)
|
||||
## CHECK IF X-LITELM-API-KEY IS PASSED IN - supercedes Authorization header
|
||||
|
|
@ -2985,6 +2999,10 @@ async def user_api_key_auth_builder(
|
|||
valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit")
|
||||
valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit")
|
||||
valid_token.end_user_tpd_limit = end_user_params.get("end_user_tpd_limit")
|
||||
if "end_user_model_tpm_limit" in end_user_params:
|
||||
valid_token.end_user_model_tpm_limit = end_user_params["end_user_model_tpm_limit"]
|
||||
if "end_user_model_rpm_limit" in end_user_params:
|
||||
valid_token.end_user_model_rpm_limit = end_user_params["end_user_model_rpm_limit"]
|
||||
valid_token.allowed_model_region = end_user_params.get("allowed_model_region")
|
||||
|
||||
return await validate_resolved_virtual_key(
|
||||
|
|
@ -3670,6 +3688,18 @@ async def run_centralized_common_checks(
|
|||
user_custom_auth,
|
||||
)
|
||||
|
||||
if not sub_call and getattr(request.state, "deferred_model_body_check", False) is True:
|
||||
body_model: Final = request_data.get("model")
|
||||
is_request_body_safe(
|
||||
request_body=request_data,
|
||||
general_settings=_GENERAL_SETTINGS_ADAPTER.validate_python(general_settings),
|
||||
llm_router=llm_router,
|
||||
model=body_model if isinstance(body_model, str) else "",
|
||||
route=route,
|
||||
team_id=user_api_key_auth_obj.team_id,
|
||||
defer_s3_buckets=checks_team_s3_buckets_in_handler(request),
|
||||
)
|
||||
|
||||
# Public routes (e.g. /health/liveness) are exempt from
|
||||
# auth in the builder — the wrapper must not retroactively apply
|
||||
# authz on top, or k8s readiness probes and other unauthenticated
|
||||
|
|
@ -4134,6 +4164,10 @@ async def _apply_key_end_user_default_budget_to_token(
|
|||
valid_token.end_user_tpd_limit = default_budget.tpd_limit
|
||||
if not keep_token_limits or valid_token.end_user_model_max_budget is None:
|
||||
valid_token.end_user_model_max_budget = default_budget.model_max_budget
|
||||
if not keep_token_limits or valid_token.end_user_model_tpm_limit is None:
|
||||
valid_token.end_user_model_tpm_limit = default_budget.model_tpm_limit # rebind-ok: in-place like siblings
|
||||
if not keep_token_limits or valid_token.end_user_model_rpm_limit is None:
|
||||
valid_token.end_user_model_rpm_limit = default_budget.model_rpm_limit # rebind-ok: in-place like siblings
|
||||
|
||||
|
||||
async def _reserve_budget_after_common_checks(
|
||||
|
|
@ -4502,6 +4536,7 @@ async def user_api_key_auth(
|
|||
azure_apim_header=azure_apim_header,
|
||||
request_data=request_data,
|
||||
custom_litellm_key_header=custom_litellm_key_header,
|
||||
defer_team_checks=True,
|
||||
)
|
||||
except Exception:
|
||||
# The body was read first, so a caller who sent both a malformed body and
|
||||
|
|
@ -4981,7 +5016,7 @@ async def authorize_internal_virtual_key(
|
|||
)
|
||||
)
|
||||
route: Final = get_request_route(request=request)
|
||||
await pre_db_read_auth_checks(request_data=request_data, request=request, route=route)
|
||||
await pre_db_read_auth_checks(request_data=request_data, request=request, route=route, defer_team_checks=True)
|
||||
auth: Final = await validate_resolved_virtual_key(
|
||||
request=request,
|
||||
request_data=request_data,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Final, Literal, Protocol, cast
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -20,6 +21,7 @@ from litellm.litellm_core_utils.core_helpers import attribute_or_none, has_attri
|
|||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, set_hidden_param
|
||||
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.auth_utils import enforce_team_s3_bucket_allowlist
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
|
||||
from litellm.proxy.batches_endpoints.litellm_executed_batches import (
|
||||
|
|
@ -81,6 +83,7 @@ if TYPE_CHECKING:
|
|||
from prisma.models import LiteLLM_ManagedObjectTable
|
||||
|
||||
router: Final = APIRouter()
|
||||
_REQUEST_BODY_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
class _UserBatchLister(Protocol):
|
||||
|
|
@ -285,6 +288,7 @@ async def create_batch(
|
|||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
general_settings_view,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
|
|
@ -294,6 +298,13 @@ async def create_batch(
|
|||
data: dict = {}
|
||||
try:
|
||||
data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
enforce_team_s3_bucket_allowlist(
|
||||
request_body=_REQUEST_BODY_ADAPTER.validate_python(data),
|
||||
general_settings=general_settings_view(),
|
||||
llm_router=llm_router,
|
||||
team_metadata=user_api_key_dict.team_metadata,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
)
|
||||
_raise_when_routing_headers_disagree(request, data.get("model"))
|
||||
verbose_proxy_logger.debug(
|
||||
"Request received by LiteLLM:\n%s",
|
||||
|
|
@ -384,6 +395,7 @@ async def create_batch(
|
|||
prepare_data_with_credentials(
|
||||
data=_create_batch_data,
|
||||
credentials=credentials,
|
||||
keep_request_buckets=True,
|
||||
)
|
||||
|
||||
# Create batch using model credentials
|
||||
|
|
@ -488,6 +500,7 @@ async def create_batch(
|
|||
prepare_data_with_credentials(
|
||||
data=_create_batch_data,
|
||||
credentials=credentials,
|
||||
keep_request_buckets=True,
|
||||
)
|
||||
|
||||
# Create batch using model credentials
|
||||
|
|
@ -506,6 +519,7 @@ async def create_batch(
|
|||
llm_router=llm_router,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
keep_request_buckets=True,
|
||||
)
|
||||
_raise_not_found_when_openai_fallback_unservable(
|
||||
requested_provider=requested_provider,
|
||||
|
|
|
|||
|
|
@ -452,7 +452,7 @@ class LiteLLMExecutedBatchRunner:
|
|||
) -> LiteLLMBatch:
|
||||
endpoint: Final = _validate_endpoint(create_request.get("endpoint"))
|
||||
content: Final = await self._download_input(unified_input_file_id, user_api_key_dict)
|
||||
parsed: Final = parse_batch_input(content, endpoint, self._body_rejection(model))
|
||||
parsed: Final = parse_batch_input(content, endpoint, self._body_rejection(model, user_api_key_dict.team_id))
|
||||
if isinstance(parsed, InvalidBatchInput):
|
||||
raise batch_error(400, f"Invalid batch input file: {parsed.describe()}")
|
||||
llm_batch_id: Final = f"{LITELLM_EXECUTED_BATCH_ID_PREFIX}{uuid_module.uuid4().hex}"
|
||||
|
|
@ -533,7 +533,7 @@ class LiteLLMExecutedBatchRunner:
|
|||
return failed
|
||||
return await self.batches.load_batch(batch.id) or batch
|
||||
|
||||
def _body_rejection(self, model: str) -> BodyRejection:
|
||||
def _body_rejection(self, model: str, team_id: str | None) -> BodyRejection:
|
||||
def reject(body: Mapping[str, object]) -> str | None:
|
||||
try:
|
||||
is_request_body_safe(
|
||||
|
|
@ -541,6 +541,7 @@ class LiteLLMExecutedBatchRunner:
|
|||
general_settings=dict(self.general_settings),
|
||||
llm_router=self.llm_router,
|
||||
model=model,
|
||||
team_id=team_id,
|
||||
)
|
||||
except ValueError as e:
|
||||
return str(e)
|
||||
|
|
|
|||
|
|
@ -260,6 +260,16 @@ Delete a credential:
|
|||
lite credentials delete <credential-name>
|
||||
```
|
||||
|
||||
#### Connect a GitHub Copilot Credential
|
||||
|
||||
Connect your own GitHub Copilot account to a per-user GitHub Copilot credential, the same as the Connect button in the UI. Run it with your own virtual key, since the connection belongs to the user that key resolves to:
|
||||
|
||||
```bash
|
||||
lite --api-key sk-your-key credentials connect <credential-name>
|
||||
```
|
||||
|
||||
It prints a code and a GitHub link, then waits. Open the link, enter the code, and approve. The command exits 0 once GitHub confirms the login, and exits 1 if the code expires, the login is denied, or the GitHub account has no Copilot seat.
|
||||
|
||||
### Keys Management
|
||||
|
||||
The CLI provides commands for managing API keys on your LiteLLM proxy server:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import json
|
||||
from collections.abc import Sequence
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
import click
|
||||
|
|
@ -8,9 +11,30 @@ import rich
|
|||
from rich.table import Table
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.models.credentials import UserConnectionPollResponse, UserConnectionPollStatus
|
||||
|
||||
from ...credentials import CredentialsManagementClient
|
||||
from ...exceptions import UnauthorizedError
|
||||
from ._cli_context import cli_context_values
|
||||
|
||||
_SLOW_DOWN_STEP_SECONDS: Final = 5
|
||||
_CONNECT_FAILURES: Final[MappingProxyType[UserConnectionPollStatus, str]] = MappingProxyType(
|
||||
{
|
||||
"expired": "The code expired before it was approved. Run the command again for a new code",
|
||||
"denied": "The GitHub login was denied",
|
||||
"no_copilot_seat": "That GitHub account has no GitHub Copilot seat",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PollClock:
|
||||
now: Callable[[], float]
|
||||
sleep: Callable[[float], None]
|
||||
|
||||
|
||||
SYSTEM_CLOCK: Final = PollClock(now=time.monotonic, sleep=time.sleep)
|
||||
|
||||
|
||||
class _CredentialInfo(TypedDict):
|
||||
custom_llm_provider: ReadOnly[NotRequired[str]]
|
||||
|
|
@ -179,3 +203,80 @@ def get(ctx: click.Context, credential_name: str):
|
|||
client: Final = CredentialsManagementClient(context["base_url"], context["api_key"])
|
||||
response: Final = client.get(credential_name)
|
||||
_print_json(response)
|
||||
|
||||
|
||||
def _next_poll_interval(poll: UserConnectionPollResponse, current: int) -> int:
|
||||
if poll.interval is not None:
|
||||
return poll.interval
|
||||
return current + _SLOW_DOWN_STEP_SECONDS if poll.status == "slow_down" else current
|
||||
|
||||
|
||||
def _poll_until_settled(
|
||||
client: CredentialsManagementClient,
|
||||
credential_name: str,
|
||||
flow_handle: str,
|
||||
first_interval: int,
|
||||
expires_at: float,
|
||||
clock: PollClock,
|
||||
) -> UserConnectionPollResponse:
|
||||
interval = first_interval # rebind-ok: GitHub can ask for a longer wait on any poll
|
||||
while clock.now() + interval < expires_at:
|
||||
clock.sleep(interval)
|
||||
if (poll := client.poll_user_connection(credential_name, flow_handle)).status not in ("pending", "slow_down"):
|
||||
return poll
|
||||
interval = _next_poll_interval(poll, interval)
|
||||
return UserConnectionPollResponse(status="expired")
|
||||
|
||||
|
||||
def connect_user_credential(
|
||||
client: CredentialsManagementClient,
|
||||
credential_name: str,
|
||||
clock: PollClock,
|
||||
) -> UserConnectionPollResponse:
|
||||
"""Run the GitHub device login for the calling key's user and return the final poll result.
|
||||
|
||||
Polling stops once the next poll would land after the code expires, since the proxy rejects polls on an expired
|
||||
code. The countdown starts before the code is requested, so it never runs past the proxy's own."""
|
||||
requested_at: Final = clock.now()
|
||||
started: Final = client.start_user_connection(credential_name)
|
||||
click.echo(f"Go to {started.verification_uri} and enter code {started.user_code}")
|
||||
click.echo("Waiting for approval...")
|
||||
expires_at: Final = requested_at + started.expires_in
|
||||
return _poll_until_settled(client, credential_name, started.flow_handle, started.interval, expires_at, clock)
|
||||
|
||||
|
||||
def _report_http_error(response: requests.Response) -> None:
|
||||
click.echo(f"Error: HTTP {response.status_code}", err=True)
|
||||
try:
|
||||
error_body: Final[_JsonBodyView] = {"body": response.json()}
|
||||
_print_json(error_body["body"])
|
||||
except json.JSONDecodeError:
|
||||
click.echo(response.text, err=True)
|
||||
|
||||
|
||||
@credentials.command()
|
||||
@click.argument("credential_name")
|
||||
@click.pass_context
|
||||
def connect(ctx: click.Context, credential_name: str) -> None:
|
||||
"""Connect your own GitHub Copilot account to a per-user credential, without the UI"""
|
||||
context: Final = cli_context_values(ctx)
|
||||
client: Final = CredentialsManagementClient(context["base_url"], context["api_key"])
|
||||
try:
|
||||
result: Final = connect_user_credential(client, credential_name, SYSTEM_CLOCK)
|
||||
except UnauthorizedError as e:
|
||||
if isinstance(e.orig_exception, requests.exceptions.HTTPError) and e.orig_exception.response is not None:
|
||||
_report_http_error(e.orig_exception.response)
|
||||
raise click.Abort()
|
||||
except requests.exceptions.HTTPError as e:
|
||||
_report_http_error(e.response)
|
||||
raise click.Abort()
|
||||
except requests.exceptions.RequestException as e:
|
||||
click.echo(
|
||||
f"Error: could not reach the proxy at {context['base_url']} ({type(e).__name__}). Run the command again",
|
||||
err=True,
|
||||
)
|
||||
raise click.Abort()
|
||||
if result.status != "connected":
|
||||
click.echo(f"Error: {_CONNECT_FAILURES.get(result.status, result.status)}", err=True)
|
||||
raise click.exceptions.Exit(1)
|
||||
click.echo(f"Connected as {result.github_login} for credential '{credential_name}'")
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ from urllib.parse import quote
|
|||
import requests
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.models.credentials import UserConnectionPollResponse, UserConnectionStartResponse
|
||||
|
||||
from .exceptions import UnauthorizedError
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
|
@ -217,3 +219,33 @@ class CredentialsManagementClient:
|
|||
if e.response.status_code == 401:
|
||||
raise UnauthorizedError(e)
|
||||
raise
|
||||
|
||||
def _send_json(self, request: requests.Request) -> object:
|
||||
try:
|
||||
response: Final = requests.Session().send(request.prepare(), timeout=self._timeout)
|
||||
response.raise_for_status()
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise UnauthorizedError(e)
|
||||
raise
|
||||
|
||||
def _user_connection_url(self, credential_name: str, action: str) -> str:
|
||||
return f"{self._base_url}/credentials/{quote(credential_name, safe='')}/user_connection/{action}"
|
||||
|
||||
def start_user_connection(self, credential_name: str) -> UserConnectionStartResponse:
|
||||
"""Start the GitHub device login that connects the calling key's user to a per-user credential."""
|
||||
request: Final = requests.Request(
|
||||
"POST", self._user_connection_url(credential_name, "start"), headers=self._get_headers()
|
||||
)
|
||||
return UserConnectionStartResponse.model_validate(self._send_json(request))
|
||||
|
||||
def poll_user_connection(self, credential_name: str, flow_handle: str) -> UserConnectionPollResponse:
|
||||
"""Check once whether the user approved the device login that ``start_user_connection`` began."""
|
||||
request: Final = requests.Request(
|
||||
"POST",
|
||||
self._user_connection_url(credential_name, "poll"),
|
||||
headers=self._get_headers(),
|
||||
json={"flow_handle": flow_handle},
|
||||
)
|
||||
return UserConnectionPollResponse.model_validate(self._send_json(request))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import contextlib
|
|||
import json
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
|
@ -380,7 +381,7 @@ def resolve_litellm_call_id(client_call_id: str | None) -> str:
|
|||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def should_return_raw_model_name(request_data: dict[str, object]) -> bool:
|
||||
def should_return_raw_model_name(request_data: Mapping[str, object]) -> bool:
|
||||
return any(
|
||||
isinstance(metadata, dict) and metadata.get(RETURN_RAW_MODEL_NAME_METADATA_KEY) is True
|
||||
for metadata in (request_data.get("metadata"), request_data.get("litellm_metadata"))
|
||||
|
|
@ -4865,6 +4866,15 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
_finalize_streaming_generator_cleanup = finalize_streaming_generator_cleanup
|
||||
|
||||
@staticmethod
|
||||
def record_first_chunk_overhead(
|
||||
logging_obj: LiteLLMLoggingObj | None, chunk_received_at: float, is_provider_output: bool = True
|
||||
) -> None:
|
||||
"""The proxy's own work on a chunk; the logging object keeps the first one for the stamped overhead.
|
||||
A keepalive ping carries no provider output, so it is never that first chunk."""
|
||||
if logging_obj is not None and is_provider_output:
|
||||
logging_obj.set_first_chunk_overhead(time.perf_counter() - chunk_received_at)
|
||||
|
||||
@staticmethod
|
||||
async def async_streaming_data_generator(
|
||||
response: object,
|
||||
|
|
@ -4922,12 +4932,12 @@ class ProxyBaseLLMRequestProcessing:
|
|||
response=response,
|
||||
request_data=request_data,
|
||||
)
|
||||
release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(
|
||||
stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None
|
||||
)
|
||||
timing_sink: Final = stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None
|
||||
release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(timing_sink)
|
||||
try:
|
||||
str_so_far = ""
|
||||
async for chunk in guarded_stream:
|
||||
chunk_received_at = time.perf_counter()
|
||||
# ``.format(chunk)`` was previously evaluated for every chunk
|
||||
# regardless of log level; gate it behind the level check.
|
||||
if debug_enabled:
|
||||
|
|
@ -4968,6 +4978,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
delivered_chunk = delivered_chunk or chunk != STREAM_SSE_KEEPALIVE_PING_BYTES
|
||||
serialized = serialize_chunk(chunk)
|
||||
recent_tail = advance_sse_tail(recent_tail, serialized)
|
||||
ProxyBaseLLMRequestProcessing.record_first_chunk_overhead(
|
||||
timing_sink, chunk_received_at, is_provider_output=chunk != STREAM_SSE_KEEPALIVE_PING_BYTES
|
||||
)
|
||||
yield serialized
|
||||
held_tail: Final = flush_tail() if flush_tail is not None else b""
|
||||
if held_tail:
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
1706
litellm/proxy/common_utils/codex_bedrock_runtime_models_0.166.0.json
Normal file
1706
litellm/proxy/common_utils/codex_bedrock_runtime_models_0.166.0.json
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -6,7 +6,8 @@ feature on) and decodes the body as its own ``ModelsResponse``, ``{"models": [Mo
|
|||
never the OpenAI list shape. Codex then ignores its own bundled catalog, so this body is all it knows
|
||||
about the provider's models. It is built from the rows ``/v1/models`` already lists: a model Codex
|
||||
knows, by its id or by the upstream model its deployments call once the provider's own prefixes are
|
||||
stripped, keeps Codex's own entry for it, minus any service tier some deployment behind it prices tokens
|
||||
stripped, keeps Codex's own entry for it, out of the catalog Codex itself uses for the provider those
|
||||
deployments call (its Bedrock catalogs for ``bedrock`` and ``bedrock_mantle``, its OpenAI one otherwise), minus any service tier some deployment behind it prices tokens
|
||||
but not that tier for; any other model gets an entry translated from the capabilities its deployments' model info
|
||||
declares (see ``codex_model_translation``). ``model_info.service_tiers`` becomes the entry's
|
||||
``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier``
|
||||
|
|
@ -20,6 +21,15 @@ https://github.com/openai/codex/blob/rust-v0.166.0/codex-rs/models-manager/model
|
|||
as ``codex_bundled_models_0.166.0.json``, and ``codex_base_instructions.md`` is
|
||||
``codex-rs/models-manager/prompt.md`` at the same tag. Refresh both by downloading those two paths at
|
||||
a newer tag.
|
||||
|
||||
Codex builds its Bedrock catalogs in code rather than shipping them as files, so the ones for Bedrock's
|
||||
runtime and Mantle endpoints are vendored from Codex 0.166.0's own output, with the ``base_instructions``
|
||||
copy of each entry's ``model_messages.instructions_template`` that it adds for older clients dropped:
|
||||
``codex -c 'model_provider="amazon-bedrock-runtime"' debug models | jq 'del(.models[].base_instructions)'``
|
||||
as ``codex_bedrock_runtime_models_0.166.0.json``, and ``model_provider="amazon-bedrock"`` the same way as
|
||||
``codex_bedrock_mantle_models_0.166.0.json``. Refresh them at the same release as the bundled catalog,
|
||||
which they take each entry's ``minimal_client_version`` from, and recheck
|
||||
``CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION`` against the fields the new output leaves out.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -34,7 +44,6 @@ from pathlib import Path
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
from packaging.version import Version
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -49,6 +58,7 @@ from litellm.proxy.common_utils.codex_model_translation import (
|
|||
no_priced_deployment_lacks_tier,
|
||||
tier_price_keys,
|
||||
)
|
||||
from litellm.proxy.common_utils.codex_version import codex_version_at_least
|
||||
from litellm.types.llms.base_model import LiteLLMBaseModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -58,7 +68,16 @@ if TYPE_CHECKING:
|
|||
|
||||
CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
|
||||
CODEX_BUNDLED_MODELS_PATH: Final = Path(__file__).with_name("codex_bundled_models_0.166.0.json")
|
||||
CodexBedrockEndpoint: TypeAlias = Literal["bedrock", "bedrock_mantle"]
|
||||
CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION: Final = "0.148.0"
|
||||
CODEX_BEDROCK_MODELS_PATHS: Final[Mapping[CodexBedrockEndpoint, Path]] = MappingProxyType(
|
||||
{
|
||||
"bedrock": Path(__file__).with_name("codex_bedrock_runtime_models_0.166.0.json"),
|
||||
"bedrock_mantle": Path(__file__).with_name("codex_bedrock_mantle_models_0.166.0.json"),
|
||||
}
|
||||
)
|
||||
CODEX_CATALOG_BYTE_LIMIT: Final = 1024 * 1024
|
||||
_FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT: Final = "0.166.0-alpha.1"
|
||||
CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"})
|
||||
|
||||
_BODY_PREFIX: Final = '{"models":['
|
||||
|
|
@ -129,8 +148,8 @@ _NATIVE_ALIAS_DEFAULTS: Final[CodexAliasDefaults] = "deployment"
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class CodexCatalogRow:
|
||||
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
|
||||
input limit, the upstream model behind it and the provider its deployment declares, the reasoning
|
||||
effort its deployments pin, and the
|
||||
input limit, the upstream model behind it and the provider its deployment declares, every routable
|
||||
deployment's `upstreams` (model and declared provider), the reasoning effort its deployments pin, and the
|
||||
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
|
||||
value per deployment behind the id, None where a deployment sets none, and
|
||||
`configured_max_input_tokens` one `model_info.max_input_tokens` per deployment the same way.
|
||||
|
|
@ -142,6 +161,7 @@ class CodexCatalogRow:
|
|||
max_input_tokens: int | None = None
|
||||
upstream_model: str | None = None
|
||||
upstream_provider: str | None = None
|
||||
upstreams: tuple[tuple[str, str | None], ...] = ()
|
||||
reasoning_effort: str | None = None
|
||||
display_name: str | None = None
|
||||
service_tiers: tuple[object, ...] = ()
|
||||
|
|
@ -158,6 +178,40 @@ def bundled_codex_models() -> Mapping[str, CodexStockModel]:
|
|||
return MappingProxyType({model.slug: model for model in catalog.models})
|
||||
|
||||
|
||||
def _with_minimal_client_version(model: CodexStockModel, built_from: str | tuple[int, ...] | None) -> CodexStockModel:
|
||||
source_minimum: Final = built_from if not isinstance(built_from, tuple) else ".".join(map(str, built_from))
|
||||
minimum: Final = (
|
||||
source_minimum
|
||||
if codex_version_at_least(source_minimum, CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION)
|
||||
else CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION
|
||||
)
|
||||
return model.model_copy(update={"minimal_client_version": minimum})
|
||||
|
||||
|
||||
@cache
|
||||
def bundled_bedrock_codex_models(endpoint: CodexBedrockEndpoint) -> Mapping[str, CodexStockModel]:
|
||||
"""Codex's catalog for a Bedrock endpoint by slug, read once from the vendored file. `codex debug models`
|
||||
prints no `minimal_client_version`, so each entry takes the later of the bundled entry's Codex built it from
|
||||
and `CODEX_BEDROCK_MODELS_MINIMAL_CLIENT_VERSION`, the first release that decodes an entry without the fields
|
||||
`debug models` leaves out, such as `supports_parallel_tool_calls`."""
|
||||
from litellm.llms.bedrock.common_utils import bedrock_model_name_without_provider
|
||||
|
||||
catalog: Final = CodexStockCatalog.model_validate_json(
|
||||
CODEX_BEDROCK_MODELS_PATHS[endpoint].read_text(encoding="utf-8")
|
||||
)
|
||||
built_from: Final = MappingProxyType(
|
||||
{slug: model.minimal_client_version for slug, model in bundled_codex_models().items()}
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
model.slug: _with_minimal_client_version(
|
||||
model, built_from.get(bedrock_model_name_without_provider(model.slug))
|
||||
)
|
||||
for model in catalog.models
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def codex_base_instructions() -> str:
|
||||
return CODEX_BASE_INSTRUCTIONS_PATH.read_text(encoding="utf-8")
|
||||
|
|
@ -274,15 +328,67 @@ def _provider_model_name(upstream_model: str, upstream_provider: str | None) ->
|
|||
return model if name_for is None else name_for(model)
|
||||
|
||||
|
||||
def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _StockMatch:
|
||||
"""Codex's entry for a row, and whether the row lists it under another name than Codex's (an alias)."""
|
||||
|
||||
model: CodexStockModel
|
||||
alias: bool
|
||||
|
||||
|
||||
def _on_endpoint(
|
||||
declared: Sequence[tuple[str | None, str]], endpoint: CodexBedrockEndpoint
|
||||
) -> tuple[CodexBedrockEndpoint, str] | None:
|
||||
return next(((endpoint, model) for provider, model in declared if provider == endpoint), None)
|
||||
|
||||
|
||||
def _bedrock_upstream(row: CodexCatalogRow) -> tuple[CodexBedrockEndpoint, str] | None:
|
||||
"""The Bedrock endpoint a request to the row can reach whose Codex catalog promises the least, with the model
|
||||
a deployment on it calls. A request can route to any deployment, and Codex's Bedrock entries leave out what
|
||||
Bedrock does not support yet, so one Bedrock deployment puts the whole row on its catalog, the runtime one
|
||||
(no tool search) ahead of Mantle's."""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider
|
||||
|
||||
first: Final = () if row.upstream_model is None else ((row.upstream_model, row.upstream_provider),)
|
||||
declared: Final = tuple((declared_provider(model, provider), model) for model, provider in (*first, *row.upstreams))
|
||||
on_bedrock: Final = (_on_endpoint(declared, endpoint) for endpoint in CODEX_BEDROCK_MODELS_PATHS)
|
||||
return next((found for found in on_bedrock if found is not None), None)
|
||||
|
||||
|
||||
def _bedrock_stock_for(row_id: str, endpoint: CodexBedrockEndpoint, upstream_model: str) -> _StockMatch | None:
|
||||
"""Codex's entry for the row's id or the Bedrock model as written, else for the same model under any of
|
||||
Bedrock's routing and region prefixes (`eu.openai.gpt-6.1-sol` takes `global.openai.gpt-6.1-sol`'s)."""
|
||||
from litellm.llms.bedrock.common_utils import bedrock_model_name_without_provider
|
||||
|
||||
stock: Final = bundled_bedrock_codex_models(endpoint)
|
||||
names: Final = (row_id, upstream_model.removeprefix(f"{endpoint}/"))
|
||||
exact: Final = next((stock[name] for name in names if name in stock), None)
|
||||
model_names: Final = tuple(bedrock_model_name_without_provider(name) for name in names)
|
||||
same_model: Final = (
|
||||
next((model for slug, model in stock.items() if bedrock_model_name_without_provider(slug) == name), None)
|
||||
for name in model_names
|
||||
)
|
||||
found: Final = exact if exact is not None else next((model for model in same_model if model is not None), None)
|
||||
if found is None:
|
||||
return None
|
||||
return _StockMatch(found, model_names[0] != bedrock_model_name_without_provider(found.slug))
|
||||
|
||||
|
||||
def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> _StockMatch | None:
|
||||
"""Codex's entry for the row's id, else for its upstream model as written, else for that model's name
|
||||
once its provider's own prefixes are stripped (`bedrock/us.openai.gpt-6.1-sol` is `gpt-6.1-sol`)."""
|
||||
once its provider's own prefixes are stripped (`openai/gpt-6.1-sol` is `gpt-6.1-sol`), out of Codex's
|
||||
Bedrock catalog when a deployment behind the row is on Bedrock and `stock` otherwise. A Bedrock model that
|
||||
catalog does not know has no entry, since `stock`'s would promise what Bedrock does not support."""
|
||||
bedrock: Final = _bedrock_upstream(row)
|
||||
if bedrock is not None:
|
||||
return _bedrock_stock_for(row.id, *bedrock)
|
||||
names: Final = (
|
||||
(row.id,)
|
||||
if row.upstream_model is None
|
||||
else (row.id, row.upstream_model, _provider_model_name(row.upstream_model, row.upstream_provider))
|
||||
)
|
||||
return next((stock[name] for name in names if name in stock), None)
|
||||
found: Final = next((stock[name] for name in names if name in stock), None)
|
||||
return None if found is None else _StockMatch(found, row.id != found.slug)
|
||||
|
||||
|
||||
def _readable_by(stock_model: CodexStockModel, client_version: str | None) -> bool:
|
||||
|
|
@ -292,10 +398,7 @@ def _readable_by(stock_model: CodexStockModel, client_version: str | None) -> bo
|
|||
minimum: Final = stock_model.minimal_client_version
|
||||
if minimum is None or client_version is None:
|
||||
return True
|
||||
try:
|
||||
return Version(client_version) >= Version(minimum if isinstance(minimum, str) else ".".join(map(str, minimum)))
|
||||
except ValueError:
|
||||
return False
|
||||
return codex_version_at_least(client_version, minimum if isinstance(minimum, str) else ".".join(map(str, minimum)))
|
||||
|
||||
|
||||
def _tier_fields(stock_model: CodexStockModel, tiers: tuple[CodexServiceTier, ...]) -> Mapping[str, object]:
|
||||
|
|
@ -391,7 +494,7 @@ def _entry(
|
|||
client_version: str | None,
|
||||
) -> CodexCatalogEntry:
|
||||
matched: Final = _stock_for(row, stock)
|
||||
stock_model: Final = matched if matched is not None and _readable_by(matched, client_version) else None
|
||||
stock_model: Final = matched.model if matched is not None and _readable_by(matched.model, client_version) else None
|
||||
stock_tiers: Final = MappingProxyType(
|
||||
{tier.id: tier for tier in (stock_model.service_tiers if stock_model is not None else ())}
|
||||
)
|
||||
|
|
@ -412,7 +515,7 @@ def _entry(
|
|||
upgrade: Final = (
|
||||
stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
|
||||
)
|
||||
is_alias: Final = row.id != stock_model.slug
|
||||
is_alias: Final = matched is not None and matched.alias
|
||||
deployment_decides: Final = is_alias and alias_defaults == "deployment"
|
||||
stock_name: Final = row.id if is_alias else stock_model.display_name
|
||||
listing: Final = {
|
||||
|
|
@ -497,6 +600,16 @@ class CodexCatalogBody:
|
|||
left_out: tuple[str, ...]
|
||||
|
||||
|
||||
def codex_catalog_byte_limit(client_version: str | None) -> int | None:
|
||||
"""None, no limit, for a Codex that reads a catalog of any size since https://github.com/openai/codex/pull/56525.
|
||||
An older Codex, or one sending no version, drops a catalog over 1 MiB. 0.165.0 still has the limit."""
|
||||
return (
|
||||
None
|
||||
if codex_version_at_least(client_version, _FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT)
|
||||
else CODEX_CATALOG_BYTE_LIMIT
|
||||
)
|
||||
|
||||
|
||||
def codex_models_response_json(
|
||||
rows: Sequence[CodexCatalogRow],
|
||||
*,
|
||||
|
|
@ -555,7 +668,7 @@ def merged_codex_catalog(
|
|||
*,
|
||||
stock: Mapping[str, CodexStockModel] | None = None,
|
||||
instructions: str | None = None,
|
||||
byte_limit: int = CODEX_CATALOG_BYTE_LIMIT,
|
||||
byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
|
||||
client_version: str | None = None,
|
||||
) -> str | None:
|
||||
try:
|
||||
|
|
@ -569,7 +682,7 @@ def merged_codex_catalog(
|
|||
entries: Final = _entries(added, stock, instructions, first_priority, _NATIVE_ALIAS_DEFAULTS, client_version)
|
||||
added_models: Final = tuple(_entry_json(entry) for entry in entries)
|
||||
serialized: Final = tuple(_compact_json(model) for model in added_models)
|
||||
budget: Final = byte_limit - len(_compact_json({**top, "models": upstream}))
|
||||
budget: Final = None if byte_limit is None else byte_limit - len(_compact_json({**top, "models": upstream}))
|
||||
kept: Final = tuple(sorted(_kept_positions(entries, serialized, budget)))
|
||||
if len(kept) < len(entries):
|
||||
verbose_proxy_logger.warning(
|
||||
|
|
@ -659,6 +772,7 @@ def _catalog_row(
|
|||
upstream_provider=(
|
||||
registry.resolve(lookup_id, kind="upstream_provider", team_id=team_id) if registry is not None else None
|
||||
),
|
||||
upstreams=registry.resolve(lookup_id, kind="upstreams", team_id=team_id) if registry is not None else (),
|
||||
display_name=registry.group(label_id, field="display_name") if registry is not None else None,
|
||||
reasoning_effort=(
|
||||
_shared_pinned_effort(registry.group(lookup_id, field="pinned_reasoning_efforts", team_id=team_id))
|
||||
|
|
@ -714,17 +828,19 @@ def codex_model_list_body(
|
|||
) -> str:
|
||||
"""The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out. `stock` and
|
||||
`instructions` are the requesting Codex release's catalog and prompt, the bundled ones when None."""
|
||||
byte_limit: Final = codex_catalog_byte_limit(client_version)
|
||||
body: Final = codex_models_response_json(
|
||||
codex_catalog_rows(rows, entries, llm_router, team_id, label_entries),
|
||||
stock=stock,
|
||||
instructions=instructions,
|
||||
byte_limit=byte_limit,
|
||||
client_version=client_version,
|
||||
)
|
||||
if body.left_out:
|
||||
verbose_proxy_logger.warning(
|
||||
"Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, "
|
||||
"then model_list order; list the models Codex users need first in model_list",
|
||||
CODEX_CATALOG_BYTE_LIMIT,
|
||||
byte_limit,
|
||||
", ".join(body.left_out),
|
||||
)
|
||||
return body.json
|
||||
|
|
|
|||
|
|
@ -15,10 +15,10 @@ from functools import cache
|
|||
from types import NoneType
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from packaging.version import Version
|
||||
from pydantic import ConfigDict, Field, StringConstraints, field_serializer
|
||||
|
||||
from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort
|
||||
from litellm.proxy.common_utils.codex_version import codex_version_at_least
|
||||
from litellm.types.llms.base_model import LiteLLMBaseModel
|
||||
|
||||
TierId: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
|
||||
|
|
@ -27,7 +27,7 @@ CodexInputModality: TypeAlias = Literal["text", "image"]
|
|||
_TEXT_ONLY: Final[tuple[CodexInputModality, ...]] = ("text",)
|
||||
_TEXT_AND_IMAGE: Final[tuple[CodexInputModality, ...]] = ("text", "image")
|
||||
TOKEN_PRICE_KEY: Final = "input_cost_per_token"
|
||||
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0")
|
||||
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = "0.138.0"
|
||||
_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
|
||||
_PREFERRED_DEFAULT_EFFORT: Final = "medium"
|
||||
_CODEX_UNSET_EFFORT: Final = "none"
|
||||
|
|
@ -95,7 +95,7 @@ class CodexFallbackModel(LiteLLMBaseModel):
|
|||
apply_patch_tool_type: Literal["freeform"] | None = None
|
||||
truncation_policy: CodexTruncationPolicy = CodexTruncationPolicy()
|
||||
experimental_supported_tools: tuple[()] = ()
|
||||
input_modalities: tuple[CodexInputModality, ...] = _TEXT_AND_IMAGE
|
||||
input_modalities: tuple[CodexInputModality, ...] = _TEXT_ONLY
|
||||
context_window: int | None
|
||||
base_instructions: str
|
||||
supports_search_tool: bool = False
|
||||
|
|
@ -148,8 +148,8 @@ def no_priced_deployment_lacks_tier(tier_id: str, deployments: Sequence[CodexDep
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class CodexCapabilities:
|
||||
"""What a request to a listed name can count on whichever deployment serves it: a capability is False
|
||||
when one deployment lacks it, True when one has it and none lacks it, None when none says, and the
|
||||
context window is the smallest one any deployment declares."""
|
||||
when one deployment lacks it, True only when none lacks it, None otherwise, and the context window is
|
||||
the smallest one any deployment declares."""
|
||||
|
||||
context_window: int | None = None
|
||||
supports_vision: bool | None = None
|
||||
|
|
@ -163,6 +163,13 @@ def _combined(flags: Iterable[bool | None]) -> bool | None:
|
|||
return None if not said else False not in said
|
||||
|
||||
|
||||
def _declared_by_every_deployment(flags: Iterable[bool | None]) -> bool | None:
|
||||
said: Final = frozenset(flags)
|
||||
if False in said:
|
||||
return False
|
||||
return None if not said or None in said else True
|
||||
|
||||
|
||||
def combined_capabilities(
|
||||
deployments: Sequence[CodexDeploymentCapabilities], listed_context_window: int | None = None
|
||||
) -> CodexCapabilities:
|
||||
|
|
@ -175,7 +182,7 @@ def combined_capabilities(
|
|||
)
|
||||
return CodexCapabilities(
|
||||
context_window=min(windows) if windows else listed_context_window,
|
||||
supports_vision=_combined(deployment.supports_vision for deployment in deployments),
|
||||
supports_vision=_declared_by_every_deployment(deployment.supports_vision for deployment in deployments),
|
||||
supports_function_calling=_combined(deployment.supports_function_calling for deployment in deployments),
|
||||
supports_parallel_function_calling=_combined(
|
||||
deployment.supports_parallel_function_calling for deployment in deployments
|
||||
|
|
@ -184,13 +191,6 @@ def combined_capabilities(
|
|||
)
|
||||
|
||||
|
||||
def _parses_custom_efforts(client_version: str | None) -> bool:
|
||||
try:
|
||||
return client_version is not None and Version(client_version) >= _FIRST_CODEX_WITH_CUSTOM_EFFORTS
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def codex_reasoning_levels(
|
||||
efforts: Sequence[str], client_version: str | None
|
||||
) -> tuple[tuple[CodexFallbackReasoningLevel, ...], str | None]:
|
||||
|
|
@ -202,7 +202,7 @@ def codex_reasoning_levels(
|
|||
group accepts it, so a user who picks no level sends what Codex sent before, else medium, else the
|
||||
nearest weaker declared level, so picking no level never costs more reasoning than Codex's usual medium,
|
||||
else the weakest declared level, else the first one."""
|
||||
parses_any: Final = _parses_custom_efforts(client_version)
|
||||
parses_any: Final = codex_version_at_least(client_version, _FIRST_CODEX_WITH_CUSTOM_EFFORTS)
|
||||
offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES)
|
||||
if not offered:
|
||||
return (), None
|
||||
|
|
@ -230,10 +230,10 @@ def codex_fallback_entry(
|
|||
) -> CodexFallbackModel:
|
||||
"""Codex's entry for a model it does not know, from the capabilities its deployments share.
|
||||
|
||||
A capability no deployment declares keeps Codex's own default for an unknown model, except the freeform
|
||||
`apply_patch` tool, which goes out only to a model that calls tools: `supports_reasoning_summary_parameter`
|
||||
is left out, which Codex reads as true, and the fields only Codex before 0.150.0 reads keep the false
|
||||
it requires.
|
||||
A capability no deployment declares keeps Codex's own default for an unknown model, except image input
|
||||
and the freeform `apply_patch` tool, which go out only to a model known to read images or call tools:
|
||||
`supports_reasoning_summary_parameter` is left out, which Codex reads as true, and the fields only Codex
|
||||
before 0.150.0 reads keep the false it requires.
|
||||
"""
|
||||
levels, default_level = codex_reasoning_levels(reasoning_efforts, client_version)
|
||||
return CodexFallbackModel(
|
||||
|
|
@ -247,7 +247,7 @@ def codex_fallback_entry(
|
|||
supports_reasoning_summaries=capabilities.supports_reasoning is True,
|
||||
supports_parallel_tool_calls=capabilities.supports_parallel_function_calling is True,
|
||||
apply_patch_tool_type="freeform" if capabilities.supports_function_calling is True else None,
|
||||
input_modalities=_TEXT_ONLY if capabilities.supports_vision is False else _TEXT_AND_IMAGE,
|
||||
input_modalities=_TEXT_AND_IMAGE if capabilities.supports_vision is True else _TEXT_ONLY,
|
||||
context_window=capabilities.context_window,
|
||||
base_instructions=instructions,
|
||||
supports_search_tool=supports_search_tool,
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue