diff --git a/.github/linters/ruff-strict.toml b/.github/linters/ruff-strict.toml index 406306ae809..5fe809cb764 100644 --- a/.github/linters/ruff-strict.toml +++ b/.github/linters/ruff-strict.toml @@ -2,7 +2,7 @@ extend = "../../pyproject.toml" [lint] preview = true -select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D202", "D403", "D417", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"] +select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D202", "D403", "D417", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR0916", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"] extend-select = [] # Overrides the inherited list: rules this gate enforces itself must NOT be external here, # so this config's RUF100 flags their stale `# noqa` directives. What remains external is @@ -44,6 +44,7 @@ convention = "google" [lint.pylint] max-args = 5 +max-bool-expr = 3 [lint.flake8-tidy-imports.banned-api] "typing.Any".msg = "Use a concrete type. Frozen slots=True dataclass (preferred) / NamedTuple / ReadOnly TypedDict for payloads." diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 8ed1876a081..88acf50837f 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -178,6 +178,7 @@ jobs: windows-sdk: name: windows-sdk + if: github.event_name != 'pull_request' runs-on: windows-2025 timeout-minutes: 30 env: @@ -213,6 +214,7 @@ jobs: proxy-build-from-pip: name: proxy-build-from-pip + if: github.event_name != 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 10 services: @@ -352,7 +354,7 @@ jobs: smoke-passed: name: smoke passed - needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip] + needs: [dashboard-build, core-checks, all-extras] if: always() runs-on: ubuntu-latest timeout-minutes: 2 diff --git a/.github/workflows/test-rust-db.yml b/.github/workflows/test-rust-db.yml index 116d6785287..3a8af7f170f 100644 --- a/.github/workflows/test-rust-db.yml +++ b/.github/workflows/test-rust-db.yml @@ -91,6 +91,8 @@ jobs: - name: Install Python dependencies working-directory: . + env: + SQLX_OFFLINE: "true" run: uv sync --frozen --extra extra-proxy - name: Deploy Prisma migrations diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f27b58149ab..50bcc6c2b42 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -126,7 +126,7 @@ jobs: name: ${{ matrix.shard }} needs: [rust-bridge, assert-shard-coverage] if: ${{ !cancelled() }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner || 'ubuntu-latest' }} timeout-minutes: ${{ matrix.job-timeout-minutes }} permissions: contents: read @@ -220,6 +220,7 @@ jobs: dist: loadscope - shard: llms-providers + runner: ubuntu-latest-16-cores test-path: |- tests/unit/llms/test_api_base_trailing_slash_routes.py tests/unit/llms/test_cache_control_and_reasoning.py @@ -380,7 +381,7 @@ jobs: tests/unit/llms/zai --ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py python-version: "3.12" - workers: 4 + workers: 8 reruns: 0 timeout-minutes: 20 job-timeout-minutes: 60 @@ -398,10 +399,11 @@ jobs: dist: loadscope - shard: root + runner: ubuntu-latest-16-cores test-path: >- tests/unit/test_*.py python-version: "3.12" - workers: 4 + workers: 8 reruns: 2 timeout-minutes: 20 job-timeout-minutes: 60 @@ -522,6 +524,7 @@ jobs: dist: loadscope - shard: proxy-endpoints + runner: ubuntu-latest-16-cores test-path: |- tests/unit/proxy/management_endpoints tests/unit/proxy/management_helpers @@ -539,13 +542,14 @@ jobs: --ignore=tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py --ignore=tests/unit/proxy/management_helpers/test_audit_logs_proxy.py python-version: "3.12" - workers: 4 + workers: 8 reruns: 0 timeout-minutes: 20 job-timeout-minutes: 60 dist: loadscope - shard: proxy-feature-endpoints + runner: ubuntu-latest-16-cores test-path: |- tests/unit/proxy/guardrails --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py @@ -578,7 +582,7 @@ jobs: tests/unit/proxy/config_resolvers tests/unit/proxy/utils python-version: "3.12" - workers: 4 + workers: 8 reruns: 0 timeout-minutes: 20 job-timeout-minutes: 60 diff --git a/litellm/batches/batch_line_items.py b/litellm/batches/batch_line_items.py index 8f1e3efaaf3..a6522214dba 100644 --- a/litellm/batches/batch_line_items.py +++ b/litellm/batches/batch_line_items.py @@ -24,7 +24,7 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - from litellm.integrations.s3_v3 import S3Logger + from litellm.integrations.s3_v2 import S3Logger BatchFileProvider: TypeAlias = Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] @@ -242,7 +242,7 @@ async def _deliver( def _resolve_logger(callback: "litellm.CALLBACK_TYPES | S3Logger") -> "CustomLogger | S3Logger | None": - from litellm.integrations.s3_v3 import S3Logger + from litellm.integrations.s3_v2 import S3Logger match callback: case CustomLogger() | S3Logger(): diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index f0d400b7b01..50ffae41147 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -169,7 +169,7 @@ class DualCache(BaseCache): key, parent_otel_span: Span | None = None, local_only: bool = False, - **kwargs, + **kwargs: object, ): # Try to fetch from in-memory cache first try: @@ -190,7 +190,7 @@ class DualCache(BaseCache): result = redis_result - print_verbose(f"get cache: cache result: {result}") + print_verbose("get cache: cache result: %s", result) return result except Exception as e: log_redis_failure( @@ -245,16 +245,16 @@ class DualCache(BaseCache): key, parent_otel_span: Span | None = None, local_only: bool = False, - **kwargs, + **kwargs: object, ): # Try to fetch from in-memory cache first try: - print_verbose(f"async get cache: cache key: {key}; local_only: {local_only}") + print_verbose("async get cache: cache key: %s; local_only: %s", key, local_only) result = None if self.in_memory_cache is not None: in_memory_result: Final = await self.in_memory_cache.async_get_cache(key, **kwargs) - print_verbose(f"in_memory_result: {in_memory_result}") + print_verbose("in_memory_result: %s", in_memory_result) if in_memory_result is not None: result = in_memory_result @@ -268,7 +268,7 @@ class DualCache(BaseCache): result = redis_result - print_verbose(f"get cache: cache result: {result}") + print_verbose("get cache: cache result: %s", result) return result except Exception as e: log_redis_failure( @@ -451,8 +451,8 @@ class DualCache(BaseCache): with_traceback=True, ) - async def async_set_cache(self, key, value, local_only: bool = False, **kwargs): - print_verbose(f"async set cache: cache key: {key}; local_only: {local_only}; value: {value}") + async def async_set_cache(self, key, value, local_only: bool = False, **kwargs: object): + print_verbose("async set cache: cache key: %s; local_only: %s; value: %s", key, local_only, value) try: if self.in_memory_cache is not None: if "ttl" not in kwargs and self.default_in_memory_ttl is not None: @@ -473,7 +473,7 @@ class DualCache(BaseCache): """ Batch write values to the cache """ - print_verbose(f"async batch set cache: cache keys: {cache_list}; local_only: {local_only}") + print_verbose("async batch set cache: cache keys: %s; local_only: %s", cache_list, local_only) try: if self.in_memory_cache is not None: if "ttl" not in kwargs and self.default_in_memory_ttl is not None: diff --git a/litellm/constants.py b/litellm/constants.py index 68331e3199e..4d239472542 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2362,6 +2362,7 @@ MODEL_BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "model_batch_enqueued_tok MAX_BATCH_FILE_RECORDS_KEY: Final = "max_batch_file_records" MAX_BATCH_FILE_UPLOADS_PER_DAY_KEY: Final = "max_batch_file_uploads_per_day" MAX_FILE_DOWNLOADS_PER_MINUTE_KEY: Final = "max_file_downloads_per_minute" +MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY: Final = "model_limit_max_borrow_multiple" ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS: Final = ( BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, MAX_BATCH_FILE_RECORDS_KEY, diff --git a/litellm/enterprise/pyproject.toml b/litellm/enterprise/pyproject.toml index 352aaa5d4dc..56098cb1028 100644 --- a/litellm/enterprise/pyproject.toml +++ b/litellm/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.85" +version = "0.1.86" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.85" +version = "0.1.86" version_files = [ "pyproject.toml:^version", "../../pyproject.toml:litellm-enterprise==", diff --git a/litellm/files/main.py b/litellm/files/main.py index 3c73f6d5389..8d57d90c57f 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -35,7 +35,6 @@ FileDeleteProvider = Literal[ ] FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral", "xai"] import litellm -from litellm import get_secret_str from litellm.files.streaming import FileContentStreamingResponse from litellm.files.types import FileContentProvider, FileContentStreamingResult, FileRetrieveProvider from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider @@ -47,7 +46,6 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.common_utils import get_openai_credentials from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI -from litellm.llms.vertex_ai.files.handler import VertexAIFilesHandler from litellm.llms.xai.batches.handler import XAIBatchesHandler from litellm.llms.xai.batches.transformation import is_xai_batch_results_id from litellm.types.llms.openai import ( @@ -102,7 +100,6 @@ def _file_content_logging_obj(kwargs: dict[str, object], _is_async: bool) -> Lit openai_files_instance: Final = OpenAIFilesAPI() azure_files_instance: Final = AzureOpenAIFilesAPI() -vertex_ai_files_instance: Final = VertexAIFilesHandler() bedrock_files_instance: Final = BedrockFilesHandler() xai_batch_results_instance: Final = XAIBatchesHandler() ################################################# @@ -986,27 +983,6 @@ def file_content( client=client, litellm_params=litellm_params_dict, ) - elif custom_llm_provider == "vertex_ai": - api_base: Final = optional_params.api_base or "" - vertex_ai_project: Final = ( - optional_params.vertex_project or litellm.vertex_project or get_secret_str("VERTEXAI_PROJECT") - ) - vertex_ai_location: Final = ( - optional_params.vertex_location or litellm.vertex_location or get_secret_str("VERTEXAI_LOCATION") - ) - vertex_credentials: Final = optional_params.vertex_credentials or get_secret_str("VERTEXAI_CREDENTIALS") - - response = vertex_ai_files_instance.file_content( - _is_async=_is_async, - file_content_request=_file_content_request, - api_base=api_base, - vertex_credentials=vertex_credentials, - vertex_project=vertex_ai_project, - vertex_location=vertex_ai_location, - timeout=timeout, - max_retries=optional_params.max_retries, - litellm_params=litellm_params_dict, - ) elif custom_llm_provider == "bedrock": response = bedrock_files_instance.file_content( _is_async=_is_async, diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index ea570bc19c9..3d4de310753 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from enum import Enum -from functools import lru_cache +from functools import cache, lru_cache from typing import Annotated, Final from pydantic import AliasChoices, ConfigDict, Field, TypeAdapter, ValidationError, field_validator, model_validator @@ -218,7 +218,8 @@ class OpenTelemetryV2Config(BaseSettings): description=( "``llm_only`` keeps just the model-call spans on the operator's own Langfuse " "exporter (the spec whose owner is ``langfuse_otel``). Other exporters and " - "key/team destinations are not affected." + "key/team destinations are not affected. Destinations default to " + "``langfuse_destination_span_scope`` instead." ), ) excluded_services: Annotated[frozenset[str], NoDecode] = Field( @@ -317,9 +318,7 @@ class OpenTelemetryV2Config(BaseSettings): @field_validator("langfuse_span_scope", mode="before") @classmethod def _normalize_langfuse_span_scope(cls, value: object) -> object: - if isinstance(value, str): - return value.strip().lower() - return value + return _normalized_span_scope(value) @field_validator( "baggage_promoted_keys", @@ -417,6 +416,44 @@ def capture_message_content_from_env() -> str: return _CaptureMessageContentSettings().capture_message_content +def _normalized_span_scope(value: object) -> object: + return value.strip().lower() if isinstance(value, str) else value + + +class _LangfuseDestinationSpanScopeSettings(BaseSettings): + model_config = SettingsConfigDict(extra="ignore", frozen=True) + + langfuse_destination_span_scope: OtelSpanScope = Field( + default="full", + validation_alias=AliasChoices("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE"), + ) + + @field_validator("langfuse_destination_span_scope", mode="before") + @classmethod + def _normalize_scope(cls, value: object) -> object: + return _normalized_span_scope(value) + + +def langfuse_destination_span_scope(configured: object = None) -> OtelSpanScope: + return _cached_langfuse_destination_span_scope(None if configured is None else str(configured)) + + +@cache +def _cached_langfuse_destination_span_scope(configured: str | None) -> OtelSpanScope: + try: + return _LangfuseDestinationSpanScopeSettings.model_validate( + {"LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE": configured} if configured is not None else {} + ).langfuse_destination_span_scope + except ValidationError as error: + verbose_logger.warning( + "Invalid LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE or " + "callback_settings.otel.langfuse_destination_span_scope. " + "Allowed values are 'full' and 'llm_only'. Falling back to 'full'. %s", + error, + ) + return "full" + + _EXCLUDED_SERVICES_INPUT: Final[TypeAdapter[str | tuple[object, ...]]] = TypeAdapter(str | tuple[object, ...]) diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index a14d1b69b5a..f14e92686d8 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -161,10 +161,12 @@ def _resource_defaults(callback_name: str) -> Mapping[str, str]: return backend() if backend is not None else _NO_ATTRS -def _span_scope(callback_name: str, params: StandardCallbackDynamicParams) -> OtelSpanScope: +def _span_scope( + callback_name: str, params: StandardCallbackDynamicParams, langfuse_destination_span_scope: OtelSpanScope +) -> OtelSpanScope: if callback_name != "langfuse_otel": return "full" - return params.get("langfuse_span_scope") or "full" + return params.get("langfuse_span_scope") or langfuse_destination_span_scope def destination_capable_backends() -> frozenset[str]: @@ -199,6 +201,7 @@ def destination_for( capture_message_content: str | None = None, excluded_db_systems: frozenset[str] | None = None, metadata_keys: tuple[str, ...] | None = None, + langfuse_destination_span_scope: OtelSpanScope = "full", ) -> OtelDestination | None: """The destination ``params`` names for ``callback_name``, or ``None``. @@ -228,7 +231,7 @@ def destination_for( resource_defaults=_resource_defaults(callback_name), callback_name=callback_name, protocol=protocol, - span_scope=_span_scope(callback_name, params), + span_scope=_span_scope(callback_name, params, langfuse_destination_span_scope), capture_message_content=capture_message_content, excluded_db_systems=excluded_db_systems, metadata_keys=metadata_keys or (), diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 43abc08d2df..10be7cbff13 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -181,6 +181,18 @@ class _CustomerBudgetRow(Protocol): def litellm_budget_table(self) -> _JoinedBudgetRow | None: ... +def _dynamic_limiter_holder_labels( + kind: Literal["team", "key", "customer_org"], holder_id: str, holder_alias: str | None +) -> Mapping[str, str | None]: + match kind: + case "team": + return {"team": holder_id, "team_alias": holder_alias} + case "key": + return {"hashed_api_key": holder_id, "api_key_alias": holder_alias} + case "customer_org": + return {"customer_org_id": holder_id, "customer_org_alias": holder_alias} + + def _customer_budget_metrics_enabled() -> bool: return litellm.enable_end_user_cost_tracking_prometheus_only is True and not litellm.disable_end_user_cost_tracking @@ -4201,14 +4213,17 @@ class PrometheusLogger(CustomLogger): def set_dynamic_limiter_share( self, model_group: str, - team: str, - team_alias: str | None, + holder_kind: Literal["team", "key", "customer_org"], + holder_id: str, + holder_alias: str | None, priority: str, share_tokens: int, borrowed_tokens: int, ) -> None: enum_values: Final = UserAPIKeyLabelValues( - model_group=model_group, team=team, team_alias=team_alias, priority=priority + model_group=model_group, + priority=priority, + **_dynamic_limiter_holder_labels(holder_kind, holder_id, holder_alias), ) self._set_labeled_gauge( self.litellm_dynamic_limiter_share_tokens, "litellm_dynamic_limiter_share_tokens", enum_values, share_tokens @@ -4239,8 +4254,9 @@ class PrometheusLogger(CustomLogger): def increment_dynamic_limiter_throttled( self, model_group: str, - team: str, - team_alias: str | None, + holder_kind: Literal["team", "key", "customer_org"], + holder_id: str, + holder_alias: str | None, priority: str, rate_limit_type: str, reason: str, @@ -4250,11 +4266,10 @@ class PrometheusLogger(CustomLogger): "litellm_dynamic_limiter_throttled_total", UserAPIKeyLabelValues( model_group=model_group, - team=team, - team_alias=team_alias, priority=priority, rate_limit_type=rate_limit_type, reason=reason, + **_dynamic_limiter_holder_labels(holder_kind, holder_id, holder_alias), ), ) diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 98892a7fdf4..45a6f887650 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -1,3 +1,481 @@ -from litellm.integrations.s3_v3 import S3BatchUploadError, S3Logger +import asyncio +import copy +import json +import os +from collections.abc import Callable, Collection, Mapping +from typing import TYPE_CHECKING, Final -__all__ = ["S3BatchUploadError", "S3Logger"] +from typing_extensions import Unpack + +import litellm +from litellm._logging import verbose_logger +from litellm.constants import ( + COMPLETION_HTTP_FALLBACK_SECONDS, + DEFAULT_S3_BATCH_SIZE, + DEFAULT_S3_FLUSH_INTERVAL_SECONDS, + DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY, + DEFAULT_S3_MAX_CONCURRENT_UPLOADS, + HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, + S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS, +) +from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger_v2.call import CallContext +from litellm.integrations.custom_logger_v2.derivations import ( + cold_storage_object_key, + log_id, + logged_key_identity, + mapping, + payload_copy, +) +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.field_sources import HookArgs +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on +from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch +from litellm.integrations.custom_logger_v2.rust import RustLogger +from litellm.integrations.custom_logger_v2.selection import Selection +from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds +from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document +from litellm.secret_managers.main import get_secret, get_secret_str +from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity +from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload + +if TYPE_CHECKING: + from litellm.rust_bridge._native import NativeS3Log + +CALLBACK_NAME: Final = "s3_v2" +STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files" +_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4 +_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600 + + +class S3BatchUploadError(Exception): + def __init__(self, failed: int, total: int) -> None: + self.failed: Final = failed + self.total: Final = total + super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush") + + +def _raise_batch_upload_error(failed: int, total: int) -> None: + raise S3BatchUploadError(failed=failed, total=total) + + +def batch_upload_failed(failure: tuple[int, int, int] | None) -> None: + if failure is None: + return + queued, failed, total = failure + try: + _raise_batch_upload_error(failed, total) + except S3BatchUploadError: + verbose_logger.exception( + "CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued + ) + + +def _cold_storage_is_s3_v2() -> bool: + return litellm.cold_storage_custom_logger == "s3_v2" + + +def _region_from_environment() -> str | None: + region_name: Final = get_secret_str("AWS_REGION_NAME", None) + region: Final = get_secret_str("AWS_REGION", None) + return region if region is not None else region_name + + +class _Host: + def __init__(self) -> None: + self.failures: Final = CustomLogger() + + def callback_failed(self) -> None: + self.failures.handle_callback_failure(callback_name="S3Logger") + + def web_identity_token(self, reference: str) -> str: + if reference.startswith("os.environ/") or reference in os.environ: + raise ValueError("Invalid web identity token reference.") + match _read_secret(get_secret, reference): + case None: + raise ValueError("OIDC token could not be retrieved from secret manager.") + case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH: + raise ValueError( + "Parameter validation failed:\nInvalid length for parameter WebIdentityToken, " + f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}" + ) + case str() as token: + return token + case other: + raise ValueError( + "Parameter validation failed:\nInvalid type for parameter WebIdentityToken, " + f"value: {other}, type: {type(other)}, valid types: {str}" + ) + + +def _read_secret(read: Callable[[str], object], reference: str) -> object: + return read(reference) + + +def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust + configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None) + source: Final[Mapping[str, object] | Mapping[object, object]] = ( + params_source if params_source is not None else (configured or {}) + ) + return { + key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value) + for key, value in source.items() + if isinstance(key, str) + } + + +def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]: + return { + "batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE, + "max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS, + "max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY, + "upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS, + "download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS, + "download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, + "web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY), + } + + +def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float: + return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS + + +_CONFIGURED_FIRST: Final = frozenset( + { + "s3_bucket_name", + "s3_path", + "s3_region_name", + "s3_api_version", + "s3_endpoint_url", + "s3_aws_access_key_id", + "s3_aws_secret_access_key", + "s3_aws_session_token", + "s3_aws_session_name", + "s3_aws_profile_name", + "s3_aws_role_name", + "s3_aws_web_identity_token", + "s3_aws_sts_endpoint", + "s3_config", + "s3_server_side_encryption", + "s3_sse_kms_key_id", + } +) +_CONSTRUCTOR_ENABLES: Final = frozenset( + { + "s3_use_team_prefix", + "s3_strip_base64_files", + "s3_use_key_prefix", + "s3_use_virtual_hosted_style", + "s3_adaptive_concurrency", + "s3_batch_file_upload", + } +) +_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"}) +_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"}) +_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"}) +_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = { + "s3_use_ssl": True, + "s3_drop_on_terminal_error": True, + "s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS, + "s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS, + "s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS, + "s3_batch_size": DEFAULT_S3_BATCH_SIZE, +} + + +def _blank(value: object) -> bool: + return value is None or value == "" + + +def _constructor_wins(key: str, configured: object, value: object) -> bool: + return ( + (key in _CONFIGURED_FIRST and not configured) + or (key in _CONFIGURED_UNLESS_NONE and configured is None) + or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured)) + or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None) + ) + + +_KEEP_CONFIGURED: Final = object() + + +def _constructor_value(key: str, configured: object, value: object) -> object: + if _constructor_wins(key, configured, value): + return value + if key in _CONSTRUCTOR_ENABLES and value: + return True + if key == "s3_max_queue_size" and value is not None: + return resolve_s3_max_queue_size( + configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE) + ) + if key == "s3_max_retry_age_seconds": + fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS) + resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback) + return 0 if resolved is None else resolved + return _KEEP_CONFIGURED + + +def _constructed_params( + params_source: Mapping[str, object] | None, settings: S3LoggerSettings +) -> dict[str, object]: # mutable-ok: handed to Rust + params: Final = _resolved_params(params_source) + overrides: Final = { + key: replaced + for key, value in settings.items() + if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None) + and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED + } + return {**params, **overrides} + + +class _Base64Stripper(CustomLogger): + def strip(self, payload: StandardLoggingPayload) -> None: + _ = self._strip_base64_from_messages_sync(payload) + + def stripped(self, messages: list[object]) -> list[dict[str, object]]: + return self._process_messages(messages=[_shallow_copy(message) for message in messages]) + + +def _shallow_copy(message: object) -> object: + entries: Final = mapping(message) + return copy.copy(entries) if entries is not None and isinstance(message, dict) else message + + +_STRIPPER: Final = _Base64Stripper() + + +def legacy_should_log(kwargs: Mapping[str, object]) -> bool: + from litellm.litellm_core_utils import litellm_logging + + if litellm.global_disable_no_log_param: + return True + litellm_params: Final = kwargs.get("litellm_params", {}) + lookup: Final = getattr(litellm_params, "get", None) + if callable(lookup) and lookup("no-log", False) is True: + return False + return not litellm_logging.callback_disabled_dynamically( + callback=CALLBACK_NAME, + litellm_params=litellm_params, + standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check + ) + + +def _payload_messages(payload: Mapping[str, object]) -> list[object]: + raw: Final = payload.get("messages") + return [*raw] if isinstance(raw, list) else [] + + +class S3StripBase64Files(CustomLoggerV2): + selection = Selection(name=STRIP_CALLBACK_NAME) + + def __init__(self, s3_logger_key: str) -> None: + self.s3_logger_key: Final = s3_logger_key + + @on( + Hook.log_prepare, + reads={Field.deprecated_kwargs, Field.standard_logging_payload}, + writes={Field.standard_logging_payload}, + ) + async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None: + from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE + + kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None)) + payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None)) + if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs): + return None + messages: Final = _payload_messages(payload) + if not messages: + return None + return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)}) + + +async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None: + while True: + await asyncio.sleep(interval) + try: + batch_upload_failed(await sink.flush()) + except (asyncio.CancelledError, KeyboardInterrupt, SystemExit): + raise + except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does + verbose_logger.exception("CustomLogger: periodic flush failed: %s", e) + + +def _excluding_fields(kwargs: object) -> Mapping[str, object] | None: + excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None) + details: Final = mapping(kwargs) + logged: Final = None if details is None else mapping(details.get("standard_logging_object")) + if details is None or logged is None or not isinstance(excluded, Collection) or not excluded: + return None + if logged.get("status") == "failure": + return None + return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}} + + +def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object: + source: Final = mapping(kwargs) if details is None else details + return None if source is None else source.get("standard_logging_object") + + +def _legacy_kwargs(view: object) -> object: + return getattr(view, Field.deprecated_kwargs.value, None) + + +_read_attribute: Final[Callable[[object, str], object]] = getattr + + +class _ExcludedView: + def __init__(self, view: object, details: Mapping[str, object]) -> None: + args: Final = HookArgs(request=details) + self.view: Final = view + self.fields: Final[Mapping[str, object]] = { + Field.deprecated_kwargs.value: details, + Field.standard_logging_payload_ref.value: payload_copy(args), + Field.log_id.value: log_id(args), + Field.cold_storage_object_key.value: cold_storage_object_key(args), + Field.logged_key_identity.value: logged_key_identity(args), + } + + def __getattr__(self, name: str) -> object: + if name not in self.fields: + return _read_attribute(self.view, name) + value: Final = self.fields[name] + if value is None: + raise AttributeError(name) + return value + + +_ATTEMPT_ENDED_READS: Final = frozenset( + { + Field.deprecated_kwargs, + Field.attempt_timing, + Field.log_id, + Field.cold_storage_object_key, + Field.standard_logging_payload_ref, + Field.logged_key_identity, + } +) + + +class S3Logger(RustLogger): + selection = Selection(name=CALLBACK_NAME, honors_no_log=True) + + def __init__( + self, + s3_callback_params_override: Mapping[str, object] | None = None, + **settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords + ) -> None: + from litellm.rust_bridge._native import NativeS3Log + + host: Final = _Host() + params: Final = _constructed_params(s3_callback_params_override, settings) + sink: Final = NativeS3Log( + params, + _constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)), + host, + _cold_storage_is_s3_v2(), + None if params.get("s3_region_name") else _region_from_environment(), + ) + self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name") + self.sink: Final = sink + self.s3_host: Final = host + self.periodic_flush_task: Final = asyncio.create_task( + _periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS))) + ) + super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None}) + self.v2_companion: Final = ( + S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self)) + if sink.strip_base64_files + else None + ) + + @on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True) + async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None: + return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx) + + async def run_rust( + self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object] + ) -> Patch | Abort | None: + from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE + + kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None) + if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict + return None + payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict + if payload is not None and not isinstance(payload, Mapping): + return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload + try: + raw: Final = _legacy_kwargs(view) + details: Final = _excluding_fields(raw) + logged: Final = _logged_payload(raw, details) + if self.sink.strip_base64_files and isinstance(logged, Mapping): + _STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does + return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx) + except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on + return self._failed(e) + + def _failed(self, error: Exception) -> None: + verbose_logger.error("s3 Layer Error - %s", error, exc_info=error) + self.s3_host.callback_failed() + + @property + def s3_path(self) -> str | None: + return self.sink.s3_path + + @property + def s3_bucket_name(self) -> str | None: + return self.sink.s3_bucket_name + + def resolve_partition_granularity(self) -> S3PartitionGranularity: + return "hour" if self.sink.partition_granularity() == "hour" else "day" + + async def flush_queue(self) -> None: + batch_upload_failed(await self.sink.flush()) + + async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None: + try: + audit_log_id: Final = audit_log.get("id", "unknown") + batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}")) + except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger + verbose_logger.exception("S3 audit log error: %s", e) + + async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None: + object_name: Final = batch_line_items_object_name(payload) + s3_path: Final = self.s3_path + s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name + body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload) + delivered: Final = await self.sink.upload_line_items( + s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:]) + ) + if not delivered: + verbose_logger.error( + "s3 logging: the %d line items of part %d of batch %s were not stored at %s", + len(payload["lines"]), + payload["part"], + payload["batch_id"], + s3_object_key, + ) + + async def _download_object_from_s3(self, s3_object_key: str) -> object: + try: + downloaded: Final = await self.sink.download(s3_object_key) + return None if downloaded is None else json.loads(downloaded) + except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger + verbose_logger.exception("Error downloading from S3: %s", e) + return None + + async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object: + try: + return await self._download_object_from_s3(object_key) + except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger + verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e) + return None + + +__all__ = [ + "S3BatchUploadError", + "S3Logger", + "S3StripBase64Files", + "legacy_should_log", +] diff --git a/litellm/integrations/s3_v3.py b/litellm/integrations/s3_v3.py deleted file mode 100644 index 45a6f887650..00000000000 --- a/litellm/integrations/s3_v3.py +++ /dev/null @@ -1,481 +0,0 @@ -import asyncio -import copy -import json -import os -from collections.abc import Callable, Collection, Mapping -from typing import TYPE_CHECKING, Final - -from typing_extensions import Unpack - -import litellm -from litellm._logging import verbose_logger -from litellm.constants import ( - COMPLETION_HTTP_FALLBACK_SECONDS, - DEFAULT_S3_BATCH_SIZE, - DEFAULT_S3_FLUSH_INTERVAL_SECONDS, - DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY, - DEFAULT_S3_MAX_CONCURRENT_UPLOADS, - HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, - S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS, -) -from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name -from litellm.integrations.custom_batch_logger import CustomBatchLogger -from litellm.integrations.custom_logger import CustomLogger -from litellm.integrations.custom_logger_v2.call import CallContext -from litellm.integrations.custom_logger_v2.derivations import ( - cold_storage_object_key, - log_id, - logged_key_identity, - mapping, - payload_copy, -) -from litellm.integrations.custom_logger_v2.enums import Field, Hook -from litellm.integrations.custom_logger_v2.field_sources import HookArgs -from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on -from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch -from litellm.integrations.custom_logger_v2.rust import RustLogger -from litellm.integrations.custom_logger_v2.selection import Selection -from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds -from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document -from litellm.secret_managers.main import get_secret, get_secret_str -from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity -from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload - -if TYPE_CHECKING: - from litellm.rust_bridge._native import NativeS3Log - -CALLBACK_NAME: Final = "s3_v2" -STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files" -_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4 -_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600 - - -class S3BatchUploadError(Exception): - def __init__(self, failed: int, total: int) -> None: - self.failed: Final = failed - self.total: Final = total - super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush") - - -def _raise_batch_upload_error(failed: int, total: int) -> None: - raise S3BatchUploadError(failed=failed, total=total) - - -def batch_upload_failed(failure: tuple[int, int, int] | None) -> None: - if failure is None: - return - queued, failed, total = failure - try: - _raise_batch_upload_error(failed, total) - except S3BatchUploadError: - verbose_logger.exception( - "CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued - ) - - -def _cold_storage_is_s3_v2() -> bool: - return litellm.cold_storage_custom_logger == "s3_v2" - - -def _region_from_environment() -> str | None: - region_name: Final = get_secret_str("AWS_REGION_NAME", None) - region: Final = get_secret_str("AWS_REGION", None) - return region if region is not None else region_name - - -class _Host: - def __init__(self) -> None: - self.failures: Final = CustomLogger() - - def callback_failed(self) -> None: - self.failures.handle_callback_failure(callback_name="S3Logger") - - def web_identity_token(self, reference: str) -> str: - if reference.startswith("os.environ/") or reference in os.environ: - raise ValueError("Invalid web identity token reference.") - match _read_secret(get_secret, reference): - case None: - raise ValueError("OIDC token could not be retrieved from secret manager.") - case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH: - raise ValueError( - "Parameter validation failed:\nInvalid length for parameter WebIdentityToken, " - f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}" - ) - case str() as token: - return token - case other: - raise ValueError( - "Parameter validation failed:\nInvalid type for parameter WebIdentityToken, " - f"value: {other}, type: {type(other)}, valid types: {str}" - ) - - -def _read_secret(read: Callable[[str], object], reference: str) -> object: - return read(reference) - - -def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust - configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None) - source: Final[Mapping[str, object] | Mapping[object, object]] = ( - params_source if params_source is not None else (configured or {}) - ) - return { - key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value) - for key, value in source.items() - if isinstance(key, str) - } - - -def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]: - return { - "batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE, - "max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS, - "max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY, - "upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS, - "download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS, - "download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, - "web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY), - } - - -def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float: - return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS - - -_CONFIGURED_FIRST: Final = frozenset( - { - "s3_bucket_name", - "s3_path", - "s3_region_name", - "s3_api_version", - "s3_endpoint_url", - "s3_aws_access_key_id", - "s3_aws_secret_access_key", - "s3_aws_session_token", - "s3_aws_session_name", - "s3_aws_profile_name", - "s3_aws_role_name", - "s3_aws_web_identity_token", - "s3_aws_sts_endpoint", - "s3_config", - "s3_server_side_encryption", - "s3_sse_kms_key_id", - } -) -_CONSTRUCTOR_ENABLES: Final = frozenset( - { - "s3_use_team_prefix", - "s3_strip_base64_files", - "s3_use_key_prefix", - "s3_use_virtual_hosted_style", - "s3_adaptive_concurrency", - "s3_batch_file_upload", - } -) -_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"}) -_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"}) -_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"}) -_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = { - "s3_use_ssl": True, - "s3_drop_on_terminal_error": True, - "s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS, - "s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS, - "s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS, - "s3_batch_size": DEFAULT_S3_BATCH_SIZE, -} - - -def _blank(value: object) -> bool: - return value is None or value == "" - - -def _constructor_wins(key: str, configured: object, value: object) -> bool: - return ( - (key in _CONFIGURED_FIRST and not configured) - or (key in _CONFIGURED_UNLESS_NONE and configured is None) - or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured)) - or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None) - ) - - -_KEEP_CONFIGURED: Final = object() - - -def _constructor_value(key: str, configured: object, value: object) -> object: - if _constructor_wins(key, configured, value): - return value - if key in _CONSTRUCTOR_ENABLES and value: - return True - if key == "s3_max_queue_size" and value is not None: - return resolve_s3_max_queue_size( - configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE) - ) - if key == "s3_max_retry_age_seconds": - fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS) - resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback) - return 0 if resolved is None else resolved - return _KEEP_CONFIGURED - - -def _constructed_params( - params_source: Mapping[str, object] | None, settings: S3LoggerSettings -) -> dict[str, object]: # mutable-ok: handed to Rust - params: Final = _resolved_params(params_source) - overrides: Final = { - key: replaced - for key, value in settings.items() - if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None) - and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED - } - return {**params, **overrides} - - -class _Base64Stripper(CustomLogger): - def strip(self, payload: StandardLoggingPayload) -> None: - _ = self._strip_base64_from_messages_sync(payload) - - def stripped(self, messages: list[object]) -> list[dict[str, object]]: - return self._process_messages(messages=[_shallow_copy(message) for message in messages]) - - -def _shallow_copy(message: object) -> object: - entries: Final = mapping(message) - return copy.copy(entries) if entries is not None and isinstance(message, dict) else message - - -_STRIPPER: Final = _Base64Stripper() - - -def legacy_should_log(kwargs: Mapping[str, object]) -> bool: - from litellm.litellm_core_utils import litellm_logging - - if litellm.global_disable_no_log_param: - return True - litellm_params: Final = kwargs.get("litellm_params", {}) - lookup: Final = getattr(litellm_params, "get", None) - if callable(lookup) and lookup("no-log", False) is True: - return False - return not litellm_logging.callback_disabled_dynamically( - callback=CALLBACK_NAME, - litellm_params=litellm_params, - standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check - ) - - -def _payload_messages(payload: Mapping[str, object]) -> list[object]: - raw: Final = payload.get("messages") - return [*raw] if isinstance(raw, list) else [] - - -class S3StripBase64Files(CustomLoggerV2): - selection = Selection(name=STRIP_CALLBACK_NAME) - - def __init__(self, s3_logger_key: str) -> None: - self.s3_logger_key: Final = s3_logger_key - - @on( - Hook.log_prepare, - reads={Field.deprecated_kwargs, Field.standard_logging_payload}, - writes={Field.standard_logging_payload}, - ) - async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None: - from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE - - kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None)) - payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None)) - if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs): - return None - messages: Final = _payload_messages(payload) - if not messages: - return None - return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)}) - - -async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None: - while True: - await asyncio.sleep(interval) - try: - batch_upload_failed(await sink.flush()) - except (asyncio.CancelledError, KeyboardInterrupt, SystemExit): - raise - except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does - verbose_logger.exception("CustomLogger: periodic flush failed: %s", e) - - -def _excluding_fields(kwargs: object) -> Mapping[str, object] | None: - excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None) - details: Final = mapping(kwargs) - logged: Final = None if details is None else mapping(details.get("standard_logging_object")) - if details is None or logged is None or not isinstance(excluded, Collection) or not excluded: - return None - if logged.get("status") == "failure": - return None - return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}} - - -def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object: - source: Final = mapping(kwargs) if details is None else details - return None if source is None else source.get("standard_logging_object") - - -def _legacy_kwargs(view: object) -> object: - return getattr(view, Field.deprecated_kwargs.value, None) - - -_read_attribute: Final[Callable[[object, str], object]] = getattr - - -class _ExcludedView: - def __init__(self, view: object, details: Mapping[str, object]) -> None: - args: Final = HookArgs(request=details) - self.view: Final = view - self.fields: Final[Mapping[str, object]] = { - Field.deprecated_kwargs.value: details, - Field.standard_logging_payload_ref.value: payload_copy(args), - Field.log_id.value: log_id(args), - Field.cold_storage_object_key.value: cold_storage_object_key(args), - Field.logged_key_identity.value: logged_key_identity(args), - } - - def __getattr__(self, name: str) -> object: - if name not in self.fields: - return _read_attribute(self.view, name) - value: Final = self.fields[name] - if value is None: - raise AttributeError(name) - return value - - -_ATTEMPT_ENDED_READS: Final = frozenset( - { - Field.deprecated_kwargs, - Field.attempt_timing, - Field.log_id, - Field.cold_storage_object_key, - Field.standard_logging_payload_ref, - Field.logged_key_identity, - } -) - - -class S3Logger(RustLogger): - selection = Selection(name=CALLBACK_NAME, honors_no_log=True) - - def __init__( - self, - s3_callback_params_override: Mapping[str, object] | None = None, - **settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords - ) -> None: - from litellm.rust_bridge._native import NativeS3Log - - host: Final = _Host() - params: Final = _constructed_params(s3_callback_params_override, settings) - sink: Final = NativeS3Log( - params, - _constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)), - host, - _cold_storage_is_s3_v2(), - None if params.get("s3_region_name") else _region_from_environment(), - ) - self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name") - self.sink: Final = sink - self.s3_host: Final = host - self.periodic_flush_task: Final = asyncio.create_task( - _periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS))) - ) - super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None}) - self.v2_companion: Final = ( - S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self)) - if sink.strip_base64_files - else None - ) - - @on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True) - async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None: - return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx) - - async def run_rust( - self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object] - ) -> Patch | Abort | None: - from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE - - kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None) - if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict - return None - payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict - if payload is not None and not isinstance(payload, Mapping): - return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload - try: - raw: Final = _legacy_kwargs(view) - details: Final = _excluding_fields(raw) - logged: Final = _logged_payload(raw, details) - if self.sink.strip_base64_files and isinstance(logged, Mapping): - _STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does - return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx) - except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on - return self._failed(e) - - def _failed(self, error: Exception) -> None: - verbose_logger.error("s3 Layer Error - %s", error, exc_info=error) - self.s3_host.callback_failed() - - @property - def s3_path(self) -> str | None: - return self.sink.s3_path - - @property - def s3_bucket_name(self) -> str | None: - return self.sink.s3_bucket_name - - def resolve_partition_granularity(self) -> S3PartitionGranularity: - return "hour" if self.sink.partition_granularity() == "hour" else "day" - - async def flush_queue(self) -> None: - batch_upload_failed(await self.sink.flush()) - - async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None: - try: - audit_log_id: Final = audit_log.get("id", "unknown") - batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}")) - except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger - verbose_logger.exception("S3 audit log error: %s", e) - - async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None: - object_name: Final = batch_line_items_object_name(payload) - s3_path: Final = self.s3_path - s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name - body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload) - delivered: Final = await self.sink.upload_line_items( - s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:]) - ) - if not delivered: - verbose_logger.error( - "s3 logging: the %d line items of part %d of batch %s were not stored at %s", - len(payload["lines"]), - payload["part"], - payload["batch_id"], - s3_object_key, - ) - - async def _download_object_from_s3(self, s3_object_key: str) -> object: - try: - downloaded: Final = await self.sink.download(s3_object_key) - return None if downloaded is None else json.loads(downloaded) - except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger - verbose_logger.exception("Error downloading from S3: %s", e) - return None - - async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object: - try: - return await self._download_object_from_s3(object_key) - except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger - verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e) - return None - - -__all__ = [ - "S3BatchUploadError", - "S3Logger", - "S3StripBase64Files", - "legacy_should_log", -] diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index 535fef8fcec..74badc2c8c1 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -46,7 +46,7 @@ from litellm.integrations.opik.opik import OpikLogger from litellm.integrations.pointfive import PointFiveLogger from litellm.integrations.posthog import PostHogLogger from litellm.integrations.prometheus import PrometheusLogger -from litellm.integrations.s3_v3 import S3Logger +from litellm.integrations.s3_v2 import S3Logger from litellm.integrations.sqs import SQSLogger from litellm.integrations.ternary.ternary_logger import TernaryLogger from litellm.integrations.vantage.vantage_logger import VantageLogger diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index 2abb1c7e4da..d518fb85b79 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -171,8 +171,12 @@ def is_annual_budget_duration(duration: str) -> bool: return _parse_duration(normalize_duration(duration)) == (12, "mo") +def normalize_budget_window_start(moment: datetime) -> datetime: + return moment.replace(microsecond=moment.microsecond // 1000 * 1000) + + def get_annual_budget_window_start(reset_at: datetime, timezone_str: str) -> datetime: - boundary: Final = _setup_timezone(reset_at, timezone_str)[0] + boundary: Final = _setup_timezone(normalize_budget_window_start(reset_at), timezone_str)[0] return _shift_months(boundary, -12).astimezone(timezone.utc) @@ -180,24 +184,25 @@ def get_budget_window_start(duration: str, reset_at: datetime) -> datetime: """Start of the budget period that ends at `reset_at`, under the same rules `get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar months and a duration it does not recognize resets at the next midnight.""" + boundary: Final = normalize_budget_window_start(reset_at) value, unit = _parse_duration(normalize_duration(duration)) if value is None: - return reset_at - timedelta(days=1) + return boundary - timedelta(days=1) match unit: case "mo": - return _shift_months(reset_at, -value) + return _shift_months(boundary, -value) case "d": - return _shift_months(reset_at, -1) if value == 30 else reset_at - timedelta(days=value) + return _shift_months(boundary, -1) if value == 30 else boundary - timedelta(days=value) case "w": - return reset_at - timedelta(weeks=value) + return boundary - timedelta(weeks=value) case "h": - return reset_at - timedelta(hours=value) + return boundary - timedelta(hours=value) case "m": - return reset_at - timedelta(minutes=value) + return boundary - timedelta(minutes=value) case "s": - return reset_at - timedelta(seconds=value) + return boundary - timedelta(seconds=value) case _: - return reset_at - timedelta(days=1) + return boundary - timedelta(days=1) def _setup_timezone(current_time: datetime, timezone_str: str = "UTC") -> tuple[datetime, tzinfo]: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 15a95e91c52..6ab55772394 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6536,7 +6536,7 @@ def build_custom_logger_compatible_class( _in_memory_loggers.append(_gcs_bucket_logger) return _gcs_bucket_logger elif logging_integration == "s3_v2": - from litellm.integrations.s3_v3 import S3Logger as S3V2Logger + from litellm.integrations.s3_v2 import S3Logger as S3V2Logger for callback in _in_memory_loggers: if isinstance(callback, S3V2Logger): @@ -7360,7 +7360,7 @@ def get_custom_logger_compatible_class( if isinstance(callback, GCSBucketLogger): return callback elif logging_integration == "s3_v2": - from litellm.integrations.s3_v3 import S3Logger as S3V2Logger + from litellm.integrations.s3_v2 import S3Logger as S3V2Logger for callback in _in_memory_loggers: if isinstance(callback, S3V2Logger): @@ -8067,7 +8067,7 @@ class StandardLoggingPayloadSetup: ) if custom_logger and hasattr(custom_logger, "s3_path") and getattr(custom_logger, "s3_path"): s3_path = getattr(custom_logger, "s3_path") - from litellm.integrations.s3_v3 import S3Logger as S3V2Logger + from litellm.integrations.s3_v2 import S3Logger as S3V2Logger if isinstance(custom_logger, S3V2Logger): partition_granularity = custom_logger.resolve_partition_granularity() diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index ee70152636f..79c1c82356d 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1008,9 +1008,8 @@ class ChunkProcessor: cache_creation_token_details: CacheCreationTokenDetails | None = None cost: float | None = None - for chunk in chunks: - usage_chunk = self.extract_usage_chunk(chunk) - + usage_chunks: Final = tuple(self.extract_usage_chunk(chunk) for chunk in chunks) + for usage_chunk in usage_chunks: if usage_chunk is not None: usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk) if usage_chunk_dict["prompt_tokens"] is not None and ( @@ -1091,15 +1090,12 @@ class ChunkProcessor: completion_tokens_details=None if cursor_was_reset else completion_tokens_details, prompt_tokens_details=prompt_tokens_details, cost=cost, - inference_geo=self._last_provider_pricing_field(chunks, "inference_geo"), - speed=self._last_provider_pricing_field(chunks, "speed"), + inference_geo=self._last_provider_pricing_field(usage_chunks, "inference_geo"), + speed=self._last_provider_pricing_field(usage_chunks, "speed"), ) - def _last_provider_pricing_field( - self, - chunks: Sequence["_UsageBearingChunk | ModelResponse"], - field: str, - ) -> str | None: + @staticmethod + def _last_provider_pricing_field(usage_chunks: Sequence[Usage | None], field: str) -> str | None: """ Last value of a provider-specific usage field that changes pricing but is not a declared ``Usage`` field, e.g. Anthropic's ``speed`` (fast mode multiplies @@ -1107,9 +1103,8 @@ class ChunkProcessor: """ values: Final = [ value - for chunk in chunks - if (usage_chunk := self.extract_usage_chunk(chunk)) is not None - and isinstance(value := getattr(usage_chunk, field, None), str) + for usage_chunk in usage_chunks + if usage_chunk is not None and isinstance(value := getattr(usage_chunk, field, None), str) ] return values[-1] if values else None diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index c0116454ce6..f51050114fe 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1884,9 +1884,11 @@ def _count_anthropic_content( except Exception as e: if default_token_count is not None: return default_token_count - raise (UnsupportedContentTypeError if isinstance(e, UnsupportedContentTypeError) else ValueError)( - f"Error counting field '{field_name}': {e}" - ) + raise ( + UnsupportedContentTypeError + if server_fields is not None or isinstance(e, UnsupportedContentTypeError) + else ValueError + )(f"Error counting field '{field_name}': {e}") return tokens diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 0743bd6e7e7..039e6e24f96 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -79,6 +79,18 @@ def _without_region_path(model: str) -> str: return model if region is None else model_id +_ASYNC_INVOKE_REQUEST_FIELDS: Final = frozenset({"clientRequestToken", "tags"}) + + +def _split_async_invoke_extra_body( + extra_body: Mapping[str, object], +) -> tuple[Mapping[str, object], Mapping[str, object]]: + return ( + {key: value for key, value in extra_body.items() if key in _ASYNC_INVOKE_REQUEST_FIELDS}, + {key: value for key, value in extra_body.items() if key not in _ASYNC_INVOKE_REQUEST_FIELDS}, + ) + + class BedrockEmbedding(BaseAWSLLM): _web_identity_session_policy: ClassVar[WebIdentitySessionPolicy] = BEDROCK_EMBEDDING_SESSION_POLICY @@ -296,7 +308,13 @@ class BedrockEmbedding(BaseAWSLLM): extra_body: Mapping[str, object] = MappingProxyType({}), ): responses: Final[list[dict]] = [] - for data in ({**extra_body, **item} for item in batch_data): + request_fields, model_fields = _split_async_invoke_extra_body(extra_body) + for data in ( + {**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}} + if "modelInput" in item + else {**extra_body, **item} + for item in batch_data + ): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} @@ -366,7 +384,13 @@ class BedrockEmbedding(BaseAWSLLM): extra_body: Mapping[str, object] = MappingProxyType({}), ): responses: Final[list[dict]] = [] - for data in ({**extra_body, **item} for item in batch_data): + request_fields, model_fields = _split_async_invoke_extra_body(extra_body) + for data in ( + {**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}} + if "modelInput" in item + else {**extra_body, **item} + for item in batch_data + ): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} diff --git a/litellm/llms/fal_ai/image_generation/bytedance_transformation.py b/litellm/llms/fal_ai/image_generation/bytedance_transformation.py index de7b3c9b958..50401241133 100644 --- a/litellm/llms/fal_ai/image_generation/bytedance_transformation.py +++ b/litellm/llms/fal_ai/image_generation/bytedance_transformation.py @@ -92,6 +92,7 @@ class FalAIBytedanceSeedreamV3Config(FalAIBytedanceBaseConfig): """ IMAGE_GENERATION_ENDPOINT: str = "fal-ai/bytedance/seedream/v3/text-to-image" + MODEL_FAMILY_ROOTS: tuple[str, ...] = ("fal-ai/bytedance/seedream/", "bytedance/seedream/") class FalAIBytedanceDreaminaV31Config(FalAIBytedanceBaseConfig): diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py index fc215e3d94e..344f9ca088f 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -34,6 +34,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig): """ IMAGE_GENERATION_ENDPOINT: str = "fal-ai/flux-pro/v1.1-ultra" + MODEL_FAMILY_ROOTS: tuple[str, ...] = ("fal-ai/flux-pro/",) def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]: """ diff --git a/litellm/llms/fal_ai/image_generation/transformation.py b/litellm/llms/fal_ai/image_generation/transformation.py index 79913d5c651..cbb3e8848c2 100644 --- a/litellm/llms/fal_ai/image_generation/transformation.py +++ b/litellm/llms/fal_ai/image_generation/transformation.py @@ -5,7 +5,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict -from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.litellm_core_utils.url_utils import encode_url_path_segments, join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -71,6 +71,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig): DEFAULT_BASE_URL: str = "https://fal.run" IMAGE_GENERATION_ENDPOINT: str = "" + MODEL_FAMILY_ROOTS: tuple[str, ...] = () def get_complete_url( self, @@ -89,7 +90,16 @@ class FalAIBaseConfig(BaseImageGenerationConfig): base_url: Final = api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL if not self.IMAGE_GENERATION_ENDPOINT: return base_url.rstrip("/") - return join_api_base(base_url, self.IMAGE_GENERATION_ENDPOINT) + return join_api_base(base_url, self._endpoint_for_model(model)) + + def _endpoint_for_model(self, model: str) -> str: + path: Final = encode_url_path_segments(model.strip("/"), field_name="model") + names_default_endpoint: Final = path.lower().removeprefix( + "fal-ai/" + ) == self.IMAGE_GENERATION_ENDPOINT.removeprefix("fal-ai/") + if names_default_endpoint or not path.lower().startswith(self.MODEL_FAMILY_ROOTS): + return self.IMAGE_GENERATION_ENDPOINT + return path def validate_environment( self, diff --git a/litellm/llms/fal_ai/videos/transformation.py b/litellm/llms/fal_ai/videos/transformation.py index bea3de75f71..a0285ce3bb1 100644 --- a/litellm/llms/fal_ai/videos/transformation.py +++ b/litellm/llms/fal_ai/videos/transformation.py @@ -55,7 +55,7 @@ class _ModelProfile: _SEEDANCE_PROFILE: Final[_ModelProfile] = _ModelProfile( resolutions=frozenset({"480p", "720p", "1080p", "4k"}), resolution_tiers=((480, "480p"), (720, "720p"), (1080, "1080p"), (sys.maxsize, "4k")), - default_resolution="720p", + default_resolution=None, integer_duration=False, reference_key="image_url", reference_as_list=False, diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index 4fb5a77653d..a2b18b948fd 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -1,7 +1,7 @@ """ Supports writing files to Google AI Studio Files API. -For vertex ai, check out the vertex_ai/files/handler.py file. +For vertex ai, check out the vertex_ai/files/transformation.py file. """ import re diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index c40a86b221b..756de44b6c4 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -3,6 +3,7 @@ Helper util for handling perplexity-specific cost calculation - e.g.: citation tokens, search queries """ +import math from datetime import datetime from typing import Final @@ -52,13 +53,14 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non except (ValueError, TypeError): return default + cache_read_cost_value: Final = model_info.get("cache_read_input_token_cost") rates: Final = apply_off_peak_pricing( model_info, current_time, TokenRates( input_rate=_safe_float_cast(model_info.get("input_cost_per_token")), output_rate=_safe_float_cast(model_info.get("output_cost_per_token")), - cache_read_rate=0.0, + cache_read_rate=math.nan if cache_read_cost_value is None else _safe_float_cast(cache_read_cost_value), cache_creation_rate=0.0, reasoning_rate=None, ), @@ -67,7 +69,15 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non output_cost_per_token: Final = rates.output_rate ## CALCULATE INPUT COST - prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token + prompt_tokens: Final = usage.prompt_tokens or 0 + reported_cached_tokens: Final = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0 + cached_tokens: Final = min(prompt_tokens, reported_cached_tokens) + cache_read_cost_per_token: Final = ( + input_cost_per_token if math.isnan(rates.cache_read_rate) else rates.cache_read_rate + ) + prompt_cost: float = ( + prompt_tokens - cached_tokens + ) * input_cost_per_token + cached_tokens * cache_read_cost_per_token ## ADD CITATION TOKENS COST (if present) citation_tokens: Final = getattr(usage, "citation_tokens", 0) or 0 diff --git a/litellm/llms/replicate/videos/transformation.py b/litellm/llms/replicate/videos/transformation.py index cd01c832686..02b8766f6ec 100644 --- a/litellm/llms/replicate/videos/transformation.py +++ b/litellm/llms/replicate/videos/transformation.py @@ -77,13 +77,19 @@ def _profile_for_model(model: str) -> _ModelProfile: return _MINIMAX_PROFILE if replicate_model_owner(model) == "minimax" else _GENERIC_PROFILE +def _invalid_seconds() -> ReplicateError: + return ReplicateError(status_code=400, message="Replicate seconds must be a finite number", headers={}) + + def _duration_value(seconds: object) -> int | float: if isinstance(seconds, bool) or not isinstance(seconds, (int, float, str)): - raise TypeError("Replicate seconds must be a numeric value") + raise _invalid_seconds() try: duration: Final = float(seconds) - except ValueError as error: - raise ValueError("Replicate seconds must be a numeric value") from error + except (ValueError, OverflowError) as error: + raise _invalid_seconds() from error + if not math.isfinite(duration): + raise _invalid_seconds() return int(duration) if duration.is_integer() else duration @@ -99,7 +105,7 @@ def _size_params(size: object, profile: _ModelProfile) -> Mapping[str, object]: if normalized.count("x") != 1: return MappingProxyType({}) width_text, height_text = normalized.split("x") - if not (width_text.isdigit() and height_text.isdigit()) or int(width_text) <= 0 or int(height_text) <= 0: + if not (width_text.isdecimal() and height_text.isdecimal()) or int(width_text) <= 0 or int(height_text) <= 0: return MappingProxyType({}) width: Final = int(width_text) height: Final = int(height_text) diff --git a/litellm/llms/vertex_ai/files/handler.py b/litellm/llms/vertex_ai/files/handler.py deleted file mode 100644 index 54c0fa97550..00000000000 --- a/litellm/llms/vertex_ai/files/handler.py +++ /dev/null @@ -1,239 +0,0 @@ -import asyncio -import json -import os -import time -from collections.abc import Coroutine, Mapping -from typing import Final -from urllib.parse import unquote - -import httpx - -from litellm import LlmProviders -from litellm.integrations.gcs_bucket.gcs_bucket_base import ( - GCSBucketBase, - GCSLoggingConfig, -) -from litellm.litellm_core_utils.cloud_storage_security import ( - VERTEX_AI_MANAGED_GCS_PREFIX, - should_allow_legacy_cloud_file_ids, - validate_managed_cloud_file_id, -) -from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.types.llms.openai import ( - FileContentRequest, - HttpxBinaryResponseContent, -) -from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES -from litellm.types.utils import StandardCallbackDynamicParams - -from .transformation import VertexAIFilesConfig - - -class VertexAIFilesHandler(GCSBucketBase): - """ - Handles Calling VertexAI in OpenAI Files API format v1/files/* - - This implementation uploads files on GCS Buckets - """ - - def __init__(self): - super().__init__() - self.async_httpx_client = get_async_httpx_client( - llm_provider=LlmProviders.VERTEX_AI, - ) - - def _resolve_read_gcs_config( - self, - litellm_params: Mapping[str, object] | None, - vertex_credentials: VERTEX_CREDENTIALS_TYPES | None, - ) -> tuple[str | None, str | None]: - """ - Resolve the GCS bucket and service-account credentials for the read/content path. - - Sources them from the deployment's ``litellm_params`` (``gcs_bucket_name`` / - ``bucket_name`` and ``vertex_credentials``), mirroring the write path in - ``VertexAIFilesConfig._get_configured_bucket_name``, and falls back to the - ``GCS_BATCH_BUCKET_NAME`` then ``GCS_BUCKET_NAME`` / ``GCS_PATH_SERVICE_ACCOUNT`` - env vars. This lets Vertex batch run entirely at the model-group level, so output - written to a per-model bucket is readable without setting the global env vars. - """ - params: Final[Mapping[str, object]] = litellm_params or {} - bucket_candidate: Final = params.get("gcs_bucket_name") or params.get("bucket_name") - configured_bucket_name = ( - bucket_candidate - if isinstance(bucket_candidate, str) - else os.getenv("GCS_BATCH_BUCKET_NAME") or os.getenv("GCS_BUCKET_NAME") - ) - - credentials: Final = params.get("vertex_credentials") or vertex_credentials - if isinstance(credentials, dict): - path_service_account: str | None = json.dumps(credentials) - elif isinstance(credentials, str): - path_service_account = credentials - else: - path_service_account = os.getenv("GCS_PATH_SERVICE_ACCOUNT") - - return configured_bucket_name, path_service_account - - def _extract_bucket_and_object_from_file_id( - self, - file_id: str, - configured_bucket_name: str, - litellm_params: dict | None = None, - ) -> tuple[str, str]: - """ - Validate and extract bucket name and object path from file_id. - - Expected format: gs://bucket-name/litellm-vertex-files/path/to/file - - Returns: - tuple: (bucket_name, object_path) - - bucket_name: "bucket-name" - - object_path: "litellm-vertex-files/path/to/file" - """ - return validate_managed_cloud_file_id( - file_id=file_id, - scheme="gs://", - configured_bucket_name=configured_bucket_name, - allowed_object_prefixes=(VERTEX_AI_MANAGED_GCS_PREFIX,), - allow_legacy_cloud_file_ids=should_allow_legacy_cloud_file_ids(litellm_params), - ) - - async def afile_content( - self, - file_content_request: FileContentRequest, - vertex_credentials: VERTEX_CREDENTIALS_TYPES | None, - vertex_project: str | None, - vertex_location: str | None, - timeout: float | httpx.Timeout, - max_retries: int | None, - litellm_params: dict | None = None, - ) -> HttpxBinaryResponseContent: - """ - Download file content from GCS bucket for VertexAI files. - - Args: - file_content_request: Contains file_id (URL-encoded GCS path) - vertex_credentials: VertexAI credentials - vertex_project: VertexAI project ID - vertex_location: VertexAI location - timeout: Request timeout - max_retries: Max retry attempts - - Returns: - HttpxBinaryResponseContent: Binary content wrapped in compatible response format - """ - file_id: Final = file_content_request.get("file_id") - if not file_id: - raise ValueError("file_id is required in file_content_request") - - configured_bucket_name, path_service_account = self._resolve_read_gcs_config( - litellm_params=litellm_params, - vertex_credentials=vertex_credentials, - ) - dynamic_params: Final = StandardCallbackDynamicParams( - gcs_bucket_name=configured_bucket_name, - gcs_path_service_account=path_service_account, - ) - gcs_logging_config: Final[GCSLoggingConfig] = await self.get_gcs_logging_config( - kwargs={"standard_callback_dynamic_params": dynamic_params} - ) - bucket_name, object_path = self._extract_bucket_and_object_from_file_id( - file_id=file_id, - configured_bucket_name=gcs_logging_config["bucket_name"], - litellm_params=litellm_params, - ) - - download_kwargs: Final = { - "standard_callback_dynamic_params": { - "gcs_bucket_name": bucket_name, - "gcs_path_service_account": gcs_logging_config["path_service_account"], - } - } - - file_content: Final = await self.download_gcs_object(object_name=object_path, **download_kwargs) - decoded_file_id: Final = unquote(file_id) - - if file_content is None: - raise ValueError(f"Failed to download file from GCS: {decoded_file_id}") - - mock_response: Final = httpx.Response( - status_code=200, - content=file_content, - headers={ - "content-type": "application/octet-stream", - "content-length": str(len(file_content)), - }, - request=httpx.Request(method="GET", url=decoded_file_id), - ) - - # Apply transformation to convert Vertex AI batch outputs to OpenAI format - config: Final = VertexAIFilesConfig() - - # Create a logging object for transformation - logging_obj: Final = Logging( - model="", - messages=[], - stream=False, - call_type="afile_content", - start_time=time.time(), - litellm_call_id="", - function_id="", - ) - - return config.transform_file_content_response( - raw_response=mock_response, logging_obj=logging_obj, litellm_params={} - ) - - def file_content( - self, - _is_async: bool, - file_content_request: FileContentRequest, - api_base: str | None, - vertex_credentials: VERTEX_CREDENTIALS_TYPES | None, - vertex_project: str | None, - vertex_location: str | None, - timeout: float | httpx.Timeout, - max_retries: int | None, - litellm_params: dict | None = None, - ) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]: - """ - Download file content from GCS bucket for VertexAI files. - Supports both sync and async operations. - - Args: - _is_async: Whether to run asynchronously - file_content_request: Contains file_id (URL-encoded GCS path) - api_base: API base (unused for GCS operations) - vertex_credentials: VertexAI credentials - vertex_project: VertexAI project ID - vertex_location: VertexAI location - timeout: Request timeout - max_retries: Max retry attempts - - Returns: - HttpxBinaryResponseContent or Coroutine: Binary content wrapped in compatible response format - """ - if _is_async: - return self.afile_content( - file_content_request=file_content_request, - vertex_credentials=vertex_credentials, - vertex_project=vertex_project, - vertex_location=vertex_location, - timeout=timeout, - max_retries=max_retries, - litellm_params=litellm_params, - ) - else: - return asyncio.run( - self.afile_content( - file_content_request=file_content_request, - vertex_credentials=vertex_credentials, - vertex_project=vertex_project, - vertex_location=vertex_location, - timeout=timeout, - max_retries=max_retries, - litellm_params=litellm_params, - ) - ) diff --git a/litellm/llms/vertex_ai/vertex_embeddings/bge.py b/litellm/llms/vertex_ai/vertex_embeddings/bge.py index 71135a66fa5..c13e31aa1c8 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/bge.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/bge.py @@ -10,8 +10,9 @@ Model name handling: - This module focuses on request/response transformation only """ -from typing import Final +from typing import Final, cast # noqa: TID251 # extra_body carries raw fields the TypedDict lacks +from litellm.llms.base_llm.embedding.transformation import split_extra_body from litellm.types.utils import EmbeddingResponse, Usage from .types import ( @@ -84,7 +85,12 @@ class VertexBGEConfig: vertex_text_embedding_input_list.append(embedding_input) vertex_request["instances"] = vertex_text_embedding_input_list - vertex_request["parameters"] = EmbeddingParameters(**optional_params) + body_params, extra_body = split_extra_body(optional_params) + vertex_request["parameters"] = ( + cast( # cast-ok: extra_body carries raw Vertex fields the TypedDict does not list + EmbeddingParameters, {**extra_body, **body_params} + ) + ) return vertex_request diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 333c4deca69..0ba1311c567 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -6,6 +6,9 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele from litellm.exceptions import BadRequestError, UnsupportedParamsError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.anthropic.messages.native import handler as main +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( + AmazonAnthropicClaudeMessagesConfig, +) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.rust_bridge import catalog, detectors from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target @@ -105,8 +108,11 @@ def _route_target( api_base: str | None, compacting: bool, ) -> Target: - if main._provider_messages_config(model, provider, api_base) is not None: # pyright: ignore[reportPrivateUsage] # the handler's own config lookup - return Target.MESSAGES + provider_config: Final = main._provider_messages_config(model, provider, api_base) # pyright: ignore[reportPrivateUsage] # the handler's own config lookup + if provider_config is not None: + return ( + Target.BEDROCK_INVOKE if type(provider_config) is AmazonAnthropicClaudeMessagesConfig else Target.MESSAGES + ) if main._deployment_passes_through_anthropic_messages(model_info): # pyright: ignore[reportPrivateUsage] # the handler's own passthrough opt-in return Target.MESSAGES_PASSTHROUGH if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d5a26bd3ae9..bca7e61d873 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -101932,5 +101932,31 @@ "supports_parallel_function_calling": true, "supports_reasoning": true, "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/pro/us/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K with one input image; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135; the first input image is free and each additional input image adds $0.0045 per output, which is not modeled; fal calls this pricing tentative"}, + "mode": "image_generation", + "output_cost_per_image": 0.135, + "output_cost_per_image_1K": 0.0675, + "output_cost_per_image_2K": 0.135, + "source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/pro/us/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135, so 1.5K has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.135, + "output_cost_per_image_1K": 0.0675, + "output_cost_per_image_2K": 0.135, + "source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] } } diff --git a/litellm/packaging/litellm-proxy/pyproject.toml b/litellm/packaging/litellm-proxy/pyproject.toml index 8efdca8db74..915b0e3708d 100644 --- a/litellm/packaging/litellm-proxy/pyproject.toml +++ b/litellm/packaging/litellm-proxy/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy" -version = "1.109.0" +version = "1.111.0" description = "Opt-in Rust HTTP forwarding server for the LiteLLM Python proxy" requires-python = ">=3.10, <3.16" license = "MIT" diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql index b00cf892b68..ce7171637a2 100644 --- a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql @@ -1,6 +1,6 @@ SET LOCAL lock_timeout = '10s'; CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogDelivery" ( - "delivery_id" TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, + "delivery_id" TEXT PRIMARY KEY, "request_id" TEXT NOT NULL, "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending', "attempts" INTEGER NOT NULL DEFAULT 0, diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261104020000_spendlogs_delivery_id_drop_default/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261104020000_spendlogs_delivery_id_drop_default/migration.sql new file mode 100644 index 00000000000..ee5c8652f6b --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261104020000_spendlogs_delivery_id_drop_default/migration.sql @@ -0,0 +1,2 @@ +SET LOCAL lock_timeout = '10s'; +ALTER TABLE "LiteLLM_SpendLogDelivery" ALTER COLUMN "delivery_id" DROP DEFAULT; diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261115000000_customer_hierarchy_indexes/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261115000000_customer_hierarchy_indexes/migration.sql new file mode 100644 index 00000000000..519990421b2 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261115000000_customer_hierarchy_indexes/migration.sql @@ -0,0 +1,7 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_parent_team_ids_idx" ON "LiteLLM_CustomerMemberTable" USING GIN ("parent_team_ids"); +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_budget_id_idx" ON "LiteLLM_CustomerMemberTable"("budget_id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_budget_override_id_idx" ON "LiteLLM_CustomerMemberTable"("budget_override_id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerTeamTable_budget_id_idx" ON "LiteLLM_CustomerTeamTable"("budget_id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerTeamTable_budget_override_id_idx" ON "LiteLLM_CustomerTeamTable"("budget_override_id"); diff --git a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma index db88b24dcb0..9e1f47a5d1f 100644 --- a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma @@ -85,6 +85,8 @@ model LiteLLM_CustomerTeamTable { updated_at DateTime @default(now()) @updatedAt updated_by String @@index([parent_org_id]) + @@index([budget_id]) + @@index([budget_override_id]) @@index([spend]) } @@ -107,6 +109,9 @@ model LiteLLM_CustomerMemberTable { updated_at DateTime @default(now()) @updatedAt updated_by String @@index([parent_org_id]) + @@index([parent_team_ids], type: Gin) + @@index([budget_id]) + @@index([budget_override_id]) @@index([spend]) } @@ -2308,7 +2313,7 @@ enum AccountingDeliveryState { } model LiteLLM_SpendLogDelivery { - delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + delivery_id String @id @default(uuid()) request_id String state AccountingDeliveryState @default(pending) accounting_payload Json diff --git a/litellm/proxy-extras/pyproject.toml b/litellm/proxy-extras/pyproject.toml index 1d5f3038ac5..567aecc8f69 100644 --- a/litellm/proxy-extras/pyproject.toml +++ b/litellm/proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.116" +version = "0.4.117" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.116" +version = "0.4.117" version_files = [ "pyproject.toml:^version", "../../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index e215d55ebdf..312a8c8f492 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -206,6 +206,8 @@ from .auth_utils import ( if TYPE_CHECKING: from opentelemetry.trace import Span as _Span + from litellm.proxy.spend_tracking.native_budgets import BudgetDecision + Span = _Span else: Span = Any @@ -948,12 +950,11 @@ def _global_proxy_budget_check(global_proxy_spend: float | None, skip_budget_che and RouteChecks.runs_inference_checks(route=route) and route not in MODEL_LIST_ROUTES ): - if math.isfinite(litellm.max_budget) and global_proxy_spend > litellm.max_budget: - raise litellm.BudgetExceededError( - current_cost=global_proxy_spend, - max_budget=litellm.max_budget, - entity_type=Litellm_EntityType.PROXY.value, - ) + native: Final = _native_budget_decision( + {"check": "proxy", "spend": global_proxy_spend, "max_budget": litellm.max_budget} + ) + if native.refused: + raise native.error() _GUARDRAIL_MODIFICATION_KEYS: Final[tuple] = ( @@ -1488,14 +1489,16 @@ async def common_checks( user_info=call_info, ) ) - if math.isfinite(user_budget) and user_spend >= user_budget: - raise litellm.BudgetExceededError( - current_cost=user_spend, - max_budget=user_budget, - message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}", - entity_type=Litellm_EntityType.USER.value, - entity_id=user_object.user_id, - ) + native: Final = _native_budget_decision( + { + "check": "user", + "spend": user_spend, + "max_budget": user_budget, + "user_id": user_object.user_id, + } + ) + if native.refused: + raise native.error() # Each scope reads a distinct counter key with no cross-scope ordering # dependency, so the per-scope Redis-first reads run concurrently instead @@ -2032,14 +2035,16 @@ async def check_end_user_budget( max_budget=end_user_budget, fallback_authoritative=True, ) - if end_user_spend > end_user_budget: - raise litellm.BudgetExceededError( - current_cost=end_user_spend, - max_budget=end_user_budget, - message=f"ExceededBudget: End User={end_user_obj.user_id} over budget. Spend={end_user_spend}, Budget={end_user_budget}", - entity_type=Litellm_EntityType.END_USER.value, - entity_id=end_user_obj.user_id, - ) + native: Final = _native_budget_decision( + { + "check": "end_user", + "spend": end_user_spend, + "max_budget": end_user_budget, + "end_user_id": end_user_obj.user_id, + } + ) + if native.refused: + raise native.error() _check_end_user_budget: Final = check_end_user_budget @@ -6957,13 +6962,6 @@ async def is_valid_fallback_model( return True -# The shape abbreviate_api_key writes into LiteLLM_VerificationToken.key_name. The -# last four characters are only barred from being whitespace or a control code, -# because a custom key's can be anything else, punctuation and non-ASCII included; -# a real key is at least MINIMUM_CUSTOM_KEY_LENGTH long, so it never fullmatches. -_MASKED_KEY_NAME_RE: Final = re.compile(r"sk-\.\.\.(?:[^\s\x00-\x1f\x7f-\x9f]{4})?") - - def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool: """ Throttle an over-budget key instead of blocking it, when the key opted in @@ -6982,6 +6980,17 @@ def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool: return True +def _native_budget_decision(check: Mapping[str, object]) -> "BudgetDecision": + from litellm.proxy.spend_tracking.native_budgets import native_budget_decision + + return native_budget_decision(check) + + +def _budget_exceeded_throttle_pct(valid_token: UserAPIKeyAuth) -> float | None: + pct: Final = budget_throttle_percentage() + return pct if pct is not None and should_throttle_budget_exceeded(valid_token) else None + + async def virtual_key_max_budget_check( valid_token: UserAPIKeyAuth, proxy_logging_obj: ProxyLogging, @@ -7041,25 +7050,21 @@ async def virtual_key_max_budget_check( # Defense-in-depth (GHSA-2rv4-xv66-fpjg): spend >= NaN is always False, # so a NaN max_budget would silently disable enforcement. Treat a # non-finite max_budget as "no configured limit" rather than as a bypass. - if math.isfinite(valid_token.max_budget) and spend >= valid_token.max_budget: - if _apply_budget_exceeded_throttle(valid_token): - return - # This message is returned to the caller, and key_name has no enforced - # shape (a direct DB write bypasses abbreviate_api_key), so echo it only - # when it still looks masked and fall back to the alias otherwise. - key_label: Final = valid_token.key_alias or "key" - key_descriptor: Final = ( - f"{key_label} ({valid_token.key_name})" - if valid_token.key_name and _MASKED_KEY_NAME_RE.fullmatch(valid_token.key_name) - else key_label - ) - raise litellm.BudgetExceededError( - current_cost=spend, - max_budget=valid_token.max_budget, - message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}", - entity_type=Litellm_EntityType.KEY.value, - entity_id=valid_token.token, - ) + native: Final = _native_budget_decision( + { + "check": "key_max", + "spend": spend, + "max_budget": valid_token.max_budget, + "token": valid_token.token, + "key_alias": valid_token.key_alias, + "key_name": valid_token.key_name, + "throttle_pct": _budget_exceeded_throttle_pct(valid_token), + } + ) + if native.decision == "throttle" and _apply_budget_exceeded_throttle(valid_token): + return + if native.refused: + raise native.error() _virtual_key_max_budget_check: Final = virtual_key_max_budget_check @@ -7096,17 +7101,17 @@ async def _virtual_key_multi_budget_check( window_duration=str(w["budget_duration"]), window_start=get_budget_window_start(w), ) - if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]: - raise litellm.BudgetExceededError( - current_cost=window_spend, - max_budget=w["max_budget"], - message=( - f"ExceededBudget: Key over {w['budget_duration']} budget. " - f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}" - ), - entity_type=Litellm_EntityType.KEY.value, - entity_id=valid_token.token, - ) + native = _native_budget_decision( + { + "check": "key_window", + "spend": window_spend, + "max_budget": w["max_budget"], + "budget_duration": str(w["budget_duration"]), + "token": valid_token.token, + } + ) + if native.refused: + raise native.error() async def virtual_key_soft_budget_check( @@ -7119,7 +7124,10 @@ async def virtual_key_soft_budget_check( """ - if valid_token.soft_budget and valid_token.spend >= valid_token.soft_budget: + native: Final = _native_budget_decision( + {"check": "key_soft", "spend": valid_token.spend, "soft_budget": valid_token.soft_budget} + ) + if native.alerts: verbose_proxy_logger.debug( "Crossed Soft Budget for token %s, spend %s, soft_budget %s", valid_token.token, @@ -7422,7 +7430,16 @@ async def _check_team_member_budget( max_budget=team_member_budget, ) - if not math.isfinite(team_member_budget): + native: Final = _native_budget_decision( + { + "check": "team_member", + "spend": team_member_spend, + "max_budget": team_member_budget, + "user_id": valid_token.user_id, + "team_id": team_object.team_id, + } + ) + if native.skipped: return team_member_max_budget_alert_check( @@ -7437,14 +7454,8 @@ async def _check_team_member_budget( max_budget=team_member_budget, ) - if team_member_spend >= team_member_budget: - raise litellm.BudgetExceededError( - current_cost=team_member_spend, - max_budget=team_member_budget, - message=f"Budget has been exceeded! User={valid_token.user_id} in Team={team_object.team_id} Current cost: {team_member_spend}, Max budget: {team_member_budget}", - entity_type=Litellm_EntityType.TEAM_MEMBER.value, - entity_id=f"{valid_token.user_id}:{team_object.team_id}", - ) + if native.refused: + raise native.error() async def check_team_member_model_access( @@ -7534,7 +7545,15 @@ async def team_max_budget_check( max_budget=team_object.max_budget, ) - if math.isfinite(team_object.max_budget) and spend > team_object.max_budget: + native: Final = _native_budget_decision( + { + "check": "team_max", + "spend": spend, + "max_budget": team_object.max_budget, + "team_id": team_object.team_id, + } + ) + if native.refused: if valid_token: call_info: Final = CallInfo( token=valid_token.token, @@ -7553,13 +7572,7 @@ async def team_max_budget_check( ) ) - raise litellm.BudgetExceededError( - current_cost=spend, - max_budget=team_object.max_budget, - message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}", - entity_type=Litellm_EntityType.TEAM.value, - entity_id=team_object.team_id, - ) + raise native.error() _team_max_budget_check: Final = team_max_budget_check @@ -7571,29 +7584,26 @@ async def _user_multi_budget_check(user_object: LiteLLM_UserTable | None) -> Non from litellm.proxy.proxy_server import get_current_spend for window in user_object.budget_limits: - if ( - spend := await get_current_spend( - counter_key=user_window_counter(user_object.user_id, window.budget_duration), - fallback_spend=0.0, - max_budget=window.max_budget, - window_entity_type="User", - window_entity_id=user_object.user_id, - window_duration=window.budget_duration, - window_start=get_budget_window_start(window), - ) - ) < window.max_budget: - continue - if math.isfinite(window.max_budget) and spend >= window.max_budget: - raise litellm.BudgetExceededError( - current_cost=spend, - max_budget=window.max_budget, - message=( - f"ExceededBudget: User={user_object.user_id} over {window.budget_duration} budget. " - f"Spend=${spend:.4f}, Limit=${window.max_budget:.2f}" - ), - entity_type=Litellm_EntityType.USER.value, - entity_id=user_object.user_id, - ) + spend = await get_current_spend( + counter_key=user_window_counter(user_object.user_id, window.budget_duration), + fallback_spend=0.0, + max_budget=window.max_budget, + window_entity_type="User", + window_entity_id=user_object.user_id, + window_duration=window.budget_duration, + window_start=get_budget_window_start(window), + ) + native = _native_budget_decision( + { + "check": "user_window", + "spend": spend, + "max_budget": window.max_budget, + "budget_duration": window.budget_duration, + "user_id": user_object.user_id, + } + ) + if native.refused: + raise native.error() async def _team_multi_budget_check( @@ -7623,17 +7633,17 @@ async def _team_multi_budget_check( window_duration=str(w["budget_duration"]), window_start=get_budget_window_start(w), ) - if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]: - raise litellm.BudgetExceededError( - current_cost=window_spend, - max_budget=w["max_budget"], - message=( - f"ExceededBudget: Team={team_object.team_id} over {w['budget_duration']} budget. " - f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}" - ), - entity_type=Litellm_EntityType.TEAM.value, - entity_id=team_object.team_id, - ) + native = _native_budget_decision( + { + "check": "team_window", + "spend": window_spend, + "max_budget": w["max_budget"], + "budget_duration": str(w["budget_duration"]), + "team_id": team_object.team_id, + } + ) + if native.refused: + raise native.error() async def _team_soft_budget_check( @@ -7644,12 +7654,12 @@ async def _team_soft_budget_check( """ Triggers a budget alert if the team is over it's soft budget. """ - if ( - team_object is not None - and team_object.soft_budget is not None - and team_object.spend is not None - and team_object.spend >= team_object.soft_budget - ): + if team_object is None: + return + native: Final = _native_budget_decision( + {"check": "team_soft", "spend": team_object.spend, "soft_budget": team_object.soft_budget} + ) + if native.alerts and team_object.spend is not None: verbose_proxy_logger.debug( "Crossed Soft Budget for team %s, spend %s, soft_budget %s", team_object.team_id, @@ -7737,7 +7747,15 @@ async def _project_max_budget_check( max_budget=max_budget, ) - if project_spend >= max_budget: + native: Final = _native_budget_decision( + { + "check": "project_max", + "spend": project_spend, + "max_budget": max_budget, + "project_id": project_object.project_id, + } + ) + if native.refused: if valid_token: call_info: Final = CallInfo( token=valid_token.token, @@ -7756,13 +7774,7 @@ async def _project_max_budget_check( ) ) - raise litellm.BudgetExceededError( - current_cost=project_spend, - max_budget=max_budget, - message=f"Budget has been exceeded! Project={project_object.project_id} Current cost: {project_spend}, Max budget: {max_budget}", - entity_type=Litellm_EntityType.PROJECT.value, - entity_id=project_object.project_id, - ) + raise native.error() async def _project_soft_budget_check( @@ -7782,7 +7794,10 @@ async def _project_soft_budget_check( if project_object.litellm_budget_table is not None: soft_budget = project_object.litellm_budget_table.soft_budget - if soft_budget is not None and project_object.spend is not None and project_object.spend >= soft_budget: + native: Final = _native_budget_decision( + {"check": "project_soft", "spend": project_object.spend, "soft_budget": soft_budget} + ) + if native.alerts: verbose_proxy_logger.debug( "Crossed Soft Budget for project %s, spend %s, soft_budget %s", project_object.project_id, @@ -7945,7 +7960,10 @@ async def organization_max_budget_check( ) # Check if organization spend exceeds max budget - if math.isfinite(org_max_budget) and org_spend >= org_max_budget: + native: Final = _native_budget_decision( + {"check": "organization", "spend": org_spend, "max_budget": org_max_budget, "org_id": org_id} + ) + if native.refused: # Trigger budget alert call_info: Final = CallInfo( token=valid_token.token, @@ -7964,13 +7982,7 @@ async def organization_max_budget_check( ) ) - raise litellm.BudgetExceededError( - current_cost=org_spend, - max_budget=org_max_budget, - message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_spend}, Max budget: {org_max_budget}", - entity_type=Litellm_EntityType.ORGANIZATION.value, - entity_id=org_id, - ) + raise native.error() _organization_max_budget_check: Final = organization_max_budget_check @@ -8034,15 +8046,16 @@ async def tag_max_budget_check_for_tags( max_budget=tag_object.litellm_budget_table.max_budget, fallback_authoritative=True, ) - if tag_spend <= tag_object.litellm_budget_table.max_budget: - continue - raise litellm.BudgetExceededError( - current_cost=tag_spend, - max_budget=tag_object.litellm_budget_table.max_budget, - message=f"Budget has been exceeded! Tag={tag_name} Current cost: {tag_spend}, Max budget: {tag_object.litellm_budget_table.max_budget}", - entity_type=Litellm_EntityType.TAG.value, - entity_id=tag_name, + native = _native_budget_decision( + { + "check": "tag", + "spend": tag_spend, + "max_budget": tag_object.litellm_budget_table.max_budget, + "tag": tag_name, + } ) + if native.refused: + raise native.error() async def _model_access_group_max_budget_check( @@ -8089,15 +8102,11 @@ async def _model_access_group_max_budget_check( max_budget=budget.max_budget, fallback_authoritative=True, ) - if group_spend < budget.max_budget: - continue - raise litellm.BudgetExceededError( - current_cost=group_spend, - max_budget=budget.max_budget, - message=f"Budget has been exceeded! Model access group={group} Current cost: {group_spend}, Max budget: {budget.max_budget}", - entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP.value, - entity_id=group, + native = _native_budget_decision( + {"check": "model_access_group", "spend": group_spend, "max_budget": budget.max_budget, "group": group} ) + if native.refused: + raise native.error() def is_model_allowed_by_pattern(model: str, allowed_model_pattern: str) -> bool: diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 259c46a9dc1..bba0bded82c 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -22,6 +22,7 @@ from litellm.constants import ( INVALID_VIRTUAL_KEY_ERROR_MARKER, MINIMUM_CUSTOM_KEY_LENGTH, MODEL_BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, + MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY, STANDARD_CUSTOMER_ID_HEADERS, ) from litellm.litellm_core_utils.clientside_credential_handler import ( @@ -1922,7 +1923,21 @@ class BatchLimitRequest(Protocol): def model_fields_set(self) -> Collection[str]: ... -_ADMIN_ONLY_METADATA_KEYS: Final = (*ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS, TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY) +_ADMIN_ONLY_METADATA_KEYS: Final = ( + *ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS, + TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY, + MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY, +) +_ADMIN_ONLY_METADATA_REASONS: Final[Mapping[str, str]] = MappingProxyType( + { + TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY: ( + "It lets the holder read and write those buckets with the proxy's AWS credentials." + ), + MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY: ( + "It caps how far the holder borrows idle model capacity, so the holder cannot raise or remove it." + ), + } +) def enforce_batch_limits_are_admin_only( @@ -1955,10 +1970,8 @@ def enforce_batch_limits_are_admin_only( ) if changed is None: return - reason: Final = ( - "It lets the holder read and write those buckets with the proxy's AWS credentials." - if changed == TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY - else "It limits what the holder can do with batches, so the holder cannot raise it." + reason: Final = _ADMIN_ONLY_METADATA_REASONS.get( + changed, "It limits what the holder can do with batches, so the holder cannot raise it." ) raise HTTPException( status_code=403, diff --git a/litellm/proxy/auth/customer_hierarchy_auth.py b/litellm/proxy/auth/customer_hierarchy_auth.py index 11b6ae2c488..24dd06e64d4 100644 --- a/litellm/proxy/auth/customer_hierarchy_auth.py +++ b/litellm/proxy/auth/customer_hierarchy_auth.py @@ -553,7 +553,7 @@ async def _register_organization( if isinstance(registered, Failure): if registered.code is ErrorCode.FREE_TIER_LIMIT: verbose_proxy_logger.warning( - "CUSTOMER TRACKING OFF for organization %s because the free customer portal limit was reached. " + "CUSTOMER TRACKING OFF for organization %s because the Customer Management free tier limit was reached. " "Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise", customer_org_alias, ) @@ -620,7 +620,7 @@ async def _register_member( if isinstance(registered, Failure): if registered.code is ErrorCode.FREE_TIER_LIMIT: verbose_proxy_logger.warning( - "CUSTOMER TRACKING OFF for member %s because the free customer portal limit was reached. " + "CUSTOMER TRACKING OFF for member %s because the Customer Management free tier limit was reached. " "Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise", customer_member_id, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 27d6bc9389a..357400a4bfd 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -198,6 +198,7 @@ from litellm.proxy.spend_tracking.committed_spend import ( committed_spend_tracked, read_admission_committed_spend, ) +from litellm.proxy.spend_tracking.native_budgets import native_budget_decision from litellm.proxy.spend_tracking.spend_counter_batch import ( bind_admission_counter_keys, release_spend_counter_batch, @@ -2322,12 +2323,22 @@ async def _small_preflight_failure_json(request: Request) -> Mapping[str, object } +def _is_bodyless_get(request: Request) -> bool: + return ( + request.scope.get("method") == "GET" + and request.headers.get("content-length", "0") == "0" + and request.headers.get("transfer-encoding") is None + ) + + async def _reject_from_headers_before_reading_body( request: Request, api_key: str | None, custom_litellm_key_header: str | None, route: str ) -> None: """Refuse a request its headers already condemn, a blocked key or an exhausted key limit, before reading its body. Only a key already in the cache is considered, so a miss changes nothing and the full auth path still runs. + The limit is only checked where a body is declared and the route is metered, because model listing, token + counting and other GET discovery routes never reach the limiter and must keep answering an exhausted key. A retry flood of large payloads against a limited key used to buffer every payload just to answer 429. """ if not RouteChecks.is_llm_api_route(route=route): @@ -2377,6 +2388,8 @@ async def _reject_from_headers_before_reading_body( dispatched_call_type=str(request_data["call_type"]) if request_data["call_type"] is not None else None, ) return + if RouteChecks.is_unbilled_route(route) or _is_bodyless_get(request): + return limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): return @@ -3119,7 +3132,7 @@ async def user_api_key_auth_builder( for k, v in team_obj_dict.items(): field_name = f"team_{k}" - if field_name in valid_token.__fields__: + if field_name in type(valid_token).model_fields: setattr(valid_token, field_name, v) except Exception as e: verbose_logger.debug(e) # moving from .warning to .debug as it spams logs when team missing from cache. @@ -3420,7 +3433,16 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e fallback_spend=team_member_spend, max_budget=team_member_budget, ) - if team_member_spend >= team_member_budget: + native: Final = native_budget_decision( + { + "check": "team_member_positive", + "spend": team_member_spend, + "max_budget": team_member_budget, + "user_id": valid_token.user_id, + "team_id": valid_token.team_id, + } + ) + if native.refused: # common_checks sends this alert on requests that get past here, so only the # request rejected here sends it from the builder. team_member_max_budget_alert_check( @@ -3434,17 +3456,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e spend=team_member_spend, max_budget=team_member_budget, ) - _entity_id: Final = f"{valid_token.user_id}:{valid_token.team_id}" - raise litellm.BudgetExceededError( - current_cost=team_member_spend, - max_budget=team_member_budget, - message=( - f"Budget has been exceeded! TeamMember={_entity_id} " - f"Current cost: {team_member_spend}, Max budget: {team_member_budget}" - ), - entity_type=Litellm_EntityType.TEAM_MEMBER.value, - entity_id=_entity_id, - ) + raise native.error() # Check 3. If token is expired if valid_token.expires is not None: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7aa93c253d4..1fdb51deb2c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -5,6 +5,7 @@ import json import logging import math import time +import weakref from collections.abc import ( AsyncGenerator, AsyncIterator, @@ -4461,6 +4462,9 @@ class ProxyBaseLLMRequestProcessing: ), ) + def _request_fields(self) -> Mapping[str, object]: + return self.data + def _arm_detached_stream_failure_hook( self, logging_obj: LiteLLMLoggingObj, @@ -4475,17 +4479,24 @@ class ProxyBaseLLMRequestProcessing: so the failed spend is still written and the budget reservation released; a replacement error the hook raises has no client left to reach. """ - request_data: Final = self.data + # The closure lives on the logging object until the logging object dies, so it must not + # hold the logging object, or a request stays alive until a full GC pass (#51277) + logging_obj_ref: Final = weakref.ref(logging_obj) + request_data: Final = { + key: value for key, value in self._request_fields().items() if key != "litellm_logging_obj" + } + route_type: Final = self.route_type async def _on_detached_stream_failure(exc: Exception) -> None: - if not logging_obj.claim_stream_failure_report(): + live_logging_obj: Final = logging_obj_ref() + if live_logging_obj is None or not live_logging_obj.claim_stream_failure_report(): return try: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=exc, - request_data=request_data, - dispatched_call_type=self.route_type, + request_data={**request_data, "litellm_logging_obj": live_logging_obj}, + dispatched_call_type=route_type, ) except HTTPException: return diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index d9432b23d29..0e371e53755 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -3,7 +3,7 @@ import json import math import time from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import datetime, timedelta, timezone from enum import Enum from itertools import chain @@ -36,6 +36,7 @@ from litellm.litellm_core_utils.duration_parser import ( duration_in_seconds, get_annual_budget_window_start, is_annual_budget_duration, + normalize_budget_window_start, ) from litellm.proxy._types import ( DB_RETRY_SAFE_ERROR_TYPES, @@ -378,9 +379,15 @@ def _customer_reset_key( def _customer_ids_by_reset( rows: Sequence[_CustomerScopedRow], ids: Sequence[str], now: datetime, windows: Mapping[str, str] ) -> Mapping[tuple[float | None, str | None], tuple[str, ...]]: - keys: Final = tuple(_customer_reset_key(row, now, windows) for row in rows) + capped: Final = tuple( + (row_id, row) for row_id, row in zip(ids, rows) if _customer_effective_cap(row, now) is not None + ) + keys: Final = tuple(_customer_reset_key(row, now, windows) for _, row in capped) return MappingProxyType( - {key: tuple(row_id for row_id, row_key in zip(ids, keys) if row_key == key) for key in dict.fromkeys(keys)} + { + key: tuple(row_id for (row_id, _), row_key in zip(capped, keys) if row_key == key) + for key in dict.fromkeys(keys) + } ) @@ -506,6 +513,10 @@ class _BudgetCascade: customer_ids: Mapping[CustomerSpendEntity, tuple[str, ...]] = field(default_factory=lambda: MappingProxyType({})) customer_org_alias_keys: tuple[str, ...] = () + @property + def customer_budget_ids(self) -> tuple[str, ...]: + return tuple(b.budget_id for b in self.budgets if b.max_budget is not None and b.budget_id in self.budget_ids) + @dataclass(frozen=True, slots=True) class _EndUserWalk: @@ -1280,7 +1291,11 @@ class ResetBudgetJob: _queue_budget_linked_resets(uow.projects, cascade, extra=_SPENT_ROWS_WHERE) _queue_budget_linked_resets(uow.tags, cascade, extra=_SPENT_ROWS_WHERE) _queue_budget_linked_resets(uow.model_access_groups, cascade, extra=_SPENT_ROWS_WHERE) - _queue_budget_linked_resets(uow.customer_organizations, cascade, extra=_SPENT_ROWS_WHERE) + _queue_budget_linked_resets( + uow.customer_organizations, + replace(cascade, budget_ids=cascade.customer_budget_ids), + extra=_SPENT_ROWS_WHERE, + ) for budget_id, budget_reset_at in cascade.budget_resets: uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at) @@ -1932,7 +1947,7 @@ class ResetBudgetJob: entity_type=entity_type.value, entity_id=entity_id, window_duration=budget_duration, - new_window_start=window_start, + new_window_start=normalize_budget_window_start(window_start), ) except Exception as e: # noqa: BLE001 # the row is best effort; counter resets must still land verbose_proxy_logger.warning( diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index cabeb866864..1de923db72f 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import re from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast, overload from weakref import WeakValueDictionary @@ -48,6 +49,13 @@ def _ttl(kwargs: Mapping[str, object]) -> float | None: return _TTL.validate_python(kwargs.get("ttl")) +@dataclass(frozen=True, slots=True) +class _DecodedModel: + payload: object + model_type: type[BaseModel] + model: BaseModel + + class CacheLease: __slots__ = ("__weakref__", "key", "shared") @@ -114,6 +122,7 @@ class UserApiKeyCache(DualCache): self._redis_connection: RedisCache | None = None self._shared: AuthCacheStore | None = None self._leases: Final[WeakValueDictionary[str, CacheLease]] = WeakValueDictionary() + self._decoded_models: Final[dict[str, _DecodedModel]] = {} # mutable-ok: per-key memo of validated reads self.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl) @property @@ -175,6 +184,7 @@ class UserApiKeyCache(DualCache): def set_local(self, key: str, value: object, ttl: float | None) -> None: """A memory write alone, with ``default_in_memory_ttl`` when ``ttl`` is None.""" + self._drop_decoded((key,)) if self._shared is not None: self._shared.set_local(key, value, ttl) return @@ -199,6 +209,7 @@ class UserApiKeyCache(DualCache): """Caches ``value`` unless the lease's key was evicted since it was taken; False when refused.""" if self._leases.get(lease.key) is not lease: return False + self._drop_decoded((lease.key,)) shared: Final = self._shared if shared is None: await self.async_set_cache(key=lease.key, value=value, ttl=ttl) @@ -214,9 +225,35 @@ class UserApiKeyCache(DualCache): return True def _void_leases(self, keys: Sequence[str]) -> None: + self._drop_decoded(keys) for key in keys: _ = self._leases.pop(key, None) + def _drop_decoded(self, keys: Sequence[str]) -> None: + for key in keys: + _ = self._decoded_models.pop(key, None) + + def _decoded(self, cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object: + """In Python memory, a read of the same payload object a previous read validated returns a shallow copy of + that model instead of validating again, so a caller that sets a field cannot change what the next reader + gets. The shared store hands back a new payload on every read, so it is validated each time.""" + if model_type is None or cached is None: + return cached + memo: Final = self._decoded_models.get(key) + if memo is not None and memo.payload is cached and memo.model_type is model_type: + return memo.model.model_copy() + decoded: Final = _validated(cached, key, model_type, method) + if decoded is None or decoded is cached or self._shared is not None: + return decoded + _ = self._decoded_models.pop(key, None) + self._decoded_models[key] = _DecodedModel(payload=cached, model_type=model_type, model=decoded) + if len(self._decoded_models) > self._decoded_capacity(): + _ = self._decoded_models.pop(next(iter(self._decoded_models))) + return decoded.model_copy() + + def _decoded_capacity(self) -> int: + return self.in_memory_cache.max_size_in_memory + self.key_object_cache.in_memory_cache.max_size_in_memory + def delete_local(self, key: str) -> None: """Drops this worker's memory copy alone, as a cross-worker eviction broadcast does.""" self._void_leases((key,)) @@ -271,7 +308,7 @@ class UserApiKeyCache(DualCache): if is_user_key_cache_key(key) else super().get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs) ) - return _decoded(cached, key, model_type, "get_cache") + return self._decoded(cached, key, model_type, "get_cache") @overload async def async_get_cache( @@ -316,7 +353,7 @@ class UserApiKeyCache(DualCache): key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs ) ) - return _decoded(cached, key, model_type, "async_get_cache") + return self._decoded(cached, key, model_type, "async_get_cache") @staticmethod def _get_shared_sync(shared: AuthCacheStore, key: str, local_only: bool) -> object: @@ -337,6 +374,8 @@ class UserApiKeyCache(DualCache): def set_cache(self, key: str | None, value: CacheValue, local_only: bool = False, **kwargs: object): model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) payload: Final[object] = CacheCodec.serialize(value, model_type=model_type) + if key is not None: + self._drop_decoded((key,)) shared: Final = self._shared if shared is not None: if key is None: @@ -358,6 +397,8 @@ class UserApiKeyCache(DualCache): else, or with options the pipeline does not carry, it goes to Redis directly as before.""" model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) payload: Final[object] = CacheCodec.serialize(value, model_type=model_type) + if key is not None: + self._drop_decoded((key,)) ttl: Final = kwargs.get("ttl") pipelined: Final = ( key is not None @@ -443,6 +484,7 @@ class UserApiKeyCache(DualCache): def flush_cache(self) -> None: self._leases.clear() + self._decoded_models.clear() self.budget_windows_absent_cache.flush_cache() if self._shared is not None: self._shared.flush_local() @@ -458,6 +500,7 @@ class UserApiKeyCache(DualCache): ``model_type``: ``BaseModel`` values become JSON-safe dicts; dicts/scalars unchanged. """ normalized: Final = tuple((key, CacheCodec.serialize(value, model_type=None)) for key, value in cache_list) + self._drop_decoded(tuple(key for key, _ in normalized)) shared: Final = self._shared if shared is not None: await self._set_many_shared(shared, normalized, local_only=local_only, ttl=_ttl(kwargs)) @@ -519,9 +562,7 @@ class UserApiKeyCache(DualCache): ) -def _decoded(cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object: - if model_type is None or cached is None: - return cached +def _validated(cached: object, key: str, model_type: type[BaseModel], method: str) -> BaseModel | None: decoded: Final = CacheCodec.deserialize(cached, model_type=model_type) if decoded is None: verbose_proxy_logger.error( diff --git a/litellm/proxy/db/connection_budget.py b/litellm/proxy/db/connection_budget.py new file mode 100644 index 00000000000..26fd7d91f33 --- /dev/null +++ b/litellm/proxy/db/connection_budget.py @@ -0,0 +1,56 @@ +"""Split each Postgres pool's connection budget between Prisma and the Rust core's SQLx pool. + +Migration scaffolding. Prisma and SQLx share a database only while the Python gateway still runs Prisma +queries next to ported ones. These invariants hold until Prisma is gone: + +- Operators set one `connection_limit` per pool (`database_connection_pool_limit`, a URL pin or + `database_extra_connection_params`). Nothing an operator sets names Prisma, SQLx or Rust. The Rust share + is carved out of that limit, never added on top, so a Rust pool does not grow the connection budget. +- The Rust share is the hard-coded `RUST_RESERVED_CONNECTIONS`. It is 0 until a ported query runs on the + native pool, so Prisma keeps its whole limit, untouched, and no native pool opens. +- Only a positive integer limit is split. Any other value goes to Prisma as it is, which judges it the same + way it did before the split existed. +- The split is Python's. It is about Prisma and Prisma lives only in the Python gateway, so this module is + deleted with Prisma. +- The Rust core takes a plain `max_connections` and never learns about the split. That is already its final + interface, since the Rust gateway will pass `database_connection_pool_limit` straight to SQLx. +""" + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +RUST_RESERVED_CONNECTIONS: Final = 0 + + +@dataclass(frozen=True, slots=True) +class ConnectionBudget: + total: int + python: int + rust: int + + @classmethod + def split(cls, total: int, rust_reserved: int = RUST_RESERVED_CONNECTIONS) -> "ConnectionBudget": + """Prisma keeps at least one connection, so a total at or below the reserve ends up over it.""" + return cls(total=total, python=max(1, total - rust_reserved), rust=rust_reserved) + + @classmethod + def of( + cls, connection_limit: str | float, rust_reserved: int = RUST_RESERVED_CONNECTIONS + ) -> "ConnectionBudget | None": + """The split of a positive integer limit, or None for any other value Prisma is left to judge.""" + digits: Final = str(connection_limit) + if not (digits.isascii() and digits.isdigit()) or int(digits) == 0: + return None + return cls.split(int(digits), rust_reserved) + + @property + def prisma_limit(self) -> int | None: + """Prisma's share once the Rust pool takes some of the total, or None to leave Prisma's limit untouched.""" + return None if self.rust == 0 else self.python + + def prisma_params(self, params: Mapping[str, str | int | float]) -> Mapping[str, str | int | float]: + """``params`` with Prisma's share as its ``connection_limit``, untouched while the Rust pool takes none.""" + limit: Final = self.prisma_limit + return params if limit is None else MappingProxyType({**params, "connection_limit": limit}) diff --git a/litellm/proxy/db/db_span.py b/litellm/proxy/db/db_span.py index 06b1a6aa11b..514c97c02da 100644 --- a/litellm/proxy/db/db_span.py +++ b/litellm/proxy/db/db_span.py @@ -32,7 +32,7 @@ from litellm.proxy.db.log_db_metrics import ( # noqa: F401 # legacy module exp _T = TypeVar("_T") -def _service_logging() -> ServiceLogging | None: +def proxy_service_logging() -> ServiceLogging | None: try: from litellm.proxy.proxy_server import proxy_logging_obj except ImportError: @@ -40,7 +40,9 @@ def _service_logging() -> ServiceLogging | None: return proxy_logging_obj.service_logging_obj -def _event_metadata(table: str | None, operation: str | None) -> dict[str, str]: +def db_event_metadata( + table: str | None, operation: str | None +) -> dict[str, str]: # mutable-ok: ServiceLogging hooks take event_metadata as a dict pairs: Final = (("table_name", table), ("db_operation", operation)) return {key: value for key, value in pairs if value is not None} @@ -73,9 +75,9 @@ async def db_span(call_type: str, table: str | None, operation: str | None = Non if db_io_claimed(): yield return - service_logging: Final = _service_logging() + service_logging: Final = proxy_service_logging() start_time: Final = datetime.now() - event_metadata: Final = _event_metadata(table, operation) + event_metadata: Final = db_event_metadata(table, operation) with claim_db_io() as witness: try: yield diff --git a/litellm/proxy/db/db_url_settings.py b/litellm/proxy/db/db_url_settings.py index 9db060d5c9d..217c2dca8c4 100644 --- a/litellm/proxy/db/db_url_settings.py +++ b/litellm/proxy/db/db_url_settings.py @@ -254,12 +254,28 @@ def translate_libpq_ssl_params(url: str, resolve_root_cert: RootCertResolver = p return urllib.parse.urlunsplit(parsed._replace(query=query)) -def postgres_connection_budget_message(writer_limit: str, reader_limit: str | None, num_workers: str) -> str: +PRISMA_APPLICATION_NAME: Final = "litellm-prisma" +NATIVE_APPLICATION_NAME: Final = "litellm-native" + + +def prisma_default_params( + lifetime_params: Mapping[str, str | int | float], native_connections: int +) -> Mapping[str, str | int | float]: + """Prisma's URL defaults, tagged ``PRISMA_APPLICATION_NAME`` only once a native pool shares the database.""" + if native_connections == 0: + return lifetime_params + return MappingProxyType({**lifetime_params, "application_name": PRISMA_APPLICATION_NAME}) + + +def postgres_connection_budget_message( + writer_limit: str, rust_writer_limit: int, reader_limit: str | None, rust_reader_limit: int, num_workers: str +) -> str: """The startup line that states this pod's worst-case Postgres connection demand. Prisma's ``connection_limit`` is per query engine, and every uvicorn worker owns one engine per configured database (writer, plus the reader when - ``DATABASE_URL_READ_REPLICA`` is set). The limits are read off the final URLs, + ``DATABASE_URL_READ_REPLICA`` is set) plus a Rust pool beside each whose + share (``connection_budget``) is above 0. The limits are read off the final URLs, so a reader that pins its own ``connection_limit`` or an operator override in ``database_extra_connection_params`` is counted at its real value. The server-side cap is shared by every pod, so the number an operator has to keep @@ -272,16 +288,21 @@ def postgres_connection_budget_message(writer_limit: str, reader_limit: str | No reader: Final = int(reader_limit) if reader_limit is not None else 0 except ValueError: return ( - "LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + reader " - f"connection_limit) (workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); " + "LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + " + f"{'rust writer pool + ' if rust_writer_limit else ''}reader connection_limit" + f"{' + rust reader pool' if rust_reader_limit else ''}) " + f"(workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); " "keep pods x that figure under max_connections minus superuser_reserved_connections" ) - engines: Final = ( - f"(writer connection_limit {writer} + reader connection_limit {reader})" - if reader_limit is not None - else f"writer connection_limit {writer}" + rust_reader: Final = rust_reader_limit if reader_limit is not None else 0 + terms: Final = ( + f"writer connection_limit {writer}", + *((f"rust writer pool {rust_writer_limit}",) if rust_writer_limit else ()), + *((f"reader connection_limit {reader}",) if reader_limit is not None else ()), + *((f"rust reader pool {rust_reader}",) if rust_reader else ()), ) - per_pod: Final = workers * (writer + reader) + engines: Final = terms[0] if len(terms) == 1 else f"({' + '.join(terms)})" + per_pod: Final = workers * (writer + rust_writer_limit + reader + rust_reader) return ( f"LiteLLM Proxy: Postgres connection budget per pod = {workers} worker(s) x {engines} = up to {per_pod} " "connections; keep pods x that figure under max_connections minus superuser_reserved_connections" diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index d9d0ece4468..cdafa2cf52f 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -14,6 +14,7 @@ from litellm.proxy._types import ( ProxyException, ) from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded +from litellm.rust_bridge._native import NativeDatabaseFailure from litellm.secret_managers.main import str_to_bool # Bounds the __cause__/__context__ walk in find_database_service_unavailable_error_in_chain. @@ -175,6 +176,8 @@ class PrismaDBExceptionHandler: """ import prisma.engine.errors + if isinstance(e, NativeDatabaseFailure): + return e.unavailable if isinstance(e, (*DB_CONNECTION_ERROR_TYPES, DBLookupDeadlineExceeded)): return True if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)): @@ -769,6 +772,8 @@ _DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"}) def db_write_failure_is_requeue_safe(e: Exception) -> bool: """Whether a failed write provably never applied, so its rows can be sent again without double counting.""" + if isinstance(e, NativeDatabaseFailure): + return e.requeue_safe if isinstance(e, DB_CONNECTION_ERROR_TYPES): return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) diff --git a/litellm/proxy/db/native_database.py b/litellm/proxy/db/native_database.py new file mode 100644 index 00000000000..5c322f0afec --- /dev/null +++ b/litellm/proxy/db/native_database.py @@ -0,0 +1,369 @@ +"""Settings for the native (SQLx) pool, resolved from the same URL Prisma connects with. + +``DATABASE_URL`` is final once ``db_url_settings`` and the CLI have assembled it: token +minted, libpq TLS translated to Prisma's dialect, timeouts in ``options``, pool params +added. Reading that URL keeps one resolver for both pools instead of a second pass over +the ``DATABASE_*`` env vars. + +Credentials follow the host. Each process runs one token refresh loop, owned by whatever +hosts it, and feeds every client in that process: here the worker's loop re-mints Prisma's +URL and pushes the same kind of token into the Rust pool through ``set_writer_password``. +The Rust core only accepts a password and never mints one, so a Rust gateway process +brings its own loop. Two processes minting at once is fine, since IAM and Entra tokens +are independent. What must not exist twice is the logic that resolves region, endpoint +and user, so when the Rust gateway needs it, it moves into the Rust core and this +loop calls it. +""" + +import asyncio +import shlex +import struct +import urllib.parse +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import MappingProxyType +from typing import Final, Protocol + +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, pkcs12 + +from litellm._logging import verbose_proxy_logger +from litellm._service_logger import ServiceTypes +from litellm.proxy.db.db_span import db_event_metadata +from litellm.proxy.db.db_url_settings import NATIVE_APPLICATION_NAME, PRISMA_APPLICATION_NAME +from litellm.proxy.db.pgbouncer import database_url_is_pooled, fold_libpq_option_tokens, libpq_option_setting +from litellm.proxy.db.token_auth import ( + DEFAULT_POSTGRES_PORT, + DatabaseTokenAuth, + IAMEndpoint, + mint_database_token, + parse_database_token_expiration, + parse_iam_endpoint_from_url, + resolve_database_token_auth, +) +from litellm.rust_bridge.database import ( + NativeClientIdentity, + NativeDatabaseEndpoint, + NativePoolLimits, + SslMode, +) + +TOKEN_REFRESH_BUFFER_SECONDS: Final = 180.0 +TOKEN_FALLBACK_REFRESH_SECONDS: Final = 600.0 +TOKEN_RETRY_SECONDS: Final = 30.0 +PRISMA_DEFAULT_SCHEMA: Final = "public" +PRISMA_DEFAULT_POOL_TIMEOUT_SECONDS: Final = 10.0 +PRISMA_DEFAULT_STATEMENT_CACHE_SIZE: Final = 100 +TLS_PROBE_TIMEOUT_SECONDS: Final = 5.0 +_POSTGRES_SSL_REQUEST: Final = struct.pack("!ii", 8, 80877103) + + +@dataclass(frozen=True, slots=True) +class NativeEndpointSettings: + endpoint: NativeDatabaseEndpoint + ignored_options: tuple[str, ...] + + +def _params(url: str) -> Mapping[str, str]: + return MappingProxyType(dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query))) + + +def needs_tls_probe(url: str) -> bool: + """Prisma's ``sslmode=prefer`` with ``sslaccept=strict`` verifies TLS when the server offers it + and falls back to plaintext when it does not; SQLx has no verified-or-plaintext mode, so the + caller asks the server once and pins the answer.""" + params: Final = _params(url) + return params.get("sslmode", "prefer") == "prefer" and params.get("sslaccept") == "strict" and "host" not in params + + +async def server_offers_tls(url: str, timeout: float = TLS_PROBE_TIMEOUT_SECONDS) -> bool: + """Whether the server answers a Postgres ``SSLRequest`` with ``S``; an unreachable server counts as offering it.""" + parsed: Final = urllib.parse.urlsplit(url) + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection(parsed.hostname, parsed.port or int(DEFAULT_POSTGRES_PORT)), timeout + ) + except (OSError, asyncio.TimeoutError): + return True + try: + writer.write(_POSTGRES_SSL_REQUEST) + await writer.drain() + answer: Final = await asyncio.wait_for(reader.read(1), timeout) + except (OSError, asyncio.TimeoutError): + return True + finally: + writer.close() + return answer != b"N" + + +def _tls(params: Mapping[str, str], tls_offered: bool) -> tuple[SslMode, str | None]: + sslmode: Final = params.get("sslmode", "prefer") + strict: Final = params.get("sslaccept") == "strict" + if sslmode == "disable" or (sslmode == "prefer" and strict and not tls_offered): + return "disable", None + if strict: + return "verify-full", params.get("sslcert") + return ("require" if sslmode == "require" else "prefer"), None + + +def _client_identity( + params: Mapping[str, str], ssl_mode: SslMode, read_file: Callable[[Path], bytes] +) -> NativeClientIdentity | None: + identity_path: Final = params.get("sslidentity") + if identity_path is None or ssl_mode == "disable": + return None + password: Final = params.get("sslpassword") + key, cert, chain = pkcs12.load_key_and_certificates( + read_file(Path(identity_path)), None if password is None else password.encode() + ) + if key is None or cert is None: + raise ValueError(f"sslidentity {identity_path!r} must hold a client certificate and its private key") + return NativeClientIdentity( + cert_pem=b"".join(c.public_bytes(Encoding.PEM) for c in (cert, *chain)).decode(), + key_pem=key.private_bytes(Encoding.PEM, PrivateFormat.PKCS8, NoEncryption()).decode(), + ) + + +def _session_options(params: Mapping[str, str]) -> tuple[tuple[tuple[str, str], ...], tuple[str, ...]]: + raw: Final = params.get("options", "") + try: + tokens: Final[Sequence[str]] = fold_libpq_option_tokens(tuple(shlex.split(raw))) + except ValueError: + return (), (raw,) + readable: Final = tuple((token, libpq_option_setting(token)) for token in tokens) + schema: Final = params.get("schema", PRISMA_DEFAULT_SCHEMA) + search_path: Final = () if schema == PRISMA_DEFAULT_SCHEMA else (("search_path", f'"{schema}"'),) + settings: Final = tuple( + (name, value) for name, _, value in (setting.partition("=") for _, setting in readable if setting is not None) + ) + return (*search_path, *settings), tuple(token for token, setting in readable if setting is None) + + +def _seconds(params: Mapping[str, str], key: str, default: float | None = None) -> float | None: + value: Final = params.get(key) + if value is None: + return default + return None if float(value) == 0 else float(value) + + +def _statement_cache_capacity(params: Mapping[str, str]) -> int: + if params.get("pgbouncer") == "true": + return 0 + return int(params.get("statement_cache_size", PRISMA_DEFAULT_STATEMENT_CACHE_SIZE)) + + +def _application_name(params: Mapping[str, str]) -> str: + configured: Final = params.get("application_name", PRISMA_APPLICATION_NAME) + return NATIVE_APPLICATION_NAME if configured == PRISMA_APPLICATION_NAME else configured + + +def native_endpoint( + url: str, + *, + tls_offered: bool, + max_connections: int, + read_file: Callable[[Path], bytes] = Path.read_bytes, +) -> NativeEndpointSettings: + """The native pool's endpoint for the Prisma URL ``url``, read the way Prisma reads it. + + ``host`` is Prisma's Unix socket directory. The pool takes ``max_connections``, the + Rust share of the budget (``connection_budget``), and tags its connections + ``NATIVE_APPLICATION_NAME``. An unset ``pool_timeout`` takes Prisma's default and + 0 waits forever. + ``sslidentity`` / ``sslpassword`` are a PKCS#12 client certificate, handed over as PEM. + ``options`` tokens that are not ``-c name=value`` settings cannot be sent by SQLx and + come back in ``ignored_options`` instead of failing boot. + """ + parsed: Final = urllib.parse.urlsplit(url) + params: Final = _params(url) + socket: Final = params.get("host") + if not (parsed.hostname or socket) or not parsed.username: + raise ValueError("DATABASE_URL must name a host and a user for the native pool") + ssl_mode, ssl_root_cert = _tls(params, tls_offered) + options, ignored = _session_options(params) + return NativeEndpointSettings( + endpoint=NativeDatabaseEndpoint( + host=parsed.hostname or "localhost", + port=parsed.port or int(DEFAULT_POSTGRES_PORT), + socket=socket, + username=urllib.parse.unquote(parsed.username), + password=urllib.parse.unquote(parsed.password) if parsed.password else None, + database=urllib.parse.unquote(parsed.path.lstrip("/")) or "postgres", + ssl_mode=ssl_mode, + ssl_root_cert=ssl_root_cert, + ssl_client=_client_identity(params, ssl_mode, read_file), + application_name=_application_name(params), + options=options, + statement_cache_capacity=_statement_cache_capacity(params), + pool=NativePoolLimits( + max_connections=max_connections, + acquire_timeout_seconds=_seconds(params, "pool_timeout", PRISMA_DEFAULT_POOL_TIMEOUT_SECONDS), + idle_timeout_seconds=_seconds(params, "max_idle_connection_lifetime"), + socket_timeout_seconds=_seconds(params, "socket_timeout"), + ), + ), + ignored_options=ignored, + ) + + +async def resolve_native_endpoint(url: str, max_connections: int) -> NativeEndpointSettings: + tls_offered: Final = await server_offers_tls(url) if needs_tls_probe(url) else True + return native_endpoint(url, tls_offered=tls_offered, max_connections=max_connections) + + +@dataclass(frozen=True, slots=True) +class TokenRotation: + auth: DatabaseTokenAuth + endpoint: IAMEndpoint + mint_token: Callable[[DatabaseTokenAuth, IAMEndpoint], str] = mint_database_token + + def mint(self) -> str: + return urllib.parse.unquote(self.mint_token(self.auth, self.endpoint)) + + def seconds_until_refresh(self, token: str, now: datetime) -> float: + expires_at: Final = parse_database_token_expiration(self.auth, token) + if expires_at is None: + return TOKEN_FALLBACK_REFRESH_SECONDS + return max(TOKEN_RETRY_SECONDS, (expires_at - now).total_seconds() - TOKEN_REFRESH_BUFFER_SECONDS) + + +def writer_token_rotation(url: str) -> TokenRotation | None: + """The token rotation the writer needs, or None for a static password. + + A URL pointed at the in-container PgBouncer carries PgBouncer's own credential, + and PgBouncer renews the upstream token itself. + """ + if database_url_is_pooled(): + return None + auth: Final = resolve_database_token_auth() + if auth is None: + return None + return TokenRotation(auth=auth, endpoint=parse_iam_endpoint_from_url(url)) + + +def reader_token_rotation(url: str) -> TokenRotation | None: + """The token rotation the read replica needs, or None for a static password. + + The reader never goes through the in-container PgBouncer, so only the token toggles + decide, with the reader's own region, the same as its Prisma client. + """ + auth: Final = resolve_database_token_auth(read_replica=True) + if auth is None: + return None + return TokenRotation(auth=auth, endpoint=parse_iam_endpoint_from_url(url)) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None) + + +async def _refresh(install: Callable[[str], None], rotation: TokenRotation, now: Callable[[], datetime]) -> float: + try: + token: Final = await asyncio.to_thread(rotation.mint) + except Exception as error: # noqa: BLE001 # boto3 and azure-identity raise many types; any failure means retry + verbose_proxy_logger.error( + "Native database pool keeps its current %s (%r); retrying in %.0fs.", + rotation.auth.label, + error, + TOKEN_RETRY_SECONDS, + ) + return TOKEN_RETRY_SECONDS + install(token) + return rotation.seconds_until_refresh(token, now()) + + +async def keep_token_fresh( + install: Callable[[str], None], + rotation: TokenRotation, + current_token: str, + now: Callable[[], datetime] = _utcnow, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, +) -> None: + """Hand the native pool a fresh token before the current one expires. + + Existing connections stay authenticated; only connections opened after a refresh + use the new token, the same as a re-minted Prisma URL. + """ + delay = rotation.seconds_until_refresh(current_token, now()) # rebind-ok: each refresh schedules the next + while True: + await sleep(delay) + delay = await _refresh(install, rotation, now) + + +class DatabaseServiceHooks(Protocol): + async def async_service_success_hook( + self, + service: ServiceTypes, + call_type: str, + duration: float, + parent_otel_span: None = None, + start_time: datetime | None = None, + end_time: datetime | None = None, + event_metadata: dict[str, str] | None = None, # mutable-ok: matches the ServiceLogging hook signature + ) -> None: ... + + async def async_service_failure_hook( + self, + service: ServiceTypes, + duration: float, + error: Exception, + call_type: str, + parent_otel_span: None = None, + start_time: datetime | None = None, + end_time: datetime | None = None, + event_metadata: dict[str, str] | None = None, # mutable-ok: matches the ServiceLogging hook signature + ) -> None: ... + + +class NativeDatabaseError(Exception): + """A native (SQLx) query failure, reported to service logging like a Prisma one.""" + + +@dataclass(frozen=True, slots=True) +class ServiceLoggingEvents: + """Turns native round trips into the ``ServiceTypes.DB`` events ``db_span`` emits. + + Rust calls ``record`` from its own threads once per round trip, so the hook is + handed to the proxy's event loop rather than awaited here. + """ + + loop: asyncio.AbstractEventLoop + service_logging: Callable[[], DatabaseServiceHooks | None] + + def record( + self, + call_type: str, + table: str | None, + operation: str | None, + started_at: float, + duration_seconds: float, + failure: str | None, + ) -> None: + service_logging: Final = self.service_logging() + if service_logging is None: + return + start_time: Final = datetime.fromtimestamp(started_at, tz=timezone.utc).astimezone().replace(tzinfo=None) + hook: Final = ( + service_logging.async_service_success_hook( + service=ServiceTypes.DB, + call_type=call_type, + parent_otel_span=None, + duration=duration_seconds, + start_time=start_time, + end_time=start_time + timedelta(seconds=duration_seconds), + event_metadata=db_event_metadata(table, operation), + ) + if failure is None + else service_logging.async_service_failure_hook( + error=NativeDatabaseError(failure), + service=ServiceTypes.DB, + call_type=call_type, + parent_otel_span=None, + duration=duration_seconds, + start_time=start_time, + end_time=start_time + timedelta(seconds=duration_seconds), + event_metadata=db_event_metadata(table, operation), + ) + ) + _ = self.loop.call_soon_threadsafe(self.loop.create_task, hook) diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py index c9fbabd3585..d9983ce774a 100644 --- a/litellm/proxy/db/pgbouncer.py +++ b/litellm/proxy/db/pgbouncer.py @@ -140,19 +140,27 @@ def _userlist_quote(value: str) -> str: return '"' + value.replace('"', '""') + '"' -def _option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None: - """The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else. - - Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``; a - detached ``-c`` is folded into the token that follows it first. - """ - folded: Final = tuple( +def fold_libpq_option_tokens(tokens: Sequence[str]) -> tuple[str, ...]: + """The tokens of a libpq ``options`` string with each detached ``-c`` folded into the token after it.""" + return tuple( f"-c{tokens[index + 1]}" if token == "-c" and index + 1 < len(tokens) else token for index, token in enumerate(tokens) if index == 0 or tokens[index - 1] != "-c" ) - settings: Final = tuple(token[2:] for token in folded if token.startswith(("-c", "--")) and "=" in token[2:]) - return settings if len(settings) == len(folded) else None + + +def libpq_option_setting(token: str) -> str | None: + """The ``name=value`` setting a folded ``-cname=value`` or ``--name=value`` token carries, or None.""" + return token[2:] if token.startswith(("-c", "--")) and "=" in token[2:] else None + + +def libpq_option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None: + """The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else. + + Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``. + """ + settings: Final = tuple(libpq_option_setting(token) for token in fold_libpq_option_tokens(tokens)) + return None if None in settings else tuple(setting for setting in settings if setting is not None) def _connect_query(options: str) -> str | PgBouncerError: @@ -163,7 +171,7 @@ def _connect_query(options: str) -> str | PgBouncerError: the settings are applied to each new server connection instead. Every client shares them, which is what the single ``DATABASE_URL`` gave anyway. """ - settings: Final = _option_settings(tuple(shlex.split(options))) + settings: Final = libpq_option_settings(tuple(shlex.split(options))) if settings is None: return PgBouncerError(f"cannot translate the DATABASE_URL options {options!r} into PgBouncer settings") return "; ".join( diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index e201ccbe5c1..02f79a7897e 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -34,9 +34,13 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ( map_v3_rate_limit_type, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( # noqa: F401 # legacy module exports + BORROWER_KINDS, RATE_LIMIT_UNVERIFIABLE_MESSAGE, + BorrowerKind, + BorrowingShareState, BorrowingTeamModelHold, BorrowingTeamModelLimits, + ModelLimitBorrower, PROXY_MaxParallelRequestsHandler_v3, RateLimitDescriptor, RateLimitDescriptorRateLimitObject, @@ -50,11 +54,11 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( # noqa: F401 # l WindowGuardedIncrement, _PROXY_MaxParallelRequestsHandler_v3, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _socket_turn_tpm_windows, # pyright: ignore[reportPrivateUsage] # same per-turn TPM window the regular limiter checks - borrowing_team_model_limits, callback_kwargs_mapping, claim_request_stash_for_data, get_or_create_request_stash, get_request_stash, + model_limit_borrowers, request_stash_for_callback, ) from litellm.proxy.hooks.priority_class_share import ( @@ -62,6 +66,8 @@ from litellm.proxy.hooks.priority_class_share import ( ActiveTeamRegistry, TeamShare, counter_value, + customer_member, + key_member, pool_member, team_member, team_share, @@ -81,14 +87,36 @@ if TYPE_CHECKING: from litellm.types.utils import PriorityReservationSettings +@dataclass(frozen=True, slots=True) +class _ShareHolder: + """The team, key or customer organization a share belongs to, as the dynamic limiter metrics label it.""" + + kind: BorrowerKind + entity_id: str + alias: str | None + + class _SettlementMetadata(BaseModel): - """The two fields of a request's standard logging metadata that token settlement reads.""" + """The fields of a request's standard logging metadata that token settlement reads.""" model_config = ConfigDict(frozen=True) user_api_key_auth_metadata: Mapping[str, object] | None = None user_api_key_team_id: str | None = None user_api_key_team_alias: str | None = None + user_api_key_hash: str | None = None + user_api_key_alias: str | None = None + user_api_key_customer_org_id: str | None = None + user_api_key_customer_org_alias: str | None = None + + def holder(self, kind: BorrowerKind) -> _ShareHolder | None: + holders: Final[Mapping[BorrowerKind, tuple[str | None, str | None]]] = { + "team": (self.user_api_key_team_id, self.user_api_key_team_alias), + "key": (self.user_api_key_hash, self.user_api_key_alias), + "customer_org": (self.user_api_key_customer_org_id, self.user_api_key_customer_org_alias), + } + entity_id, alias = holders[kind] + return None if entity_id is None else _ShareHolder(kind=kind, entity_id=entity_id, alias=alias) @property def priority(self) -> str | None: @@ -128,8 +156,10 @@ def _tightest_of(*shares: tuple[RateLimitDescriptor, int] | None) -> tuple[RateL return min((share for share in shares if share is not None), key=lambda share: share[1], default=None) -def _share_on_model(share: RateLimitDescriptor | None, team: str, model_group: str) -> RateLimitDescriptor | None: - return share if share is not None and share["value"] == f"{team}:{model_group}" else None +def _share_on_model( + share: RateLimitDescriptor | None, holder_id: str | None, model_group: str +) -> RateLimitDescriptor | None: + return share if share is not None and share["value"] == f"{holder_id}:{model_group}" else None def _capped_idle_share( @@ -166,16 +196,55 @@ def _settled_tokens(usage: Usage | None, rate_limit_type: str) -> int: return usage.total_tokens -TEAM_MODEL_LIMIT_AT_SATURATION: Final = "model_per_team_saturated" +def model_limit_at_saturation(kind: BorrowerKind) -> str: + return f"model_per_{kind}_saturated" + + +def model_limit_max_borrow_multiple(kind: BorrowerKind) -> str: + return f"model_per_{kind}_max_borrow_multiple" + + +TEAM_MODEL_LIMIT_AT_SATURATION: Final = model_limit_at_saturation("team") PRIORITY_CLASS_SHARE_AT_SATURATION: Final = "priority_class_share_saturated" -TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = "model_per_team_max_borrow_multiple" +TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = model_limit_max_borrow_multiple("team") +_BORROWER_LABELS: Final[Mapping[BorrowerKind, str]] = MappingProxyType( + {"team": "Team", "key": "Key", "customer_org": "Customer organization"} +) +_BORROWER_BY_REASON: Final[Mapping[str, BorrowerKind]] = MappingProxyType( + { + **{model_limit_at_saturation(kind): kind for kind in BORROWER_KINDS}, + **{model_limit_max_borrow_multiple(kind): kind for kind in BORROWER_KINDS}, + } +) +_CLASS_SHARE_HOLDER_PREFIXES: Final[Mapping[str, BorrowerKind | None]] = MappingProxyType( + { + key_member(""): "key", + customer_member("customer_org", ""): "customer_org", + customer_member("customer_member", ""): None, + } +) + + +@dataclass(frozen=True, slots=True) +class _FallbackTargetShare: + """What one borrower holds on a router fallback target: its share once saturated, its ceiling, and the tokens.""" + + kind: BorrowerKind + share: RateLimitDescriptor + enforced: RateLimitDescriptor | None + cap: RateLimitDescriptor | None + reservation: int + + @property + def descriptors(self) -> tuple[RateLimitDescriptor, ...]: + return tuple(d for d in (self.enforced, self.cap) if d is not None) @dataclass(frozen=True, slots=True) class _BorrowingTeamShare: - """The team's share and, when the team caps how far it borrows, its ceiling, both on the share's counter.""" + """A borrower's share and, when it caps how far it borrows, its ceiling, both on the share's counter.""" - limits: BorrowingTeamModelLimits + borrower: ModelLimitBorrower share: RateLimitDescriptor ceiling: RateLimitDescriptor | None @@ -183,6 +252,31 @@ class _BorrowingTeamShare: return self.share if saturated else None +def _tpm_share(descriptor: RateLimitDescriptor) -> int | None: + return (descriptor["rate_limit"] or {}).get("tokens_per_unit") + + +def _holding_shares( + borrowing: Mapping[BorrowerKind, _BorrowingTeamShare], saturated: bool +) -> Mapping[BorrowerKind, _BorrowingTeamShare]: + """The borrowers that hold tokens before the call: all of them once saturated, only the capped ones before.""" + return MappingProxyType( + {kind: share for kind, share in borrowing.items() if saturated or share.ceiling is not None} + ) + + +def _class_share_kind(holder: str | None) -> BorrowerKind | None: + """The metric labels a class share slice is published under. A customer member's slice has no label of its own.""" + if holder is None: + return None + matched: Final = next((prefix for prefix in _CLASS_SHARE_HOLDER_PREFIXES if holder.startswith(prefix)), None) + return "team" if matched is None else _CLASS_SHARE_HOLDER_PREFIXES[matched] + + +def _is_team_class_share_holder(holder: str) -> bool: + return not any(holder.startswith(prefix) for prefix in _CLASS_SHARE_HOLDER_PREFIXES) + + def _team_limit_name(rate_limit_type: str) -> str: return "model_rpm_limit" if rate_limit_type == "requests" else "model_tpm_limit" @@ -220,7 +314,7 @@ def _window_guarded( def _team_model_settlement( - stash: RequestRateLimiterStash | None, model_group: str | None, total_tokens: int, ttl: int + state: BorrowingShareState, model_group: str | None, total_tokens: int, ttl: int ) -> tuple[ReservationOperation, ...]: """Settle the counter that served the call to actual usage and return the admitting reservation it did not use. @@ -228,15 +322,13 @@ def _team_model_settlement( the admitting reservation is returned. A hold on a fallback target that did not serve stays for that attempt's own failure callback, which may still be reporting the usage it recorded. """ - if stash is None: - return () - served: Final = stash.borrowing_team_model_fallback_limits.get(model_group) if model_group is not None else None - admitted: Final = stash.borrowing_team_model_limit - cap: Final = stash.borrowing_team_model_cap - reserved_tokens: Final = stash.borrowing_team_model_reserved_tokens - cap_reserved_tokens: Final = stash.borrowing_team_model_cap_reserved_tokens - window: Final = stash.borrowing_team_model_reserved_window - cap_window: Final = stash.borrowing_team_model_cap_reserved_window + served: Final = state.fallback_limits.get(model_group) if model_group is not None else None + admitted: Final = state.limit + cap: Final = state.cap + reserved_tokens: Final = state.reserved_tokens + cap_reserved_tokens: Final = state.cap_reserved_tokens + window: Final = state.reserved_window + cap_window: Final = state.cap_reserved_window if served is None: return ( *_window_guarded(_team_model_token_increment(admitted, total_tokens - reserved_tokens, ttl), window), @@ -254,6 +346,30 @@ def _team_model_settlement( ) +def _borrowing_settlement( + stash: RequestRateLimiterStash | None, model_group: str | None, total_tokens: int, ttl: int +) -> tuple[ReservationOperation, ...]: + if stash is None: + return () + return tuple( + operation + for state in stash.borrowing_shares.values() + for operation in _team_model_settlement( + state, model_group, total_tokens, ttl + ) # comprehension-ok: flattens each borrower's settlement + ) + + +def _served_holds( + stash: RequestRateLimiterStash | None, model_group: str | None +) -> Mapping[BorrowerKind, BorrowingTeamModelHold | None]: + if stash is None or model_group is None: + return MappingProxyType({}) + return MappingProxyType( + {kind: state.fallback_limits.get(model_group) for kind, state in stash.borrowing_shares.items()} + ) + + def _reserved_window( response: RateLimitResponse, descriptor: RateLimitDescriptor | None ) -> ReservationWindowIdentity | None: @@ -520,14 +636,17 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ) -> TeamShare | None: """The caller's slice of its class share, or None when the split is off or the key has no team. A key with no team keeps the class pool but still marks its class active, so the pool's capacity is not lent away.""" - if ( - litellm.priority_reservation is None - or not _get_priority_settings().priority_class_share_splits_equally_among_active_teams - ): + settings: Final = _get_priority_settings() + if litellm.priority_reservation is None or not settings.class_share_split_entities: return None priority_class: Final = self._priority_class(priority) + holder: Final = self._class_share_holder(user_api_key_dict) member: Final = ( - pool_member(priority_class) if user_api_key_dict.team_id is None else team_member(user_api_key_dict.team_id) + pool_member(priority_class) + if holder is None + else team_member(holder) + if _is_team_class_share_holder(holder) + else holder ) from redis.exceptions import RedisError @@ -541,9 +660,37 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): e, ) return None - if user_api_key_dict.team_id is None: + if holder is None: return None - return team_share(priority_class, members, normalized_weights, _get_priority_settings().default_priority) + return team_share(priority_class, members, normalized_weights, settings.default_priority) + + def _class_share_holder(self, user_api_key_dict: UserAPIKeyAuth) -> str | None: + """Whose slice of the class share the request counts against: its team when teams are listed, so customer + headers a caller picks cannot move it off its team's slice, then customer member, then customer organization, + then key. None keeps it on the class pool.""" + entities: Final = _get_priority_settings().class_share_split_entities + hierarchy: Final = user_api_key_dict.customer_hierarchy + candidates: Final = ( + ( + "team" in entities + and user_api_key_dict.team_id is not None + and _is_team_class_share_holder(user_api_key_dict.team_id), + lambda: str(user_api_key_dict.team_id), + ), + ( + "customer_member" in entities and hierarchy is not None and hierarchy.customer_member_id is not None, + lambda: customer_member("customer_member", str(hierarchy.customer_member_id if hierarchy else "")), + ), + ( + "customer_organization" in entities and hierarchy is not None, + lambda: customer_member("customer_org", hierarchy.customer_org_id if hierarchy else ""), + ), + ( + "key" in entities and user_api_key_dict.api_key is not None, + lambda: key_member(str(user_api_key_dict.api_key)), + ), + ) + return next((holder() for applies, holder in candidates if applies), None) def _create_priority_based_descriptors( self, @@ -598,34 +745,41 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): descriptors.append( RateLimitDescriptor( key="priority_model" if share is None else PRIORITY_CLASS_SHARE_AT_SATURATION, - value=priority_key if share is None else counter_value(model, str(user_api_key_dict.team_id)), + value=priority_key + if share is None + else counter_value(model, str(self._class_share_holder(user_api_key_dict))), rate_limit=rate_limit_config, ) ) return descriptors - def _borrowing_team_share(self, model: str, user_api_key_dict: UserAPIKeyAuth) -> _BorrowingTeamShare | None: - """One counter per team and resolved model group, so an alias and its target share the team's share.""" + def _borrowing_shares( + self, model: str, user_api_key_dict: UserAPIKeyAuth + ) -> Mapping[BorrowerKind, _BorrowingTeamShare]: + """One counter per borrower and resolved model group, so an alias and its target share the borrower's share.""" rate_limited: Final = self.v3_limiter.rate_limited_model(model) or RateLimitedModel( requested=model, group=model ) - limits: Final = borrowing_team_model_limits(user_api_key_dict, rate_limited) - if limits.tpm is None and limits.rpm is None: - return None - ceiling: Final = limits.borrow_ceiling() - value: Final = f"{user_api_key_dict.team_id}:{rate_limited.group}" - return _BorrowingTeamShare( - limits=limits, - share=self._team_share_descriptor(value, limits), - ceiling=None - if ceiling is None - else self._team_share_descriptor(value, ceiling, TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE), + return MappingProxyType( + { + borrower.kind: self._borrowing_share(borrower, rate_limited.group) + for borrower in model_limit_borrowers(user_api_key_dict, rate_limited) + } ) - def _team_share_descriptor( - self, value: str, limits: BorrowingTeamModelLimits, key: str = TEAM_MODEL_LIMIT_AT_SATURATION - ) -> RateLimitDescriptor: + def _borrowing_share(self, borrower: ModelLimitBorrower, group: str) -> _BorrowingTeamShare: + ceiling: Final = borrower.limits.borrow_ceiling() + value: Final = f"{borrower.entity_id}:{group}" + return _BorrowingTeamShare( + borrower=borrower, + share=self._team_share_descriptor(value, borrower.limits, model_limit_at_saturation(borrower.kind)), + ceiling=None + if ceiling is None + else self._team_share_descriptor(value, ceiling, model_limit_max_borrow_multiple(borrower.kind)), + ) + + def _team_share_descriptor(self, value: str, limits: BorrowingTeamModelLimits, key: str) -> RateLimitDescriptor: return RateLimitDescriptor( key=key, value=value, @@ -642,14 +796,31 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): status: RateLimitStatus, saturation: float, priority: str | None, - user_api_key_dict: UserAPIKeyAuth, team_share: _BorrowingTeamShare, ) -> ProxyRateLimitError: - if status["descriptor_key"] != TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: - return self._team_limit_error(model, status, saturation, priority, user_api_key_dict) - return self._team_borrow_ceiling_error( - model, status, saturation, priority, user_api_key_dict, team_share.limits + if status["descriptor_key"] != model_limit_max_borrow_multiple(team_share.borrower.kind): + return self._team_limit_error(model, status, saturation, priority, team_share.borrower) + return self._team_borrow_ceiling_error(model, status, saturation, priority, team_share.borrower) + + def _throttled_holder(self, reason: str, user_api_key_dict: UserAPIKeyAuth) -> _ShareHolder | None: + """Whose share refused the request: the borrower the reason names, or the class share's team or key.""" + kind: Final = ( + _class_share_kind(self._class_share_holder(user_api_key_dict)) + if reason == PRIORITY_CLASS_SHARE_AT_SATURATION + else _BORROWER_BY_REASON.get(reason, "team") ) + if kind is None: + return None + hierarchy: Final = user_api_key_dict.customer_hierarchy + holders: Final[Mapping[BorrowerKind, tuple[str | None, str | None]]] = { + "team": (user_api_key_dict.team_id, user_api_key_dict.team_alias), + "key": (user_api_key_dict.api_key, user_api_key_dict.key_alias), + "customer_org": (None, None) + if hierarchy is None + else (hierarchy.customer_org_id, hierarchy.customer_org_alias), + } + entity_id, alias = holders[kind] + return None if entity_id is None else _ShareHolder(kind=kind, entity_id=entity_id, alias=alias) def _throttled( self, refusal: ProxyRateLimitError, model: str, user_api_key_dict: UserAPIKeyAuth @@ -657,14 +828,16 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): """Count a refusal that reaches the caller under the reason its ``rate_limited_by`` header carries.""" headers: Final = refusal.headers or {} reason: Final = headers.get("rate_limited_by") - if user_api_key_dict.team_id is None or reason is None: + holder: Final = None if reason is None else self._throttled_holder(reason, user_api_key_dict) + if holder is None or reason is None: return refusal for logger in _prometheus_loggers(): try: logger.increment_dynamic_limiter_throttled( model_group=model, - team=user_api_key_dict.team_id, - team_alias=user_api_key_dict.team_alias, + holder_kind=holder.kind, + holder_id=holder.entity_id, + holder_alias=holder.alias, priority=headers.get("x-litellm-priority") or "default", rate_limit_type=headers.get("rate_limit_type") or "tokens", reason=reason, @@ -678,51 +851,55 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): stash: RequestRateLimiterStash | None, model_group: str | None, settlement: _SettlementMetadata, - hold: BorrowingTeamModelHold | None, + holds: Mapping[BorrowerKind, BorrowingTeamModelHold | None], ) -> None: - """Publish the team's share on the model that served the request, the fallback target's own hold when a + """Publish each borrower's share on the model that served the request, the fallback target's own hold when a router fallback served it, and the tokens settled above that share so far this window.""" try: loggers: Final = _prometheus_loggers() - if not loggers or stash is None or model_group is None or settlement.user_api_key_team_id is None: + if not loggers or stash is None or model_group is None: return - share: Final = ( - _tightest_of(_tightest_team_share((hold.descriptor,)), _capped_idle_share(hold.share, hold.cap)) - if hold is not None - else _tightest_of( - _tightest_team_share((stash.borrowing_team_model_limit, stash.priority_class_share)), - _capped_idle_share(stash.borrowing_team_model_share, stash.borrowing_team_model_cap), - ) - ) or _tightest_team_share( - ( - hold.share + class_share_kind: Final = _class_share_kind(stash.priority_class_share_holder) + group: Final = ( + self.v3_limiter.rate_limited_model(model_group) + or RateLimitedModel(requested=model_group, group=model_group) + ).group + for kind, state in stash.borrowing_shares.items(): + holder = settlement.holder(kind) + hold = holds.get(kind) + class_share = stash.priority_class_share if kind == class_share_kind else None + share = ( + _tightest_of(_tightest_team_share((hold.descriptor,)), _capped_idle_share(hold.share, hold.cap)) if hold is not None - else _share_on_model( - stash.borrowing_team_model_share, - settlement.user_api_key_team_id, + else _tightest_of( + _tightest_team_share((state.limit, class_share)), _capped_idle_share(state.share, state.cap) + ) + ) or _tightest_team_share( + ( ( - self.v3_limiter.rate_limited_model(model_group) - or RateLimitedModel(requested=model_group, group=model_group) - ).group, - ), + hold.share + if hold is not None + else _share_on_model(state.share, None if holder is None else holder.entity_id, group) + ), + ) ) - ) - if share is None: - return - descriptor, share_tokens = share - used: Final = await self._get_saturation_value_from_cache( - RateLimitScope(descriptor["key"], descriptor["value"]).counter("tokens") - ) - used_tokens: Final = 0 if used is None else int(float(used)) - for logger in loggers: - logger.set_dynamic_limiter_share( - model_group=model_group, - team=settlement.user_api_key_team_id, - team_alias=settlement.user_api_key_team_alias, - priority=stash.dynamic_limiter_priority or "default", - share_tokens=share_tokens, - borrowed_tokens=max(0, used_tokens - share_tokens), + if holder is None or share is None: + continue + descriptor, share_tokens = share + used = await self._get_saturation_value_from_cache( + RateLimitScope(descriptor["key"], descriptor["value"]).counter("tokens") ) + used_tokens = 0 if used is None else int(float(used)) + for logger in loggers: + logger.set_dynamic_limiter_share( + model_group=model_group, + holder_kind=holder.kind, + holder_id=holder.entity_id, + holder_alias=holder.alias, + priority=stash.dynamic_limiter_priority or "default", + share_tokens=share_tokens, + borrowed_tokens=max(0, used_tokens - share_tokens), + ) except Exception as metrics_error: # noqa: BLE001 # a failing metrics sink must not change settlement verbose_proxy_logger.warning("Dynamic rate limiter share metrics failed: %s", metrics_error) @@ -732,18 +909,19 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): status: RateLimitStatus, saturation: float, priority: str | None, - user_api_key_dict: UserAPIKeyAuth, - limits: BorrowingTeamModelLimits, + borrower: ModelLimitBorrower, ) -> ProxyRateLimitError: resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) limit_name: Final = _team_limit_name(status["rate_limit_type"]) + limits: Final = borrower.limits + label: Final = _BORROWER_LABELS[borrower.kind] return ProxyRateLimitError( detail={ - "error": f"Team {limit_name} borrow cap reached for {model}: the team may use at most " + "error": f"{label} {limit_name} borrow cap reached for {model}: the {label.lower()} may use at most " f"{limits.max_borrow_multiple}x its {limit_name} while borrowing idle capacity. " - f"Team: {user_api_key_dict.team_id}, " + f"{label}: {borrower.entity_id}, " f"Rate limit type: {status['rate_limit_type']}, " - f"Team {limit_name}: {limits.share(status['rate_limit_type'])}, " + f"{label} {limit_name}: {limits.share(status['rate_limit_type'])}, " f"Borrow cap: {status['current_limit']}, " f"Remaining: {status['limit_remaining']}, " f"Model saturation: {saturation:.1%}" @@ -751,7 +929,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): headers={ "retry-after": str(self.v3_limiter.window_size), "rate_limit_type": str(status["rate_limit_type"]), - "rate_limited_by": TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE, + "rate_limited_by": model_limit_max_borrow_multiple(borrower.kind), "x-litellm-priority": priority or "default", "x-litellm-saturation": f"{saturation:.2%}", }, @@ -766,23 +944,24 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): status: RateLimitStatus, saturation: float, priority: str | None, - user_api_key_dict: UserAPIKeyAuth, + borrower: ModelLimitBorrower, ) -> ProxyRateLimitError: resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) + label: Final = _BORROWER_LABELS[borrower.kind] return ProxyRateLimitError( detail={ - "error": f"Team {_team_limit_name(status['rate_limit_type'])} reached for {model} " + "error": f"{label} {_team_limit_name(status['rate_limit_type'])} reached for {model} " "and the model is saturated, so there is no idle capacity to borrow. " - f"Team: {user_api_key_dict.team_id}, " + f"{label}: {borrower.entity_id}, " f"Rate limit type: {status['rate_limit_type']}, " - f"Team {_team_limit_name(status['rate_limit_type'])}: {status['current_limit']}, " + f"{label} {_team_limit_name(status['rate_limit_type'])}: {status['current_limit']}, " f"Remaining: {status['limit_remaining']}, " f"Model saturation: {saturation:.1%}" }, headers={ "retry-after": str(self.v3_limiter.window_size), "rate_limit_type": str(status["rate_limit_type"]), - "rate_limited_by": TEAM_MODEL_LIMIT_AT_SATURATION, + "rate_limited_by": model_limit_at_saturation(borrower.kind), "x-litellm-priority": priority or "default", "x-litellm-saturation": f"{saturation:.2%}", }, @@ -801,15 +980,19 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): share: TeamShare, ) -> ProxyRateLimitError: resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) + holder: Final = self._throttled_holder(PRIORITY_CLASS_SHARE_AT_SATURATION, user_api_key_dict) + label: Final = "Customer member" if holder is None else _BORROWER_LABELS[holder.kind] + hierarchy: Final = user_api_key_dict.customer_hierarchy + member_id: Final = None if hierarchy is None else hierarchy.customer_member_id error: Final = ProxyRateLimitError( detail={ "error": f"Priority class share reached for {model} and the model is saturated. " - f"Team: {user_api_key_dict.team_id}, " + f"{label}: {member_id if holder is None else holder.entity_id}, " f"Priority: {priority or 'default'}, " - f"Active teams in class: {share.active_teams_in_class}, " + f"Active callers in class: {share.active_teams_in_class}, " f"Active classes: {share.active_classes}, " f"Rate limit type: {status['rate_limit_type']}, " - f"Team share: {status['current_limit']}, " + f"Share: {status['current_limit']}, " f"Remaining: {status['limit_remaining']}, " f"Model saturation: {saturation:.1%}" }, @@ -854,25 +1037,24 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): connection_stash: Final = get_request_stash() if connection_stash is not None and frame_share is not None: connection_stash.priority_class_share = frame_share - borrowing: Final = self._borrowing_team_share(model=requested_model, user_api_key_dict=user_api_key_dict) - if not share_descriptors and borrowing is None: + connection_stash.priority_class_share_holder = self._class_share_holder(user_api_key_dict) + borrowing: Final = self._borrowing_shares(model=requested_model, user_api_key_dict=user_api_key_dict) + if not share_descriptors and not borrowing: + if connection_stash is not None: + self._track_frame_shares(connection_stash, borrowing, saturated=False) return saturation: Final = await self._check_model_saturation(requested_model, model_group_info) saturated: Final = saturation >= _get_priority_settings().saturation_threshold - enforced_team: Final = None if borrowing is None else borrowing.enforced(saturated) - team_cap: Final = None if borrowing is None else borrowing.ceiling - if connection_stash is not None and borrowing is not None: - connection_stash.borrowing_team_model_limit = ( - connection_stash.borrowing_team_model_limit - if enforced_team is None and connection_stash.borrowing_team_model_reserved_tokens - else enforced_team - ) - connection_stash.borrowing_team_model_share = borrowing.share - connection_stash.borrowing_team_model_cap = team_cap - descriptors: Final = ( - (share_descriptors if saturated else ()) - + (() if enforced_team is None else (enforced_team,)) - + (() if team_cap is None else (team_cap,)) + if connection_stash is not None: + self._track_frame_shares(connection_stash, borrowing, saturated) + descriptors: Final = (share_descriptors if saturated else ()) + tuple( + descriptor + for borrower_share in borrowing.values() + for descriptor in ( + borrower_share.enforced(saturated), + borrower_share.ceiling, + ) # comprehension-ok: each borrower's share then cap + if descriptor is not None ) if not descriptors: return @@ -892,18 +1074,76 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if over_limit is None: return over_class_share: Final = over_limit["descriptor_key"] == PRIORITY_CLASS_SHARE_AT_SATURATION - if share is not None and (over_class_share or borrowing is None): + if share is not None and (over_class_share or not borrowing): raise self._throttled( self._class_share_error(requested_model, over_limit, saturation, priority, user_api_key_dict, share), requested_model, user_api_key_dict, ) - if borrowing is not None: + over_kind: Final = _BORROWER_BY_REASON.get(over_limit["descriptor_key"]) + over_borrower: Final = borrowing.get(over_kind) if over_kind is not None else None + if over_borrower is not None: raise self._throttled( - self._team_share_error(requested_model, over_limit, saturation, priority, user_api_key_dict, borrowing), + self._team_share_error(requested_model, over_limit, saturation, priority, over_borrower), requested_model, user_api_key_dict, ) + resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(requested_model) + raise ProxyRateLimitError( + detail={ + "error": "Rate limit exceeded", + "descriptor_key": over_limit["descriptor_key"], + "rate_limit_type": str(over_limit["rate_limit_type"]), + }, + rate_limit_type=map_v3_rate_limit_type(over_limit["rate_limit_type"]), + headers={"retry-after": str(self.v3_limiter.window_size), "x-litellm-priority": priority or "default"}, + model=resolved_model, + llm_provider=llm_provider, + ) + + def _track_frame_shares( + self, + connection_stash: RequestRateLimiterStash, + borrowing: Mapping[BorrowerKind, _BorrowingTeamShare], + saturated: bool, + ) -> None: + for kind in BORROWER_KINDS: + borrower_share = borrowing.get(kind) + state = connection_stash.borrowing_shares[kind] + enforced = None if borrower_share is None else borrower_share.enforced(saturated) + state.limit = state.limit if enforced is None and state.reserved_tokens else enforced + state.share = None if borrower_share is None else borrower_share.share + state.cap = None if borrower_share is None else borrower_share.ceiling + + async def _admission_reservations( + self, + borrowing: Mapping[BorrowerKind, _BorrowingTeamShare], + stash: RequestRateLimiterStash, + saturated: bool, + data: Mapping[str, object], + model: str, + call_type: str | None, + user_api_key_dict: UserAPIKeyAuth, + ) -> tuple[_FallbackTargetShare, ...]: + """Record what each borrower enforces on the admitting model and size the tokens to hold on it.""" + for kind in BORROWER_KINDS: + borrower_share = borrowing.get(kind) + state = stash.borrowing_shares[kind] + state.limit = None if borrower_share is None else borrower_share.enforced(saturated) + state.share = None if borrower_share is None else borrower_share.share + state.cap = None if borrower_share is None else borrower_share.ceiling + holding: Final = _holding_shares(borrowing, saturated) + reservation: Final = await self._tightest_token_reservation(holding, data, model, call_type, user_api_key_dict) + return tuple( + _FallbackTargetShare( + kind=kind, + share=borrower_share.share, + enforced=borrower_share.enforced(saturated), + cap=borrower_share.ceiling, + reservation=reservation if kind in holding and _tpm_share(borrower_share.share) is not None else 0, + ) + for kind, borrower_share in borrowing.items() + ) def _create_model_tracking_descriptor( self, @@ -1001,6 +1241,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): get_or_create_request_stash().priority_class_share = next( (d for d in priority_descriptors if d["key"] == PRIORITY_CLASS_SHARE_AT_SATURATION), None ) + get_or_create_request_stash().priority_class_share_holder = self._class_share_holder(user_api_key_dict) get_or_create_request_stash().dynamic_limiter_priority = priority if priority_descriptors: descriptors_to_check.extend(priority_descriptors) @@ -1013,41 +1254,34 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # asyncio.Lock + in-memory fallback for single-process deployments. # All-or-nothing: if any enforced descriptor would exceed its limit, # no counter is modified and the response carries "OVER_LIMIT". - team_share: Final = self._borrowing_team_share(model=model, user_api_key_dict=user_api_key_dict) - if team_share is not None: - descriptors_to_check.append(team_share.share) - enforced_team_share: Final = None if team_share is None else team_share.enforced(should_enforce_priority) - team_cap: Final = None if team_share is None else team_share.ceiling - held_team_share: Final = enforced_team_share if enforced_team_share is not None else team_cap - get_or_create_request_stash().borrowing_team_model_limit = enforced_team_share - get_or_create_request_stash().borrowing_team_model_share = None if team_share is None else team_share.share - get_or_create_request_stash().borrowing_team_model_cap = team_cap - - enforced_descriptors: Final[list[RateLimitDescriptor]] = [model_wide_descriptor] - if priority_descriptors and should_enforce_priority: - enforced_descriptors.extend(priority_descriptors) - if enforced_team_share is not None: - enforced_descriptors.append(enforced_team_share) - if team_cap is not None: - enforced_descriptors.append(team_cap) + borrowing: Final = self._borrowing_shares(model=model, user_api_key_dict=user_api_key_dict) + request_stash: Final = get_or_create_request_stash() + held_shares: Final = await self._admission_reservations( + borrowing, request_stash, should_enforce_priority, data, model, call_type, user_api_key_dict + ) + reservations: Final = {held.kind: held.reservation for held in held_shares} per_request_increment: Final[dict[Literal["requests", "tokens"], int]] = { "requests": 1, "tokens": 0, } - team_reservation: Final = ( - await self._team_token_reservation(team_share.share, data, model, call_type, user_api_key_dict) - if team_share is not None and held_team_share is not None - else 0 + enforced_descriptors: Final[tuple[RateLimitDescriptor, ...]] = ( + model_wide_descriptor, + *(priority_descriptors if should_enforce_priority else ()), + *(d for held in held_shares for d in held.descriptors), # comprehension-ok: each borrower's share then cap + ) + increments: Final[tuple[dict[Literal["requests", "tokens"], int], ...]] = ( + per_request_increment, + *(per_request_increment for _ in (priority_descriptors if should_enforce_priority else ())), + *( + {"requests": 1, "tokens": held.reservation} + for held in held_shares + for _ in held.descriptors # comprehension-ok: one increment per held descriptor + ), ) atomic_response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( - descriptors=enforced_descriptors, - increments=[ - {"requests": 1, "tokens": team_reservation} - if descriptor is enforced_team_share or descriptor is team_cap - else per_request_increment - for descriptor in enforced_descriptors - ], + descriptors=list(enforced_descriptors), + increments=list(increments), parent_otel_span=user_api_key_dict.parent_otel_span, ) @@ -1104,11 +1338,10 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): llm_provider=llm_provider, ) - if ( - descriptor_key in (TEAM_MODEL_LIMIT_AT_SATURATION, TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE) - and team_share is not None - ): - raise self._team_share_error(model, status, saturation, priority, user_api_key_dict, team_share) + over_kind = _BORROWER_BY_REASON.get(descriptor_key) + over_borrower = borrowing.get(over_kind) if over_kind is not None else None + if over_borrower is not None: + raise self._team_share_error(model, status, saturation, priority, over_borrower) if descriptor_key == PRIORITY_CLASS_SHARE_AT_SATURATION and share is not None: raise self._class_share_error(model, status, saturation, priority, user_api_key_dict, share) @@ -1141,33 +1374,32 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): llm_provider=llm_provider, ) - # If priority is NOT enforced (saturation below threshold) but - # priority_descriptors exist, increment them for tracking only — no - # check, no rollback. This matches the prior tracking semantics. - # - # Using the non-atomic should_rate_limit (instead of - # atomic_check_and_increment_by_n) is intentional here: we don't want - # to enforce the limit, we only want to bump the counter so the - # priority allocation has accurate usage when it later becomes - # enforced. The increment-then-check semantics of should_rate_limit - # are fine because we ignore the OVER_LIMIT response. - stash: Final = get_or_create_request_stash() - stash.borrowing_team_model_reserved_tokens = team_reservation if enforced_team_share is not None else 0 - stash.borrowing_team_model_cap_reserved_tokens = team_reservation if team_cap is not None else 0 - stash.borrowing_team_model_reserved_window = ( - _reserved_window(atomic_response, enforced_team_share) if team_reservation else None + # Below saturation the priority counters are tracked, not enforced, so an + # OVER_LIMIT here is ignored. The atomic call adds one request and no + # tokens, real usage lands on the token counter at settlement. + for kind in BORROWER_KINDS: + state = request_stash.borrowing_shares[kind] + reservation = reservations.get(kind, 0) + state.reserved_tokens = reservation if state.limit is not None else 0 + state.cap_reserved_tokens = reservation if state.cap is not None else 0 + state.reserved_window = _reserved_window(atomic_response, state.limit) if reservation else None + state.cap_reserved_window = _reserved_window(atomic_response, state.cap) if reservation else None + tightest_held: Final = min( + ( + (tpm, borrower_share.share, reservations[kind]) + for kind, borrower_share in borrowing.items() + if reservations.get(kind) and (tpm := _tpm_share(borrower_share.share)) is not None + ), + key=lambda held_share: held_share[0], + default=None, ) - stash.borrowing_team_model_cap_reserved_window = ( - _reserved_window(atomic_response, team_cap) if team_reservation else None - ) - self._cap_admitted_output(None if team_share is None else team_share.share, team_reservation, data, call_type) - if priority_descriptors and share is not None and not should_enforce_priority: - get_or_create_request_stash().rate_limit_response = atomic_response - elif priority_descriptors and not should_enforce_priority: - priority_tracking_response: Final = await self.v3_limiter.should_rate_limit( - descriptors=priority_descriptors, + if tightest_held is not None: + self._cap_admitted_output(tightest_held[1], tightest_held[2], data, call_type) + if priority_descriptors and not should_enforce_priority: + priority_tracking_response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( + descriptors=list(priority_descriptors), + increments=[per_request_increment for _ in priority_descriptors], parent_otel_span=user_api_key_dict.parent_otel_span, - read_only=False, ) get_or_create_request_stash().rate_limit_response = RateLimitResponse( overall_code=atomic_response["overall_code"], @@ -1190,6 +1422,25 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if team_reservation and tpm_share is not None and self.v3_limiter.get_rate_limit_type() != "input": self.v3_limiter.bound_output_for_limit(data, tpm_share, call_type) + async def _tightest_token_reservation( + self, + holding: Mapping[BorrowerKind, _BorrowingTeamShare], + data: Mapping[str, object], + model: str, + call_type: str | None, + user_api_key_dict: UserAPIKeyAuth, + ) -> int: + """One estimate for every borrower that holds tokens, sized with the output cap of the tightest TPM share, + since that is the cap the admitted request is sent with.""" + tightest: Final = min( + (borrower_share.share for borrower_share in holding.values() if _tpm_share(borrower_share.share)), + key=lambda descriptor: _tpm_share(descriptor) or 0, + default=None, + ) + if tightest is None: + return 0 + return await self._team_token_reservation(tightest, data, model, call_type, user_api_key_dict) + async def _team_token_reservation( self, descriptor: RateLimitDescriptor, @@ -1224,59 +1475,74 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): async def async_fallback_target_allowed( self, user_api_key_dict: UserAPIKeyAuth, model: str, request_data: Mapping[str, object], *, commit: bool = True ) -> bool: - """Hold a router fallback target to the team's share on it once it saturates, and to its borrow ceiling before. + """Hold a router fallback target to each borrower's share on it once it saturates, and to its borrow ceiling + before. - With ``commit`` the attempt is counted toward the target's saturation, and toward the team's counter only + With ``commit`` the attempt is counted toward the target's saturation, and toward a borrower's counter only while the share is enforced, holding the prompt's estimated tokens there. An idle target is only checked. """ - team_share: Final = self._borrowing_team_share(model=model, user_api_key_dict=user_api_key_dict) + borrowing: Final = self._borrowing_shares(model=model, user_api_key_dict=user_api_key_dict) model_group_info: Final = self.llm_router.registry().group(model) - if team_share is None or model_group_info is None: + if commit: + self._hold_fallback_for_borrowers_without_a_share(model, borrowing) + if not borrowing or model_group_info is None: return True saturation: Final = await self._check_model_saturation(model, model_group_info) - enforced: Final = team_share.enforced(saturation >= _get_priority_settings().saturation_threshold) - cap: Final = team_share.ceiling - held: Final = enforced if enforced is not None else cap - reservation: Final = ( - await self._team_token_reservation( - team_share.share, - {**request_data, "model": model}, - model, - get_or_create_request_stash().call_type, - user_api_key_dict, - ) - if held is not None - else 0 + saturated: Final = saturation >= _get_priority_settings().saturation_threshold + holding: Final = _holding_shares(borrowing, saturated) + reservation: Final = await self._tightest_token_reservation( + holding, {**request_data, "model": model}, model, get_or_create_request_stash().call_type, user_api_key_dict + ) + targets: Final = tuple( + _FallbackTargetShare( + kind=kind, + share=borrower_share.share, + enforced=borrower_share.enforced(saturated), + cap=borrower_share.ceiling, + reservation=reservation if kind in holding and _tpm_share(borrower_share.share) is not None else 0, + ) + for kind, borrower_share in borrowing.items() ) - team_descriptors: Final = tuple(d for d in (enforced, cap) if d is not None) if not commit: - for descriptor in team_descriptors: - if not await self._share_has_room(descriptor, reservation, user_api_key_dict): - return False + for target in targets: + for descriptor in target.descriptors: + if not await self._share_has_room(descriptor, target.reservation, user_api_key_dict): + return False return True model_wide_descriptor: Final = self._create_model_tracking_descriptor( model=model, model_group_info=model_group_info, high_limit_multiplier=1 ) response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( - descriptors=[model_wide_descriptor, *team_descriptors], + descriptors=[ + model_wide_descriptor, + *( + d for target in targets for d in target.descriptors + ), # comprehension-ok: each borrower's share then cap + ], increments=[ {"requests": 1, "tokens": 0}, - *({"requests": 1, "tokens": reservation} for _ in team_descriptors), + *( + {"requests": 1, "tokens": target.reservation} + for target in targets + for _ in target.descriptors # comprehension-ok: one increment per held descriptor + ), ], parent_otel_span=user_api_key_dict.parent_otel_span, ) if response["overall_code"] == "OVER_LIMIT": return False - self._hold_fallback_share( - model, - team_share.share, - enforced, - reservation if enforced is not None else 0, - _reserved_window(response, enforced), - cap, - reservation if cap is not None else 0, - _reserved_window(response, cap), - ) + for target in targets: + self._hold_fallback_share( + get_or_create_request_stash().borrowing_shares[target.kind], + model, + target.share, + target.enforced, + target.reservation if target.enforced is not None else 0, + _reserved_window(response, target.enforced), + target.cap, + target.reservation if target.cap is not None else 0, + _reserved_window(response, target.cap), + ) return True async def _share_has_room( @@ -1291,8 +1557,20 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): for status in checked["statuses"] ) + def _hold_fallback_for_borrowers_without_a_share( + self, model: str, borrowing: Mapping[BorrowerKind, _BorrowingTeamShare] + ) -> None: + """A borrower with no limit on the target takes an empty hold, so the target's usage never settles to its + share on the admitting model.""" + resolved: Final = self.v3_limiter.rate_limited_model(model) + names: Final = (model,) if resolved is None else (model, resolved.group) + for kind, state in get_or_create_request_stash().borrowing_shares.items(): + if kind not in borrowing and model not in state.fallback_limits: + state.hold_fallback(names, BorrowingTeamModelHold(descriptor=None)) + def _hold_fallback_share( self, + stash: BorrowingShareState, model: str, share: RateLimitDescriptor, charged: RateLimitDescriptor | None, @@ -1308,8 +1586,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ``cap`` is the team's borrow ceiling on the target, charged whether or not the target is saturated. Each counter carries an earlier hold on the same target only while that hold sits in the counter's current window. """ - stash: Final = get_or_create_request_stash() - earlier: Final = stash.borrowing_team_model_fallback_limits.get(model) + earlier: Final = stash.fallback_limits.get(model) carried: Final = ( earlier.reserved_tokens if earlier is not None and window is not None and earlier.window == window else 0 ) @@ -1327,9 +1604,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): cap_window=cap_window, share=share, ) - stash.borrowing_team_model_fallback_limits = MappingProxyType( - {**stash.borrowing_team_model_fallback_limits, model: hold, share["value"].rpartition(":")[2]: hold} - ) + stash.hold_fallback((model, share["value"].rpartition(":")[2]), hold) @with_service_target(RATE_LIMITS_TARGET) async def async_pre_call_hook( @@ -1511,22 +1786,14 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): _usage_of(callback_kwargs, response_obj), self.v3_limiter.get_rate_limit_type() ) stash: Final = request_stash_for_callback(callback_kwargs) - team_model_tokens: Final = _team_model_settlement( + team_model_tokens: Final = _borrowing_settlement( stash, model_group, total_tokens, self.v3_limiter.window_size ) - served: Final = ( - None - if stash is None or model_group is None - else stash.borrowing_team_model_fallback_limits.get(model_group) - ) + served: Final = _served_holds(stash, model_group) if stash is not None: - stash.borrowing_team_model_reserved_tokens = 0 - stash.borrowing_team_model_cap_reserved_tokens = 0 - stash.borrowing_team_model_reserved_window = None - stash.borrowing_team_model_cap_reserved_window = None - stash.borrowing_team_model_fallback_limits = MappingProxyType( - {k: h for k, h in stash.borrowing_team_model_fallback_limits.items() if h is not served} - ) + for kind, state in stash.borrowing_shares.items(): + state.clear_reservation() + state.drop_fallback_hold(served.get(kind)) if not model_group: if team_model_tokens: await self.v3_limiter.async_increment_reservation_aware_tokens( @@ -1562,13 +1829,17 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): priority_key = f"{model_group}:default_pool" class_share: Final = None if stash is None else stash.priority_class_share - settled_team_id: Final = settlement_metadata.user_api_key_team_id + settled_holder: Final = ( + settlement_metadata.user_api_key_team_id + if stash is None or stash.priority_class_share_holder is None + else stash.priority_class_share_holder + ) priority_scope: Final = ( RateLimitScope("priority_model", priority_key) if class_share is None else RateLimitScope( class_share["key"], - class_share["value"] if settled_team_id is None else counter_value(model_group, settled_team_id), + class_share["value"] if settled_holder is None else counter_value(model_group, settled_holder), ) ) usage_operations: Final = ( @@ -1621,11 +1892,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): stash: Final = get_request_stash() if stash is None: return - refund: Final = _team_model_settlement(stash, None, 0, self.v3_limiter.window_size) - stash.borrowing_team_model_reserved_tokens = 0 - stash.borrowing_team_model_cap_reserved_tokens = 0 - stash.borrowing_team_model_reserved_window = None - stash.borrowing_team_model_cap_reserved_window = None + refund: Final = _borrowing_settlement(stash, None, 0, self.v3_limiter.window_size) + for state in stash.borrowing_shares.values(): + state.clear_reservation() if refund: await self.v3_limiter.async_increment_reservation_aware_tokens( pipeline_operations=refund, parent_otel_span=user_api_key_dict.parent_otel_span @@ -1644,52 +1913,21 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if stash is None: return failed_group: Final = get_model_group_from_litellm_kwargs(dict(callback_kwargs)) - failed_hold: Final = ( - stash.borrowing_team_model_fallback_limits.get(failed_group) if failed_group is not None else None - ) used: Final = _settled_tokens( _usage_of(callback_kwargs, None if isinstance(response_obj, BaseException) else response_obj), self.v3_limiter.get_rate_limit_type(), ) - own_usage: Final = used if failed_hold is None else 0 - own_delta: Final = own_usage - stash.borrowing_team_model_reserved_tokens - own_cap_delta: Final = own_usage - stash.borrowing_team_model_cap_reserved_tokens - refund: Final = ( - *_window_guarded( - _team_model_token_increment(stash.borrowing_team_model_limit, own_delta, self.v3_limiter.window_size), - stash.borrowing_team_model_reserved_window, - ), - *_window_guarded( - _team_model_token_increment(stash.borrowing_team_model_cap, own_cap_delta, self.v3_limiter.window_size), - stash.borrowing_team_model_cap_reserved_window, - ), - *( - () - if failed_hold is None - else ( - *_window_guarded( - _team_model_token_increment( - failed_hold.descriptor, used - failed_hold.reserved_tokens, self.v3_limiter.window_size - ), - failed_hold.window, - ), - *_window_guarded( - _team_model_token_increment( - failed_hold.cap, used - failed_hold.cap_reserved_tokens, self.v3_limiter.window_size - ), - failed_hold.cap_window, - ), - ) - ), + failed_holds: Final = _served_holds(stash, failed_group) + refund: Final = tuple( + operation + for kind, state in stash.borrowing_shares.items() + for operation in self._failed_attempt_settlement( + state, failed_holds.get(kind), used + ) # comprehension-ok: flattens each borrower's settlement ) - stash.borrowing_team_model_reserved_tokens = 0 - stash.borrowing_team_model_cap_reserved_tokens = 0 - stash.borrowing_team_model_reserved_window = None - stash.borrowing_team_model_cap_reserved_window = None - if failed_hold is not None: - stash.borrowing_team_model_fallback_limits = MappingProxyType( - {k: h for k, h in stash.borrowing_team_model_fallback_limits.items() if h is not failed_hold} - ) + for kind, state in stash.borrowing_shares.items(): + state.clear_reservation() + state.drop_fallback_hold(failed_holds.get(kind)) if refund: await self.v3_limiter.async_increment_reservation_aware_tokens(pipeline_operations=refund) await self._record_team_share( @@ -1697,7 +1935,36 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): failed_group, _SettlementLoggingObject.model_validate(callback_kwargs.get("standard_logging_object") or {}).metadata or _SettlementMetadata(), - failed_hold, + failed_holds, + ) + + def _failed_attempt_settlement( + self, state: BorrowingShareState, failed_hold: BorrowingTeamModelHold | None, used: int + ) -> tuple[ReservationOperation, ...]: + ttl: Final = self.v3_limiter.window_size + own_usage: Final = used if failed_hold is None else 0 + return ( + *_window_guarded( + _team_model_token_increment(state.limit, own_usage - state.reserved_tokens, ttl), state.reserved_window + ), + *_window_guarded( + _team_model_token_increment(state.cap, own_usage - state.cap_reserved_tokens, ttl), + state.cap_reserved_window, + ), + *( + () + if failed_hold is None + else ( + *_window_guarded( + _team_model_token_increment(failed_hold.descriptor, used - failed_hold.reserved_tokens, ttl), + failed_hold.window, + ), + *_window_guarded( + _team_model_token_increment(failed_hold.cap, used - failed_hold.cap_reserved_tokens, ttl), + failed_hold.cap_window, + ), + ) + ), ) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a6697004fc7..50a32c9daac 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -41,6 +41,7 @@ from litellm.caching.redis_cache import log_redis_failure from litellm.constants import ( DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY, + MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY, NO_MAX_TOKENS_TPM_RESERVATION_FRACTION, RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY, ) @@ -118,6 +119,7 @@ from litellm.types.extraction import ExtractionResponse from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType +from litellm.types.proxy.auth.customer_hierarchy import CustomerScopeAuth from litellm.types.utils import ( CallTypes, EmbeddingResponse, @@ -200,7 +202,7 @@ def inherited_team_model_limit( return team_limit -TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = "team_model_limit_max_borrow_multiple" +MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY MAX_BORROW_MULTIPLE_CEILING: Final = 1_000_000 _MAX_BORROW_MULTIPLE: Final[TypeAdapter[float]] = TypeAdapter( Annotated[float, Field(ge=1, le=MAX_BORROW_MULTIPLE_CEILING, allow_inf_nan=False)] @@ -231,30 +233,48 @@ class BorrowingTeamModelLimits: NO_BORROWING: Final = BorrowingTeamModelLimits(tpm=None, rpm=None) -def team_model_limit_max_borrow_multiple(user_api_key_dict: UserAPIKeyAuth) -> float | None: - raw: Final[object] = (user_api_key_dict.team_metadata or {}).get(TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE) +BorrowerKind: TypeAlias = Literal["team", "key", "customer_org"] +BORROWER_KINDS: Final[tuple[BorrowerKind, ...]] = ("team", "key", "customer_org") +_METADATA: Final = TypeAdapter(Mapping[str, object]) + + +@dataclass(frozen=True, slots=True) +class ModelLimitBorrower: + """Who holds a per-model limit the dynamic rate limiter enforces at saturation, and that limit.""" + + kind: BorrowerKind + entity_id: str + alias: str | None + limits: BorrowingTeamModelLimits + + +def _max_borrow_multiple(raw: object, setting: str, owner: str) -> float | None: if raw is None: return None try: return _MAX_BORROW_MULTIPLE.validate_python(raw) except ValidationError: verbose_proxy_logger.warning( - "Ignoring %s=%r on team %s: expected a number from 1 to %d, so the team borrows up to the model limit", - TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE, + "Ignoring %s=%r on %s: expected a number from 1 to %d, so the value is not used", + setting, raw, - user_api_key_dict.team_id, + owner, MAX_BORROW_MULTIPLE_CEILING, ) return None +def team_model_limit_max_borrow_multiple(user_api_key_dict: UserAPIKeyAuth) -> float | None: + return _max_borrow_multiple( + _METADATA.validate_python(user_api_key_dict.team_metadata or {}).get(MODEL_LIMIT_MAX_BORROW_MULTIPLE), + MODEL_LIMIT_MAX_BORROW_MULTIPLE, + f"team {user_api_key_dict.team_id}", + ) + + def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> BorrowingTeamModelLimits: settings: Final = litellm.priority_reservation_settings - if ( - settings is None - or not settings.team_model_limit_borrows_idle_capacity_from_other_teams - or user_api_key_dict.team_id is None - ): + if settings is None or "team" not in settings.model_limit_borrowing_entities or user_api_key_dict.team_id is None: return NO_BORROWING limits: Final = BorrowingTeamModelLimits( tpm=inherited_team_model_limit(user_api_key_dict, model, "model_tpm_limit"), @@ -263,10 +283,90 @@ def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLi if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group): return NO_BORROWING return BorrowingTeamModelLimits( - tpm=limits.tpm, rpm=limits.rpm, max_borrow_multiple=team_model_limit_max_borrow_multiple(user_api_key_dict) + tpm=limits.tpm, + rpm=limits.rpm, + max_borrow_multiple=team_model_limit_max_borrow_multiple(user_api_key_dict) + or settings.model_limit_borrowing_config.max_borrow_multiple, ) +def borrowing_key_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> BorrowingTeamModelLimits: + """The key's own per-model limits when keys are listed to borrow idle capacity.""" + settings: Final = litellm.priority_reservation_settings + if settings is None or "key" not in settings.model_limit_borrowing_entities or user_api_key_dict.api_key is None: + return NO_BORROWING + limits: Final = BorrowingTeamModelLimits( + tpm=model.limit_in(get_key_own_model_rate_limit(user_api_key_dict, "model_tpm_limit")), + rpm=model.limit_in(get_key_own_model_rate_limit(user_api_key_dict, "model_rpm_limit")), + ) + if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group): + return NO_BORROWING + return BorrowingTeamModelLimits( + tpm=limits.tpm, + rpm=limits.rpm, + max_borrow_multiple=_max_borrow_multiple( + _METADATA.validate_python( + user_api_key_dict.metadata or {} # pyright: ignore[reportUnknownMemberType] # validates the untyped key metadata + ).get(MODEL_LIMIT_MAX_BORROW_MULTIPLE), + MODEL_LIMIT_MAX_BORROW_MULTIPLE, + "a key", + ) + or settings.model_limit_borrowing_config.max_borrow_multiple, + ) + + +def _customer_org_scope(user_api_key_dict: UserAPIKeyAuth) -> CustomerScopeAuth | None: + hierarchy: Final = user_api_key_dict.customer_hierarchy + if hierarchy is None: + return None + return next((scope for scope in hierarchy.scopes if scope.key == "customer_org"), None) + + +def borrowing_customer_org_model_limits( + user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel +) -> BorrowingTeamModelLimits: + """The customer organization's per-model limits when customer organizations are listed to borrow idle + capacity. The share is one counter per organization and model, across every key the customer calls.""" + settings: Final = litellm.priority_reservation_settings + scope: Final = _customer_org_scope(user_api_key_dict) + if settings is None or "customer_organization" not in settings.model_limit_borrowing_entities or scope is None: + return NO_BORROWING + limits: Final = BorrowingTeamModelLimits( + tpm=model.limit_in(scope.model_tpm_limit), rpm=model.limit_in(scope.model_rpm_limit) + ) + if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group): + return NO_BORROWING + return BorrowingTeamModelLimits( + tpm=limits.tpm, rpm=limits.rpm, max_borrow_multiple=settings.model_limit_borrowing_config.max_borrow_multiple + ) + + +def model_limit_borrowers(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> tuple[ModelLimitBorrower, ...]: + """Every level of the request whose per-model limit on ``model`` is a share that borrows idle capacity.""" + org_scope: Final = _customer_org_scope(user_api_key_dict) + candidates: Final = ( + ModelLimitBorrower( + kind="team", + entity_id=str(user_api_key_dict.team_id), + alias=user_api_key_dict.team_alias, + limits=borrowing_team_model_limits(user_api_key_dict, model), + ), + ModelLimitBorrower( + kind="key", + entity_id=str(user_api_key_dict.api_key), + alias=user_api_key_dict.key_alias, + limits=borrowing_key_model_limits(user_api_key_dict, model), + ), + ModelLimitBorrower( + kind="customer_org", + entity_id="" if org_scope is None else org_scope.entity_id, + alias=None if org_scope is None else org_scope.alias, + limits=borrowing_customer_org_model_limits(user_api_key_dict, model), + ), + ) + return tuple(borrower for borrower in candidates if borrower.limits != NO_BORROWING) + + def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]: if router is None: return (model,) @@ -587,6 +687,34 @@ class BorrowingTeamModelHold: share: RateLimitDescriptor | None = None +@dataclass(slots=True) +class BorrowingShareState: + """One borrower's share on the admitting model for this request: what was enforced, what was held on it, and the + holds taken on router fallback targets.""" + + limit: RateLimitDescriptor | None = None + reserved_tokens: int = 0 + reserved_window: ReservationWindowIdentity | None = None + share: RateLimitDescriptor | None = None + cap: RateLimitDescriptor | None = None + cap_reserved_tokens: int = 0 + cap_reserved_window: ReservationWindowIdentity | None = None + fallback_limits: Mapping[str, BorrowingTeamModelHold] = field(default_factory=lambda: MappingProxyType({})) + + def clear_reservation(self) -> None: + self.reserved_tokens = 0 + self.cap_reserved_tokens = 0 + self.reserved_window = None + self.cap_reserved_window = None + + def hold_fallback(self, names: tuple[str, ...], hold: BorrowingTeamModelHold) -> None: + self.fallback_limits = MappingProxyType({**self.fallback_limits, **dict.fromkeys(names, hold)}) + + def drop_fallback_hold(self, hold: BorrowingTeamModelHold | None) -> None: + if hold is not None: + self.fallback_limits = MappingProxyType({k: h for k, h in self.fallback_limits.items() if h is not hold}) + + @dataclass(slots=True) class RequestRateLimiterStash: """ @@ -631,17 +759,11 @@ class RequestRateLimiterStash: otpm_reserved_window_identities: frozenset[ReservationWindowIdentity] = field(default_factory=frozenset) batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None batch_tpd_refund_ops: tuple[ReservationOperation, ...] = () - borrowing_team_model_limit: RateLimitDescriptor | None = None - borrowing_team_model_reserved_tokens: int = 0 - borrowing_team_model_reserved_window: ReservationWindowIdentity | None = None - borrowing_team_model_share: RateLimitDescriptor | None = None - borrowing_team_model_cap: RateLimitDescriptor | None = None - borrowing_team_model_cap_reserved_tokens: int = 0 - borrowing_team_model_cap_reserved_window: ReservationWindowIdentity | None = None - borrowing_team_model_fallback_limits: Mapping[str, BorrowingTeamModelHold] = field( - default_factory=lambda: MappingProxyType({}) + borrowing_shares: Mapping[BorrowerKind, BorrowingShareState] = field( + default_factory=lambda: MappingProxyType({kind: BorrowingShareState() for kind in BORROWER_KINDS}) ) priority_class_share: RateLimitDescriptor | None = None + priority_class_share_holder: str | None = None dynamic_limiter_priority: str | None = None reservation_released: bool = False io_reservation_released: bool = False @@ -2718,8 +2840,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): model: Final = self._rate_limited_model(requested_model) if model is None or user_api_key_dict.customer_hierarchy is None: return () + org_borrowing: Final = borrowing_customer_org_model_limits(user_api_key_dict, model) limits: Final = tuple( - (scope, model.limit_in(scope.model_tpm_limit), model.limit_in(scope.model_rpm_limit)) + ( + scope, + None + if scope.key == "customer_org" and org_borrowing.tpm is not None + else model.limit_in(scope.model_tpm_limit), + None + if scope.key == "customer_org" and org_borrowing.rpm is not None + else model.limit_in(scope.model_rpm_limit), + ) for scope in user_api_key_dict.customer_hierarchy.scopes ) return tuple( @@ -2779,14 +2910,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if model is None: return borrowing: Final = borrowing_team_model_limits(user_api_key_dict, model) + key_borrowing: Final = borrowing_key_model_limits(user_api_key_dict, model) model_specific_tpm_limit: Final = ( None - if borrowing.tpm is not None + if borrowing.tpm is not None or key_borrowing.tpm is not None else model.limit_in(get_key_model_tpm_limit(user_api_key_dict, model_name=model.group)) ) model_specific_rpm_limit: Final = ( None - if borrowing.rpm is not None + if borrowing.rpm is not None or key_borrowing.rpm is not None else model.limit_in(get_key_model_rpm_limit(user_api_key_dict, model_name=model.group)) ) if model_specific_tpm_limit is None and model_specific_rpm_limit is None: diff --git a/litellm/proxy/hooks/priority_class_share.py b/litellm/proxy/hooks/priority_class_share.py index 229afe09856..2fe7ff8b0ce 100644 --- a/litellm/proxy/hooks/priority_class_share.py +++ b/litellm/proxy/hooks/priority_class_share.py @@ -1,6 +1,6 @@ -"""Splits a priority class's share of a model among the teams active on it. +"""Splits a priority class's share of a model among the callers active on it. -Enabled by ``priority_reservation_settings.priority_class_share_splits_equally_among_active_teams``. +Enabled by ``priority_reservation_settings.priority_class_share_config``. """ import time @@ -40,6 +40,16 @@ def team_member(team_id: str) -> str: return f"team{MEMBER_SEPARATOR}{team_id}" +def key_member(key_hash: str) -> str: + """The registry id and counter owner of a key that takes its own slice of its class, apart from every team id.""" + return f"key{MEMBER_SEPARATOR}{key_hash}" + + +def customer_member(kind: str, entity_id: str) -> str: + """The registry id and counter owner of a customer organization or customer member taking its own slice.""" + return f"{kind}{MEMBER_SEPARATOR}{entity_id}" + + def is_pool_member(member: str) -> bool: return member.startswith(_POOL_PREFIX) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 86f16411724..4fc2a1419c8 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1595,7 +1595,11 @@ def _tenant_destination( name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None ) -> "OtelDestination | None": from litellm.integrations.otel.model.baggage import parse_destination_metadata_keys - from litellm.integrations.otel.model.config import parse_capture_message_content, parse_excluded_services + from litellm.integrations.otel.model.config import ( + langfuse_destination_span_scope, + parse_capture_message_content, + parse_excluded_services, + ) from litellm.integrations.otel.presets.destinations import destination_for callback_vars: Final = MappingProxyType( @@ -1605,6 +1609,7 @@ def _tenant_destination( ) ) ) + otel_settings: Final = as_str_mapping((litellm.callback_settings or {}).get("otel")) or {} return destination_for( name, _tenant_otel_params(callback_vars), @@ -1612,6 +1617,9 @@ def _tenant_destination( parse_capture_message_content(callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)), parse_excluded_services(callback_vars.get(EXCLUDED_SERVICES_VAR)), parse_destination_metadata_keys(callback_vars.get(BAGGAGE_METADATA_KEYS_VAR)), + langfuse_destination_span_scope=langfuse_destination_span_scope( + otel_settings.get("langfuse_destination_span_scope") + ), ) diff --git a/litellm/proxy/management_endpoints/budget_management_endpoints.py b/litellm/proxy/management_endpoints/budget_management_endpoints.py index 091b88bb046..fbf70336328 100644 --- a/litellm/proxy/management_endpoints/budget_management_endpoints.py +++ b/litellm/proxy/management_endpoints/budget_management_endpoints.py @@ -278,7 +278,12 @@ async def update_budget( return response -@router.post("/budget/new", tags=["budget management"], description=new_budget.__doc__) +@router.post( + "/budget/new", + tags=["budget management"], + description=new_budget.__doc__, + operation_id="new_budget_budget_new_post", +) async def new_budget_endpoint( budget_obj: BudgetWriteRequest, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], @@ -286,7 +291,12 @@ async def new_budget_endpoint( return await new_budget(budget_obj, user_api_key_dict) -@router.post("/budget/update", tags=["budget management"], description=update_budget.__doc__) +@router.post( + "/budget/update", + tags=["budget management"], + description=update_budget.__doc__, + operation_id="update_budget_budget_update_post", +) async def update_budget_endpoint( budget_obj: BudgetWriteRequest, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index a484f4ff302..1396f92a3c1 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -103,7 +103,7 @@ from litellm.proxy.utils import handle_exception_on_proxy, hash_password, query_ from litellm.redis.stores.spend_counters import ( user_counter, ) -from litellm.repositories.chunked_in import update_many_in +from litellm.repositories.chunked_in import find_many_in, update_many_in from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ( @@ -159,6 +159,7 @@ class _AdminTeamAndKeyRows(TypedDict): _ADMIN_TEAM_AND_KEY_ROWS_ADAPTER: Final = TypeAdapter( list[_AdminTeamAndKeyRows], config=ConfigDict(hide_input_in_errors=True) ) +_TEAM_METADATA_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _ADMIN_KEY_ROW_ADAPTER: Final = TypeAdapter( dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True) ) @@ -468,8 +469,12 @@ async def add_new_user_to_default_team( user_api_key_dict: UserAPIKeyAuth, teams: Sequence[str | NewUserRequestTeam], prisma_client: "PrismaClient", + teams_are_defaults: bool = True, ): - memberships: Final = tuple(_default_team_membership(team) for team in teams) + requested: Final = tuple(_default_team_membership(team) for team in teams) + memberships: Final = ( + await _memberships_on_team_member_budget(requested, prisma_client) if teams_are_defaults else requested + ) for membership in memberships: with contextlib.suppress(Exception): await _add_user_to_team( @@ -482,6 +487,25 @@ async def add_new_user_to_default_team( ) +async def _memberships_on_team_member_budget( + memberships: tuple[NewUserRequestTeam, ...], prisma_client: "PrismaClient" +) -> tuple[NewUserRequestTeam, ...]: + """A default team's max_budget_in_team is already the team's member budget, which also carries the reset.""" + capped_team_ids: Final = [m.team_id for m in memberships if m.max_budget_in_team is not None] + if not capped_team_ids: + return memberships + team_rows: Final = await find_many_in(_team_table(prisma_client), "team_id", capped_team_ids) + budgeted_team_ids: Final = frozenset( + row.team_id + for row in team_rows + if isinstance(row.metadata, dict) + and _TEAM_METADATA_ADAPTER.validate_python(row.metadata).get("team_member_budget_id") is not None + ) + return tuple( + m.model_copy(update={"max_budget_in_team": None}) if m.team_id in budgeted_team_ids else m for m in memberships + ) + + def _default_team_membership(team: str | NewUserRequestTeam) -> NewUserRequestTeam: if isinstance(team, str): return NewUserRequestTeam(team_id=team) @@ -666,6 +690,7 @@ async def new_user( user_api_key_dict=user_api_key_dict, teams=teams, prisma_client=prisma_client, + teams_are_defaults=data.teams is None, ) user_id: Final = cast(str | None, response.get("user_id", None)) diff --git a/litellm/proxy/management_endpoints/management_v1/customers.py b/litellm/proxy/management_endpoints/management_v1/customers.py index a678a535385..a999e49ff2e 100644 --- a/litellm/proxy/management_endpoints/management_v1/customers.py +++ b/litellm/proxy/management_endpoints/management_v1/customers.py @@ -17,6 +17,7 @@ from litellm.proxy.list_api.common import ( ) from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX from litellm.proxy.management_helpers.customer_hierarchy import ( + BudgetSort, CustomerHierarchy, CustomerPortalLimits, ErrorCode, @@ -54,13 +55,18 @@ _ERROR_DETAILS: Final[Mapping[ErrorCode, tuple[int, str, str]]] = MappingProxyTy { ErrorCode.CAPACITY_BUSY: ( 503, - "Customer portal busy", - "Customer portal capacity is being updated. Retry the request.", + "Customer Management busy", + "Customer Management capacity is being updated. Retry the request.", + ), + ErrorCode.TRANSACTION_TIMEOUT: ( + 504, + "Customer Management operation timed out", + "Customer Management operation timed out. The transaction was rolled back. Retry the request.", ), ErrorCode.FREE_TIER_LIMIT: ( 403, - "Free customer portal limit reached", - "Free customer portal limit reached: up to 5 customer organizations, 5 teams per organization and " + "Customer Management free tier limit reached", + "Customer Management free tier limit reached: up to 5 customer organizations, 5 teams per organization and " "5 members per team without a LiteLLM Enterprise license. Upgrade to Enterprise to remove the limits: " "https://www.litellm.ai/enterprise", ), @@ -213,7 +219,8 @@ AuthCache: TypeAlias = Annotated[UserApiKeyCache, Depends(get_user_api_key_cache @router.get( "/customer_portal/limits", response_model=CustomerPortalLimits, - description="Get customer portal free-tier limits and current capacity. Proxy admins only.", + summary="Get Customer Management Limits", + description="Get Customer Management free-tier limits and current capacity. Proxy admins only.", ) @management_endpoint_wrapper async def get_customer_portal_limits( @@ -407,9 +414,12 @@ async def list_customer_teams( page: Annotated[int, Query(ge=1)] = 1, page_size: Annotated[int, Query(ge=1)] = 50, parent_org_id: str | None = None, + sort: Annotated[ + BudgetSort | None, Query(description="Sort by displayed budget limit, with uncapped rows last.") + ] = None, ) -> ListResponse[Team]: size: Final = min(page_size, 100) - result: Final = resolve(await hierarchy.list_teams(parent_org_id, page, size)) + result: Final = resolve(await hierarchy.list_teams(parent_org_id, page, size, sort)) return list_response(result, request, page, size) @@ -470,7 +480,9 @@ async def get_customer_member( @router.patch( "/customer_members/{customer_member_id}", response_model=CustomerMember, - description="Update a customer member and its budgets. Proxy admins only.", + description="Update a customer member and its budgets. Supply parent_team_ids to replace its teams within the " + "same organization, with the first team as default. Identity, budget counters and historical usage are retained. " + "Historical usage stays attributed to the original team. Proxy admins only.", ) @management_endpoint_wrapper async def update_customer_member( @@ -505,7 +517,10 @@ async def delete_customer_member( "/customer_members", response_model=ListResponse[CustomerMember], dependencies=(Depends(reject_duplicate_list_params),), - description="List customer members with page-number pagination and resolved budgets. Proxy admins only.", + description=( + "List customer members with page-number pagination and resolved budgets. " + "search filters by customer member ID or alias before paginating. Proxy admins only." + ), ) @management_endpoint_wrapper async def list_customer_members( @@ -515,9 +530,17 @@ async def list_customer_members( page_size: Annotated[int, Query(ge=1)] = 50, parent_team_id: str | None = None, parent_org_id: str | None = None, + search: Annotated[str, Query(description="Case-insensitive substring of the customer member ID or alias")] = "", + sort: Annotated[ + BudgetSort | None, Query(description="Sort by displayed budget limit, with uncapped rows last.") + ] = None, ) -> ListResponse[CustomerMember]: size: Final = min(page_size, 100) - result: Final = resolve(await hierarchy.list_members(parent_team_id, page, size, parent_org_id=parent_org_id)) + result: Final = resolve( + await hierarchy.list_members( + parent_team_id, page, size, parent_org_id=parent_org_id, search=search.strip(), sort=sort + ) + ) return list_response(result, request, page, size) diff --git a/litellm/proxy/management_endpoints/team_admin_field_permissions.py b/litellm/proxy/management_endpoints/team_admin_field_permissions.py index f80aa2a2e29..f51eb1088cd 100644 --- a/litellm/proxy/management_endpoints/team_admin_field_permissions.py +++ b/litellm/proxy/management_endpoints/team_admin_field_permissions.py @@ -108,7 +108,7 @@ PROXY_CONTROLLED_TEAM_METADATA_KEYS: Final[frozenset[str]] = frozenset( "phoenix_project_name_override", "phoenix_project_name", "opik", - "team_model_limit_max_borrow_multiple", + "model_limit_max_borrow_multiple", ) ) _NOT_COLUMNS: Final[frozenset[str]] = frozenset({"team_id", "metadata"}) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 24675406567..ffe530cb5db 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -301,7 +301,10 @@ async def add_team_callbacks( - langfuse_secret: The secret for the Langfuse callback - langfuse_host: The host for the Langfuse callback - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) - - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + - langfuse_span_scope: For langfuse_otel, "full" sends the whole request trace, + "llm_only" sends only model-call spans. + Omitted follows LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE (default "full"), also configurable as + callback_settings.otel.langfuse_destination_span_scope. - capture_message_content: For OTel v2 callbacks, "no_content" or "span_only". An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 6da15cca9ad..4dd289f7789 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -238,7 +238,7 @@ from litellm.proxy.utils import ( ) from litellm.redis.stores.spend_counters import team_counter from litellm.repositories.budget_repository import BudgetRepository -from litellm.repositories.chunked_in import delete_many_in, find_many_in +from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.prisma_protocols import TableActions @@ -919,6 +919,50 @@ class TeamMemberBudgetHandler: TeamMemberBudgetHandler.clean_team_member_fields(updated_kv) return updated_kv + @staticmethod + async def member_budgets_on_team_reset_window( + team_id: str, team_member_budget_id: str | None, prisma_client: PrismaClient + ) -> tuple[str | None, tuple[str, ...]]: + """The team default's reset window, and the budgets of members who take their cap from it and reset on a copy.""" + team_default: Final = ( + await _budget_db(prisma_client).find_unique(where={"budget_id": team_member_budget_id}) + if team_member_budget_id is not None + else None + ) + team_window: Final = team_default.budget_duration if team_default is not None else None + memberships: Final = await _team_membership_db(prisma_client).find_many( + where={"team_id": team_id, "budget_id": {"not": team_member_budget_id}}, + include={"litellm_budget_table": True}, + ) + return team_window, tuple( + m.litellm_budget_table.budget_id + for m in memberships + if m.litellm_budget_table is not None + and m.litellm_budget_table.max_budget is None + and m.litellm_budget_table.budget_duration == team_window + ) + + @staticmethod + async def move_members_to_team_reset_window( + team_id: str, team_member_budget_id: str | None, budget_duration: str | None, prisma_client: PrismaClient + ) -> None: + """Runs before the team default is written, so a retry after a failure still finds the members left behind.""" + team_window, budget_ids = await TeamMemberBudgetHandler.member_budgets_on_team_reset_window( + team_id=team_id, team_member_budget_id=team_member_budget_id, prisma_client=prisma_client + ) + if team_window == budget_duration or not budget_ids: + return + _ = await update_many_in( + _budget_db(prisma_client), + "budget_id", + budget_ids, + data={ + "budget_duration": budget_duration, + "budget_reset_at": get_budget_reset_time(budget_duration) if budget_duration else None, + }, + atomicity="per_chunk_ok", + ) + @staticmethod async def backfill_team_member_budget_entries( team_id: str, @@ -2825,6 +2869,15 @@ async def update_team( }, } + _current_member_budget_id: Final = (existing_team_row.metadata or {}).get("team_member_budget_id") + if "team_member_budget_duration" in _team_member_fields_in_request: + await TeamMemberBudgetHandler.move_members_to_team_reset_window( + team_id=data.team_id, + team_member_budget_id=_current_member_budget_id if isinstance(_current_member_budget_id, str) else None, + budget_duration=data.team_member_budget_duration, + prisma_client=prisma_client, + ) + if _team_member_fields_in_request and TeamMemberBudgetHandler.should_create_budget( team_member_budget=data.team_member_budget, team_member_rpm_limit=data.team_member_rpm_limit, diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index 65b64a297c9..c5ee783bea1 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -22,7 +22,7 @@ from litellm.repositories.table_repositories import AuditLogRepository from litellm.types.utils import StandardAuditLogPayload if TYPE_CHECKING: - from litellm.integrations.s3_v3 import S3Logger + from litellm.integrations.s3_v2 import S3Logger _audit_log_callback_cache: Final[dict[str, "CustomLogger | S3Logger"]] = {} ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY: Final = "allow_litellm_changed_by_header" @@ -78,9 +78,9 @@ def _resolve_audit_log_callback(name: str) -> "CustomLogger | S3Logger | None": instance: CustomLogger | S3Logger | None audit_params: Final[Mapping[str, object] | None] = getattr(litellm, "s3_audit_callback_params", None) if name == "s3_v2" and audit_params is not None: - from litellm.integrations import s3_v3 + from litellm.integrations import s3_v2 - instance = s3_v3.S3Logger(s3_callback_params_override=audit_params) + instance = s3_v2.S3Logger(s3_callback_params_override=audit_params) else: from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, diff --git a/litellm/proxy/management_helpers/customer_hierarchy.py b/litellm/proxy/management_helpers/customer_hierarchy.py index d2a687a08cb..7510fd9571b 100644 --- a/litellm/proxy/management_helpers/customer_hierarchy.py +++ b/litellm/proxy/management_helpers/customer_hierarchy.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from enum import Enum from itertools import chain -from typing import TYPE_CHECKING, Final, Generic, Protocol, TypeVar +from typing import TYPE_CHECKING, Final, Generic, Literal, Protocol, TypeAlias, TypeVar from uuid import uuid4 from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator @@ -20,7 +20,7 @@ from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.list_api.common import escape_like from litellm.proxy.management_helpers.customer_budget_usage import customer_budget_usage from litellm.repositories.budget_repository import create_budget_row, update_budget_row -from litellm.repositories.chunked_in import count_in, delete_many_in, find_many_in +from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE, count_in, delete_many_in, find_many_in from litellm.repositories.prisma_protocols import TableActions from litellm.types.proxy.management_endpoints.customer_hierarchy import ( Budget, @@ -29,6 +29,7 @@ from litellm.types.proxy.management_endpoints.customer_hierarchy import ( BudgetTemplate, CustomerMember, MemberCreate, + MemberFields, MemberPatch, Organization, OrganizationBudget, @@ -45,9 +46,53 @@ if TYPE_CHECKING: from prisma.models import LiteLLM_BudgetWindowSpend +BudgetSort: TypeAlias = Literal["budget_asc", "budget_desc"] + + +def customer_budget_limit(budget: Budget | None) -> float | None: + if budget is None: + return None + limits: Final = budget.budget_limits or () + monthly: Final = next( + ( + limit.max_budget + for limit in limits + if limit.budget_duration.strip().lower() in ("1mo", "1month", "1months", "30d", "30days", "monthly") + ), + None, + ) + if monthly is not None: + return monthly + if budget.max_budget is not None: + return budget.max_budget + return next( + ( + limit.max_budget + for limit in limits + if limit.budget_duration.strip().lower() in ("12mo", "12month", "12months") + ), + None, + ) + + +def _budget_sort_key(budget: Budget | None, sort: BudgetSort) -> tuple[bool, float]: + limit: Final = customer_budget_limit(budget) + return limit is None, (limit or 0) * (-1 if sort == "budget_desc" else 1) + + +class _BudgetSortRow(BaseModel): + model_config = ConfigDict(frozen=True) + + entity_id: str + budget_id: str | None + budget_override_id: str | None + budget_override_expires_at: datetime | None + + class ErrorCode(str, Enum): FREE_TIER_LIMIT = "free_tier_limit" CAPACITY_BUSY = "capacity_busy" + TRANSACTION_TIMEOUT = "transaction_timeout" NOT_FOUND = "not_found" PARENT_NOT_FOUND = "parent_not_found" ALREADY_EXISTS = "already_exists" @@ -371,7 +416,7 @@ def team_response(row: PrismaRecord, now: datetime) -> Team: def override_expiry( - row: Team | CustomerMember | None, payload: TeamPatch | MemberPatch, override_id: str | None + row: Team | CustomerMember | None, payload: TeamPatch | MemberFields, override_id: str | None ) -> datetime | None: if override_id is None: return None @@ -486,6 +531,12 @@ _MEMBER_INCLUDE: Final = {"budget": True, "budget_override": True} _ID: Final = TypeAdapter(str) +def _search_filter(search: str, fields: tuple[str, ...]) -> Mapping[str, object]: + if not search: + return {} + return {"OR": tuple({field: {"contains": escape_like(search), "mode": "insensitive"}} for field in fields)} + + def _member_ids(rows: Sequence[PrismaRecord]) -> tuple[str, ...]: return tuple(_ID.validate_python(row.model_dump()["customer_member_id"]) for row in rows) @@ -995,9 +1046,20 @@ class CustomerHierarchyStore: return await self.update_budget(current_id, payload.budget_override, actor) async def update_member(self, member_id: str, payload: MemberPatch, actor: str) -> CustomerMember | Failure: + if payload.parent_team_ids is not None: + await self.lock_customer_identities() member: Final = await self.get_member(member_id) if isinstance(member, Failure): return member + if payload.parent_team_ids is not None: + teams: Final = await find_many_in(self.teams, "customer_team_id", payload.parent_team_ids) + if len(teams) != len(payload.parent_team_ids): + return Failure(ErrorCode.PARENT_NOT_FOUND) + if any(team.model_dump()["parent_org_id"] != member.parent_org_id for team in teams): + return Failure(ErrorCode.CUSTOMER_ORG_MISMATCH) + added: Final = tuple(team for team in payload.parent_team_ids if team not in member.parent_team_ids) + if await self.member_capacity_reached(added): + return Failure(ErrorCode.FREE_TIER_LIMIT) selected: Final = await self.member_template_patch(member, payload) if isinstance(selected, Failure): return selected @@ -1012,6 +1074,7 @@ class CustomerHierarchyStore: where={"customer_member_id": member_id}, data={ **alias_patch(payload), + **({"parent_team_ids": payload.parent_team_ids} if payload.parent_team_ids is not None else {}), **({"budget_id": selected} if "budget_id" in payload.model_fields_set else {}), **( { @@ -1028,16 +1091,7 @@ class CustomerHierarchyStore: return await self.get_member(member_id) async def list_organizations(self, page: int, page_size: int, search: str = "") -> Page[Organization]: - where: Final = ( - { - "OR": tuple( - {field: {"contains": escape_like(search), "mode": "insensitive"}} - for field in ("alias", "customer_org_id") - ) - } - if search - else {} - ) + where: Final = _search_filter(search, ("alias", "customer_org_id")) count: Final = await self.organizations.count(where=where) rows: Final = await self.organizations.find_many( where=where, @@ -1048,9 +1102,49 @@ class CustomerHierarchyStore: ) return Page(await self.organization_responses(rows), count) - async def list_teams(self, parent_org_id: str | None, page: int, page_size: int) -> Page[Team]: + async def list_teams( + self, parent_org_id: str | None, page: int, page_size: int, sort: BudgetSort | None = None + ) -> Page[Team]: where: Final = {"parent_org_id": parent_org_id} if parent_org_id is not None else {} count: Final = await self.teams.count(where=where) + if sort is not None: + ids: Final = await self._budget_sort_ids( + await self.db.query_raw( + "SELECT customer_team_id AS entity_id, budget_id, budget_override_id, budget_override_expires_at " + 'FROM "LiteLLM_CustomerTeamTable" WHERE ($1::text IS NULL OR parent_org_id = $1) ' + "ORDER BY created_at ASC, customer_team_id ASC", + parent_org_id, + ), + sort, + page, + page_size, + ) + chunks: Final = tuple( + [ + await self.teams.find_many( + # bounded-ok: Each slice has at most IN_LIST_CHUNK_SIZE ids. + where={"customer_team_id": {"in": list(ids[start : start + IN_LIST_CHUNK_SIZE])}}, + include=_MEMBER_INCLUDE, + ) + for start in range(0, len(ids), IN_LIST_CHUNK_SIZE) + ] + ) + rows_by_id: Final = { + _ID.validate_python(row.model_dump()["customer_team_id"]): row for row in chain.from_iterable(chunks) + } + return Page( + await customer_budget_usage( + tuple( + team_response(rows_by_id[identifier], self.now) + for identifier in ids + if identifier in rows_by_id + ), + self.window_spend, + self.now, + self.settings, + ), + count, + ) rows: Final = await self.teams.find_many( where=where, skip=(page - 1) * page_size, @@ -1060,19 +1154,72 @@ class CustomerHierarchyStore: ) return Page( await customer_budget_usage( - tuple(team_response(row, self.now) for row in rows), self.window_spend, self.now, self.settings + tuple(team_response(row, self.now) for row in rows), + self.window_spend, + self.now, + self.settings, ), count, ) async def list_members( - self, parent_team_id: str | None, page: int, page_size: int, *, parent_org_id: str | None = None + self, + parent_team_id: str | None, + page: int, + page_size: int, + *, + parent_org_id: str | None = None, + search: str = "", + sort: BudgetSort | None = None, ) -> Page[CustomerMember]: where: Final[Mapping[str, object]] = { **({"parent_team_ids": {"has": parent_team_id}} if parent_team_id is not None else {}), **({"parent_org_id": parent_org_id} if parent_org_id is not None else {}), + **_search_filter(search, ("alias", "customer_member_id")), } count: Final = await self.members.count(where=where) + if sort is not None: + ids: Final = await self._budget_sort_ids( + await self.db.query_raw( + "SELECT customer_member_id AS entity_id, budget_id, budget_override_id, budget_override_expires_at " + 'FROM "LiteLLM_CustomerMemberTable" WHERE ($1::text IS NULL OR parent_org_id = $1) ' + "AND ($2::text IS NULL OR parent_team_ids @> ARRAY[$2]::text[]) " + "AND ($3 = '' OR alias ILIKE $3 OR customer_member_id ILIKE $3) " + "ORDER BY created_at ASC, customer_member_id ASC", + parent_org_id, + parent_team_id, + f"%{escape_like(search)}%" if search else "", + ), + sort, + page, + page_size, + ) + chunks: Final = tuple( + [ + await self.members.find_many( + # bounded-ok: Each slice has at most IN_LIST_CHUNK_SIZE ids. + where={"customer_member_id": {"in": list(ids[start : start + IN_LIST_CHUNK_SIZE])}}, + include=_MEMBER_INCLUDE, + ) + for start in range(0, len(ids), IN_LIST_CHUNK_SIZE) + ] + ) + rows_by_id: Final = { + _ID.validate_python(row.model_dump()["customer_member_id"]): row for row in chain.from_iterable(chunks) + } + return Page( + await customer_budget_usage( + tuple( + member_response(rows_by_id[identifier], self.now) + for identifier in ids + if identifier in rows_by_id + ), + self.window_spend, + self.now, + self.settings, + ), + count, + ) rows: Final = await self.members.find_many( where=where, skip=(page - 1) * page_size, @@ -1082,11 +1229,36 @@ class CustomerHierarchyStore: ) return Page( await customer_budget_usage( - tuple(member_response(row, self.now) for row in rows), self.window_spend, self.now, self.settings + tuple(member_response(row, self.now) for row in rows), + self.window_spend, + self.now, + self.settings, ), count, ) + async def _budget_sort_ids(self, rows: object, sort: BudgetSort, page: int, page_size: int) -> tuple[str, ...]: + candidates: Final = TypeAdapter(tuple[_BudgetSortRow, ...]).validate_python(rows) + references: Final = chain.from_iterable((row.budget_id, row.budget_override_id) for row in candidates) + budgets: Final = { + _ID.validate_python(row.model_dump()["budget_id"]): row.model_dump() + for row in await find_many_in(self.budgets, "budget_id", (ref for ref in references if ref is not None)) + } + + def key(row: _BudgetSortRow) -> tuple[bool, float]: + template: Final = related_budget(budgets.get(row.budget_id or ""), self.now) + override: Final = related_budget( + budgets.get(row.budget_override_id or ""), + self.now, + template.max_budget if template is not None else None, + ) + return _budget_sort_key( + effective_member_budget(template, override, row.budget_override_expires_at, self.now), sort + ) + + ordered: Final = sorted(candidates, key=key) + return tuple(row.entity_id for row in ordered[(page - 1) * page_size : page * page_size]) + async def _delete_unused_budgets(self, budget_ids: Sequence[str]) -> None: where: Final[Mapping[str, object]] = {field: {"none": {}} for field in _BUDGET_REFERENCE_FIELDS} await delete_many_in( @@ -1098,8 +1270,10 @@ class CustomerHierarchyStore: chunk_size=_DELETE_BATCH_SIZE, ) - async def _delete_team_member_batch(self, team_id: str) -> bool: - rows: Final = await self.members.find_many(where={"parent_team_ids": {"has": team_id}}, take=_DELETE_BATCH_SIZE) + async def _delete_team_member_batch(self, team_ids: tuple[str, ...]) -> bool: + rows: Final = await self.members.find_many( + where={"parent_team_ids": {"has_some": list(team_ids)}}, take=_DELETE_BATCH_SIZE + ) members: Final = tuple(_MemberDeletionRow.model_validate(row) for row in rows) if not members: return False @@ -1111,9 +1285,9 @@ class CustomerHierarchyStore: await self._delete_unused_budgets(budget_ids) return True - async def _delete_team_members(self, team_id: str) -> None: + async def _delete_team_members(self, team_ids: tuple[str, ...]) -> None: while True: - if not await self._delete_team_member_batch(team_id): + if not await self._delete_team_member_batch(team_ids): return async def _delete_organization_team_batch(self, organization_id: str) -> bool: @@ -1121,9 +1295,8 @@ class CustomerHierarchyStore: teams: Final = tuple(_TeamDeletionRow.model_validate(row) for row in rows) if not teams: return False - for team in teams: - await self._delete_team_members(team.customer_team_id) team_ids: Final = tuple(team.customer_team_id for team in teams) + await self._delete_team_members(team_ids) team_budget_ids: Final = chain.from_iterable((team.budget_id, team.budget_override_id) for team in teams) budget_ids: Final = tuple(budget_id for budget_id in team_budget_ids if budget_id is not None) await delete_many_in(self.teams, "customer_team_id", team_ids, atomicity="caller_transaction") @@ -1154,11 +1327,12 @@ class CustomerHierarchyStore: return None async def delete_team(self, team_id: str, *, cascade: bool = False) -> None | Failure: + await self.lock_customer_identities() team: Final = await self.get_team(team_id) if isinstance(team, Failure): return team if cascade: - await self._delete_team_members(team_id) + await self._delete_team_members((team_id,)) elif await self.members.count(where={"parent_team_ids": {"has": team_id}}): return Failure(ErrorCode.HAS_CHILDREN) await self.teams.delete(where={"customer_team_id": team_id}) @@ -1166,6 +1340,7 @@ class CustomerHierarchyStore: return None async def delete_organization(self, organization_id: str, *, cascade: bool = False) -> None | Failure: + await self.lock_customer_identities() organization: Final = await self.get_organization(organization_id) if isinstance(organization, Failure): return organization @@ -1196,7 +1371,13 @@ class CustomerHierarchy: *, operation_type: TransactionOperation = TransactionOperation.READ_WRITE, ) -> T | Failure: - from prisma.errors import DataError, ForeignKeyViolationError, RecordNotFoundError, UniqueViolationError + from prisma.errors import ( + DataError, + ForeignKeyViolationError, + RecordNotFoundError, + TransactionExpiredError, + UniqueViolationError, + ) try: async with self.db.tx() as tx: @@ -1205,6 +1386,8 @@ class CustomerHierarchy: return Failure(ErrorCode.ALREADY_EXISTS) except RecordNotFoundError: return Failure(ErrorCode.NOT_FOUND) + except TransactionExpiredError: + return Failure(ErrorCode.TRANSACTION_TIMEOUT) except ForeignKeyViolationError: return Failure( ErrorCode.HAS_CHILDREN if operation_type is TransactionOperation.DELETE else ErrorCode.PARENT_NOT_FOUND @@ -1278,15 +1461,24 @@ class CustomerHierarchy: return await self.transaction(lambda store: store.list_organizations(page, page_size, search)) async def list_teams( - self, parent_org_id: str | None = None, page: int = 1, page_size: int = 50 + self, parent_org_id: str | None = None, page: int = 1, page_size: int = 50, sort: BudgetSort | None = None ) -> Page[Team] | Failure: - return await self.transaction(lambda store: store.list_teams(parent_org_id, page, page_size)) + return await self.transaction(lambda store: store.list_teams(parent_org_id, page, page_size, sort)) async def list_members( - self, parent_team_id: str | None = None, page: int = 1, page_size: int = 50, *, parent_org_id: str | None = None + self, + parent_team_id: str | None = None, + page: int = 1, + page_size: int = 50, + *, + parent_org_id: str | None = None, + search: str = "", + sort: BudgetSort | None = None, ) -> Page[CustomerMember] | Failure: return await self.transaction( - lambda store: store.list_members(parent_team_id, page, page_size, parent_org_id=parent_org_id) + lambda store: store.list_members( + parent_team_id, page, page_size, parent_org_id=parent_org_id, search=search, sort=sort + ) ) async def add_template(self, organization_id: str, payload: BudgetTemplate, actor: str) -> Organization | Failure: diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 21d352aa45a..30034ef1a4e 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -460,6 +460,8 @@ async def _resolve_member_budget_id( changes reach them; ``/team/member_update`` clones that row on first write. A lone ``budget_duration`` clones the default with the reset window overridden, or creates a window-only budget when there is no team default. + ``allowed_models`` with no cap creates a budget that takes the default's reset + window and leaves the cap to the default. With nothing set the member gets no budget, though ``add_new_member`` still writes its membership row. """ has_explicit_limit: Final = max_budget_in_team is not None or allowed_models is not None @@ -493,14 +495,34 @@ async def _resolve_member_budget_id( budget_data["max_budget"] = max_budget_in_team if allowed_models is not None: budget_data["allowed_models"] = allowed_models - if budget_duration is not None: - budget_data["budget_duration"] = budget_duration - budget_data["budget_reset_at"] = get_budget_reset_time(budget_duration=budget_duration) + member_budget_duration: Final = budget_duration or ( + await _team_default_budget_duration(prisma_client, default_team_budget_id, tx) + if max_budget_in_team is None and default_team_budget_id is not None + else None + ) + if member_budget_duration is not None: + budget_data["budget_duration"] = member_budget_duration + budget_data["budget_reset_at"] = get_budget_reset_time(budget_duration=member_budget_duration) budget_table: Final[_PrismaBudgetTable] = _budget_table(prisma_client, tx) response: Final = await budget_table.create(data=budget_data) return response.budget_id +async def _team_default_budget_duration( + prisma_client: PrismaClient, default_team_budget_id: str, tx: MemberWriteTx | None +) -> str | None: + """The reset period a member with models of their own and no cap of their own must carry. + + Their cap is read from the team default at auth time, but spend is only reset for a + member whose own budget row has a reset date. + """ + default_budget: Final = await _budget_table(prisma_client, tx).find_unique( + where={"budget_id": default_team_budget_id} + ) + duration: Final = default_budget.model_dump().get("budget_duration") if default_budget is not None else None + return duration if isinstance(duration, str) and duration else None + + async def _append_team_id_if_absent( prisma_client: PrismaClient, user_id: str, team_id: str, tx: MemberWriteTx | None = None ) -> None: diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index f7ecda99d9d..90e6ff369e7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -598,6 +598,13 @@ def _fal_target(endpoint: str) -> httpx.URL: ) +def _fal_upstream_endpoint(target: httpx.URL) -> str: + base_url: Final = httpx.URL(fal_ai_queue_base()) + base_path: Final = base_url.copy_with(path=base_url.path).raw_path.partition(b"?")[0].decode("ascii").strip("/") + upstream_path: Final = target.raw_path.partition(b"?")[0].decode("ascii").lstrip("/") + return upstream_path.removeprefix(f"{base_path}/") if base_path else upstream_path + + @router.api_route( "/fal_ai/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], @@ -619,10 +626,14 @@ async def fal_ai_proxy_route( status_code=401, detail="FAL_AI_API_KEY is not set and no fal_ai pass-through deployment credentials are configured", ) - if "/requests/" not in endpoint and fal_ai_passthrough_cost(endpoint, await read_request_body(request)) is None: + upstream_endpoint: Final = _fal_upstream_endpoint(updated_url) + if ( + "/requests/" not in upstream_endpoint + and fal_ai_passthrough_cost(upstream_endpoint, await read_request_body(request)) is None + ): raise HTTPException( status_code=400, - detail=f"fal_ai/{endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai", + detail=f"fal_ai/{upstream_endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai", ) endpoint_func: Final = create_pass_through_route( endpoint=endpoint, diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index f252e69d739..a56f6d9f00d 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -79,7 +79,7 @@ def _build_db_connection_url_params( socket_timeout: float | None = None, disable_prepared_statements: bool = False, extra_params: dict | None = None, -) -> dict: +) -> dict[str, str | int | float]: """Build the Prisma DATABASE_URL query params controlling connection pool behavior. `connect_timeout` / `socket_timeout` map to the Prisma URL params of the same @@ -91,7 +91,7 @@ def _build_db_connection_url_params( `extra_params` is an untyped passthrough — keys it provides win over the named arguments above, so it can be used to override any default we set here. """ - params: Final[dict] = { + params: Final[dict[str, str | int | float]] = { "connection_limit": connection_limit, } if pool_timeout is not None: @@ -1637,11 +1637,13 @@ def run_server( db_connection_timeout = LiteLLMDatabaseConnectionPool.database_connection_pool_timeout.value if os.getenv("DATABASE_URL", None) is not None or os.getenv("DIRECT_URL", None) is not None: + from litellm.proxy.db.connection_budget import ConnectionBudget from litellm.proxy.db.db_url_settings import ( DISABLE_PREPARED_STATEMENTS_ENV_VAR, add_missing_query_params, idle_lifetime_params, postgres_connection_budget_message, + prisma_default_params, reader_shareable_params, translate_libpq_ssl_params, unsupported_db_scheme, @@ -1676,7 +1678,13 @@ def run_server( disable_prepared_statements=disable_prepared_statements, extra_params=db_extra_connection_params, ) + writer_budget: Final = ConnectionBudget.of(connection_url_params["connection_limit"]) + rust_writer_connections: Final = 0 if writer_budget is None else writer_budget.rust + prisma_url_params: Final = ( + connection_url_params if writer_budget is None else writer_budget.prisma_params(connection_url_params) + ) lifetime_params: Final = idle_lifetime_params(general_settings.get("database_max_idle_connection_lifetime")) + prisma_defaults: Final = prisma_default_params(lifetime_params, rust_writer_connections) if os.getenv("DATABASE_URL", None) is not None: database_url = get_secret("DATABASE_URL", default_value=None) resolved_url: Final[str | None] = str(database_url) if database_url else None @@ -1693,10 +1701,10 @@ def run_server( ) modified_url = append_query_params( writer_url, - connection_url_params, + dict(prisma_url_params), ) os.environ["DATABASE_URL"] = translate_libpq_ssl_params( - add_missing_query_params(modified_url, lifetime_params) + add_missing_query_params(modified_url, prisma_defaults) ) if os.getenv("DIRECT_URL", None) is not None: database_url = os.getenv("DIRECT_URL") @@ -1710,6 +1718,14 @@ def run_server( # on the writer. Anything pinned on the replica URL wins, unlike the # writer where the config is applied on top. read_replica_url: Final[str | None] = os.getenv("DATABASE_URL_READ_REPLICA") + reader_budget: Final = ( + ConnectionBudget.of( + _url_query_value(read_replica_url, "connection_limit") or connection_url_params["connection_limit"] + ) + if read_replica_url + else None + ) + reader_prisma_limit: Final = None if reader_budget is None else reader_budget.prisma_limit if read_replica_url: reader_options: Final[str] = _pg_options_with_timeouts( _url_query_value(read_replica_url, "options"), @@ -1717,27 +1733,34 @@ def run_server( db_lock_timeout, db_idle_in_transaction_timeout, ) + reader_url: Final = ( + _with_query_value(read_replica_url, "options", reader_options) + if reader_options + else read_replica_url + ) os.environ["DATABASE_URL_READ_REPLICA"] = translate_libpq_ssl_params( add_missing_query_params( add_missing_query_params( - _with_query_value(read_replica_url, "options", reader_options) - if reader_options - else read_replica_url, - reader_shareable_params(connection_url_params), + reader_url + if reader_prisma_limit is None + else _with_query_value(reader_url, "connection_limit", str(reader_prisma_limit)), + reader_shareable_params(prisma_url_params), ), - lifetime_params, + prisma_defaults, ) ) print_startup_message( postgres_connection_budget_message( writer_limit=_url_query_value(os.getenv("DATABASE_URL"), "connection_limit") or f"{db_connection_pool_limit}", + rust_writer_limit=rust_writer_connections, reader_limit=( _url_query_value(os.getenv("DATABASE_URL_READ_REPLICA"), "connection_limit") or f"{db_connection_pool_limit}" ) if read_replica_url else None, + rust_reader_limit=0 if reader_budget is None else reader_budget.rust, num_workers=f"{1 if run_hypercorn else num_workers}", ) ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b161fef588a..cf0306ab005 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -213,7 +213,7 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import ( snapshot_tuning_baselines, tuning_limit_violation, ) -from litellm.rust_bridge.catalog import Target +from litellm.rust_bridge.catalog import NATIVE_MESSAGES_TARGETS, Target from litellm.types.utils import ( PRICING_OVERRIDES_KEY, CallTypes, @@ -681,6 +681,7 @@ from litellm.proxy.config_resolvers.settings_rules import ( from litellm.proxy.config_resolvers.sso import SSO_FIELD_ENV_VARS from litellm.proxy.container_endpoints.endpoints import router as container_router from litellm.proxy.credential_endpoints.endpoints import router as credential_router +from litellm.proxy.db.connection_budget import RUST_RESERVED_CONNECTIONS from litellm.proxy.db.create_views import SupportsRawQueries from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import ( @@ -1620,7 +1621,12 @@ async def _connect_to_count_stored_values(resources: AsyncExitStack) -> Supports async def _own_database(resources: AsyncExitStack, client: PrismaClient | None) -> database_runtime.Database | None: if client is None: return None - database: Final = await database_runtime.Database.open(client) + database: Final = await database_runtime.Database.open( + client, + os.environ["DATABASE_URL"], + os.environ.get("DATABASE_URL_READ_REPLICA") or None, + RUST_RESERVED_CONNECTIONS, + ) resources.push_async_callback(database.aclose) return database @@ -7446,7 +7452,7 @@ class ProxyConfig: if key == "request_timeout": litellm.request_timeout_explicitly_set = True if key in {"s3_audit_callback_params", "s3_callback_params"}: - from litellm.integrations.s3_v3 import S3Logger as S3V2Logger + from litellm.integrations.s3_v2 import S3Logger as S3V2Logger from litellm.litellm_core_utils.litellm_logging import ( _in_memory_loggers, ) @@ -16124,7 +16130,6 @@ def _count_context_management( _GATEWAY_EDIT_ROUTES: Final = frozenset({Target.CHAT_COMPLETIONS, Target.RESPONSES}) -_MESSAGES_ROUTES: Final = frozenset({Target.MESSAGES, Target.MESSAGES_PASSTHROUGH}) def _messages_route(deployment: _TokenCountDeployment, request_model: str) -> Target: @@ -16663,7 +16668,7 @@ async def _count_request_tokens( local_view: Final = ( None - if route not in _MESSAGES_ROUTES or context_edits is None or request_messages is None + if route not in NATIVE_MESSAGES_TARGETS or context_edits is None or request_messages is None else await _bridged_request_view( request_messages, system, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index db88b24dcb0..9e1f47a5d1f 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -85,6 +85,8 @@ model LiteLLM_CustomerTeamTable { updated_at DateTime @default(now()) @updatedAt updated_by String @@index([parent_org_id]) + @@index([budget_id]) + @@index([budget_override_id]) @@index([spend]) } @@ -107,6 +109,9 @@ model LiteLLM_CustomerMemberTable { updated_at DateTime @default(now()) @updatedAt updated_by String @@index([parent_org_id]) + @@index([parent_team_ids], type: Gin) + @@index([budget_id]) + @@index([budget_override_id]) @@index([spend]) } @@ -2308,7 +2313,7 @@ enum AccountingDeliveryState { } model LiteLLM_SpendLogDelivery { - delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + delivery_id String @id @default(uuid()) request_id String state AccountingDeliveryState @default(pending) accounting_payload Json diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 0bde66b5cf8..2ceaba4171c 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -23,6 +23,7 @@ from litellm.litellm_core_utils.duration_parser import ( duration_in_seconds, get_annual_budget_window_start, is_annual_budget_duration, + normalize_budget_window_start, ) from litellm.litellm_core_utils.duration_parser import ( get_budget_window_start as calendar_budget_window_start, @@ -30,7 +31,6 @@ from litellm.litellm_core_utils.duration_parser import ( from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate from litellm.models.budget import BudgetLimit from litellm.proxy._types import ( - Litellm_EntityType, LiteLLM_TeamMembership, LiteLLM_TeamTable, LiteLLM_UserTable, @@ -60,6 +60,11 @@ from litellm.proxy.spend_tracking.input_tokens import ( models_counted_for, openai_chat_pdf_file_tokens, ) +from litellm.proxy.spend_tracking.native_budgets import ( + new_reservation_id, + reserve_native, + settle_native_reservation, +) from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.redis.stores.spend_counters import ( @@ -145,21 +150,6 @@ def _customer_window_counter( ) -_COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = { - "Key": Litellm_EntityType.KEY.value, - "Team": Litellm_EntityType.TEAM.value, - "TeamMember": Litellm_EntityType.TEAM_MEMBER.value, - "User": Litellm_EntityType.USER.value, - "EndUser": Litellm_EntityType.END_USER.value, - "Tag": Litellm_EntityType.TAG.value, - "Model access group": Litellm_EntityType.MODEL_ACCESS_GROUP.value, - "Organization": Litellm_EntityType.ORGANIZATION.value, - "Project": Litellm_EntityType.PROJECT.value, - "customer_org": Litellm_EntityType.CUSTOMER_ORG.value, - "customer_team": Litellm_EntityType.CUSTOMER_TEAM.value, - "customer_member": Litellm_EntityType.CUSTOMER_MEMBER.value, -} - _CACHED_VALUE: Final = TypeAdapter(object) _WINDOW_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) @@ -203,7 +193,11 @@ def get_reserved_counter_keys(budget_reservation: dict | None) -> set: _lease_renewals: Final[set[asyncio.Task[None]]] = set() # mutable-ok: asyncio only weak-refs pending tasks -def _start_reservation_lease_renewal(budget_reservation: Mapping[str, object], counter_keys: frozenset[str]) -> None: +def start_reservation_lease_renewal( + budget_reservation: Mapping[str, object], + counter_keys: frozenset[str], + request_task: asyncio.Task[object] | None = None, +) -> None: """A reservation lives inside spend counter keys that expire on their Redis TTL. Renew the TTL while the request is in flight so a request longer than the TTL does not drop its reservation and admit concurrent requests against the DB floor on any worker.""" @@ -217,7 +211,7 @@ def _start_reservation_lease_renewal(budget_reservation: Mapping[str, object], c budget_reservation=budget_reservation, counter_keys=counter_keys, interval=redis_ttl / 2, - request_task=asyncio.current_task(), + request_task=request_task or asyncio.current_task(), ) ) _lease_renewals.add(task) @@ -243,80 +237,6 @@ async def _renew_reservation_lease( await refresh_spend_counter_ttl(counter_key=counter_key) -def _key_reservation_should_release_for_throttle(counter_key: str, valid_token: UserAPIKeyAuth | None) -> bool: - """ - Whether an over-budget key's own ``max_budget`` reservation should be - released rather than blocked, because the key opted into throttling: the - rate limiter slows it instead. Only the key's own ``max_budget`` counter is - exempt; team/user/window counters still enforce normally, and under-budget - requests never reach this branch so their concurrent-overspend protection is - untouched. - """ - if valid_token is None: - return False - return counter_key == key_counter(str(valid_token.token)) and should_throttle_budget_exceeded(valid_token) - - -async def _apply_over_budget_reservation_policy( - counter: _BudgetCounter, - valid_token: UserAPIKeyAuth | None, - entry: dict[str, float | str], - applied_entries: list[dict[str, float | str]], - reservation_cost: float, - current_spend: float, - fail_closed_budget_enforcement: bool = False, -) -> float: - """ - Decide what to do when a counter is over budget, and return the reservation - cost to carry into the next counter. Three outcomes: an over-budget key that - opted into throttling releases its own reservation (the rate limiter slows - it) and keeps the cost; a partially-remaining budget resizes the reservation - down to what is left, unless strict enforcement is on, because the known - estimate already does not fit; anything else hard-blocks by raising. - """ - if _key_reservation_should_release_for_throttle(counter.counter_key, valid_token): - await _release_applied_entries_best_effort(entries=[entry], default_reserved_cost=reservation_cost) - applied_entries.remove(entry) - return reservation_cost - - remaining_before_reservation: Final = counter.max_budget - (current_spend - reservation_cost) - if remaining_before_reservation <= 1e-12: - _raise_counter_budget_exceeded(counter=counter, current_cost=current_spend) - if fail_closed_budget_enforcement and current_spend - counter.max_budget > 1e-12: - _raise_counter_budget_exceeded( - counter=counter, - current_cost=current_spend - reservation_cost, - estimated_cost=reservation_cost, - ) - await _resize_applied_reservation( - entries=applied_entries, - current_reserved_cost=reservation_cost, - new_reserved_cost=remaining_before_reservation, - ) - return remaining_before_reservation - - -def _raise_counter_budget_exceeded( - counter: _BudgetCounter, - current_cost: float, - estimated_cost: float | None = None, -) -> NoReturn: - estimate_detail: Final = "" if estimated_cost is None else f"Estimated request cost: {estimated_cost}, " - raise litellm.BudgetExceededError( - current_cost=current_cost, - max_budget=counter.max_budget, - message=( - "Budget has been exceeded! " - f"{counter.entity_type}={counter.entity_id} " - f"Current cost: {current_cost}, " - f"{estimate_detail}" - f"Max budget: {counter.max_budget}" - ), - entity_type=_COUNTER_ENTITY_TYPES.get(counter.entity_type), - entity_id=counter.spend_log_entity_id or counter.entity_id, - ) - - async def reserve_budget_for_request( request_body: dict, route: str, @@ -365,7 +285,7 @@ async def reserve_budget_for_request( path_params=path_params, ) - reservation_cost = estimate_request_max_cost( + reservation_cost: Final = estimate_request_max_cost( request_body=request_body, route=route, llm_router=llm_router, @@ -377,51 +297,64 @@ async def reserve_budget_for_request( # In that case we fall back to read-time enforcement only. if reservation_cost is None or reservation_cost <= 0: return None - - applied_entries: Final[list[dict[str, float | str]]] = [] - try: - with _counters_batch_scope(frozenset(counter.counter_key for counter in counters)): - reservable: Final = await _initialize_reservation_counters( - counters=counters, - fail_closed_budget_enforcement=fail_closed_budget_enforcement, - ) - reservation_cost = await _reserve_reservable_counters( - reservable=reservable, - valid_token=valid_token, - applied_entries=applied_entries, - reservation_cost=reservation_cost, - fail_closed_budget_enforcement=fail_closed_budget_enforcement, - ) - except Exception: - await _release_applied_entries_best_effort( - entries=applied_entries, - default_reserved_cost=reservation_cost, - ) - raise - - if not applied_entries: - return None - - input_cost: Final = estimate_request_input_cost( - request_body=request_body, - route=route, - llm_router=llm_router, + return await _native_budget_plan( + counters=counters, + valid_token=valid_token, + reservation_cost=reservation_cost, input_token_counts=input_token_counts, - path_params=path_params, + input_cost=estimate_request_input_cost( + request_body=request_body, + route=route, + llm_router=llm_router, + input_token_counts=input_token_counts, + path_params=path_params, + ), + fail_closed_budget_enforcement=fail_closed_budget_enforcement, ) - budget_reservation: Final = { - "reserved_cost": reservation_cost, - "entries": applied_entries, - "finalized": False, - "callback_bound": False, - "input_cost": min(float(input_cost or 0.0), reservation_cost), - "input_tokens": max(input_token_counts.values(), default=None), - } - _start_reservation_lease_renewal( - budget_reservation=budget_reservation, - counter_keys=frozenset(get_reserved_counter_keys(budget_reservation=budget_reservation)), + + +async def _native_budget_plan( + counters: Sequence[_BudgetCounter], + valid_token: UserAPIKeyAuth, + reservation_cost: float, + input_token_counts: Mapping[str, int], + input_cost: float | None, + fail_closed_budget_enforcement: bool, +) -> dict[str, object] | None: # mutable-ok: settlement stamps finalized on the reservation it returns + with _counters_batch_scope(frozenset(counter.counter_key for counter in counters)): + reservable: Final = await _initialize_reservation_counters( + counters=counters, + fail_closed_budget_enforcement=fail_closed_budget_enforcement, + ) + if not reservable: + return None + input_tokens: Final = max(input_token_counts.values(), default=None) + return await reserve_native( + { + "counters": [ + { + "counter_key": counter.counter_key, + "max_budget": counter.max_budget, + "fallback_spend": counter.fallback_spend, + "entity_type": counter.entity_type, + "entity_id": counter.entity_id, + "spend_log_entity_id": counter.spend_log_entity_id, + } + for counter in reservable + ], + "reservation_cost": float(reservation_cost), + "input_cost": float(input_cost or 0.0), + "input_tokens": input_tokens, + "throttle_counter": key_counter(str(valid_token.token)) + if should_throttle_budget_exceeded(valid_token) + else None, + "fail_closed": fail_closed_budget_enforcement, + "status_code": int(litellm.budget_exceeded_status_code), + "reservation_id": new_reservation_id(), + }, + valid_token, + {counter.counter_key: counter.reset_at.isoformat() for counter in reservable if counter.reset_at is not None}, ) - return budget_reservation async def reconcile_budget_reservation( @@ -430,19 +363,11 @@ async def reconcile_budget_reservation( finalize: bool = True, apply_consistent: bool = True, ) -> tuple[PendingSpendIncrement, ...]: - """Settle every reserved counter on ``actual_cost``. With ``apply_consistent`` False the adjustments for - counters that still hold the reservation are returned instead of written, so the caller can pipeline them with - its own increments and then call ``stamp_budget_reservation_actual_cost``.""" if not budget_reservation or budget_reservation.get("finalized") is True: return () - - reserved_cost: Final = float(budget_reservation.get("reserved_cost") or 0.0) - actual: Final = float(actual_cost or 0.0) - pending: Final = await _set_reserved_entries_actual_cost( - entries=budget_reservation.get("entries") or [], - actual_cost=actual, - default_reserved_cost=reserved_cost, - apply_consistent=apply_consistent, + _ = float(budget_reservation.get("reserved_cost") or 0.0) + pending: Final = await settle_native_reservation( + budget_reservation, float(actual_cost or 0.0), apply_consistent=apply_consistent ) if finalize: budget_reservation["finalized"] = True @@ -1088,121 +1013,6 @@ async def _initialize_reservation_counter(counter: _BudgetCounter) -> None: raise _CounterReservationUnavailable -async def _reserve_reservable_counters( - reservable: Sequence[_BudgetCounter], - valid_token: UserAPIKeyAuth | None, - applied_entries: list[dict[str, float | str]], - reservation_cost: float, - fail_closed_budget_enforcement: bool, -) -> float: - """Charge the counters group by group (see ``_reservation_groups``), settling the over-budget policy on each - group before the next is charged, and hand back the reservation cost the policy left standing.""" - current_spend_by_counter_key: Final = { - counter.counter_key: await _get_current_counter_value(counter=counter) for counter in reservable - } - for group in _reservation_groups( - counters=reservable, - current_spend_by_counter_key=current_spend_by_counter_key, - reservation_cost=reservation_cost, - ): - charged_cost = reservation_cost - entries = tuple(_counter_to_reservation_entry(counter=counter, reserved_cost=charged_cost) for counter in group) - applied_entries.extend(entries) - reserved_values = await _reserve_counters(counters=group, entries=entries, reservation_cost=charged_cost) - if reserved_values is None: - for entry in entries: - applied_entries.remove(entry) - if fail_closed_budget_enforcement: - _raise_reservation_unavailable(counter_key=group[0].counter_key) - continue - for counter, entry, reserved_value in zip(group, entries, reserved_values): - if entry not in applied_entries: - continue - if reserved_value is not None: - current_spend = reserved_value - (charged_cost - reservation_cost) - else: - current_spend = current_spend_by_counter_key[counter.counter_key] + reservation_cost - if current_spend > counter.max_budget: - reservation_cost = await _apply_over_budget_reservation_policy( - counter=counter, - valid_token=valid_token, - entry=entry, - applied_entries=applied_entries, - reservation_cost=reservation_cost, - current_spend=current_spend, - fail_closed_budget_enforcement=fail_closed_budget_enforcement, - ) - return reservation_cost - - -def _reservation_groups( - counters: Sequence[_BudgetCounter], - current_spend_by_counter_key: Mapping[str, float], - reservation_cost: float, -) -> tuple[tuple[_BudgetCounter, ...], ...]: - """Every counter the batch read says still has room for the estimate is charged in one pipeline. As soon as one - does not, the counters are charged one at a time so the over-budget policy settles each before the next is - touched, and a rejection charges nothing after it.""" - if not counters: - return () - if all( - current_spend_by_counter_key[counter.counter_key] + reservation_cost <= counter.max_budget - for counter in counters - ): - return (tuple(counters),) - return tuple((counter,) for counter in counters) - - -async def _reserve_counters( - counters: Sequence[_BudgetCounter], - entries: Sequence[dict[str, float | str]], - reservation_cost: float, -) -> tuple[float | None, ...] | None: - """One INCRBYFLOAT pipeline reserves every counter. When it fails each counter is dropped, and one that cannot - be dropped is released instead in case its increment landed, so nothing is left to release by the caller.""" - from litellm.proxy.proxy_server import invalidate_spend_counter, run_spend_counter_pipeline - - if not counters: - return () - try: - reserved: Final = await run_spend_counter_pipeline( - pending=tuple( - PendingSpendIncrement(counter_key=counter.counter_key, increment=reservation_cost) - for counter in counters - ) - ) - except Exception: - verbose_proxy_logger.warning( - "Skipping budget reservation for %s because spend counter reservation failed", - tuple(counter.counter_key for counter in counters), - exc_info=True, - ) - for counter, entry in zip(counters, entries): - try: - await invalidate_spend_counter(counter_key=counter.counter_key) - except Exception: - verbose_proxy_logger.warning( - "Failed to invalidate spend counter after budget reservation failure for %s", - counter.counter_key, - exc_info=True, - ) - await _release_applied_entries_best_effort( - entries=[entry], - default_reserved_cost=reservation_cost, - ) - return None - return tuple(reserved) + (None,) * (len(counters) - len(reserved)) - - -async def _get_current_counter_value(counter: _BudgetCounter) -> float: - from litellm.proxy.proxy_server import get_current_spend - - return await get_current_spend( - counter_key=counter.counter_key, - fallback_spend=counter.fallback_spend, - ) - - def _counters_batch_scope(counter_keys: frozenset[str]) -> spend_counter_batch_scope: """Each counter is read once, then written, so one MGET up front serves every read in the loop.""" from litellm.proxy.proxy_server import spend_counter_store @@ -1210,176 +1020,17 @@ def _counters_batch_scope(counter_keys: frozenset[str]) -> spend_counter_batch_s return spend_counter_batch_scope(spend_counter_store, counter_keys=counter_keys) -@dataclass(frozen=True, slots=True) -class _EntryAdjustment: - entry: dict[str, float | str] - counter_key: str - target_adjustment: float - adjustment: float - actual_cost: float - - def _entry_settled_cost(entry: Mapping[str, object], actual_cost: float, settlement_time: datetime | None) -> float: reset_at: Final = _coerce_datetime(entry.get("window_reset_at")) return 0.0 if settlement_time is not None and reset_at is not None and reset_at <= settlement_time else actual_cost -def _entry_adjustment( - entry: dict[str, float | str], - actual_cost: float, - default_reserved_cost: float, - settlement_time: datetime | None = None, -) -> _EntryAdjustment | None: - counter_key: Final = entry.get("counter_key") - if counter_key is None: - return None - settled_cost: Final = _entry_settled_cost(entry, actual_cost, settlement_time) - target_adjustment: Final = settled_cost - _get_entry_reserved_cost( - entry=entry, default_reserved_cost=default_reserved_cost - ) - adjustment: Final = target_adjustment - float(entry.get("applied_adjustment") or 0.0) - if adjustment == 0: - return None - return _EntryAdjustment( - entry=entry, - counter_key=str(counter_key), - target_adjustment=target_adjustment, - adjustment=adjustment, - actual_cost=settled_cost, - ) - - -async def _set_reserved_entries_actual_cost( - entries: list[dict], - actual_cost: float, - default_reserved_cost: float, - reseed_on_inconsistent: bool = True, - apply_consistent: bool = True, -) -> tuple[PendingSpendIncrement, ...]: - """Every reserved counter is read from one MGET and the consistent adjustments go out in one pipeline, or are - returned unwritten when ``apply_consistent`` is False. A counter that was flushed or reseeded since reservation - is settled on its own after the pipeline.""" - from litellm.proxy.proxy_server import increment_spend_counters_pipeline - - settlement_time: Final = datetime.now(timezone.utc) if reseed_on_inconsistent else None - with _counters_batch_scope(frozenset(str(entry["counter_key"]) for entry in entries if "counter_key" in entry)): - adjustments: Final = tuple( - adjustment - for entry in entries - if (adjustment := _entry_adjustment(entry, actual_cost, default_reserved_cost, settlement_time)) is not None - ) - consistent: Final = tuple( - await asyncio.gather( - *( - _counter_can_apply_adjustment(counter_key=item.counter_key, adjustment=item.adjustment) - for item in adjustments - ) - ) - ) - inconsistent: Final = tuple(item for item, ok in zip(adjustments, consistent) if not ok) - if inconsistent and not reseed_on_inconsistent: - # Pre-call admission resize: the in-flight reservation cost is not yet - # persisted, so the DB floor would discard it. Keep the original - # fail-closed behavior (raise -> reserve_budget_for_request releases and - # denies) rather than admitting against an inconsistent counter. - raise RuntimeError( - f"Cannot resize budget reservation against inconsistent counter {inconsistent[0].counter_key}" - ) - applicable: Final = tuple(item for item, ok in zip(adjustments, consistent) if ok) - applicable_pending: Final = tuple( - PendingSpendIncrement(counter_key=item.counter_key, increment=item.adjustment) for item in applicable - ) - if apply_consistent: - await increment_spend_counters_pipeline(pending=applicable_pending) - for item in inconsistent: - await _reseed_reserved_entry(item=item, actual_cost=item.actual_cost) - for item in adjustments if apply_consistent else inconsistent: - item.entry["applied_adjustment"] = item.target_adjustment - return () if apply_consistent else applicable_pending - - -async def _reseed_reserved_entry(item: _EntryAdjustment, actual_cost: float) -> None: - """Post-call reconcile / release of a counter that was flushed, expired or reseeded between reservation and - reconcile: the optimistic delta no longer applies, so reseed from the DB floor and add the settled cost, since - increment_spend_counters skips reserved keys. The reconcile runs before this request's spend is enqueued to the - DB, so the reseeded floor excludes it.""" +async def reseed_reserved_counter(counter_key: str, actual_cost: float) -> None: from litellm.proxy.proxy_server import increment_spend_counter_cache, reseed_spend_counter_from_db - reseeded: Final = await reseed_spend_counter_from_db(counter_key=item.counter_key) + reseeded: Final = await reseed_spend_counter_from_db(counter_key=counter_key) if reseeded and actual_cost > 0: - await increment_spend_counter_cache(counter_key=item.counter_key, increment=actual_cost) - - -async def _counter_can_apply_adjustment( - counter_key: str, - adjustment: float, -) -> bool: - from litellm.proxy.proxy_server import read_spend_counter_cache_value - - try: - current_value, _ = await read_spend_counter_cache_value(counter_key=counter_key) - except (TypeError, ValueError): - return False - if current_value is None: - return False - - return not (adjustment < 0 and current_value + adjustment < -1e-12) - - -async def _release_applied_entries_best_effort( - entries: list[dict], - default_reserved_cost: float, -) -> None: - for entry in entries: - try: - await _set_reserved_entries_actual_cost( - entries=[entry], - actual_cost=0.0, - default_reserved_cost=default_reserved_cost, - ) - except Exception: - counter_key = entry.get("counter_key") - verbose_proxy_logger.exception("Failed to release partial budget reservation during exception cleanup") - if counter_key is None: - continue - try: - from litellm.proxy.proxy_server import invalidate_spend_counter - - await invalidate_spend_counter(counter_key=counter_key) - except Exception: - verbose_proxy_logger.exception( - "Failed to invalidate partial budget reservation counter during exception cleanup" - ) - - -async def _resize_applied_reservation( - entries: list[dict], - current_reserved_cost: float, - new_reserved_cost: float, -) -> None: - await _set_reserved_entries_actual_cost( - entries=entries, - actual_cost=new_reserved_cost, - default_reserved_cost=current_reserved_cost, - reseed_on_inconsistent=False, - ) - for entry in entries: - entry["reserved_cost"] = new_reserved_cost - entry["applied_adjustment"] = 0.0 - - -def _counter_to_reservation_entry( - counter: _BudgetCounter, - reserved_cost: float, -) -> dict[str, float | str]: - return { - "counter_key": counter.counter_key, - "entity_type": counter.entity_type, - "entity_id": counter.entity_id, - "reserved_cost": reserved_cost, - "applied_adjustment": 0.0, - **({"window_reset_at": counter.reset_at.isoformat()} if counter.reset_at is not None else {}), - } + await increment_spend_counter_cache(counter_key=counter_key, increment=actual_cost) def _get_entry_reserved_cost(entry: dict, default_reserved_cost: float) -> float: @@ -1416,10 +1067,10 @@ def get_budget_window_start(window: object, reset_settings: BudgetResetSettings reset_at = _coerce_datetime(window_dict.get("reset_at")) if reset_at is None: - return datetime.now(timezone.utc) - timedelta(seconds=duration_seconds) + return normalize_budget_window_start(datetime.now(timezone.utc) - timedelta(seconds=duration_seconds)) if reset_at.tzinfo is None: reset_at = reset_at.replace(tzinfo=timezone.utc) - return reset_at - timedelta(seconds=duration_seconds) + return normalize_budget_window_start(reset_at - timedelta(seconds=duration_seconds)) def _coerce_datetime(value: object) -> datetime | None: diff --git a/litellm/proxy/spend_tracking/native_budgets.py b/litellm/proxy/spend_tracking/native_budgets.py new file mode 100644 index 00000000000..bea06fff19f --- /dev/null +++ b/litellm/proxy/spend_tracking/native_budgets.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +import asyncio +import uuid +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING, Annotated, Final, Literal + +from fastapi import HTTPException, status +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from pydantic import Field as PydanticField + +import litellm +from litellm.integrations.custom_logger_v2.derivations import ( + budget_reservation, + mapping, + write_budget_reservation, +) +from litellm.integrations.custom_logger_v2.field_sources import HookArgs +from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth +from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope +from litellm.redis.stores.spend_counters import SpendCounterStore +from litellm.types.integrations.custom_logger_v2 import BudgetPlan, BudgetReservation, coerce_record +from litellm.types.proxy.native_budgets import BudgetCheck, NativeSettlement + +if TYPE_CHECKING: + from litellm.rust_bridge._native import NativeBudgets + + +def new_reservation_id() -> str: + return uuid.uuid4().hex + + +class _BudgetExceededFields(BaseModel): + model_config = ConfigDict(frozen=True) + + current_cost: float + max_budget: float + message: str + entity_type: str | None + entity_id: str | None + + +class _DetailFields(BaseModel): + model_config = ConfigDict(frozen=True) + + detail: str + + +class _MessageFields(BaseModel): + model_config = ConfigDict(frozen=True) + + message: str + + +class _Held(BaseModel): + model_config = ConfigDict(frozen=True) + + admission: Literal["held"] + reservation: Mapping[str, object] + + +class _Skipped(BaseModel): + model_config = ConfigDict(frozen=True) + + admission: Literal["skipped"] + + +class _Refused(BaseModel): + model_config = ConfigDict(frozen=True) + + admission: Literal["refused"] + kind: Literal["budget_exceeded", "budget_unavailable", "budget_reservation_failed"] + fields: Mapping[str, object] + + +_ADMISSION: Final[TypeAdapter[_Held | _Skipped | _Refused]] = TypeAdapter( + Annotated[_Held | _Skipped | _Refused, PydanticField(discriminator="admission")] +) +_SETTLEMENT: Final = TypeAdapter(NativeSettlement) +_CHECK: Final[TypeAdapter[BudgetCheck]] = TypeAdapter(BudgetCheck) +_HELD_ENTRIES: Final = TypeAdapter(Iterable[object]) + + +def _refusal(refused: _Refused) -> Exception: + match refused.kind: + case "budget_exceeded": + exceeded: Final = _BudgetExceededFields.model_validate(refused.fields) + return litellm.BudgetExceededError( + current_cost=exceeded.current_cost, + max_budget=exceeded.max_budget, + message=exceeded.message, + entity_type=exceeded.entity_type, + entity_id=exceeded.entity_id, + ) + case "budget_unavailable": + return HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=_DetailFields.model_validate(refused.fields).detail, + ) + case "budget_reservation_failed": + return ProxyException( + message="Authentication Error, " + _MessageFields.model_validate(refused.fields).message, + type=ProxyErrorTypes.auth_error, + param=None, + code=status.HTTP_401_UNAUTHORIZED, + ) + + +def _with_window_reset(entry: object, window_resets: Mapping[str, str]) -> object: + held: Final = mapping(entry) + if held is None: + return entry + reset: Final = window_resets.get(str(held.get("counter_key"))) + return entry if reset is None else {**held, "window_reset_at": reset} + + +def _with_window_resets(value: Mapping[str, object], window_resets: Mapping[str, str]) -> Mapping[str, object]: + entries: Final = value.get("entries") + if not window_resets or not isinstance(entries, list | tuple): + return value + return { + **value, + "entries": [_with_window_reset(entry, window_resets) for entry in _HELD_ENTRIES.validate_python(entries)], + } + + +def _planned_counters(plan: BudgetPlan) -> tuple[tuple[str, float], ...]: + return tuple( + (counter.counter_key, float(counter.fallback_spend or 0.0)) + for counter in plan.counters or () + if counter.counter_key is not None + ) + + +async def observed_counter_spend(plan: BudgetPlan) -> tuple[Mapping[str, object], ...]: + from litellm.proxy.proxy_server import get_current_spend, spend_counter_store + + counters: Final = _planned_counters(plan) + with spend_counter_batch_scope(spend_counter_store, counter_keys=frozenset(key for key, _ in counters)): + return tuple( + [ + { + "counter_key": counter_key, + "spend": await get_current_spend(counter_key=counter_key, fallback_spend=fallback_spend), + } + for counter_key, fallback_spend in counters + ] + ) + + +_engine: Final[list[tuple[SpendCounterStore, NativeBudgets]]] = [] # mutable-ok: the engine for the proxy's store + + +def native_budgets() -> NativeBudgets | None: + from litellm.proxy.proxy_server import spend_counter_store + + if _engine and _engine[0][0] is spend_counter_store: + return _engine[0][1] + from litellm.rust_bridge._native import NativeBudgets, NativeSpendCounters + + handle: Final = spend_counter_store.handle + if not isinstance(handle, NativeSpendCounters): + return None + budgets: Final = NativeBudgets(handle) + _engine[:] = [(spend_counter_store, budgets)] + return budgets + + +async def reserve_native( + plan: Mapping[str, object], + identity: UserAPIKeyAuth, + window_resets: Mapping[str, str], +) -> dict[str, object] | None: # mutable-ok: settlement stamps the reservation it returns + budgets: Final = native_budgets() + if budgets is None: + return None + record: Final = coerce_record(BudgetPlan, plan) + admission: Final = _ADMISSION.validate_python(await budgets.reserve(record, await observed_counter_spend(record))) + if isinstance(admission, _Refused): + raise _refusal(admission) + if isinstance(admission, _Skipped): + return None + held: Final = _with_window_resets(admission.reservation, window_resets) + reservation: Final[dict[str, object]] = { # mutable-ok: settlement stamps the reservation it returns + "reserved_cost": 0.0, + "entries": [], + "finalized": False, + "callback_bound": False, + "input_cost": 0.0, + "input_tokens": None, + } + identity.budget_reservation = reservation + write_budget_reservation(HookArgs(auth=identity), held) + from litellm.proxy.spend_tracking.budget_reservation import start_reservation_lease_renewal + + start_reservation_lease_renewal( + budget_reservation=reservation, + counter_keys=frozenset( + key for key in map(_held_counter_key, _held_entries(reservation.get("entries") or ())) if key is not None + ), + request_task=asyncio.current_task(), + ) + return reservation + + +@dataclass(frozen=True, slots=True) +class _Reserved: + budget_reservation: object + + +def _reservation_record(reservation: object) -> BudgetReservation | None: + return budget_reservation(HookArgs(auth=_Reserved(reservation))) + + +def _window_reset(entry: object) -> str | None: + held: Final = mapping(entry) + reset: Final = None if held is None else held.get("window_reset_at") + if isinstance(reset, datetime): + return reset.isoformat() + return reset if isinstance(reset, str) else None + + +def _held_entries(raw: object) -> tuple[object, ...]: + try: + return tuple(_HELD_ENTRIES.validate_python(raw)) + except ValidationError: + raise TypeError(f"'{type(raw).__name__}' object is not iterable") from None + + +def _held_counter_key(entry: object) -> str | None: + held: Final = mapping(entry) + counter_key: Final = None if held is None else held.get("counter_key") + return counter_key if isinstance(counter_key, str) else None + + +async def _observed_spend(counter_key: str | None) -> float | None: + if counter_key is None: + return None + from litellm.proxy.proxy_server import read_spend_counter_cache_value + + try: + value, _ = await read_spend_counter_cache_value(counter_key=counter_key) + except (TypeError, ValueError): + return None + return value + + +async def settle_native_reservation( + reservation: Mapping[str, object], + actual_cost: float, + apply_consistent: bool = True, +) -> tuple[PendingSpendIncrement, ...]: + budgets: Final = native_budgets() + if budgets is None: + return () + entries: Final = _held_entries(reservation.get("entries") or ()) + record: Final = _reservation_record(reservation) + if record is None: + return () + from litellm.proxy.proxy_server import spend_counter_store + from litellm.proxy.spend_tracking.budget_reservation import reseed_reserved_counter + + counter_keys: Final = tuple(map(_held_counter_key, entries)) + with spend_counter_batch_scope( + spend_counter_store, counter_keys=frozenset(key for key in counter_keys if key is not None) + ): + observed: Final = tuple([await _observed_spend(counter_key) for counter_key in counter_keys]) + settled: Final = _SETTLEMENT.validate_python( + await budgets.settle(record, tuple(map(_window_reset, entries)), observed, actual_cost, apply_consistent) + ) + for reseed in settled.reseed: + await reseed_reserved_counter(reseed.counter_key, reseed.actual_cost) + for entry, target in zip(entries, settled.adjustments): + if target is not None and isinstance(entry, dict): + entry["applied_adjustment"] = target + return tuple( + PendingSpendIncrement(counter_key=pending.counter_key, increment=pending.increment) + for pending in settled.pending + ) + + +class BudgetDecision(BaseModel): + model_config = ConfigDict(frozen=True) + + decision: Literal["skip", "pass", "alert", "throttle", "refuse"] + pct: int | float | None = None + current_cost: int | float | None = None + max_budget: int | float | None = None + message: str | None = None + entity_type: str | None = None + entity_id: str | None = None + + @property + def skipped(self) -> bool: + return self.decision == "skip" + + @property + def alerts(self) -> bool: + return self.decision == "alert" + + @property + def refused(self) -> bool: + return self.decision in ("refuse", "throttle") + + def error(self) -> litellm.BudgetExceededError: + return litellm.BudgetExceededError( + current_cost=self.current_cost if self.current_cost is not None else 0.0, + max_budget=self.max_budget if self.max_budget is not None else 0.0, + message=self.message, + entity_type=self.entity_type, + entity_id=self.entity_id, + ) + + +def native_budget_decision(check: Mapping[str, object]) -> BudgetDecision: + from litellm.rust_bridge._native import budget_check_decision + + return BudgetDecision.model_validate(budget_check_decision(_CHECK.validate_python(check))) diff --git a/litellm/redis/stores/spend_counters/__init__.py b/litellm/redis/stores/spend_counters/__init__.py index 64d741d6127..1b1fd58524c 100644 --- a/litellm/redis/stores/spend_counters/__init__.py +++ b/litellm/redis/stores/spend_counters/__init__.py @@ -78,6 +78,10 @@ class SpendCounterStore: if redis.client is not None: self._handle.attach(redis.client) + @property + def handle(self) -> SpendCountersHandle: + return self._handle + @property def redis_ttl(self) -> float | None: """The TTL every Redis write gives a counter, None when there is no Redis.""" diff --git a/litellm/repositories/database.py b/litellm/repositories/database.py index d60a2099477..45849a17a4b 100644 --- a/litellm/repositories/database.py +++ b/litellm/repositories/database.py @@ -1,34 +1,110 @@ import asyncio -from collections.abc import AsyncGenerator, Coroutine +import urllib.parse +from collections.abc import AsyncGenerator, Callable, Coroutine from contextlib import asynccontextmanager from dataclasses import dataclass from typing import TYPE_CHECKING, Final -from litellm.rust_bridge._native import NativeDatabase +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.db_span import proxy_service_logging +from litellm.proxy.db.native_database import ( + NativeEndpointSettings, + ServiceLoggingEvents, + TokenRotation, + keep_token_fresh, + reader_token_rotation, + resolve_native_endpoint, + writer_token_rotation, +) +from litellm.rust_bridge._native import NativeDatabase, NativeDatabaseFailure if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient +def _token_refresh( + install: Callable[[str], None], rotation: TokenRotation | None, url: str +) -> "tuple[asyncio.Task[None], ...]": + if rotation is None: + return () + current: Final = urllib.parse.unquote(urllib.parse.urlsplit(url).password or "") + return (asyncio.create_task(keep_token_fresh(install, rotation, current)),) + + +def _log_ignored_options(env_var: str, settings: NativeEndpointSettings) -> None: + if settings.ignored_options: + verbose_proxy_logger.error( + "Native database pool cannot send the %s options %s; only -c name=value settings carry over", + env_var, + settings.ignored_options, + ) + + +async def _warn_if_unreachable(native: NativeDatabase) -> None: + try: + await native.health_check() + except NativeDatabaseFailure as error: + verbose_proxy_logger.warning( + "Native database pool cannot reach the database, so operations ported to it will fail until it" + " connects. Prisma is checked separately. Cause: %s", + error, + ) + + @dataclass(frozen=True, slots=True) class Database: prisma: "PrismaClient" - native: NativeDatabase + native: NativeDatabase | None + tasks: "tuple[asyncio.Task[None], ...]" = () + boot_probe: "asyncio.Task[None] | None" = None @classmethod - async def open(cls, prisma: "PrismaClient") -> "Database": + async def open( + cls, prisma: "PrismaClient", writer_url: str, reader_url: str | None, native_connections: int + ) -> "Database": + if native_connections == 0: + return cls(prisma=prisma, native=None) try: - native: Final = await NativeDatabase.connect() + writer: Final = await resolve_native_endpoint(writer_url, native_connections) + reader: Final = ( + None if reader_url is None else await resolve_native_endpoint(reader_url, native_connections) + ) + native: Final = await NativeDatabase.connect( + ServiceLoggingEvents(loop=asyncio.get_running_loop(), service_logging=proxy_service_logging), + writer.endpoint, + None if reader is None else reader.endpoint, + ) except BaseException: await prisma.aclose() raise - return cls(prisma=prisma, native=native) + _log_ignored_options("DATABASE_URL", writer) + if reader is not None: + _log_ignored_options("DATABASE_URL_READ_REPLICA", reader) + writer_refresh: Final = _token_refresh( + native.set_writer_password, writer_token_rotation(writer_url), writer_url + ) + reader_refresh: Final = ( + () + if reader_url is None + else _token_refresh(native.set_reader_password, reader_token_rotation(reader_url), reader_url) + ) + return cls( + prisma=prisma, + native=native, + tasks=(*writer_refresh, *reader_refresh), + boot_probe=asyncio.create_task(_warn_if_unreachable(native)), + ) async def aclose(self) -> None: + background: Final = (*self.tasks, *(() if self.boot_probe is None else (self.boot_probe,))) + for task in background: + task.cancel() + await asyncio.gather(*background, return_exceptions=True) try: await self.prisma.aclose() finally: - await self.native.close() + if self.native is not None: + await self.native.close() @asynccontextmanager diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index fe6ff6d6806..e8d9eeb6174 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -4434,6 +4434,13 @@ def _log_attempt(plan: Plan, model_name: object, error: BaseException | None = N verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", served_name, model_name) +def _params_base_model(core: RustCore, deployment_id: str) -> str | None: + """``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``.""" + record: Final = as_str_mapping(core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core + base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model") + return base_model if isinstance(base_model, str) else None + + class _Driver: """Answers the attempt and fallback steps of one call, and shapes its result. @@ -4496,7 +4503,7 @@ class _Driver: self._route_phase: contextlib.ExitStack | None = None self._route_model: str = "" self._walk: Final = WalkMetadata( - request, plan.metadata_field, self._params_base_model, live=fresh and not plan.single_attempt + request, plan.metadata_field, partial(_params_base_model, core), live=fresh and not plan.single_attempt ) self.outcomes: Final = FallbackOutcomes(request) # The fallback level the walk is at, from its fallback events. @@ -4548,12 +4555,6 @@ class _Driver: endpoints.counters.retract_success(self._plan, call) endpoints.attempt_failed(self._plan, error, deployment, call, call.group) - def _params_base_model(self, deployment_id: str) -> str | None: - """``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``.""" - record: Final = as_str_mapping(self._core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core - base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model") - return base_model if isinstance(base_model, str) else None - def _logged_error(self, event: Mapping[str, object]) -> BaseException | None: """The error a ``retry_logged`` event records.""" return self._cause_error(event) @@ -5544,6 +5545,7 @@ class _Driver: def finish(self, step: Mapping[str, object]) -> object: """The response with the router's headers, in the order ``make_call`` and the wrappers above it stamp them: a usage read that raises leaves the counts off.""" + self._clear_errors() if not self._plan.sets_headers: self.log_outcomes_sync() return self._last_response @@ -5564,6 +5566,7 @@ class _Driver: async def finish_async(self, step: Mapping[str, object]) -> object: """``finish``, with the usage read on the call's own loop.""" + self._clear_errors() if not self._plan.sets_headers: await await_all(self.outcomes.take_pending()) return self._last_response @@ -5585,6 +5588,18 @@ class _Driver: return self._request.get("messages") is not None or self._request.get("input") is not None def failed(self, step: Mapping[str, object]) -> BaseException: + try: + return self._failure(step) + finally: + self._clear_errors() + + def _clear_errors(self) -> None: + self._last_error = None + self._level_errors.clear() + self._hop_first.clear() + self._level_picks.clear() + + def _failure(self, step: Mapping[str, object]) -> BaseException: model: Final = str(self._request.get("model")) primary: Final = self._level_errors.get(self._base_level) if primary is None: diff --git a/litellm/router_rust/invoke/streaming.py b/litellm/router_rust/invoke/streaming.py index 797f74e0b26..6234c9783b0 100644 --- a/litellm/router_rust/invoke/streaming.py +++ b/litellm/router_rust/invoke/streaming.py @@ -1575,6 +1575,7 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p def __init__(self, async_generator: AsyncGenerator[object, None]) -> None: # pyright: ignore[reportMissingSuperCall] # no httpx response of its own self._async_generator: Final = async_generator + self.attempt: Final = attempt self._billing_source: object = source_iterator self._last_sequence_number = -1 self.response = getattr(source_iterator, "response", None) # pyright: ignore[reportAttributeAccessIssue] # copied from the source @@ -1699,10 +1700,12 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p if response_id_exposed: _raise_released_stream_error(attempt, e) async with contextlib.aclosing( - _responses_fallback_attempt(attempt, e, source_iterator, wrapper.adopt_fallback_headers, held) - ) as fallback_stream: - async for fallback_item in fallback_stream: - yield fallback_item + _responses_recover_stream_error( + attempt, e, source_iterator, wrapper, wrapper.adopt_fallback_headers, held + ) + ) as recovered_stream: + async for recovered_item in recovered_stream: + yield recovered_item except Exception: for held_event in held: yield held_event @@ -1791,6 +1794,99 @@ def _responses_continuation_request( } +def _responses_stream_retries( + attempt: StreamAttempt, e: MidStreamFallbackError, source_iterator: "BaseResponsesAPIStreamingIterator" +) -> bool: + """Whether a broken Responses stream retries in its own group first: it broke before any output and + before a tool call, with an error the retry rule takes (408, 409, 429, 5xx, or no status), as a + ``/v1/messages`` stream does. After output the continuation fallback takes over instead.""" + if not _mid_stream_error_before_output(e) or source_iterator.started_tool_call: + return False + status_code: Final = _stream_raised_error_status(_mid_stream_retry_trigger(e)) + return status_code is None or attempt.should_retry_status(status_code) + + +async def _responses_recover_stream_error( + attempt: StreamAttempt, + e: MidStreamFallbackError, + source_iterator: "BaseResponsesAPIStreamingIterator", + wrapper: object, + adopt_headers: Callable[[object], _Prepared], + held: tuple[object, ...], +) -> AsyncGenerator[object, None]: + """A Responses stream that broke before any output is retried in its own group, then falls back + with the newest failure once the retries are spent. The primary's held lifecycle events are dropped + when a retry or a fallback answers.""" + last_error: MidStreamFallbackError = e # rebind-ok: the newest failure once a retry also failed + if _responses_stream_retries(attempt, e, source_iterator): + if hasattr(source_iterator, "aclose"): + with anyio.CancelScope(shield=True), contextlib.suppress(Exception): + await source_iterator.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + await attempt.failure_recorded() + trigger: Final = _mid_stream_retry_trigger(e) + retried: Final = await attempt.retry(trigger) + if not isinstance(retried, BaseException): + async for retried_item in _responses_yield_retried(retried, wrapper, adopt_headers): + yield retried_item + return + if retried is not trigger: + last_error = _stream_pre_content_error( # rebind-ok: the retry failed again, so its failure is the newest + retried if isinstance(retried, Exception) else e, e.model or "", e.llm_provider + ) + async with contextlib.aclosing( + _responses_fallback_attempt(attempt, last_error, source_iterator, adopt_headers, held) + ) as fallback_stream: + async for fallback_item in fallback_stream: + yield fallback_item + + +async def _responses_yield_retried( + retried: tuple[object, int, int], wrapper: object, adopt_headers: Callable[[object], _Prepared] +) -> AsyncGenerator[object, None]: + """Hands a same-group retry's stream to the client through the wrapper, with the retry's headers, and + closes it after. A retried stream that breaks again counts on from the retries already made, and when + it recovers onto another source (its own fallback) the wrapper and the events follow that source.""" + response, attempted, budget = retried + inner_attempt: Final = getattr(response, "attempt", None) + if isinstance(inner_attempt, StreamAttempt): + inner_attempt.attempted_retries = attempted + inner_attempt.committed_budget = budget + + served_as: Final = _prepare_fallback_hidden_params(wrapper)[1].get("x-litellm-model-group") + + def follow_source() -> _Prepared: + prepared: Final = adopt_headers(response) + if served_as is not None: + _ = prepared[1].setdefault("x-litellm-model-group", served_as) + newest: Final = prepared[1].get("x-litellm-attempted-retries") + _ = add_retry_headers_to_response( + response=wrapper, + attempted_retries=max(attempted, newest) if isinstance(newest, int) else attempted, + max_retries=budget, + ) + return prepared + + followed: _Prepared = follow_source() # rebind-ok: refreshed when the retried stream changes its source + source: object = getattr(response, "_billing_source", None) # rebind-ok: the retried stream's current source + try: + if hasattr(response, "__aiter__"): + async for retried_item in cast("AsyncIterator[object]", response): # cast-ok: __aiter__ checked above + if (current_source := getattr(response, "_billing_source", None)) is not source: + source = current_source + followed = follow_source() + _apply_fallback_hidden_params_to_item(retried_item, followed) + yield retried_item + else: + yield response + finally: + if hasattr(response, "aclose"): + with anyio.CancelScope(shield=True): + try: + await response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + except Exception as exc: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing retry: %s", exc) + + async def _responses_fallback_attempt( attempt: StreamAttempt, e: MidStreamFallbackError, diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py index 48387c5e605..946820f0100 100644 --- a/litellm/router_rust/surface/registry.py +++ b/litellm/router_rust/surface/registry.py @@ -133,6 +133,7 @@ from litellm.types.utils import ( ) from litellm.utils import ( ModelInfoLookupMiss, + base_fields_replaced_by, catalog_with_runtime_registrations, get_potential_model_names, get_secret, @@ -340,10 +341,11 @@ def _inherited_off_peak_base_rates( backend_entry: Final = _backend_cost_entry(host, backend_model, provider) if backend_entry is None: return {} + superseded: Final = base_fields_replaced_by(model_info) return { field: copy.deepcopy(value) for field, value in backend_entry.items() - if model_info.get(field) is None and value is not None + if model_info.get(field) is None and value is not None and field not in superseded } diff --git a/litellm/rust/Cargo.lock b/litellm/rust/Cargo.lock index 811bda4f798..dc09a8c06a0 100644 --- a/litellm/rust/Cargo.lock +++ b/litellm/rust/Cargo.lock @@ -258,6 +258,8 @@ checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319" dependencies = [ "aws-credential-types", "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", @@ -269,11 +271,14 @@ dependencies = [ "aws-types", "bytes", "fastrand", + "hex", "http 1.4.2", + "sha1 0.10.7", "time", "tokio", "tracing", "url", + "zeroize", ] [[package]] @@ -428,6 +433,58 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-sso" +version = "1.111.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.113.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sts" version = "1.108.0" @@ -3455,6 +3512,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", + "chrono", "futures-util", "litellm-auth-types", "litellm-error", @@ -3467,6 +3525,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "tempfile", "thiserror 2.0.19", "tokio", "url", @@ -3705,6 +3764,22 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "litellm-callbacks-budgets" +version = "0.1.0" +dependencies = [ + "litellm-callbacks", + "litellm-coroutine", + "litellm-python-compat", + "litellm-redis-store", + "litellm-tracing", + "rstest", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "litellm-callbacks-legacy-python" version = "0.1.0" @@ -3836,6 +3911,10 @@ name = "litellm-db" version = "0.1.0" dependencies = [ "chrono", + "futures-util", + "litellm-circuit-breaker", + "litellm-error", + "litellm-secret-value", "macro_rules_attribute", "rstest", "serde", @@ -3844,6 +3923,7 @@ dependencies = [ "strum", "thiserror 2.0.19", "tokio", + "url", ] [[package]] @@ -3983,6 +4063,7 @@ dependencies = [ "litellm-auth", "litellm-cache-memory", "litellm-cache-response", + "litellm-core-utils", "litellm-error", "litellm-gateway-auth", "litellm-host", @@ -4248,6 +4329,7 @@ dependencies = [ "litellm-inference-testing", "litellm-llms", "litellm-llms-types", + "litellm-python-compat", "litellm-router-types", "litellm-secret-value", "litellm-secrets", @@ -4370,6 +4452,7 @@ dependencies = [ "aws-smithy-types", "base64 0.22.1", "bytes", + "crc32fast", "data-url", "fancy-regex 0.19.2", "futures-util", @@ -4399,6 +4482,7 @@ dependencies = [ "thiserror 2.0.19", "time", "tokio", + "tokio-util", "url", ] @@ -4467,6 +4551,7 @@ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ "async-trait", + "aws-credential-types", "aws-sdk-secretsmanager", "azure_core", "bytes", @@ -4480,11 +4565,13 @@ dependencies = [ "litellm-cache", "litellm-cache-response", "litellm-callbacks", + "litellm-callbacks-budgets", "litellm-callbacks-legacy-python", "litellm-callbacks-python", "litellm-callbacks-s3-log", "litellm-circuit-breaker", "litellm-core-utils", + "litellm-db", "litellm-error", "litellm-host", "litellm-host-python", diff --git a/litellm/rust/Cargo.toml b/litellm/rust/Cargo.toml index 3fc881102c1..1585150e725 100644 --- a/litellm/rust/Cargo.toml +++ b/litellm/rust/Cargo.toml @@ -38,6 +38,7 @@ litellm-host-native = { path = "crates/host-native" } litellm-callbacks = { path = "crates/callbacks" } litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" } litellm-callbacks-python = { path = "crates/callbacks-python" } +litellm-callbacks-budgets = { path = "crates/callbacks-budgets" } litellm-callbacks-s3-log = { path = "crates/callbacks-s3-log" } litellm-framer = { path = "crates/framer" } litellm-auth = { path = "crates/auth" } diff --git a/litellm/rust/crates/auth-aws/Cargo.toml b/litellm/rust/crates/auth-aws/Cargo.toml index 1a7392afc52..cac0d62e5c1 100644 --- a/litellm/rust/crates/auth-aws/Cargo.toml +++ b/litellm/rust/crates/auth-aws/Cargo.toml @@ -16,12 +16,13 @@ litellm-secret-value.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } sha2.workspace = true +chrono.workspace = true thiserror.workspace = true tokio.workspace = true url.workspace = true veil.workspace = true -aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] } +aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process", "sso"] } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] } aws-sigv4 = "1.5.1" @@ -33,6 +34,7 @@ litellm-secrets.workspace = true aws-smithy-types = "1.6.4" aws-runtime = { version = "1.9.4", default-features = false } rstest.workspace = true +tempfile = "3" litellm-http = { workspace = true, features = ["test-support"] } bytes.workspace = true futures-util.workspace = true diff --git a/litellm/rust/crates/auth-aws/src/aws.rs b/litellm/rust/crates/auth-aws/src/aws.rs index a94a4e8d66b..c088ebc4546 100644 --- a/litellm/rust/crates/auth-aws/src/aws.rs +++ b/litellm/rust/crates/auth-aws/src/aws.rs @@ -2,6 +2,7 @@ use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use aws_credential_types::provider::{SharedCredentialsProvider, error::CredentialsError}; +use aws_smithy_runtime_api::client::http::SharedHttpClient; use litellm_secret_value::SecretValue; use moka::{Expiry, future::Cache}; use std::{future::Future, pin::Pin, sync::Arc}; @@ -16,19 +17,28 @@ use aws_sigv4::sign::v4; use aws_smithy_runtime_api::client::identity::Identity; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; -use litellm_auth_types::AwsParams; +use litellm_auth_types::{AwsParams, AwsSessionTag, ParamSpec}; use super::constants::{ - AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_ROLE_ARN, - AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN, - AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE, - DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES, + AWS_ACCESS_KEY_ID, AWS_DEFAULT_REGION, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION, + AWS_REGION_NAME, AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, + AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, + AWS_WEB_IDENTITY_TOKEN_FILE, BEDROCK_SESSION_POLICY, DEFAULT_BEDROCK_REGION, + DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES, WEB_IDENTITY_SESSION_SECONDS, }; use crate::error::{CredentialFlow, Error, SigningStage}; +use crate::region::{session_region, sts_endpoint_region}; const CREDENTIAL_CACHE_TTL: Duration = Duration::from_secs(600); const EXPIRY_BUFFER: Duration = Duration::from_secs(60); +pub type WebIdentityTokenFuture<'a> = + Pin, litellm_error::Source>> + Send + 'a>>; + +pub trait WebIdentityTokenSource: Send + Sync + std::fmt::Debug { + fn web_identity_token<'a>(&'a self, reference: &'a str) -> WebIdentityTokenFuture<'a>; +} + #[derive(Clone)] pub struct AwsAuthService { providers: Cache, @@ -38,10 +48,7 @@ pub struct AwsAuthService { impl Default for AwsAuthService { fn default() -> Self { - Self::new(Arc::new(NativeProviderLoader { - token: None, - policy: None, - })) + Self::new(Arc::new(NativeProviderLoader::default())) } } @@ -59,13 +66,21 @@ impl AwsAuthService { } } + pub fn with_web_identity_tokens(tokens: Arc) -> Self { + Self::new(Arc::new(NativeProviderLoader { + tokens: Some(tokens), + ..NativeProviderLoader::default() + })) + } + pub fn for_web_identity( resolver: WebIdentityTokenResolver, session_policy: Option, ) -> Self { Self::new(Arc::new(NativeProviderLoader { - token: Some(resolver), - policy: session_policy, + tokens: Some(Arc::new(ResolverWebIdentityTokens(resolver))), + session_policy, + ..NativeProviderLoader::default() })) } } @@ -96,19 +111,99 @@ trait ProviderLoader: Send + Sync { fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_>; } +#[derive(Clone, Default)] struct NativeProviderLoader { - token: Option, - policy: Option, + tokens: Option>, + session_policy: Option, + http_client: Option, + ambient: Option, +} + +impl std::fmt::Debug for NativeProviderLoader { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeProviderLoader") + .finish_non_exhaustive() + } +} + +struct ResolverWebIdentityTokens(WebIdentityTokenResolver); + +impl std::fmt::Debug for ResolverWebIdentityTokens { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ResolverWebIdentityTokens") + .finish_non_exhaustive() + } +} + +impl WebIdentityTokenSource for ResolverWebIdentityTokens { + fn web_identity_token<'a>(&'a self, reference: &'a str) -> WebIdentityTokenFuture<'a> { + let resolve = Arc::clone(&self.0); + let reference = reference.to_owned(); + Box::pin(async move { + tokio::task::spawn_blocking(move || resolve(&reference)) + .await + .map_err(litellm_error::Source::new)? + .map(|token| Some(SecretValue::new(token))) + .map_err(|message| litellm_error::Source::new(std::io::Error::other(message))) + }) + } } impl ProviderLoader for NativeProviderLoader { fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> { - Box::pin(build_provider( - config, - flow, - self.token.clone(), - self.policy.clone(), - )) + Box::pin(build_provider(config, flow, self)) + } +} + +impl NativeProviderLoader { + async fn ambient(&self) -> SharedCredentialsProvider { + match &self.ambient { + Some(provider) => provider.clone(), + None => SharedCredentialsProvider::new( + aws_config::default_provider::credentials::DefaultCredentialsChain::builder() + .build() + .await, + ), + } + } + + async fn sts_config( + &self, + config: &AwsAuthConfig, + region: Option, + ) -> aws_types::SdkConfig { + let loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); + let loader = match region { + Some(region) => loader.region(aws_types::region::Region::new(region)), + None => loader, + }; + let loader = match config.sts_endpoint.clone() { + Some(endpoint) => loader.endpoint_url(endpoint), + None => loader, + }; + let loader = match self.http_client.clone() { + Some(client) => loader.http_client(client), + None => loader, + }; + let loader = match (&config.access_key_id, &config.secret_access_key) { + (Some(key), Some(secret)) => loader.credentials_provider(Credentials::new( + key.expose(), + secret.expose(), + config + .session_token + .as_ref() + .map(|token| token.expose().to_owned()), + None, + "litellm-role-source", + )), + _ => match &self.ambient { + Some(ambient) => loader.credentials_provider(ambient.clone()), + None => loader, + }, + }; + loader.load().await } } @@ -124,6 +219,8 @@ pub struct AwsAuthConfig { pub web_identity_token: Option, pub sts_endpoint: Option, pub external_id: Option, + pub session_tags: Option>, + pub session_policy: Option<&'static str>, } impl AwsAuthConfig { @@ -149,8 +246,20 @@ impl AwsAuthConfig { external_id: self .external_id .or_else(|| env_lookup(AWS_EXTERNAL_ID).map(SecretValue::new)), + session_tags: self.session_tags, + session_policy: self.session_policy, } } + + fn sts_region(&self, env_lookup: &(dyn Fn(&str) -> Option + Sync)) -> Option { + let present = |value: Option| value.filter(|value| !value.is_empty()); + self.sts_endpoint + .as_deref() + .and_then(sts_endpoint_region) + .or_else(|| present(env_lookup(AWS_REGION))) + .or_else(|| present(env_lookup(AWS_DEFAULT_REGION))) + .or_else(|| present(self.region_name.clone())) + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -159,11 +268,13 @@ pub enum AwsAuthFlow { token: SecretValue, role: String, session_name: String, + sts_region: Option, }, AssumeRole { role: String, session_name: Option, web_identity_role: Option, + sts_region: Option, }, Profile { name: String, @@ -213,6 +324,7 @@ pub fn classify_auth( .filter(|token_file| !token_file.is_empty()) .and_then(|_| env_lookup(AWS_ROLE_ARN)); let config = config.with_environment(env_lookup); + let sts_region = config.sts_region(env_lookup); if let (Some(token), Some(role), Some(session_name)) = ( config.web_identity_token.clone(), config.role_name.clone(), @@ -222,6 +334,7 @@ pub fn classify_auth( token, role, session_name, + sts_region, }; } if let Some(role) = config.role_name.clone() { @@ -229,6 +342,7 @@ pub fn classify_auth( role, session_name: config.session_name.clone(), web_identity_role, + sts_region, }; } if let Some(name) = config.profile_name { @@ -291,6 +405,11 @@ impl AwsAuthService { ) -> Result { let resolved = config.clone().with_environment(env_lookup); let flow = classify_auth(config, env_lookup); + if let AwsAuthFlow::WebIdentity { token, .. } = &flow + && names_the_environment(token.expose(), env_lookup) + { + return Err(Error::WebIdentityTokenReference); + } self.fresh_credentials(&resolved, &flow).await } @@ -358,6 +477,13 @@ impl AwsAuthService { } } +fn names_the_environment( + reference: &str, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> bool { + reference.starts_with("os.environ/") || env_lookup(reference).is_some() +} + fn fresh(credentials: &Credentials) -> bool { credentials .expiry() @@ -380,8 +506,7 @@ impl AwsAuthFlow { async fn build_provider( resolved: AwsAuthConfig, flow: AwsAuthFlow, - token_resolver: Option, - policy: Option, + loader: &NativeProviderLoader, ) -> Result { match flow { AwsAuthFlow::SessionToken { @@ -428,70 +553,145 @@ async fn build_provider( role, session_name, web_identity_role, + sts_region, } => { - if is_already_running_as_role(&role, web_identity_role.as_deref(), &resolved).await? { - return Ok(SharedCredentialsProvider::new( - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await, - )); + let running_as = RunningAs { + role: role.clone(), + config: resolved.clone(), + sts_region: sts_region.clone(), + loader: loader.clone(), + }; + if running_as.is_target(web_identity_role.as_deref()).await { + return Ok(loader.ambient().await); } - let sdk_config = sdk_config(resolved.clone()).await; + let sdk_config = loader.sts_config(&resolved, sts_region).await; let builder = aws_config::sts::AssumeRoleProvider::builder(role) .session_name(session_name.unwrap_or_else(default_session_name)); let builder = match resolved.external_id { Some(id) => builder.external_id(id.expose()), None => builder, }; - Ok(SharedCredentialsProvider::new( - builder.configure(&sdk_config).build().await, - )) + let builder = match resolved.session_tags { + Some(tags) => builder.tags(tags.into_iter().map(|tag| (tag.key, tag.value))), + None => builder, + }; + Ok(SharedCredentialsProvider::new(AssumedRole { + provider: builder.configure(&sdk_config).build().await, + fallback: web_identity_role.is_none().then_some(running_as), + })) } AwsAuthFlow::WebIdentity { token, role, session_name, + sts_region, } => Ok(SharedCredentialsProvider::new(WebIdentityProvider { - policy, - client: aws_sdk_sts::Client::new(&sdk_config(resolved).await), + client: aws_sdk_sts::Client::new(&loader.sts_config(&resolved, sts_region).await), + tokens: loader.tokens.clone(), token, - token_resolver, role, session_name, + policy: loader + .session_policy + .clone() + .or_else(|| resolved.session_policy.map(str::to_owned)), })), } } -async fn sdk_config(config: AwsAuthConfig) -> aws_types::SdkConfig { - let loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - let loader = match config.region_name { - Some(region) => loader.region(aws_types::region::Region::new(region)), - None => loader, - }; - let loader = match config.sts_endpoint { - Some(endpoint) => loader.endpoint_url(endpoint), - None => loader, - }; - let loader = match (config.access_key_id, config.secret_access_key) { - (Some(key), Some(secret)) => loader.credentials_provider(Credentials::new( - key.expose(), - secret.expose(), - config.session_token.map(|token| token.expose().to_owned()), - None, - "litellm-role-source", - )), - _ => loader, - }; - loader.load().await +#[derive(Debug)] +struct RunningAs { + role: String, + config: AwsAuthConfig, + sts_region: Option, + loader: NativeProviderLoader, +} + +impl RunningAs { + async fn is_target(&self, web_identity_role: Option<&str>) -> bool { + if self.config.access_key_id.is_some() || self.config.secret_access_key.is_some() { + return false; + } + if role_identity(&self.role).is_none() { + return false; + } + if let Some(current_role) = web_identity_role { + return same_role_arns(&self.role, current_role); + } + let caller = AwsAuthConfig { + sts_endpoint: self.config.sts_endpoint.clone(), + ..Default::default() + }; + let sdk_config = self + .loader + .sts_config(&caller, self.sts_region.clone()) + .await; + aws_sdk_sts::Client::new(&sdk_config) + .get_caller_identity() + .send() + .await + .ok() + .and_then(|response| response.arn) + .is_some_and(|caller| same_role_arns(&self.role, &caller)) + } +} + +#[derive(Debug)] +struct AssumedRole { + provider: aws_config::sts::AssumeRoleProvider, + fallback: Option, +} + +impl ProvideCredentials for AssumedRole { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::new(async { + let error = match self.provider.provide_credentials().await { + Ok(credentials) => return Ok(credentials), + Err(error) => error, + }; + match &self.fallback { + Some(running_as) + if format!("{error:?}").contains("AccessDenied") + && running_as.is_target(None).await => + { + running_as + .loader + .ambient() + .await + .provide_credentials() + .await + } + _ => Err(error), + } + }) + } } struct WebIdentityProvider { client: aws_sdk_sts::Client, + tokens: Option>, token: SecretValue, - token_resolver: Option, - policy: Option, role: String, session_name: String, + policy: Option, +} + +impl WebIdentityProvider { + async fn token(&self) -> Result { + let Some(tokens) = &self.tokens else { + return Ok(self.token.clone()); + }; + tokens + .web_identity_token(self.token.expose()) + .await + .map_err(Error::WebIdentityToken)? + .ok_or(Error::MissingWebIdentityToken) + } } impl std::fmt::Debug for WebIdentityProvider { @@ -514,31 +714,18 @@ impl ProvideCredentials for WebIdentityProvider { Self: 'a, { aws_credential_types::provider::future::ProvideCredentials::new(async { - let token = match &self.token_resolver { - Some(resolve) => { - let resolve = Arc::clone(resolve); - let reference = self.token.expose().to_owned(); - tokio::task::spawn_blocking(move || resolve(&reference)) - .await - .map_err(|error| error.to_string()) - .and_then(|token| token) - .map_err(|message| { - CredentialsError::provider_error(std::io::Error::other(message)) - })? - } - None => self.token.expose().to_owned(), - }; - let request = self + let token = self + .token() + .await + .map_err(CredentialsError::provider_error)?; + let response = self .client .assume_role_with_web_identity() .role_arn(&self.role) .role_session_name(&self.session_name) - .web_identity_token(token); - let request = match &self.policy { - Some(policy) => request.policy(policy).duration_seconds(3600), - None => request, - }; - let response = request + .web_identity_token(token.expose()) + .duration_seconds(WEB_IDENTITY_SESSION_SECONDS) + .set_policy(self.policy.clone()) .send() .await .map_err(CredentialsError::provider_error)?; @@ -558,42 +745,6 @@ impl ProvideCredentials for WebIdentityProvider { } } -async fn is_already_running_as_role( - role: &str, - web_identity_role: Option<&str>, - config: &AwsAuthConfig, -) -> Result { - if config.access_key_id.is_some() || config.secret_access_key.is_some() { - return Ok(false); - } - if role_identity(role).is_none() { - return Ok(false); - } - if let Some(current_role) = web_identity_role { - return Ok(same_role_arns(role, current_role)); - } - - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = config.region_name.clone() { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = config.sts_endpoint.clone() { - loader = loader.endpoint_url(endpoint); - } - let sdk_config = loader.load().await; - let response = match aws_sdk_sts::Client::new(&sdk_config) - .get_caller_identity() - .send() - .await - { - Ok(response) => response, - Err(_) => return Ok(false), - }; - Ok(response - .arn() - .is_some_and(|caller| same_role_arns(role, caller))) -} - fn default_session_name() -> String { let seconds = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -814,75 +965,141 @@ pub fn resolve_bedrock_region( model_region: Option<&str>, params: &AwsParams, env_lookup: &dyn Fn(&str) -> Option, +) -> String { + bedrock_region(model_region, params, env_lookup, &|name| { + std::env::var(name).ok() + }) +} + +fn bedrock_region( + model_region: Option<&str>, + params: &AwsParams, + env_lookup: &dyn Fn(&str) -> Option, + process_env: &dyn Fn(&str) -> Option, ) -> String { resolve_aws_region(model_region, params, env_lookup) + .or_else(|| session_region(env_lookup, process_env)) .unwrap_or_else(|| DEFAULT_BEDROCK_REGION.to_string()) } +fn configured( + params: &AwsParams, + spec: &ParamSpec, + env_lookup: &dyn Fn(&str) -> Option, +) -> Option { + spec.wire + .iter() + .find_map(|name| params.get(name)) + .or_else(|| spec.env.iter().find_map(|name| env_lookup(name))) +} + pub fn aws_auth_config( params: &AwsParams, env_lookup: &dyn Fn(&str) -> Option, ) -> AwsAuthConfig { + let secret = |spec: &ParamSpec| configured(params, spec, env_lookup).map(SecretValue::new); + let setting = |spec: &ParamSpec| configured(params, spec, env_lookup); AwsAuthConfig { - access_key_id: params - .resolve(&AwsParams::ACCESS_KEY_ID, env_lookup) - .map(SecretValue::new), - secret_access_key: params - .resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup) - .map(SecretValue::new), - session_token: params - .resolve(&AwsParams::SESSION_TOKEN, env_lookup) - .map(SecretValue::new), - region_name: params.resolve(&AwsParams::REGION, env_lookup), - session_name: params.resolve(&AwsParams::SESSION_NAME, env_lookup), - profile_name: params.resolve(&AwsParams::PROFILE_NAME, env_lookup), - role_name: params.resolve(&AwsParams::ROLE_NAME, env_lookup), - web_identity_token: params - .resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup) - .map(SecretValue::new), - sts_endpoint: params.resolve(&AwsParams::STS_ENDPOINT, env_lookup), - external_id: params - .resolve(&AwsParams::EXTERNAL_ID, env_lookup) - .map(SecretValue::new), + access_key_id: secret(&AwsParams::ACCESS_KEY_ID), + secret_access_key: secret(&AwsParams::SECRET_ACCESS_KEY), + session_token: secret(&AwsParams::SESSION_TOKEN), + region_name: setting(&AwsParams::REGION), + session_name: setting(&AwsParams::SESSION_NAME), + profile_name: setting(&AwsParams::PROFILE_NAME), + role_name: setting(&AwsParams::ROLE_NAME), + web_identity_token: secret(&AwsParams::WEB_IDENTITY_TOKEN), + sts_endpoint: setting(&AwsParams::STS_ENDPOINT), + external_id: secret(&AwsParams::EXTERNAL_ID), + session_tags: params.aws_session_tags.clone().map(|mut tags| { + tags.sort_by(|left, right| left.key.cmp(&right.key)); + tags + }), + session_policy: None, } } +pub fn web_identity_session_policy(service: &str) -> Option<&'static str> { + matches!( + service, + "bedrock" | "bedrock-agentcore" | "bedrock-mantle" | "aws-external-anthropic" + ) + .then_some(BEDROCK_SESSION_POLICY) +} + /// Where the credentials that sign a request come from, decided when the request is /// prepared and resolved when it is sent. #[derive(Clone, Debug, PartialEq)] pub enum AwsCredentialSource { HostSupplied(Credentials), - Chain(AwsAuthConfig), + Chain(Box), } impl AwsCredentialSource { pub fn from_params(params: &AwsParams, env_lookup: &dyn Fn(&str) -> Option) -> Self { match host_supplied_credentials(params) { - Some(credentials) => Self::HostSupplied(credentials), - None => Self::Chain(aws_auth_config(params, env_lookup)), + Some(credentials) if names_only_static_keys(params) => Self::HostSupplied(credentials), + _ => Self::Chain(Box::new(aws_auth_config(params, env_lookup))), } } pub async fn resolve( self, auth: &AwsAuthService, + region: &str, + service: &str, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result { - match self { - Self::HostSupplied(credentials) => Ok(credentials), - Self::Chain(config) => auth.resolve_credentials(config, env_lookup).await, - } + let config = match self { + Self::HostSupplied(credentials) => AwsAuthConfig { + access_key_id: Some(SecretValue::new(credentials.access_key_id())), + secret_access_key: Some(SecretValue::new(credentials.secret_access_key())), + session_token: credentials.session_token().map(SecretValue::new), + ..AwsAuthConfig::default() + }, + Self::Chain(config) => *config, + }; + auth.resolve_credentials( + AwsAuthConfig { + region_name: Some(region.to_owned()), + session_policy: config + .session_policy + .or_else(|| web_identity_session_policy(service)), + ..config + }, + env_lookup, + ) + .await } } -/// Credentials a host resolved through its own chain and handed down verbatim. -/// -/// A host with its own resolution (LiteLLM's Python `BaseAWSLLM`, which reads -/// profiles, STS and boto sessions) passes the result here so the core signs -/// with exactly those. Without this the core would re-derive from ambient -/// state, where an unrelated `AWS_ROLE_NAME` or `AWS_PROFILE_NAME` in the -/// environment outranks explicit keys in [`classify_auth`] and the two sides -/// would sign as different principals. +fn names_only_static_keys(params: &AwsParams) -> bool { + let AwsParams { + aws_session_token, + aws_session_name, + aws_profile_name, + aws_role_name, + aws_web_identity_token, + aws_sts_endpoint, + aws_external_id, + aws_session_tags, + .. + } = params; + aws_session_token + .as_deref() + .is_none_or(|token| !token.trim().is_empty()) + && [ + aws_session_name, + aws_profile_name, + aws_role_name, + aws_web_identity_token, + aws_sts_endpoint, + aws_external_id, + ] + .iter() + .all(|value| value.is_none()) + && aws_session_tags.is_none() +} + pub fn host_supplied_credentials(params: &AwsParams) -> Option { let value = |value: &Option| { value @@ -1023,16 +1240,17 @@ mod tests { #[case] secret_key: Option<&str>, ) { let role = "arn:aws:iam::123456789012:role/target"; - let config = AwsAuthConfig { - access_key_id: access_key.map(SecretValue::new), - secret_access_key: secret_key.map(SecretValue::new), - ..Default::default() + let running_as = RunningAs { + role: role.to_string(), + config: AwsAuthConfig { + access_key_id: access_key.map(SecretValue::new), + secret_access_key: secret_key.map(SecretValue::new), + ..Default::default() + }, + sts_region: None, + loader: NativeProviderLoader::default(), }; - assert!( - !is_already_running_as_role(role, Some(role), &config) - .await - .unwrap() - ); + assert!(!running_as.is_target(Some(role)).await); } #[test] @@ -1045,7 +1263,7 @@ mod tests { recorded.lock().unwrap().insert(name.to_string()); None }; - resolve_aws_region(None, &AwsParams::default(), &env); + bedrock_region(None, &AwsParams::default(), &env, &|_| None); aws_auth_config(&AwsParams::default(), &env); assert!( seen.lock() @@ -1057,7 +1275,8 @@ mod tests { #[rstest::rstest] #[case::param_wins(Some("from-params"), &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some("from-params"))] - #[case::blank_param_falls_to_environment(Some(" "), &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some("from-env"))] + #[case::blank_param_counts_as_set(Some(" "), &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some(" "))] + #[case::blank_environment_counts_as_set(None, &[(AWS_SECRET_ACCESS_KEY, "")], Some(""))] #[case::absent_param_falls_to_environment(None, &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some("from-env"))] #[case::nothing(None, &[], None)] fn auth_config_reads_each_spec_from_params_then_the_environment( @@ -1096,7 +1315,7 @@ mod tests { #[case::call_params(Some("us-east-2"), Some("eu-west-1"), &[(AWS_REGION_NAME, "ap-south-1")], Some("eu-west-1"))] #[case::model_region(Some("us-east-2"), None, &[(AWS_REGION_NAME, "ap-south-1")], Some("us-east-2"))] #[case::aws_region_name(None, None, &[(AWS_REGION_NAME, "ap-south-1")], Some("ap-south-1"))] - #[case::aws_region(None, None, &[(AWS_REGION, "sa-east-1")], Some("sa-east-1"))] + #[case::aws_region(None, None, &[(AWS_REGION, "sa-east-1"), (AWS_DEFAULT_REGION, "eu-north-1")], Some("sa-east-1"))] #[case::nothing(None, None, &[], None)] fn a_region_comes_from_the_call_then_the_model_then_the_environment( #[case] model_region: Option<&str>, @@ -1120,11 +1339,32 @@ mod tests { expected ); assert_eq!( - resolve_bedrock_region(model_region, ¶ms, &env), + bedrock_region(model_region, ¶ms, &env, &|_| None), expected.unwrap_or(DEFAULT_BEDROCK_REGION) ); } + #[rstest::rstest] + #[case::default_region(&[(AWS_DEFAULT_REGION, "eu-north-1")], "eu-north-1")] + #[case::blank_default_region(&[(AWS_DEFAULT_REGION, "")], DEFAULT_BEDROCK_REGION)] + #[case::aws_region_outranks_it(&[(AWS_DEFAULT_REGION, "eu-north-1"), (AWS_REGION, "sa-east-1")], "sa-east-1")] + fn bedrock_falls_back_to_the_boto3_session_region( + #[case] environment: &[(&str, &str)], + #[case] expected: &str, + ) { + let env = |key: &str| { + environment + .iter() + .find(|(name, _)| *name == key) + .map(|(_, value)| value.to_string()) + }; + + assert_eq!( + bedrock_region(None, &AwsParams::default(), &env, &|_| None), + expected + ); + } + fn parity_inputs() -> (String, Vec, HeaderMap) { ( "https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke" @@ -1490,7 +1730,7 @@ mod tests { #[rstest::rstest] #[case::session(AwsAuthFlow::SessionToken { access_key_id: "credential-value".into(), secret_access_key: "credential-value".into(), session_token: "credential-value".into() })] #[case::static_keys(AwsAuthFlow::StaticKeys { access_key_id: "credential-value".into(), secret_access_key: "credential-value".into(), region_name: "region".into() })] - #[case::web_identity(AwsAuthFlow::WebIdentity { token: "credential-value".into(), role: "role".into(), session_name: "session".into() })] + #[case::web_identity(AwsAuthFlow::WebIdentity { token: "credential-value".into(), role: "role".into(), session_name: "session".into(), sts_region: None })] fn auth_flow_debug_never_discloses_credentials(#[case] flow: AwsAuthFlow) { let output = format!("{flow:?}"); assert!(!output.contains("credential-value")); @@ -1673,6 +1913,449 @@ mod tests { assert_eq!(report.matches("profile not found").count(), 1, "{report}"); } + #[derive(Clone, Debug, Default)] + struct Sts { + requests: Arc>>, + deny_assume_role: bool, + caller_arns: Arc>>>, + } + + #[derive(Clone, Debug)] + struct StsRequest { + host: String, + authorization: Option, + form: std::collections::BTreeMap, + } + + impl StsRequest { + fn action(&self) -> &str { + self.form["Action"].as_str() + } + } + + const STS_XMLNS: &str = "https://sts.amazonaws.com/doc/2011-06-15/"; + + impl Sts { + fn response(&self, request: &StsRequest) -> (u16, String) { + let credentials = |prefix: &str| { + format!( + "{prefix}-access{prefix}-secret{prefix}-session2100-01-01T00:00:00Zarn:aws:sts::123456789012:assumed-role/{prefix}/sid:s" + ) + }; + let error = |code: &str| { + format!( + "Sender{code}deniedr" + ) + }; + match request.action() { + "AssumeRole" if self.deny_assume_role => (403, error("AccessDenied")), + "AssumeRole" => ( + 200, + format!( + "{}", + credentials("role") + ), + ), + "AssumeRoleWithWebIdentity" => ( + 200, + format!( + "{}", + credentials("web") + ), + ), + "GetCallerIdentity" => match self.caller_arns.lock().unwrap().remove(0) { + Some(arn) => ( + 200, + format!( + "{arn}u123456789012" + ), + ), + None => (400, error("InvalidClientTokenId")), + }, + action => panic!("unexpected STS action {action}"), + } + } + + fn sent(&self) -> Vec { + self.requests.lock().unwrap().clone() + } + } + + impl aws_smithy_runtime_api::client::http::HttpConnector for Sts { + fn call( + &self, + request: aws_smithy_runtime_api::client::orchestrator::HttpRequest, + ) -> aws_smithy_runtime_api::client::http::HttpConnectorFuture { + let recorded = StsRequest { + host: url::Url::parse(request.uri()) + .unwrap() + .host_str() + .unwrap() + .to_string(), + authorization: request.headers().get("authorization").map(str::to_string), + form: url::form_urlencoded::parse(request.body().bytes().unwrap_or_default()) + .into_owned() + .collect(), + }; + let (status, body) = self.response(&recorded); + self.requests.lock().unwrap().push(recorded); + aws_smithy_runtime_api::client::http::HttpConnectorFuture::new(async move { + Ok( + aws_smithy_runtime_api::client::orchestrator::HttpResponse::new( + status.try_into().unwrap(), + aws_smithy_types::body::SdkBody::from(body), + ), + ) + }) + } + } + + #[derive(Debug)] + struct Tokens(Option<&'static str>); + + impl WebIdentityTokenSource for Tokens { + fn web_identity_token<'a>(&'a self, reference: &'a str) -> WebIdentityTokenFuture<'a> { + Box::pin(async move { + assert_eq!(reference, "oidc/file/var/run/secrets/token"); + Ok(self.0.map(SecretValue::new)) + }) + } + } + + fn sts_service(sts: &Sts, tokens: Option>) -> AwsAuthService { + use aws_smithy_runtime_api::client::http::{SharedHttpConnector, http_client_fn}; + let connector = sts.clone(); + AwsAuthService::new(Arc::new(NativeProviderLoader { + tokens, + session_policy: None, + http_client: Some(SharedHttpClient::new(http_client_fn(move |_, _| { + SharedHttpConnector::new(connector.clone()) + }))), + ambient: Some(SharedCredentialsProvider::new(Credentials::new( + "ambient-access", + "ambient-secret", + None, + None, + "ambient", + ))), + })) + } + + fn lookup( + environment: &'static [(&'static str, &'static str)], + ) -> impl Fn(&str) -> Option + Sync { + |key: &str| { + environment + .iter() + .find(|(name, _)| *name == key) + .map(|(_, value)| value.to_string()) + } + } + + const TARGET_ROLE: &str = "arn:aws:iam::123456789012:role/target"; + + #[rstest::rstest] + #[case::regional_endpoint(Some("https://sts.eu-west-1.amazonaws.com"), &[(AWS_REGION, "ap-south-1")], "sts.eu-west-1.amazonaws.com", "eu-west-1")] + #[case::aws_region(None, &[(AWS_REGION, "ap-south-1"), (AWS_DEFAULT_REGION, "ca-central-1")], "sts.ap-south-1.amazonaws.com", "ap-south-1")] + #[case::aws_default_region(None, &[(AWS_DEFAULT_REGION, "ca-central-1")], "sts.ca-central-1.amazonaws.com", "ca-central-1")] + #[case::the_signing_region(None, &[], "sts.us-east-1.amazonaws.com", "us-east-1")] + #[case::custom_endpoint_signs_for_the_signing_region(Some("https://sts.internal.test"), &[], "sts.internal.test", "us-east-1")] + #[tokio::test] + async fn params_keys_assume_the_role_with_tags_in_the_python_sts_region( + #[case] sts_endpoint: Option<&str>, + #[case] environment: &'static [(&'static str, &'static str)], + #[case] host: &str, + #[case] sts_region: &str, + ) { + let sts = Sts::default(); + let params = AwsParams { + aws_access_key_id: Some("source-access".into()), + aws_secret_access_key: Some("source-secret".into()), + aws_role_name: Some(TARGET_ROLE.into()), + aws_session_name: Some("attributed".into()), + aws_sts_endpoint: sts_endpoint.map(str::to_string), + aws_session_tags: Some(vec![ + AwsSessionTag { + key: "team".into(), + value: "genai".into(), + }, + AwsSessionTag { + key: "cost".into(), + value: "42".into(), + }, + ]), + ..AwsParams::default() + }; + + let credentials = AwsCredentialSource::from_params(¶ms, &no_env) + .resolve( + &sts_service(&sts, None), + "us-east-1", + BEDROCK_SERVICE, + &lookup(environment), + ) + .await + .unwrap(); + + assert_eq!(credentials.access_key_id(), "role-access"); + let [request] = sts.sent().try_into().unwrap(); + assert_eq!(request.action(), "AssumeRole"); + assert_eq!(request.host, host); + assert!( + request + .authorization + .as_deref() + .unwrap() + .contains("Credential=source-access/") + ); + assert!( + request + .authorization + .unwrap() + .contains(&format!("/{sts_region}/sts/aws4_request")) + ); + assert_eq!(request.form["RoleArn"], TARGET_ROLE); + assert_eq!(request.form["RoleSessionName"], "attributed"); + assert_eq!( + [ + request.form["Tags.member.1.Key"].as_str(), + request.form["Tags.member.1.Value"].as_str(), + request.form["Tags.member.2.Key"].as_str(), + request.form["Tags.member.2.Value"].as_str(), + ], + ["cost", "42", "team", "genai"] + ); + } + + #[rstest::rstest] + #[case::caller_is_the_target_role( + vec![None, Some("arn:aws:sts::123456789012:assumed-role/target/i-1")], + Ok("ambient-access") + )] + #[case::caller_is_another_role( + vec![None, Some("arn:aws:sts::123456789012:assumed-role/other/i-1")], + Err(()) + )] + #[tokio::test] + async fn an_access_denied_assume_role_falls_back_only_when_already_running_as_the_role( + #[case] caller_arns: Vec>, + #[case] expected: Result<&str, ()>, + ) { + let sts = Sts { + deny_assume_role: true, + caller_arns: Arc::new(std::sync::Mutex::new(caller_arns)), + ..Sts::default() + }; + let source = AwsCredentialSource::Chain(Box::new(AwsAuthConfig { + role_name: Some(TARGET_ROLE.into()), + ..AwsAuthConfig::default() + })); + + let result = source + .resolve( + &sts_service(&sts, None), + "us-east-1", + BEDROCK_SERVICE, + &no_env, + ) + .await; + + assert_eq!( + result + .as_ref() + .map(Credentials::access_key_id) + .map_err(|_| ()), + expected + ); + assert_eq!( + sts.sent() + .iter() + .map(StsRequest::action) + .collect::>(), + ["GetCallerIdentity", "AssumeRole", "GetCallerIdentity"] + ); + } + + fn web_identity_source(token: &str) -> AwsCredentialSource { + AwsCredentialSource::from_params( + &AwsParams { + aws_access_key_id: Some("ignored-access".into()), + aws_secret_access_key: Some("ignored-secret".into()), + aws_web_identity_token: Some(token.into()), + aws_role_name: Some(TARGET_ROLE.into()), + aws_session_name: Some("oidc-session".into()), + ..AwsParams::default() + }, + &no_env, + ) + } + + #[rstest::rstest] + #[case::bedrock_carries_the_session_policy(BEDROCK_SERVICE, Some(BEDROCK_SESSION_POLICY))] + #[case::another_service_does_not("textract", None)] + #[tokio::test] + async fn a_web_identity_reference_is_resolved_before_sts_sees_it( + #[case] service: &'static str, + #[case] policy: Option<&str>, + ) { + let sts = Sts::default(); + + let credentials = web_identity_source("oidc/file/var/run/secrets/token") + .resolve( + &sts_service(&sts, Some(Arc::new(Tokens(Some("resolved.jwt.value"))))), + "us-east-1", + service, + &no_env, + ) + .await + .unwrap(); + + assert_eq!(credentials.access_key_id(), "web-access"); + let [request] = sts.sent().try_into().unwrap(); + assert_eq!(request.action(), "AssumeRoleWithWebIdentity"); + assert_eq!(request.form["WebIdentityToken"], "resolved.jwt.value"); + assert_eq!(request.form["RoleArn"], TARGET_ROLE); + assert_eq!(request.form["RoleSessionName"], "oidc-session"); + assert_eq!(request.form["DurationSeconds"], "3600"); + assert_eq!(request.form.get("Policy").map(String::as_str), policy); + } + + #[rstest::rstest] + #[case::environment_prefix("os.environ/OIDC_TOKEN")] + #[case::environment_name("OIDC_TOKEN")] + #[tokio::test] + async fn a_web_identity_reference_to_the_environment_is_refused(#[case] token: &str) { + let sts = Sts::default(); + + let error = web_identity_source(token) + .resolve( + &sts_service(&sts, Some(Arc::new(Tokens(Some("unused"))))), + "us-east-1", + BEDROCK_SERVICE, + &lookup(&[("OIDC_TOKEN", "raw.jwt.value")]), + ) + .await + .unwrap_err(); + + assert_eq!(error, Error::WebIdentityTokenReference); + assert!(sts.sent().is_empty()); + } + + #[tokio::test] + async fn a_web_identity_reference_that_resolves_to_nothing_never_reaches_sts() { + let sts = Sts::default(); + + let error = web_identity_source("oidc/file/var/run/secrets/token") + .resolve( + &sts_service(&sts, Some(Arc::new(Tokens(None)))), + "us-east-1", + BEDROCK_SERVICE, + &no_env, + ) + .await + .unwrap_err(); + + assert!(matches!( + litellm_error::find::(&error), + Some(Error::Credentials { + flow: CredentialFlow::WebIdentity, + .. + }) + )); + assert!( + litellm_error::Report::new(&error) + .to_string() + .contains("resolved to no token") + ); + assert!(sts.sent().is_empty()); + } + + fn recorded_flow( + source: AwsCredentialSource, + environment: &'static [(&'static str, &'static str)], + ) -> (AwsAuthConfig, AwsAuthFlow) { + let loader = Arc::new(RecordingLoader::default()); + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(source.resolve( + &AwsAuthService::new(loader.clone()), + "eu-central-1", + BEDROCK_SERVICE, + &lookup(environment), + )) + .unwrap(); + loader.loads.lock().unwrap().remove(0) + } + + fn params_keys(extra: AwsParams) -> AwsParams { + AwsParams { + aws_access_key_id: Some("params-access".into()), + aws_secret_access_key: Some("params-secret".into()), + ..extra + } + } + + #[rstest::rstest] + #[case::keys_alone_sign_as_static_keys_in_the_signing_region( + params_keys(AwsParams::default()), + &[], + AwsAuthFlow::StaticKeys { access_key_id: "params-access".into(), secret_access_key: "params-secret".into(), region_name: "eu-central-1".into() } + )] + #[case::keys_pick_up_an_ambient_session_token( + params_keys(AwsParams::default()), + &[(AWS_SESSION_TOKEN, "ambient-token")], + AwsAuthFlow::SessionToken { access_key_id: "params-access".into(), secret_access_key: "params-secret".into(), session_token: "ambient-token".into() } + )] + #[case::keys_with_a_role_assume_it( + params_keys(AwsParams { aws_role_name: Some(TARGET_ROLE.into()), ..AwsParams::default() }), + &[], + AwsAuthFlow::AssumeRole { role: TARGET_ROLE.into(), session_name: None, web_identity_role: None, sts_region: Some("eu-central-1".into()) } + )] + #[case::keys_under_an_ambient_role_assume_it( + params_keys(AwsParams::default()), + &[(AWS_ROLE_NAME, TARGET_ROLE)], + AwsAuthFlow::AssumeRole { role: TARGET_ROLE.into(), session_name: None, web_identity_role: None, sts_region: Some("eu-central-1".into()) } + )] + #[case::keys_with_a_profile_use_the_profile( + params_keys(AwsParams { aws_profile_name: Some("work".into()), ..AwsParams::default() }), + &[], + AwsAuthFlow::Profile { name: "work".into() } + )] + #[case::a_blank_role_is_still_a_role( + params_keys(AwsParams { aws_role_name: Some(String::new()), ..AwsParams::default() }), + &[], + AwsAuthFlow::AssumeRole { role: String::new(), session_name: None, web_identity_role: None, sts_region: Some("eu-central-1".into()) } + )] + fn params_keys_follow_python_credential_precedence( + #[case] params: AwsParams, + #[case] environment: &'static [(&'static str, &'static str)], + #[case] expected: AwsAuthFlow, + ) { + let (config, flow) = recorded_flow( + AwsCredentialSource::from_params(¶ms, &no_env), + environment, + ); + + assert_eq!(flow, expected); + assert_eq!( + config.access_key_id.as_ref().map(SecretValue::expose), + Some("params-access") + ); + assert_eq!(config.region_name.as_deref(), Some("eu-central-1")); + } + + #[rstest::rstest] + #[case::bedrock(BEDROCK_SERVICE, Some(BEDROCK_SESSION_POLICY))] + #[case::claude_platform("aws-external-anthropic", Some(BEDROCK_SESSION_POLICY))] + #[case::textract("textract", None)] + fn the_session_policy_follows_the_signing_service( + #[case] service: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(web_identity_session_policy(service), expected); + } + #[ignore] #[tokio::test] async fn live_bedrock_invoke_model_returns_200() -> Result<(), Box> { diff --git a/litellm/rust/crates/auth-aws/src/bedrock_session_policy.json b/litellm/rust/crates/auth-aws/src/bedrock_session_policy.json new file mode 100644 index 00000000000..f80abfb9511 --- /dev/null +++ b/litellm/rust/crates/auth-aws/src/bedrock_session_policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream","bedrock:CountTokens","bedrock:Rerank","bedrock:Retrieve","bedrock:ListKnowledgeBases","bedrock:InvokeAgent","bedrock:ApplyGuardrail","bedrock:GetGuardrail","bedrock:ListGuardrails"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"BedrockAgentCoreLiteLLM","Effect":"Allow","Action":["bedrock-agentcore:InvokeAgentRuntime","bedrock-agentcore:InvokeAgentRuntimeForUser","bedrock-agentcore:InvokeGateway"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformLiteLLM","Effect":"Allow","Action":["aws-external-anthropic:CreateInference","aws-external-anthropic:CreateBatchInference","aws-external-anthropic:CancelBatchInference","aws-external-anthropic:DeleteBatchInference","aws-external-anthropic:CountTokens","aws-external-anthropic:Get*","aws-external-anthropic:List*"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformOutboundIdentityLiteLLM","Effect":"Allow","Action":["sts:GetWebIdentityToken","sts:TagGetWebIdentityToken"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringEquals":{"aws:CalledViaLast":"aws-external-anthropic.amazonaws.com"}}},{"Sid":"BedrockMantleLiteLLM","Effect":"Allow","Action":["bedrock-mantle:CreateInference","bedrock-mantle:CountTokens"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}}]} \ No newline at end of file diff --git a/litellm/rust/crates/auth-aws/src/constants.rs b/litellm/rust/crates/auth-aws/src/constants.rs index 0e4662eda9e..7f1ad330976 100644 --- a/litellm/rust/crates/auth-aws/src/constants.rs +++ b/litellm/rust/crates/auth-aws/src/constants.rs @@ -42,5 +42,6 @@ pub const SIGV4_COMPUTED_HEADER_NAMES: &[&str] = &[ pub const BEDROCK_SERVICE: &str = "bedrock"; pub const DEFAULT_SESSION_NAME_PREFIX: &str = "litellm-session"; pub const DEFAULT_BEDROCK_REGION: &str = "us-west-2"; -pub const BEDROCK_RUNTIME_ENDPOINT_TEMPLATE: &str = - "https://bedrock-runtime.{region}.amazonaws.com"; +pub const BEDROCK_SESSION_POLICY: &str = include_str!("bedrock_session_policy.json"); +pub const WEB_IDENTITY_SESSION_SECONDS: i32 = 3600; +pub const AWS4_AUTHORIZATION_PREFIX: &str = "AWS4-HMAC-SHA256"; diff --git a/litellm/rust/crates/auth-aws/src/error.rs b/litellm/rust/crates/auth-aws/src/error.rs index 5445de1e6c3..96caddee754 100644 --- a/litellm/rust/crates/auth-aws/src/error.rs +++ b/litellm/rust/crates/auth-aws/src/error.rs @@ -1,3 +1,4 @@ +use aws_credential_types::provider::error::CredentialsError; use litellm_error::Source; #[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] @@ -10,6 +11,14 @@ pub enum Error { #[source] source: Source, }, + #[error( + "the AWS web identity token must name a secret or an oidc/ provider, not an environment variable" + )] + WebIdentityTokenReference, + #[error("the AWS web identity token reference resolved to no token")] + MissingWebIdentityToken, + #[error("could not read the AWS web identity token")] + WebIdentityToken(#[source] Source), #[error("AWS web identity response had no credentials")] MissingWebIdentityCredentials, #[error("AWS web identity response had an invalid expiration")] @@ -49,6 +58,20 @@ pub enum SigningStage { } impl Error { + pub fn is_empty_default_chain(&self) -> bool { + let Self::Credentials { + flow: CredentialFlow::DefaultChain, + source, + } = self + else { + return false; + }; + matches!( + source.downcast_ref::(), + Some(CredentialsError::CredentialsNotLoaded(_)) + ) + } + pub(crate) fn credentials( flow: CredentialFlow, ) -> impl FnOnce(E) -> Self { @@ -73,3 +96,40 @@ impl From for litellm_auth_types::Error { Self::provider(error) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[rstest::rstest] + #[case::empty_default_chain( + CredentialFlow::DefaultChain, + CredentialsError::not_loaded("no providers in chain provided credentials"), + true + )] + #[case::failed_default_chain( + CredentialFlow::DefaultChain, + CredentialsError::provider_error("IMDS answered 500"), + false + )] + #[case::empty_profile( + CredentialFlow::Profile, + CredentialsError::not_loaded("the profile has no credentials"), + false + )] + fn only_a_default_chain_that_loaded_nothing_is_empty( + #[case] flow: CredentialFlow, + #[case] failure: CredentialsError, + #[case] empty: bool, + ) { + assert_eq!( + Error::credentials(flow)(failure).is_empty_default_chain(), + empty + ); + } + + #[test] + fn a_web_identity_failure_is_not_an_empty_default_chain() { + assert!(!Error::MissingWebIdentityToken.is_empty_default_chain()); + } +} diff --git a/litellm/rust/crates/auth-aws/src/lib.rs b/litellm/rust/crates/auth-aws/src/lib.rs index c8c5b5e8653..b467af27a6c 100644 --- a/litellm/rust/crates/auth-aws/src/lib.rs +++ b/litellm/rust/crates/auth-aws/src/lib.rs @@ -2,10 +2,12 @@ mod aws; pub mod constants; mod error; mod provider; +mod region; mod signer; pub use aws::*; pub use aws_credential_types::Credentials; pub use error::{CredentialFlow, Error, SigningStage}; pub use provider::AwsCredentialsProvider; -pub use signer::SigV4Signer; +pub use region::{AwsPartition, aws_partition, bedrock_runtime_endpoint}; +pub use signer::{CallerSigningHeaders, SigV4Signer}; diff --git a/litellm/rust/crates/auth-aws/src/region.rs b/litellm/rust/crates/auth-aws/src/region.rs new file mode 100644 index 00000000000..ab5f49a26ef --- /dev/null +++ b/litellm/rust/crates/auth-aws/src/region.rs @@ -0,0 +1,239 @@ +use std::path::PathBuf; + +use crate::constants::AWS_DEFAULT_REGION; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AwsPartition { + pub partition: &'static str, + pub dns_suffix: &'static str, +} + +const COMMERCIAL_PARTITION: AwsPartition = AwsPartition { + partition: "aws", + dns_suffix: "amazonaws.com", +}; + +const PARTITIONS_BY_REGION_PREFIX: [(&str, AwsPartition); 6] = [ + ( + "cn-", + AwsPartition { + partition: "aws-cn", + dns_suffix: "amazonaws.com.cn", + }, + ), + ( + "us-gov-", + AwsPartition { + partition: "aws-us-gov", + dns_suffix: "amazonaws.com", + }, + ), + ( + "us-isob-", + AwsPartition { + partition: "aws-iso-b", + dns_suffix: "sc2s.sgov.gov", + }, + ), + ( + "us-isof-", + AwsPartition { + partition: "aws-iso-f", + dns_suffix: "csp.hci.ic.gov", + }, + ), + ( + "us-iso-", + AwsPartition { + partition: "aws-iso", + dns_suffix: "c2s.ic.gov", + }, + ), + ( + "eu-isoe-", + AwsPartition { + partition: "aws-iso-e", + dns_suffix: "cloud.adc-e.uk", + }, + ), +]; + +pub fn aws_partition(region: &str) -> AwsPartition { + PARTITIONS_BY_REGION_PREFIX + .iter() + .find(|(prefix, _)| region.starts_with(prefix)) + .map_or(COMMERCIAL_PARTITION, |(_, partition)| *partition) +} + +pub fn bedrock_runtime_endpoint(region: &str) -> String { + format!( + "https://bedrock-runtime.{region}.{}", + aws_partition(region).dns_suffix + ) +} + +pub(crate) fn session_region( + env_lookup: &dyn Fn(&str) -> Option, + process_env: &dyn Fn(&str) -> Option, +) -> Option { + env_lookup(AWS_DEFAULT_REGION) + .filter(|region| !region.is_empty()) + .or_else(|| profile_region(process_env)) +} + +fn profile_region(process_env: &dyn Fn(&str) -> Option) -> Option { + let profile = process_env("AWS_PROFILE") + .or_else(|| process_env("AWS_DEFAULT_PROFILE")) + .filter(|profile| !profile.is_empty()) + .unwrap_or_else(|| "default".to_string()); + let home = process_env("HOME"); + let path = match process_env("AWS_CONFIG_FILE").filter(|path| !path.is_empty()) { + Some(path) => match (path.strip_prefix("~/"), &home) { + (Some(rest), Some(home)) => PathBuf::from(home).join(rest), + _ => PathBuf::from(path), + }, + None => PathBuf::from(home?).join(".aws").join("config"), + }; + config_file_region(&std::fs::read_to_string(path).ok()?, &profile) +} + +fn config_file_region(contents: &str, profile: &str) -> Option { + let named = format!("profile {profile}"); + let selects = |section: &str| section == named || (profile == "default" && section == profile); + let mut selected = false; + for line in contents.lines().map(str::trim) { + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { + continue; + } + if let Some(section) = line + .strip_prefix('[') + .and_then(|line| line.strip_suffix(']')) + { + selected = selects(§ion.split_whitespace().collect::>().join(" ")); + continue; + } + if selected + && let Some((key, value)) = line.split_once('=') + && key.trim() == "region" + { + return Some(value.trim().to_string()).filter(|region| !region.is_empty()); + } + } + None +} + +pub(crate) fn sts_endpoint_region(endpoint: &str) -> Option { + let url = url::Url::parse(endpoint).ok()?; + let labels: Vec<&str> = url.host_str()?.split('.').collect(); + (0..labels.len()).find_map(|index| { + let ["sts" | "sts-fips", region, rest @ ..] = &labels[index..] else { + return None; + }; + let rest = rest.join("."); + let valid_region = !region.is_empty() + && region + .chars() + .all(|char| char.is_ascii_lowercase() || char.is_ascii_digit() || char == '-'); + (valid_region + && (rest.starts_with("amazonaws.com") || rest.starts_with("vpce.amazonaws.com"))) + .then(|| region.to_string()) + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::commercial("us-east-1", "https://bedrock-runtime.us-east-1.amazonaws.com")] + #[case::china("cn-north-1", "https://bedrock-runtime.cn-north-1.amazonaws.com.cn")] + #[case::gov_cloud("us-gov-west-1", "https://bedrock-runtime.us-gov-west-1.amazonaws.com")] + #[case::iso("us-iso-east-1", "https://bedrock-runtime.us-iso-east-1.c2s.ic.gov")] + #[case::iso_b( + "us-isob-east-1", + "https://bedrock-runtime.us-isob-east-1.sc2s.sgov.gov" + )] + #[case::iso_f( + "us-isof-south-1", + "https://bedrock-runtime.us-isof-south-1.csp.hci.ic.gov" + )] + #[case::iso_e( + "eu-isoe-west-1", + "https://bedrock-runtime.eu-isoe-west-1.cloud.adc-e.uk" + )] + fn the_runtime_endpoint_uses_the_partition_dns_suffix( + #[case] region: &str, + #[case] expected: &str, + ) { + assert_eq!(bedrock_runtime_endpoint(region), expected); + } + + #[rstest] + #[case::china("cn-northwest-1", "aws-cn")] + #[case::gov_cloud("us-gov-east-1", "aws-us-gov")] + #[case::commercial("eu-central-1", "aws")] + fn a_region_names_its_partition(#[case] region: &str, #[case] expected: &str) { + assert_eq!(aws_partition(region).partition, expected); + } + + #[rstest] + #[case::regional("https://sts.eu-west-1.amazonaws.com", Some("eu-west-1"))] + #[case::fips("https://sts-fips.us-east-2.amazonaws.com/", Some("us-east-2"))] + #[case::china("https://sts.cn-north-1.amazonaws.com.cn", Some("cn-north-1"))] + #[case::vpc_endpoint( + "https://vpce-0abc-1234.sts.ap-south-1.vpce.amazonaws.com", + Some("ap-south-1") + )] + #[case::global("https://sts.amazonaws.com", None)] + #[case::custom_host("https://sts.internal.example.com", None)] + #[case::no_scheme("sts.eu-west-1.amazonaws.com", None)] + fn the_sts_region_is_read_from_a_regional_endpoint( + #[case] endpoint: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(sts_endpoint_region(endpoint).as_deref(), expected); + } + + const CONFIG: &str = "[default]\nregion = eu-west-3\n\n[profile work]\noutput = json\nregion=ap-northeast-1\n\n[profile empty]\nregion =\n"; + + #[rstest] + #[case::default_profile("default", Some("eu-west-3"))] + #[case::named_profile("work", Some("ap-northeast-1"))] + #[case::blank_region("empty", None)] + #[case::unknown_profile("missing", None)] + fn the_profile_region_comes_from_its_config_section( + #[case] profile: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(config_file_region(CONFIG, profile).as_deref(), expected); + } + + #[rstest] + #[case::default_region_wins(Some("sa-east-1"), None, Some("sa-east-1"))] + #[case::blank_default_region_reads_the_profile(Some(""), Some("work"), Some("ap-northeast-1"))] + #[case::profile_from_the_environment(None, Some("work"), Some("ap-northeast-1"))] + #[case::default_profile(None, None, Some("eu-west-3"))] + fn the_session_region_reads_the_default_region_then_the_profile( + #[case] default_region: Option<&str>, + #[case] profile: Option<&str>, + #[case] expected: Option<&str>, + ) { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(file.path(), CONFIG).unwrap(); + let config = file.path().to_string_lossy().into_owned(); + let env = |name: &str| { + (name == AWS_DEFAULT_REGION) + .then(|| default_region.map(str::to_string)) + .flatten() + }; + let process_env = |name: &str| match name { + "AWS_CONFIG_FILE" => Some(config.clone()), + "AWS_PROFILE" => profile.map(str::to_string), + _ => None, + }; + + assert_eq!(session_region(&env, &process_env).as_deref(), expected); + } +} diff --git a/litellm/rust/crates/auth-aws/src/signer.rs b/litellm/rust/crates/auth-aws/src/signer.rs index 2d9ad9ba123..12867f0c785 100644 --- a/litellm/rust/crates/auth-aws/src/signer.rs +++ b/litellm/rust/crates/auth-aws/src/signer.rs @@ -1,11 +1,18 @@ +use std::collections::BTreeMap; use std::time::SystemTime; use aws_credential_types::Credentials; -use reqwest::{Request, header::HeaderMap}; +use aws_sigv4::sign::v4; +use chrono::{DateTime, Utc}; +use reqwest::{ + Request, + header::{HeaderMap, HeaderName, HeaderValue}, +}; +use sha2::{Digest, Sha256}; use crate::{ - AwsAuthService, AwsCredentialSource, Error, SigningScope, aws_signature_headers, - is_sigv4_computed_header, sign_headers, + AwsAuthService, AwsCredentialSource, Error, SigningScope, SigningStage, aws_signature_headers, + constants::AWS4_AUTHORIZATION_PREFIX, is_sigv4_computed_header, sign_headers, }; use litellm_http::outbound::RequestSigner; @@ -15,6 +22,14 @@ pub struct SigV4Signer { service: &'static str, credentials: Credentials, clock: fn() -> SystemTime, + caller_headers: CallerSigningHeaders, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum CallerSigningHeaders { + #[default] + Dropped, + Botocore, } impl SigV4Signer { @@ -24,6 +39,7 @@ impl SigV4Signer { service, credentials, clock: SystemTime::now, + caller_headers: CallerSigningHeaders::Dropped, } } @@ -31,6 +47,18 @@ impl SigV4Signer { Self { clock, ..self } } + pub fn with_caller_headers(self, caller_headers: CallerSigningHeaders) -> Self { + Self { + caller_headers, + ..self + } + } + + fn keeps_caller_security_token(&self) -> bool { + self.caller_headers == CallerSigningHeaders::Botocore + && self.credentials.session_token().is_none() + } + pub async fn resolve( auth: &AwsAuthService, region: String, @@ -38,48 +66,246 @@ impl SigV4Signer { credentials: AwsCredentialSource, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result { - Ok(Self::new( - region, - service, - credentials.resolve(auth, env_lookup).await?, - )) + let credentials = credentials + .resolve(auth, ®ion, service, env_lookup) + .await?; + Ok(Self::new(region, service, credentials)) } } -impl RequestSigner for SigV4Signer { - fn sign(&self, request: &Request) -> Result { - if let Some(name) = request - .headers() - .keys() - .find(|name| is_sigv4_computed_header(name.as_str())) - { - return Err(litellm_http::Error::computed_header(name.as_str())); +const CALLER_HEADERS_THE_SIGNER_REPLACES: &[&str] = &["x-amz-date", "x-amz-security-token", "date"]; + +const CALLER_DATE_HEADERS: &[&str] = &["x-amz-date", "date"]; + +const SECURITY_TOKEN: &str = "x-amz-security-token"; + +const BOTOCORE_UNSIGNED_HEADERS: &[&str] = &["expect", "user-agent", "x-amzn-trace-id"]; + +fn signable(headers: &HeaderMap, keep_security_token: bool) -> HeaderMap { + aws_signature_headers(headers) + .iter() + .filter(|(name, _)| { + !is_sigv4_computed_header(name.as_str()) + || (keep_security_token && name.as_str() == SECURITY_TOKEN) + }) + .map(|(name, value)| (name.clone(), value.clone())) + .collect() +} + +fn hex_sha256(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn quote_path(path: &str) -> String { + path.bytes() + .map(|byte| match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' | b'/' => { + char::from(byte).to_string() + } + other => format!("%{other:02X}"), + }) + .collect() +} + +fn canonical_uri(path: &str) -> String { + if path.is_empty() { + return "/".to_string(); + } + let segments = path.split('/').fold(Vec::new(), |mut kept, segment| { + match segment { + "" | "." => {} + ".." => { + kept.pop(); + } + other => kept.push(other), } + kept + }); + let leading = if path.starts_with('/') { "/" } else { "" }; + let trailing = if path.ends_with('/') && !segments.is_empty() { + "/" + } else { + "" + }; + quote_path(&format!("{leading}{}{trailing}", segments.join("/"))) +} + +fn canonical_query(query: Option<&str>) -> String { + let mut pairs: Vec<(&str, &str)> = query + .filter(|query| !query.is_empty()) + .map(|query| { + query + .split('&') + .map(|pair| pair.split_once('=').unwrap_or((pair, ""))) + .collect() + }) + .unwrap_or_default(); + pairs.sort_unstable(); + pairs + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join("&") +} + +fn canonical_host(url: &reqwest::Url) -> String { + let host = url.host_str().unwrap_or_default(); + match url.port() { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + } +} + +fn header_pair(name: &str, value: &str) -> Result<(HeaderName, HeaderValue), Error> { + Ok(( + HeaderName::from_bytes(name.as_bytes()).map_err(Error::signing(SigningStage::Output))?, + HeaderValue::from_str(value).map_err(Error::signing(SigningStage::Output))?, + )) +} + +fn sign_with_date_header( + request: &Request, + body: &[u8], + headers: &HeaderMap, + scope: SigningScope<'_>, +) -> Result { + let time: DateTime = scope.time.into(); + let amz_date = time.format("%Y%m%dT%H%M%SZ").to_string(); + let http_date = time.format("%a, %d %b %Y %H:%M:%S -0000").to_string(); + let session_token = scope.credentials.session_token(); + let mut canonical: BTreeMap> = BTreeMap::new(); + for (name, value) in headers { + if BOTOCORE_UNSIGNED_HEADERS.contains(&name.as_str()) { + continue; + } + let value = value.to_str().map_err(|_| Error::NonAsciiHeader { + name: name.to_string(), + })?; + canonical + .entry(name.as_str().to_string()) + .or_default() + .push(value.split_whitespace().collect::>().join(" ")); + } + canonical.insert("date".into(), vec![http_date.clone()]); + if let Some(token) = session_token { + canonical.insert(SECURITY_TOKEN.into(), vec![token.to_string()]); + } + canonical + .entry("host".into()) + .or_insert_with(|| vec![canonical_host(request.url())]); + let payload = headers + .get("x-amz-content-sha256") + .and_then(|value| value.to_str().ok()) + .map_or_else(|| hex_sha256(body), str::to_string); + let canonical_headers: String = canonical + .iter() + .map(|(name, values)| format!("{name}:{}\n", values.join(","))) + .collect(); + let signed_headers = canonical.keys().cloned().collect::>().join(";"); + let canonical_request = format!( + "{}\n{}\n{}\n{canonical_headers}\n{signed_headers}\n{payload}", + request.method().as_str(), + canonical_uri(request.url().path()), + canonical_query(request.url().query()), + ); + let credential_scope = format!( + "{}/{}/{}/aws4_request", + &amz_date[..8], + scope.region, + scope.service + ); + let string_to_sign = format!( + "{AWS4_AUTHORIZATION_PREFIX}\n{amz_date}\n{credential_scope}\n{}", + hex_sha256(canonical_request.as_bytes()) + ); + let signing_key = v4::generate_signing_key( + scope.credentials.secret_access_key(), + scope.time, + scope.region, + scope.service, + ); + let signature = v4::calculate_signature(signing_key, string_to_sign.as_bytes()); + let authorization = format!( + "{AWS4_AUTHORIZATION_PREFIX} Credential={}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}", + scope.credentials.access_key_id() + ); + [ + Some(("date", http_date.as_str())), + session_token.map(|token| (SECURITY_TOKEN, token)), + Some(("authorization", authorization.as_str())), + ] + .into_iter() + .flatten() + .map(|(name, value)| header_pair(name, value)) + .collect() +} + +fn overrides_the_signature(headers: &HeaderMap) -> bool { + headers + .get(reqwest::header::AUTHORIZATION) + .is_some_and(|value| { + !value + .as_bytes() + .starts_with(AWS4_AUTHORIZATION_PREFIX.as_bytes()) + }) +} + +impl RequestSigner for SigV4Signer { + fn owned_headers(&self) -> &'static [&'static str] { + if self.keeps_caller_security_token() { + CALLER_DATE_HEADERS + } else { + CALLER_HEADERS_THE_SIGNER_REPLACES + } + } + + fn sign(&self, request: &Request) -> Result { let body = match request.body() { None => &[], Some(body) => body .as_bytes() .ok_or_else(litellm_http::Error::unsignable_body)?, }; - sign_headers( - request.method().as_str(), - request.url().as_str(), - body, - &aws_signature_headers(request.headers()), - SigningScope { - region: &self.region, - service: self.service, - credentials: &self.credentials, - time: (self.clock)(), - }, - ) + let headers = signable(request.headers(), self.keeps_caller_security_token()); + let scope = SigningScope { + region: &self.region, + service: self.service, + credentials: &self.credentials, + time: (self.clock)(), + }; + let signed = match self.caller_headers == CallerSigningHeaders::Botocore + && request.headers().contains_key(reqwest::header::DATE) + { + true => sign_with_date_header(request, body, &headers, scope), + false => sign_headers( + request.method().as_str(), + request.url().as_str(), + body, + &headers, + scope, + ), + } .map_err(|error| match error { Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error), Error::ExpiredCredentials | Error::Credentials { .. } + | Error::WebIdentityTokenReference + | Error::MissingWebIdentityToken + | Error::WebIdentityToken(_) | Error::MissingWebIdentityCredentials | Error::WebIdentityExpiration(_) | Error::Signing { .. } => litellm_http::Error::signature(error), + })?; + Ok(match overrides_the_signature(request.headers()) { + true => signed + .iter() + .filter(|(name, _)| *name != reqwest::header::AUTHORIZATION) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(), + false => signed, }) } } @@ -176,22 +402,166 @@ mod tests { assert!(original.contains("/us-east-1/textract/aws4_request")); } - #[test] - fn a_forwarded_computed_header_is_refused_instead_of_sent_twice() { - let error = signed( + fn sent(headers: &[(&str, &str)], credentials: Credentials) -> Request { + signed( json( URL, - &[("authorization".into(), "Bearer caller".into())], + &headers + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>(), &json_value!({}), None, ) .unwrap(), - &signer("textract"), + &SigV4Signer::new("us-east-1".into(), "bedrock", credentials).with_clock(fixed_clock), ) - .unwrap_err(); + .unwrap() + } - assert!(error.is_computed_header()); - assert!(error.to_string().contains("authorization"), "{error}"); + fn header<'a>(request: &'a Request, name: &str) -> Vec<&'a str> { + request + .headers() + .get_all(name) + .iter() + .map(|value| value.to_str().unwrap()) + .collect() + } + + #[rstest::rstest] + #[case::caller_date("date", None)] + #[case::caller_amz_date("x-amz-date", Some("20231114T221320Z"))] + #[case::caller_security_token_without_a_session("x-amz-security-token", None)] + fn a_header_the_signer_computes_is_dropped_from_the_callers_copy( + #[case] name: &str, + #[case] expected: Option<&str>, + ) { + let request = sent( + &[(name, "caller-value")], + Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"), + ); + + assert_eq!( + header(&request, name), + expected.into_iter().collect::>() + ); + assert!(header(&request, "authorization")[0].starts_with("AWS4-HMAC-SHA256 ")); + } + + #[test] + fn a_session_token_replaces_the_callers_security_token() { + let request = sent( + &[("X-Amz-Security-Token", "caller-value")], + Credentials::new( + "AKIDEXAMPLE", + "secret", + Some("session".into()), + None, + "test", + ), + ); + + assert_eq!(header(&request, "x-amz-security-token"), ["session"]); + } + + #[rstest::rstest] + #[case::bearer_overrides_the_signature("Bearer gateway", "Bearer gateway")] + #[case::a_stale_signature_is_replaced( + "AWS4-HMAC-SHA256 Credential=stale", + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;host;x-amz-date" + )] + fn a_callers_authorization_overrides_the_signature_unless_it_is_sigv4( + #[case] caller: &str, + #[case] expected: &str, + ) { + let request = sent( + &[("Authorization", caller)], + Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"), + ); + let authorization = header(&request, "authorization"); + + assert_eq!(authorization.len(), 1); + assert_eq!( + authorization[0].split(", Signature=").next(), + Some(expected) + ); + assert_eq!(header(&request, "x-amz-date"), ["20231114T221320Z"]); + } + + const BEDROCK_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke"; + const ARN_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A1%3Ainference-profile%2Fx/invoke?b=2&a=1"; + const CALLER_SIGNING_HEADERS: &[(&str, &str)] = &[ + ("date", "Wed, 01 Jan 2020 00:00:00 GMT"), + ("x-amz-date", "20200101T000000Z"), + ("x-amz-security-token", "caller-token"), + ]; + const CALLER_TOKEN: &[(&str, &str)] = &[("x-amz-security-token", "caller-token")]; + const CALLER_DATE: &[(&str, &str)] = &[("Date", "Wed, 01 Jan 2020 00:00:00 GMT")]; + const HTTP_DATE: &str = "Tue, 14 Nov 2023 22:13:20 -0000"; + const AMZ_DATE: &str = "20231114T221320Z"; + + #[rstest::rstest] + #[case::date_and_caller_token(URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "1b3652d523d5296b3904a80981e936a201167da9cb7bd200c2da8171dadb2c08")] + #[case::caller_token_only(URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "0fbd1a8e98efbfbcdb6fa9d665fe89ffffcdd8a7656865d63def21a5344d02dd")] + #[case::date_only(URL, CALLER_DATE, None, (Some(HTTP_DATE), None, None), "content-type;date;host", "cccdac7b04f20ebfe37a156c4e806a14e6c0a4b0aad11ca67e25dca337237085")] + #[case::session_replaces_the_caller_token_under_a_date(URL, CALLER_SIGNING_HEADERS, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "3406b8200cae95c8b66037dc5a9a838e1fe4da94b57b89fada3c05cde8a2c15c")] + #[case::session_replaces_the_caller_token(URL, CALLER_TOKEN, Some("session"), (None, Some(AMZ_DATE), Some("session")), "content-type;host;x-amz-date;x-amz-security-token", "d75425952d9f5ef7465a6ef3acabf15908c0edfd8ce2529cc3f0f8d2e9232e76")] + #[case::colon_in_the_model_path(BEDROCK_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "47d75a0dcb00499a025fc0f9f010f6dce89c7661eb54cb97c3a293e9c585b153")] + #[case::encoded_arn_and_query(ARN_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "8fb7d97a0ad5df9d1c3bbfc55897266059a5a2f35a1aaed9687f370047e78363")] + #[case::encoded_arn_and_query_without_a_date(ARN_URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "f5eb993af90c7896203bf6b3b85c0087087f712a0634e44ac0a04c9d87d6fe12")] + #[case::encoded_arn_and_query_with_a_session(ARN_URL, CALLER_DATE, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "15e8d5a9f64eb64366363f951db64de2a747959573a8cda9b5faa0fcbabc713c")] + fn botocore_caller_headers_sign_like_botocore( + #[case] url: &str, + #[case] caller: &[(&str, &str)], + #[case] session: Option<&str>, + #[case] sent: (Option<&str>, Option<&str>, Option<&str>), + #[case] signed_headers: &str, + #[case] signature: &str, + ) { + let request = signed( + json( + url, + &caller + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>(), + &json_value!({}), + None, + ) + .unwrap(), + &SigV4Signer::new( + "us-east-1".into(), + "bedrock", + Credentials::new( + "AKIDEXAMPLE", + "secret", + session.map(str::to_string), + None, + "test", + ), + ) + .with_clock(fixed_clock) + .with_caller_headers(CallerSigningHeaders::Botocore), + ) + .unwrap(); + + assert_eq!( + ( + header(&request, "date"), + header(&request, "x-amz-date"), + header(&request, "x-amz-security-token"), + header(&request, "authorization"), + ), + ( + sent.0.into_iter().collect::>(), + sent.1.into_iter().collect::>(), + sent.2.into_iter().collect::>(), + vec![format!( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders={signed_headers}, Signature={signature}" + ) + .as_str()], + ) + ); } #[test] diff --git a/litellm/rust/crates/auth-types/src/lib.rs b/litellm/rust/crates/auth-types/src/lib.rs index ab3bd3febbf..189655e6933 100644 --- a/litellm/rust/crates/auth-types/src/lib.rs +++ b/litellm/rust/crates/auth-types/src/lib.rs @@ -52,6 +52,6 @@ pub use credential::{ }; pub use error::{Error, ErrorDetail}; pub use http::CredentialPlacement; -pub use params::{AwsParams, ParamKind, ParamSpec, VertexParams}; +pub use params::{AwsParams, AwsSessionTag, ParamKind, ParamSpec, VertexParams}; pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; diff --git a/litellm/rust/crates/auth-types/src/params/aws.rs b/litellm/rust/crates/auth-types/src/params/aws.rs index de21fbf94c9..c3489c2d234 100644 --- a/litellm/rust/crates/auth-types/src/params/aws.rs +++ b/litellm/rust/crates/auth-types/src/params/aws.rs @@ -21,6 +21,14 @@ const fn spec( } } +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct AwsSessionTag { + #[serde(rename = "Key")] + pub key: String, + #[serde(rename = "Value")] + pub value: String, +} + /// The `aws_*` fields of Python's `GenericLiteLLMParams`. #[derive(Redact, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct AwsParams { @@ -50,11 +58,15 @@ pub struct AwsParams { pub aws_external_id: Option, #[serde(default)] pub aws_bedrock_runtime_endpoint: Option, + #[serde(default)] + pub aws_session_tags: Option>, } impl AwsParams { pub const ROLE_ARN_ENV: &str = "AWS_ROLE_ARN"; pub const WEB_IDENTITY_TOKEN_FILE_ENV: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; + pub const DEFAULT_REGION_ENV: &str = "AWS_DEFAULT_REGION"; + pub const SESSION_TAGS: &str = "aws_session_tags"; pub const ACCESS_KEY_ID: ParamSpec = spec( "access key id", @@ -144,6 +156,7 @@ impl AwsParams { Self::SPECS .iter() .flat_map(|spec| spec.wire.iter().copied()) + .chain([Self::SESSION_TAGS]) } /// The environment names the specs read, for hosts that resolve secrets up front. @@ -153,6 +166,7 @@ impl AwsParams { .iter() .flat_map(|spec| spec.env.iter().copied()) .chain([ + AwsParams::DEFAULT_REGION_ENV, AwsParams::ROLE_ARN_ENV, AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV, ]) @@ -205,6 +219,9 @@ impl AwsParams { aws_sts_endpoint: value("aws_sts_endpoint"), aws_external_id: value("aws_external_id"), aws_bedrock_runtime_endpoint: value("aws_bedrock_runtime_endpoint"), + aws_session_tags: optional_params + .get(Self::SESSION_TAGS) + .and_then(|tags| serde_json::from_value(tags.clone()).ok()), } } } diff --git a/litellm/rust/crates/auth-types/src/params/mod.rs b/litellm/rust/crates/auth-types/src/params/mod.rs index daad1aff8a8..634bc297f03 100644 --- a/litellm/rust/crates/auth-types/src/params/mod.rs +++ b/litellm/rust/crates/auth-types/src/params/mod.rs @@ -1,7 +1,7 @@ mod aws; mod vertex; -pub use aws::AwsParams; +pub use aws::{AwsParams, AwsSessionTag}; pub use vertex::VertexParams; /// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads diff --git a/litellm/rust/crates/auth-types/tests/aws_params.rs b/litellm/rust/crates/auth-types/tests/aws_params.rs index b438984c199..5c806a7aa7b 100644 --- a/litellm/rust/crates/auth-types/tests/aws_params.rs +++ b/litellm/rust/crates/auth-types/tests/aws_params.rs @@ -1,11 +1,16 @@ use litellm_auth_types::AwsParams; use rstest::rstest; -use serde_json::{Map, Value}; +use serde_json::{Map, Value, json}; #[rstest] fn every_named_param_roundtrips_without_loss() { let filled: Map = AwsParams::fields() - .map(|name| (name.to_string(), Value::from(format!("value-of-{name}")))) + .map(|name| match name { + AwsParams::SESSION_TAGS => { + (name.to_string(), json!([{"Key": "team", "Value": "genai"}])) + } + _ => (name.to_string(), Value::from(format!("value-of-{name}"))), + }) .collect(); let typed = AwsParams::from_optional_params(&filled); let serialized = serde_json::to_value(&typed).unwrap(); diff --git a/litellm/rust/crates/callbacks-budgets/Cargo.toml b/litellm/rust/crates/callbacks-budgets/Cargo.toml new file mode 100644 index 00000000000..03eeb1af47d --- /dev/null +++ b/litellm/rust/crates/callbacks-budgets/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "litellm-callbacks-budgets" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-callbacks.workspace = true +litellm-python-compat.workspace = true +litellm-redis-store.workspace = true +litellm-tracing.workspace = true +serde.workspace = true +serde_json.workspace = true +tracing.workspace = true + +[dev-dependencies] +litellm-coroutine.workspace = true +rstest.workspace = true +tokio = { workspace = true, features = ["macros", "rt", "time", "test-util"] } diff --git a/litellm/rust/crates/callbacks-budgets/src/admission.rs b/litellm/rust/crates/callbacks-budgets/src/admission.rs new file mode 100644 index 00000000000..47f7fcbe641 --- /dev/null +++ b/litellm/rust/crates/callbacks-budgets/src/admission.rs @@ -0,0 +1,128 @@ +use std::sync::Arc; + +use litellm_callbacks::{BudgetPlan, BudgetReservation, ReservationEntry}; +use litellm_redis_store::spend_counters::SpendCounters; +use litellm_tracing::Logger; +use serde::Serialize; +use serde_json::{Map, Value, json}; + +use crate::{ + engine::{self, BudgetCounterSpend, EngineError, Reservation, Settle, Settlement, UNAVAILABLE}, + plan::Plan, + window::Moment, +}; + +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "admission", rename_all = "snake_case")] +pub enum Admission { + Skipped, + Held { + reservation: BudgetReservation, + }, + Refused { + kind: &'static str, + fields: Map, + }, +} + +pub struct Budgets { + store: Arc, + logger: Option, +} + +fn refusal(error: &EngineError) -> Admission { + let (kind, fields) = match error { + EngineError::Exceeded(exceeded) => ( + "budget_exceeded", + Map::from_iter([ + ("current_cost".to_owned(), json!(exceeded.current_cost)), + ("max_budget".to_owned(), json!(exceeded.max_budget)), + ("message".to_owned(), json!(exceeded.message)), + ("entity_type".to_owned(), json!(exceeded.entity_type)), + ("entity_id".to_owned(), json!(exceeded.entity_id)), + ]), + ), + EngineError::Unavailable => ( + "budget_unavailable", + Map::from_iter([("detail".to_owned(), json!(UNAVAILABLE))]), + ), + other => ( + "budget_reservation_failed", + Map::from_iter([("message".to_owned(), json!(other.to_string()))]), + ), + }; + Admission::Refused { kind, fields } +} + +fn reservation_record(reservation: &Reservation) -> BudgetReservation { + BudgetReservation { + reserved_cost: Some(reservation.reserved_cost), + entries: Some( + reservation + .entries + .iter() + .map(|entry| { + Some(ReservationEntry { + counter_key: Some(entry.counter_key.clone()), + entity_type: Some(entry.entity_type.clone()), + entity_id: Some(entry.entity_id.clone()), + reserved_cost: Some(entry.reserved_cost), + applied_adjustment: Some(entry.applied_adjustment), + ..ReservationEntry::default() + }) + }) + .collect(), + ), + input_cost: Some(reservation.input_cost), + input_tokens: reservation.input_tokens, + ..BudgetReservation::default() + } +} + +impl Budgets { + pub fn new(store: Arc) -> Self { + Self { + store, + logger: None, + } + } + + pub fn with_logger(self, logger: Logger) -> Self { + Self { + logger: Some(logger), + ..self + } + } + + async fn logged(&self, future: impl Future) -> T { + match &self.logger { + Some(logger) => logger.instrument(future).await, + None => future.await, + } + } + + pub async fn settle(&self, request: &Settle<'_>) -> Result { + self.logged(engine::settle(&self.store, request, &Moment::now())) + .await + } + + pub async fn admit( + &self, + plan: BudgetPlan, + observed: &[BudgetCounterSpend], + ) -> Result { + let plan = Plan::try_from(plan).map_err(|error| error.to_string())?; + Ok( + match self + .logged(engine::reserve(&self.store, &plan, observed)) + .await + { + Ok(None) => Admission::Skipped, + Ok(Some(reservation)) => Admission::Held { + reservation: reservation_record(&reservation), + }, + Err(error) => refusal(&error), + }, + ) + } +} diff --git a/litellm/rust/crates/callbacks-budgets/src/checks.rs b/litellm/rust/crates/callbacks-budgets/src/checks.rs new file mode 100644 index 00000000000..e679bdc67dd --- /dev/null +++ b/litellm/rust/crates/callbacks-budgets/src/checks.rs @@ -0,0 +1,496 @@ +use litellm_python_compat::{number::PyNumber, unicode::isspace}; +use serde::{Deserialize, Serialize}; + +fn is_masked_key_name(name: &str) -> bool { + let Some(rest) = name.strip_prefix("sk-...") else { + return false; + }; + rest.is_empty() + || (rest.chars().count() == 4 + && rest + .chars() + .all(|c| !isspace(c) && !matches!(c, '\u{0}'..='\u{1f}' | '\u{7f}'..='\u{9f}'))) +} + +fn py_str(value: Option<&str>) -> &str { + value.unwrap_or("None") +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(tag = "check", rename_all = "snake_case", deny_unknown_fields)] +pub enum Check { + KeyMax { + spend: PyNumber, + max_budget: PyNumber, + token: Option, + key_alias: Option, + key_name: Option, + throttle_pct: Option, + }, + KeyWindow { + spend: PyNumber, + max_budget: PyNumber, + budget_duration: String, + token: Option, + }, + KeySoft { + spend: PyNumber, + soft_budget: Option, + }, + TeamMax { + spend: PyNumber, + max_budget: PyNumber, + team_id: Option, + }, + TeamWindow { + spend: PyNumber, + max_budget: PyNumber, + budget_duration: String, + team_id: Option, + }, + UserWindow { + spend: PyNumber, + max_budget: PyNumber, + budget_duration: String, + user_id: String, + }, + TeamSoft { + spend: Option, + soft_budget: Option, + }, + ProjectMax { + spend: PyNumber, + max_budget: PyNumber, + project_id: Option, + }, + ProjectSoft { + spend: Option, + soft_budget: Option, + }, + Organization { + spend: PyNumber, + max_budget: PyNumber, + org_id: String, + }, + Tag { + spend: PyNumber, + max_budget: PyNumber, + tag: String, + }, + ModelAccessGroup { + spend: PyNumber, + max_budget: PyNumber, + group: String, + }, + User { + spend: PyNumber, + max_budget: PyNumber, + user_id: Option, + }, + TeamMember { + spend: PyNumber, + max_budget: PyNumber, + user_id: Option, + team_id: Option, + }, + TeamMemberPositive { + spend: PyNumber, + max_budget: PyNumber, + user_id: Option, + team_id: Option, + }, + EndUser { + spend: PyNumber, + max_budget: PyNumber, + end_user_id: Option, + }, + Proxy { + spend: PyNumber, + max_budget: PyNumber, + }, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Refusal { + pub current_cost: PyNumber, + pub max_budget: PyNumber, + pub message: String, + pub entity_type: &'static str, + pub entity_id: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "decision", rename_all = "snake_case")] +pub enum Decision { + Skip, + Pass, + Alert, + Throttle { pct: PyNumber }, + Refuse(Refusal), +} + +fn refuse( + spend: PyNumber, + max_budget: PyNumber, + message: String, + entity_type: &'static str, + entity_id: Option, +) -> Decision { + Decision::Refuse(Refusal { + current_cost: spend, + max_budget, + message, + entity_type, + entity_id, + }) +} + +fn current(spend: PyNumber, max_budget: PyNumber, label: &str) -> String { + format!( + "Budget has been exceeded! {label} Current cost: {}, Max budget: {}", + spend.text(), + max_budget.text() + ) +} + +fn window(spend: PyNumber, max_budget: PyNumber) -> String { + format!("Spend=${}, Limit=${}", spend.fixed(4), max_budget.fixed(2)) +} + +fn at_or_over(spend: PyNumber, max_budget: PyNumber) -> bool { + max_budget.value().is_finite() && spend.value() >= max_budget.value() +} + +fn over(spend: PyNumber, max_budget: PyNumber) -> bool { + max_budget.value().is_finite() && spend.value() > max_budget.value() +} + +fn crossed(spend: Option, soft_budget: Option) -> Decision { + match (spend, soft_budget) { + (Some(spend), Some(soft)) if spend.value() >= soft.value() => Decision::Alert, + _ => Decision::Pass, + } +} + +pub fn decide(check: &Check) -> Decision { + match check.clone() { + Check::KeyMax { + spend, + max_budget, + token, + key_alias, + key_name, + throttle_pct, + } => { + if !at_or_over(spend, max_budget) { + return Decision::Pass; + } + if let Some(pct) = throttle_pct { + return Decision::Throttle { pct }; + } + let label = key_alias.filter(|alias| !alias.is_empty()); + let label = label.as_deref().unwrap_or("key"); + let descriptor = match key_name.as_deref() { + Some(name) if !name.is_empty() && is_masked_key_name(name) => { + format!("{label} ({name})") + } + _ => label.to_owned(), + }; + refuse( + spend, + max_budget, + current(spend, max_budget, &format!("Key={descriptor}")), + "key", + token, + ) + } + Check::KeyWindow { + spend, + max_budget, + budget_duration, + token, + } => { + if !at_or_over(spend, max_budget) { + return Decision::Pass; + } + refuse( + spend, + max_budget, + format!( + "ExceededBudget: Key over {budget_duration} budget. {}", + window(spend, max_budget) + ), + "key", + token, + ) + } + Check::KeySoft { spend, soft_budget } => match soft_budget { + Some(soft) if soft.value() != 0.0 && spend.value() >= soft.value() => Decision::Alert, + _ => Decision::Pass, + }, + Check::TeamMax { + spend, + max_budget, + team_id, + } => { + if !over(spend, max_budget) { + return Decision::Pass; + } + let label = format!("Team={}", py_str(team_id.as_deref())); + refuse( + spend, + max_budget, + current(spend, max_budget, &label), + "team", + team_id, + ) + } + Check::TeamWindow { + spend, + max_budget, + budget_duration, + team_id, + } => { + if !at_or_over(spend, max_budget) { + return Decision::Pass; + } + refuse( + spend, + max_budget, + format!( + "ExceededBudget: Team={} over {budget_duration} budget. {}", + py_str(team_id.as_deref()), + window(spend, max_budget) + ), + "team", + team_id, + ) + } + Check::UserWindow { + spend, + max_budget, + budget_duration, + user_id, + } => { + if !at_or_over(spend, max_budget) { + return Decision::Pass; + } + refuse( + spend, + max_budget, + format!( + "ExceededBudget: User={user_id} over {budget_duration} budget. {}", + window(spend, max_budget) + ), + "user", + Some(user_id), + ) + } + Check::TeamSoft { spend, soft_budget } | Check::ProjectSoft { spend, soft_budget } => { + crossed(spend, soft_budget) + } + Check::ProjectMax { + spend, + max_budget, + project_id, + } => { + if !max_budget.value().is_finite() { + return Decision::Skip; + } + if spend.value() < max_budget.value() || spend.value().is_nan() { + return Decision::Pass; + } + let label = format!("Project={}", py_str(project_id.as_deref())); + refuse( + spend, + max_budget, + current(spend, max_budget, &label), + "project", + project_id, + ) + } + Check::Organization { + spend, + max_budget, + org_id, + } => { + if !at_or_over(spend, max_budget) { + return Decision::Pass; + } + refuse( + spend, + max_budget, + current(spend, max_budget, &format!("Organization={org_id}")), + "organization", + Some(org_id), + ) + } + Check::Tag { + spend, + max_budget, + tag, + } => { + if spend.value() <= max_budget.value() { + return Decision::Pass; + } + refuse( + spend, + max_budget, + current(spend, max_budget, &format!("Tag={tag}")), + "tag", + Some(tag), + ) + } + Check::ModelAccessGroup { + spend, + max_budget, + group, + } => { + if max_budget.value() <= 0.0 { + return Decision::Skip; + } + if spend.value() < max_budget.value() { + return Decision::Pass; + } + refuse( + spend, + max_budget, + current(spend, max_budget, &format!("Model access group={group}")), + "model_access_group", + Some(group), + ) + } + Check::User { + spend, + max_budget, + user_id, + } => { + if !at_or_over(spend, max_budget) { + return Decision::Pass; + } + refuse( + spend, + max_budget, + format!( + "ExceededBudget: User={} over budget. Spend={}, Budget={}", + py_str(user_id.as_deref()), + spend.text(), + max_budget.text() + ), + "user", + user_id, + ) + } + Check::TeamMember { + spend, + max_budget, + user_id, + team_id, + } => { + if !max_budget.value().is_finite() { + return Decision::Skip; + } + if spend.value() < max_budget.value() || spend.value().is_nan() { + return Decision::Pass; + } + let (user, team) = (py_str(user_id.as_deref()), py_str(team_id.as_deref())); + refuse( + spend, + max_budget, + current(spend, max_budget, &format!("User={user} in Team={team}")), + "team_member", + Some(format!("{user}:{team}")), + ) + } + Check::TeamMemberPositive { + spend, + max_budget, + user_id, + team_id, + } => { + if spend.value() < max_budget.value() + || spend.value().is_nan() + || max_budget.value().is_nan() + { + return Decision::Pass; + } + let entity = format!( + "{}:{}", + py_str(user_id.as_deref()), + py_str(team_id.as_deref()) + ); + refuse( + spend, + max_budget, + current(spend, max_budget, &format!("TeamMember={entity}")), + "team_member", + Some(entity), + ) + } + Check::EndUser { + spend, + max_budget, + end_user_id, + } => { + if spend.value() <= max_budget.value() + || spend.value().is_nan() + || max_budget.value().is_nan() + { + return Decision::Pass; + } + refuse( + spend, + max_budget, + format!( + "ExceededBudget: End User={} over budget. Spend={}, Budget={}", + py_str(end_user_id.as_deref()), + spend.text(), + max_budget.text() + ), + "end_user", + end_user_id, + ) + } + Check::Proxy { spend, max_budget } => { + if !over(spend, max_budget) { + return Decision::Pass; + } + refuse( + spend, + max_budget, + format!( + "Budget has been exceeded! Current cost: {}, Max budget: {}", + spend.text(), + max_budget.text() + ), + "proxy", + None, + ) + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("sk-...", true)] + #[case("sk-...abcd", true)] + #[case("sk-...abc", false)] + #[case("sk-...ab d", false)] + #[case("sk-...abcde", false)] + #[case("sk-...ab\u{85}d", false)] + #[case("sk-...ab\u{3000}d", false)] + #[case("sk-...ab\u{202f}d", false)] + #[case("sk-...ab\u{2028}d", false)] + #[case("sk-...ab\u{200b}d", true)] + #[case("sk-...ab\u{180e}d", true)] + #[case("sk-...ab\u{feff}d", true)] + #[case("my-key", false)] + fn masked_key_names_match_the_proxy_pattern(#[case] name: &str, #[case] masked: bool) { + assert_eq!(is_masked_key_name(name), masked); + } +} diff --git a/litellm/rust/crates/callbacks-budgets/src/engine.rs b/litellm/rust/crates/callbacks-budgets/src/engine.rs new file mode 100644 index 00000000000..cf5773183fd --- /dev/null +++ b/litellm/rust/crates/callbacks-budgets/src/engine.rs @@ -0,0 +1,992 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +use litellm_callbacks::{BudgetReservation, ReservationEntry}; +use litellm_python_compat::repr::float_repr; +use litellm_redis_store::{ + Number, + spend_counters::{NotASpendCounter, SpendCounter, SpendCounters}, +}; +use serde::{Deserialize, Serialize}; + +use crate::{ + plan::{Counter, Plan, entity_type_value}, + window::{self, Moment, ReservationError}, +}; + +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct BudgetCounterSpend { + pub counter_key: Option, + pub spend: Option, +} + +const EPSILON: f64 = 1e-12; + +static ENTRY_IDS: AtomicUsize = AtomicUsize::new(0); + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Entry { + pub counter_key: String, + pub entity_type: String, + pub entity_id: String, + pub reserved_cost: f64, + pub applied_adjustment: f64, + #[serde(skip)] + id: usize, + #[serde(skip)] + counter: SpendCounter, +} + +impl Entry { + pub fn counter(&self) -> &SpendCounter { + &self.counter + } + + fn new(counter: &Counter, reserved_cost: f64) -> Self { + Self { + counter_key: counter.planned.counter_key.clone(), + entity_type: counter.planned.entity_type.clone(), + entity_id: counter.planned.entity_id.clone(), + reserved_cost, + applied_adjustment: 0.0, + id: ENTRY_IDS.fetch_add(1, Ordering::Relaxed), + counter: counter.counter.clone(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Reservation { + pub reserved_cost: f64, + pub entries: Vec, + pub input_cost: f64, + pub input_tokens: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Exceeded { + pub current_cost: f64, + pub max_budget: f64, + pub message: String, + pub entity_type: Option<&'static str>, + pub entity_id: String, +} + +#[derive(Debug)] +pub enum EngineError { + Exceeded(Exceeded), + Unavailable, + Inconsistent(String), + Store(litellm_redis_store::Error), + Plan(NotASpendCounter), + Reservation(ReservationError), +} + +impl std::fmt::Display for EngineError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Exceeded(exceeded) => f.write_str(&exceeded.message), + Self::Unavailable => f.write_str(UNAVAILABLE), + Self::Inconsistent(message) => f.write_str(message), + Self::Store(error) => write!(f, "{error}"), + Self::Plan(error) => write!(f, "{error}"), + Self::Reservation(error) => write!(f, "{error}"), + } + } +} + +impl From for EngineError { + fn from(error: ReservationError) -> Self { + Self::Reservation(error) + } +} + +impl From for EngineError { + fn from(error: litellm_redis_store::Error) -> Self { + Self::Store(error) + } +} + +pub const UNAVAILABLE: &str = "Budget enforcement unavailable: the budget reservation could not be written to the spend counter backend, and fail_closed_budget_enforcement is enabled, so the request was rejected to avoid exceeding the configured budget. Retry shortly."; + +fn exceeded(counter: &Counter, current_cost: f64, estimated_cost: Option) -> EngineError { + let planned = &counter.planned; + let estimate = estimated_cost + .map(|cost| format!("Estimated request cost: {}, ", float_repr(cost))) + .unwrap_or_default(); + EngineError::Exceeded(Exceeded { + current_cost, + max_budget: planned.max_budget, + message: format!( + "Budget has been exceeded! {}={} Current cost: {}, {}Max budget: {}", + planned.entity_type, + planned.entity_id, + float_repr(current_cost), + estimate, + float_repr(planned.max_budget), + ), + entity_type: entity_type_value(&planned.entity_type), + entity_id: planned + .spend_log_entity_id + .clone() + .unwrap_or_else(|| planned.entity_id.clone()), + }) +} + +fn observed_spend(observed: &[BudgetCounterSpend], counter: &Counter) -> f64 { + observed + .iter() + .find(|spend| spend.counter_key.as_deref() == Some(counter.key())) + .and_then(|spend| spend.spend) + .unwrap_or(counter.planned.fallback_spend) +} + +fn applies_to(value: Option, adjustment: f64) -> bool { + match value { + None => false, + Some(value) => !(adjustment < 0.0 && value + adjustment < -EPSILON), + } +} + +async fn can_apply( + store: &SpendCounters, + counter: &SpendCounter, + adjustment: f64, +) -> Result { + Ok(applies_to( + store.read(counter).await?.value().map(Number::as_f64), + adjustment, + )) +} + +async fn increment_or_invalidate( + store: &SpendCounters, + applicable: Vec<(SpendCounter, Number)>, +) -> Result<(), EngineError> { + if applicable.is_empty() { + return Ok(()); + } + if let Err(error) = store.increment_many(&applicable).await { + let counters: Vec = applicable.into_iter().map(|(c, _)| c).collect(); + store.invalidate(&counters).await; + return Err(error.into()); + } + Ok(()) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Inconsistent { + Drop, + Refuse, +} + +struct Adjustment { + index: usize, + counter: SpendCounter, + counter_key: String, + target: f64, + adjustment: f64, +} + +async fn apply( + store: &SpendCounters, + adjustments: &[Adjustment], + inconsistent: Inconsistent, +) -> Result, EngineError> { + let mut consistent = Vec::with_capacity(adjustments.len()); + for adjustment in adjustments { + consistent.push(can_apply(store, &adjustment.counter, adjustment.adjustment).await?); + } + if inconsistent == Inconsistent::Refuse + && let Some((adjustment, _)) = adjustments + .iter() + .zip(&consistent) + .find(|(_, consistent)| !**consistent) + { + return Err(EngineError::Inconsistent(format!( + "Cannot resize budget reservation against inconsistent counter {}", + adjustment.counter_key + ))); + } + let applicable: Vec<(SpendCounter, Number)> = adjustments + .iter() + .zip(&consistent) + .filter(|(_, consistent)| **consistent) + .map(|(adjustment, _)| { + ( + adjustment.counter.clone(), + Number::Float(adjustment.adjustment), + ) + }) + .collect(); + increment_or_invalidate(store, applicable).await?; + if inconsistent == Inconsistent::Drop { + for (adjustment, _) in adjustments + .iter() + .zip(&consistent) + .filter(|(_, consistent)| !**consistent) + { + store + .invalidate(std::slice::from_ref(&adjustment.counter)) + .await; + } + } + Ok(consistent) +} + +pub(crate) async fn apply_actual( + store: &SpendCounters, + entries: &mut [Entry], + actual: f64, + inconsistent: Inconsistent, +) -> Result<(), EngineError> { + let adjustments: Vec = entries + .iter() + .enumerate() + .filter_map(|(index, entry)| { + let target = actual - entry.reserved_cost; + let adjustment = target - entry.applied_adjustment; + (adjustment != 0.0).then(|| Adjustment { + index, + counter: entry.counter.clone(), + counter_key: entry.counter_key.clone(), + target, + adjustment, + }) + }) + .collect(); + apply(store, &adjustments, inconsistent).await?; + for adjustment in adjustments { + entries[adjustment.index].applied_adjustment = adjustment.target; + } + Ok(()) +} + +pub(crate) async fn release_best_effort(store: &SpendCounters, entries: &mut [Entry]) { + for entry in entries { + if apply_actual(store, std::slice::from_mut(entry), 0.0, Inconsistent::Drop) + .await + .is_err() + { + tracing::error!( + counter = entry.counter_key, + "Failed to release partial budget reservation during exception cleanup" + ); + store.invalidate(std::slice::from_ref(&entry.counter)).await; + } + } +} + +struct Admission<'a> { + store: &'a SpendCounters, + observed: &'a [BudgetCounterSpend], + plan: &'a Plan, + applied: Vec, + cost: f64, +} + +impl Admission<'_> { + async fn charge(&mut self, counters: &[Counter]) -> Result<(), EngineError> { + let fits = counters.iter().all(|counter| { + observed_spend(self.observed, counter) + self.cost <= counter.planned.max_budget + }); + let groups: Vec<&[Counter]> = if fits { + vec![counters] + } else { + counters.chunks(1).collect() + }; + for group in groups { + self.charge_group(group).await?; + } + Ok(()) + } + + async fn charge_group(&mut self, group: &[Counter]) -> Result<(), EngineError> { + let charged = self.cost; + let entries: Vec = group.iter().map(|c| Entry::new(c, charged)).collect(); + let ids: Vec = entries.iter().map(|entry| entry.id).collect(); + self.applied.extend(entries); + let increments: Vec<(SpendCounter, Number)> = group + .iter() + .map(|counter| (counter.counter.clone(), Number::Float(charged))) + .collect(); + let totals = match self.store.increment_many(&increments).await { + Ok(totals) => totals, + Err(error) => { + tracing::warn!(%error, "Skipping budget reservation because spend counter reservation failed"); + let counters: Vec = increments.into_iter().map(|(c, _)| c).collect(); + self.store.invalidate(&counters).await; + self.applied.retain(|entry| !ids.contains(&entry.id)); + if self.plan.fail_closed { + return Err(EngineError::Unavailable); + } + return Ok(()); + } + }; + for ((counter, id), total) in group.iter().zip(ids).zip(totals) { + if !self.applied.iter().any(|entry| entry.id == id) { + continue; + } + let current = total.as_f64() - (charged - self.cost); + if current > counter.planned.max_budget { + self.over_budget(counter, id, current).await?; + } + } + Ok(()) + } + + async fn over_budget( + &mut self, + counter: &Counter, + id: usize, + current: f64, + ) -> Result<(), EngineError> { + if self.plan.throttle_counter.as_deref() == Some(counter.key()) { + if let Some(position) = self.applied.iter().position(|entry| entry.id == id) { + let mut entry = self.applied.remove(position); + release_best_effort(self.store, std::slice::from_mut(&mut entry)).await; + } + return Ok(()); + } + let remaining = counter.planned.max_budget - (current - self.cost); + if remaining <= EPSILON { + return Err(exceeded(counter, current, None)); + } + if self.plan.fail_closed && current - counter.planned.max_budget > EPSILON { + return Err(exceeded(counter, current - self.cost, Some(self.cost))); + } + apply_actual( + self.store, + &mut self.applied, + remaining, + Inconsistent::Refuse, + ) + .await?; + for entry in &mut self.applied { + entry.reserved_cost = remaining; + entry.applied_adjustment = 0.0; + } + self.cost = remaining; + Ok(()) + } +} + +pub async fn reserve( + store: &SpendCounters, + plan: &Plan, + observed: &[BudgetCounterSpend], +) -> Result, EngineError> { + let counters = plan.counters().map_err(EngineError::Plan)?; + let mut admission = Admission { + store, + observed, + plan, + applied: Vec::new(), + cost: plan.reservation_cost, + }; + if let Err(error) = admission.charge(&counters).await { + release_best_effort(store, &mut admission.applied).await; + return Err(error); + } + if admission.applied.is_empty() { + return Ok(None); + } + Ok(Some(Reservation { + reserved_cost: admission.cost, + input_cost: plan.input_cost.min(admission.cost), + input_tokens: plan.input_tokens, + entries: admission.applied, + })) +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Reseed { + pub counter_key: String, + pub actual_cost: f64, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct PendingIncrement { + pub counter_key: String, + pub increment: f64, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Settlement { + pub adjustments: Vec>, + pub reseed: Vec, + pub pending: Vec, +} + +pub struct Settle<'a> { + pub reservation: &'a BudgetReservation, + pub window_resets: &'a [Option], + pub observed: &'a [Option], + pub actual: f64, + pub apply_consistent: bool, +} + +fn rejected(rejected: Option<&[String]>, member: &str) -> bool { + rejected.is_some_and(|names| names.iter().any(|name| name == member)) +} + +fn held_adjustment( + index: usize, + entry: &ReservationEntry, + window_reset: Option<&str>, + actual: f64, + default_reserved_cost: f64, + now: &Moment, +) -> Result, EngineError> { + let names = entry.rejected.as_deref(); + if rejected(names, "counter_key") { + return Err(ReservationError::type_error( + "str() of a reservation entry counter_key raises", + ) + .into()); + } + let Some(counter_key) = &entry.counter_key else { + return Ok(None); + }; + let settled_cost = match window_reset.map(window::parse_reset).transpose()?.flatten() { + Some(reset_at) if reset_at <= *now => 0.0, + _ => actual, + }; + let reserved_cost = match entry.reserved_cost { + Some(cost) if !rejected(names, "reserved_cost") => cost, + _ => default_reserved_cost, + }; + if rejected(names, "applied_adjustment") { + return Err(ReservationError::value_error( + "could not convert the reservation entry applied_adjustment to float", + ) + .into()); + } + let target = settled_cost - reserved_cost; + let adjustment = target - entry.applied_adjustment.unwrap_or(0.0); + if adjustment == 0.0 { + return Ok(None); + } + Ok(Some(( + Adjustment { + index, + counter: SpendCounter::parse(counter_key).map_err(EngineError::Plan)?, + counter_key: counter_key.clone(), + target, + adjustment, + }, + settled_cost, + ))) +} + +pub async fn settle( + store: &SpendCounters, + request: &Settle<'_>, + now: &Moment, +) -> Result { + let entries = request.reservation.entries.as_deref().unwrap_or_default(); + if entries.iter().any(Option::is_none) { + return Err( + ReservationError::type_error("a budget reservation entry is not a mapping").into(), + ); + } + let default_reserved_cost = request.reservation.reserved_cost.unwrap_or(0.0); + let mut adjustments = Vec::new(); + let mut settled_costs = Vec::new(); + for (index, entry) in entries.iter().enumerate() { + let Some(entry) = entry else { continue }; + let window_reset = request.window_resets.get(index).and_then(Option::as_deref); + if let Some((adjustment, settled_cost)) = held_adjustment( + index, + entry, + window_reset, + request.actual, + default_reserved_cost, + now, + )? { + adjustments.push(adjustment); + settled_costs.push(settled_cost); + } + } + let consistent: Vec = adjustments + .iter() + .map(|adjustment| { + applies_to( + request.observed.get(adjustment.index).copied().flatten(), + adjustment.adjustment, + ) + }) + .collect(); + if request.apply_consistent { + let applicable = adjustments + .iter() + .zip(&consistent) + .filter(|(_, consistent)| **consistent) + .map(|(adjustment, _)| { + ( + adjustment.counter.clone(), + Number::Float(adjustment.adjustment), + ) + }) + .collect(); + increment_or_invalidate(store, applicable).await?; + } + let mut settlement = Settlement { + adjustments: vec![None; entries.len()], + reseed: Vec::new(), + pending: Vec::new(), + }; + for ((adjustment, settled_cost), consistent) in + adjustments.iter().zip(settled_costs).zip(consistent) + { + if !consistent { + settlement.reseed.push(Reseed { + counter_key: adjustment.counter_key.clone(), + actual_cost: settled_cost, + }); + } else if !request.apply_consistent { + settlement.pending.push(PendingIncrement { + counter_key: adjustment.counter_key.clone(), + increment: adjustment.adjustment, + }); + continue; + } + settlement.adjustments[adjustment.index] = Some(adjustment.target); + } + Ok(settlement) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use litellm_redis_store::{ + Number, + spend_counters::{SpendCounter, SpendCounters}, + }; + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::{ + plan::{Plan, PlannedCounter}, + window::ErrorKind, + }; + + fn counter(key: &str, max_budget: f64, entity_type: &str, entity_id: &str) -> PlannedCounter { + PlannedCounter { + counter_key: key.to_owned(), + max_budget, + fallback_spend: 0.0, + entity_type: entity_type.to_owned(), + entity_id: entity_id.to_owned(), + spend_log_entity_id: None, + } + } + + fn plan(counters: Vec, cost: f64) -> Plan { + Plan { + counters, + reservation_cost: cost, + input_cost: cost / 4.0, + input_tokens: Some(12), + throttle_counter: None, + fail_closed: false, + status_code: 400, + } + } + + fn store(seed: &[(&str, f64)]) -> Arc { + let store = Arc::new(SpendCounters::new(None)); + for (key, value) in seed { + store + .set_local( + &SpendCounter::parse(key).unwrap(), + Number::Float(*value), + None, + ) + .unwrap(); + } + store + } + + fn value(store: &SpendCounters, key: &str) -> Option { + store + .get_local(&SpendCounter::parse(key).unwrap()) + .unwrap() + .map(Number::as_f64) + } + + const KEY: &str = "spend:key:hashed"; + const TEAM: &str = "spend:team:t1"; + + #[rstest] + #[tokio::test] + async fn counters_that_fit_are_charged_the_whole_estimate() { + let store = store(&[(KEY, 1.0), (TEAM, 2.0)]); + let plan = plan( + vec![ + counter(KEY, 10.0, "Key", "hashed"), + counter(TEAM, 10.0, "Team", "t1"), + ], + 3.0, + ); + let reservation = reserve(&store, &plan, &[]).await.unwrap().unwrap(); + assert_eq!(reservation.reserved_cost, 3.0); + assert_eq!(reservation.input_cost, 0.75); + assert_eq!(value(&store, KEY), Some(4.0)); + assert_eq!(value(&store, TEAM), Some(5.0)); + assert_eq!( + serde_json::to_value(&reservation).unwrap()["entries"][1], + json!({"counter_key": TEAM, "entity_type": "Team", "entity_id": "t1", "reserved_cost": 3.0, "applied_adjustment": 0.0}) + ); + } + + #[rstest] + #[tokio::test] + async fn a_partial_fit_shrinks_every_applied_entry_to_what_is_left() { + let store = store(&[(KEY, 1.0), (TEAM, 9.0)]); + let plan = plan( + vec![ + counter(KEY, 10.0, "Key", "hashed"), + counter(TEAM, 10.0, "Team", "t1"), + ], + 3.0, + ); + let reservation = reserve(&store, &plan, &[]).await.unwrap().unwrap(); + assert_eq!(reservation.reserved_cost, 1.0); + assert_eq!(value(&store, KEY), Some(2.0)); + assert_eq!(value(&store, TEAM), Some(10.0)); + } + + #[rstest] + #[case::spent( + 10.0, + false, + "Budget has been exceeded! Team=t1 Current cost: 11.0, Max budget: 10.0", + 11.0 + )] + #[case::fail_closed( + 9.5, + true, + "Budget has been exceeded! Team=t1 Current cost: 9.5, Estimated request cost: 1.0, Max budget: 10.0", + 9.5 + )] + #[rstest] + #[tokio::test] + async fn an_over_budget_counter_refuses_and_releases_what_was_charged( + #[case] team_spend: f64, + #[case] fail_closed: bool, + #[case] message: &str, + #[case] current_cost: f64, + ) { + let store = store(&[(KEY, 1.0), (TEAM, team_spend)]); + let plan = Plan { + fail_closed, + ..plan( + vec![ + counter(KEY, 10.0, "Key", "hashed"), + counter(TEAM, 10.0, "Team", "t1"), + ], + 1.0, + ) + }; + let Err(EngineError::Exceeded(exceeded)) = reserve(&store, &plan, &[]).await else { + panic!("expected a refusal"); + }; + assert_eq!(exceeded.message, message); + assert_eq!(exceeded.current_cost, current_cost); + assert_eq!(exceeded.entity_type, Some("team")); + assert_eq!(value(&store, KEY), Some(1.0)); + assert_eq!(value(&store, TEAM), Some(team_spend)); + } + + #[rstest] + #[tokio::test] + async fn a_throttled_key_releases_its_own_entry_and_keeps_the_rest() { + let store = store(&[(KEY, 10.0), (TEAM, 0.0)]); + let plan = Plan { + throttle_counter: Some(KEY.to_owned()), + ..plan( + vec![ + counter(KEY, 10.0, "Key", "hashed"), + counter(TEAM, 10.0, "Team", "t1"), + ], + 1.0, + ) + }; + let reservation = reserve(&store, &plan, &[]).await.unwrap().unwrap(); + assert_eq!(reservation.entries.len(), 1); + assert_eq!(reservation.entries[0].counter_key, TEAM); + assert_eq!(value(&store, KEY), Some(10.0)); + assert_eq!(value(&store, TEAM), Some(1.0)); + } + + fn record(reservation: &Reservation) -> BudgetReservation { + serde_json::from_value(serde_json::to_value(reservation).unwrap()).unwrap() + } + + fn stamp(record: &mut BudgetReservation, settlement: &Settlement) { + for (entry, target) in record + .entries + .iter_mut() + .flatten() + .zip(&settlement.adjustments) + { + if let (Some(entry), Some(target)) = (entry, target) { + entry.applied_adjustment = Some(*target); + } + } + } + + async fn observed(store: &SpendCounters, record: &BudgetReservation) -> Vec> { + let mut observed = Vec::new(); + for entry in record.entries.iter().flatten() { + let counter = entry + .as_ref() + .and_then(|entry| entry.counter_key.as_deref()) + .and_then(|key| SpendCounter::parse(key).ok()); + observed.push(match counter { + Some(counter) => store + .read(&counter) + .await + .unwrap() + .value() + .map(Number::as_f64), + None => None, + }); + } + observed + } + + async fn settle_as( + store: &SpendCounters, + record: &BudgetReservation, + resets: &[Option], + actual: f64, + apply_consistent: bool, + ) -> Result { + let observed = observed(store, record).await; + settle( + store, + &Settle { + reservation: record, + window_resets: resets, + observed: &observed, + actual, + apply_consistent, + }, + &Moment::from_unix_micros(NOW), + ) + .await + } + + async fn settle_now( + store: &SpendCounters, + record: &BudgetReservation, + resets: &[Option], + actual: f64, + ) -> Result { + settle_as(store, record, resets, actual, true).await + } + + #[rstest] + #[tokio::test] + async fn a_settlement_that_leaves_writes_to_the_caller_hands_back_the_consistent_increments() { + let store = store(&[(KEY, 1.0), (TEAM, 0.0)]); + let plan = plan( + vec![ + counter(KEY, 10.0, "Key", "hashed"), + counter(TEAM, 10.0, "Team", "t1"), + ], + 2.0, + ); + let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap()); + store + .invalidate(&[SpendCounter::parse(TEAM).unwrap()]) + .await; + let settlement = settle_as(&store, &record, &[], 0.5, false).await.unwrap(); + assert_eq!(value(&store, KEY), Some(3.0)); + assert_eq!( + settlement.pending, + vec![PendingIncrement { + counter_key: KEY.to_owned(), + increment: -1.5 + }] + ); + assert_eq!( + settlement.reseed, + vec![Reseed { + counter_key: TEAM.to_owned(), + actual_cost: 0.5 + }] + ); + assert_eq!(settlement.adjustments, vec![None, Some(-1.5)]); + } + + #[rstest] + #[tokio::test] + async fn a_counter_the_caller_saw_missing_is_reseeded_whatever_the_store_holds_now() { + let store = store(&[(KEY, 1.0)]); + let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0); + let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap()); + let settlement = settle( + &store, + &Settle { + reservation: &record, + window_resets: &[], + observed: &[None], + actual: 0.5, + apply_consistent: true, + }, + &Moment::from_unix_micros(NOW), + ) + .await + .unwrap(); + assert_eq!(value(&store, KEY), Some(3.0)); + assert_eq!( + settlement.reseed, + vec![Reseed { + counter_key: KEY.to_owned(), + actual_cost: 0.5 + }] + ); + } + + const NOW: i128 = 1_763_035_200_000_000; + + #[rstest] + #[case::success(0.4, 1.4)] + #[case::refund(0.0, 1.0)] + #[tokio::test] + async fn settling_moves_each_counter_from_the_hold_to_the_actual_cost( + #[case] actual: f64, + #[case] after: f64, + ) { + let store = store(&[(KEY, 1.0)]); + let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0); + let mut record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap()); + let settlement = settle_now(&store, &record, &[], actual).await.unwrap(); + assert_eq!(settlement.adjustments, vec![Some(actual - 2.0)]); + stamp(&mut record, &settlement); + let again = settle_now(&store, &record, &[], actual).await.unwrap(); + assert_eq!(again.adjustments, vec![None]); + assert_eq!(value(&store, KEY), Some(after)); + } + + #[rstest] + #[case::missing(None, 0.4, None)] + #[case::would_go_negative(Some(0.5), 0.0, Some(0.5))] + #[tokio::test] + async fn settling_an_inconsistent_counter_reports_it_and_leaves_it_alone( + #[case] before_settle: Option, + #[case] actual: f64, + #[case] after: Option, + ) { + let store = store(&[(KEY, 1.0), (TEAM, 0.0)]); + let plan = plan( + vec![ + counter(KEY, 10.0, "Key", "hashed"), + counter(TEAM, 10.0, "Team", "t1"), + ], + 2.0, + ); + let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap()); + let key = SpendCounter::parse(KEY).unwrap(); + match before_settle { + Some(spend) => store.set_local(&key, Number::Float(spend), None).unwrap(), + None => store.invalidate(std::slice::from_ref(&key)).await, + } + let settlement = settle_now(&store, &record, &[], actual).await.unwrap(); + assert_eq!( + settlement.reseed, + vec![Reseed { + counter_key: KEY.to_owned(), + actual_cost: actual + }] + ); + assert_eq!(value(&store, KEY), after); + assert_eq!(value(&store, TEAM), Some(2.0 + (actual - 2.0))); + assert_eq!(settlement.adjustments, vec![Some(actual - 2.0); 2]); + } + + #[rstest] + #[case::reset_before_settlement("2025-11-13T11:59:59+00:00", 0.0)] + #[case::reset_at_settlement("2025-11-13T12:00:00Z", 0.0)] + #[case::still_open("2025-11-13T12:00:00.000001+00:00", 0.4)] + #[case::unparsable("next month", 0.4)] + #[tokio::test] + async fn an_entry_whose_window_reset_before_settlement_settles_at_no_cost( + #[case] reset: &str, + #[case] settled: f64, + ) { + let store = store(&[(KEY, 1.0), (TEAM, 0.0)]); + let plan = plan( + vec![ + counter(KEY, 10.0, "Key", "hashed"), + counter(TEAM, 10.0, "Team", "t1"), + ], + 2.0, + ); + let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap()); + store + .invalidate(&[SpendCounter::parse(TEAM).unwrap()]) + .await; + let settlement = settle_now(&store, &record, &[Some(reset.to_owned()), None], 0.4) + .await + .unwrap(); + assert_eq!(value(&store, KEY), Some(1.0 + settled)); + assert_eq!( + settlement.adjustments, + vec![Some(settled - 2.0), Some(0.4 - 2.0)] + ); + assert_eq!( + settlement.reseed, + vec![Reseed { + counter_key: TEAM.to_owned(), + actual_cost: 0.4 + }] + ); + } + + fn held(entries: serde_json::Value) -> BudgetReservation { + serde_json::from_value(json!({"reserved_cost": 2.0, "entries": entries})).unwrap() + } + + #[rstest] + #[case::not_a_mapping(json!([{"counter_key": KEY, "reserved_cost": 2.0}, null]), None, ErrorKind::Type)] + #[case::naive_reset(json!([{"counter_key": KEY, "reserved_cost": 2.0}]), Some("2025-11-13T12:00:00"), ErrorKind::Type)] + #[case::applied_adjustment_float_refuses(json!([{"counter_key": KEY, "rejected": ["applied_adjustment"]}]), None, ErrorKind::Value)] + #[tokio::test] + async fn an_entry_python_raises_on_fails_the_settlement_before_any_counter_moves( + #[case] entries: serde_json::Value, + #[case] reset: Option<&str>, + #[case] kind: ErrorKind, + ) { + let store = store(&[(KEY, 3.0)]); + let Err(EngineError::Reservation(error)) = + settle_now(&store, &held(entries), &[reset.map(str::to_owned)], 0.5).await + else { + panic!("expected a reservation error"); + }; + assert_eq!(error.kind, kind); + assert_eq!(value(&store, KEY), Some(3.0)); + } + + #[rstest] + #[case::no_counter_key(json!([{"reserved_cost": 2.0}]), 3.0, vec![None])] + #[case::reserved_cost_float_refuses(json!([{"counter_key": KEY, "rejected": ["reserved_cost"]}]), 1.5, vec![Some(-1.5)])] + #[case::reserved_cost_missing(json!([{"counter_key": KEY}]), 1.5, vec![Some(-1.5)])] + #[case::own_reserved_cost(json!([{"counter_key": KEY, "reserved_cost": 1.0, "applied_adjustment": 0.25}]), 2.25, vec![Some(-0.5)])] + #[tokio::test] + async fn each_entry_member_is_read_as_budgets_reads_it( + #[case] entries: serde_json::Value, + #[case] after: f64, + #[case] adjustments: Vec>, + ) { + let store = store(&[(KEY, 3.0)]); + let settlement = settle_now(&store, &held(entries), &[], 0.5).await.unwrap(); + assert_eq!(settlement.adjustments, adjustments); + assert_eq!(value(&store, KEY), Some(after)); + } +} diff --git a/litellm/rust/crates/callbacks-budgets/src/lib.rs b/litellm/rust/crates/callbacks-budgets/src/lib.rs new file mode 100644 index 00000000000..0a7b356e62a --- /dev/null +++ b/litellm/rust/crates/callbacks-budgets/src/lib.rs @@ -0,0 +1,16 @@ +mod admission; +mod checks; +mod engine; +mod plan; +mod window; + +pub use admission::{Admission, Budgets}; +pub use checks::{Check, Decision, Refusal, decide}; +pub use engine::{ + BudgetCounterSpend, EngineError, Entry, Exceeded, PendingIncrement, Reseed, Reservation, + Settle, Settlement, UNAVAILABLE, reserve, settle, +}; +pub use litellm_callbacks::{BudgetPlan, BudgetReservation}; +pub use litellm_python_compat::number::PyNumber; +pub use plan::{Plan, PlannedCounter}; +pub use window::{ErrorKind, Moment, ReservationError}; diff --git a/litellm/rust/crates/callbacks-budgets/src/plan.rs b/litellm/rust/crates/callbacks-budgets/src/plan.rs new file mode 100644 index 00000000000..1286fc0cddc --- /dev/null +++ b/litellm/rust/crates/callbacks-budgets/src/plan.rs @@ -0,0 +1,114 @@ +use litellm_callbacks::{BudgetPlan, BudgetPlanCounter}; +use litellm_redis_store::spend_counters::{NotASpendCounter, SpendCounter}; + +#[derive(Clone, Debug, PartialEq)] +pub struct PlannedCounter { + pub counter_key: String, + pub max_budget: f64, + pub fallback_spend: f64, + pub entity_type: String, + pub entity_id: String, + pub spend_log_entity_id: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Plan { + pub counters: Vec, + pub reservation_cost: f64, + pub input_cost: f64, + pub input_tokens: Option, + pub throttle_counter: Option, + pub fail_closed: bool, + pub status_code: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MissingMember(pub &'static str); + +impl std::fmt::Display for MissingMember { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "the budget plan has no {}", self.0) + } +} + +fn required(value: Option, member: &'static str) -> Result { + value.ok_or(MissingMember(member)) +} + +impl TryFrom for PlannedCounter { + type Error = MissingMember; + + fn try_from(counter: BudgetPlanCounter) -> Result { + Ok(Self { + counter_key: required(counter.counter_key, "counters[].counter_key")?, + max_budget: required(counter.max_budget, "counters[].max_budget")?, + fallback_spend: required(counter.fallback_spend, "counters[].fallback_spend")?, + entity_type: required(counter.entity_type, "counters[].entity_type")?, + entity_id: required(counter.entity_id, "counters[].entity_id")?, + spend_log_entity_id: counter.spend_log_entity_id, + }) + } +} + +impl TryFrom for Plan { + type Error = MissingMember; + + fn try_from(plan: BudgetPlan) -> Result { + Ok(Self { + counters: required(plan.counters, "counters")? + .into_iter() + .map(PlannedCounter::try_from) + .collect::>()?, + reservation_cost: required(plan.reservation_cost, "reservation_cost")?, + input_cost: required(plan.input_cost, "input_cost")?, + input_tokens: plan.input_tokens, + throttle_counter: plan.throttle_counter, + fail_closed: plan.fail_closed.unwrap_or_default(), + status_code: required(plan.status_code, "status_code")?, + }) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct Counter { + pub planned: PlannedCounter, + pub counter: SpendCounter, +} + +impl Counter { + pub fn key(&self) -> &str { + &self.planned.counter_key + } +} + +impl Plan { + pub(crate) fn counters(&self) -> Result, NotASpendCounter> { + self.counters + .iter() + .map(|planned| { + Ok(Counter { + counter: SpendCounter::parse(&planned.counter_key)?, + planned: planned.clone(), + }) + }) + .collect() + } +} + +pub(crate) fn entity_type_value(entity_type: &str) -> Option<&'static str> { + match entity_type { + "Key" => Some("key"), + "Team" => Some("team"), + "TeamMember" => Some("team_member"), + "User" => Some("user"), + "EndUser" => Some("end_user"), + "Tag" => Some("tag"), + "Model access group" => Some("model_access_group"), + "Organization" => Some("organization"), + "Project" => Some("project"), + "customer_org" => Some("customer_org"), + "customer_team" => Some("customer_team"), + "customer_member" => Some("customer_member"), + _ => None, + } +} diff --git a/litellm/rust/crates/callbacks-budgets/src/window.rs b/litellm/rust/crates/callbacks-budgets/src/window.rs new file mode 100644 index 00000000000..bf4275c1cd7 --- /dev/null +++ b/litellm/rust/crates/callbacks-budgets/src/window.rs @@ -0,0 +1,109 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +use litellm_python_compat::datetime::{IsoDateTime, fromisoformat}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ErrorKind { + Type, + Value, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReservationError { + pub kind: ErrorKind, + pub message: String, +} + +impl ReservationError { + pub(crate) fn type_error(message: impl Into) -> Self { + Self { + kind: ErrorKind::Type, + message: message.into(), + } + } + + pub(crate) fn value_error(message: impl Into) -> Self { + Self { + kind: ErrorKind::Value, + message: message.into(), + } + } +} + +impl std::fmt::Display for ReservationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Moment(i128); + +impl Moment { + pub fn now() -> Self { + let since = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + Self(i128::try_from(since.as_micros()).unwrap_or(i128::MAX)) + } + + pub const fn from_unix_micros(micros: i128) -> Self { + Self(micros) + } +} + +/// A reservation entry's `window_reset_at`, read as the proxy's `_coerce_datetime` reads it: +/// `datetime.fromisoformat(text.replace("Z", "+00:00"))`, `None` where that raises. +pub(crate) fn parse_reset(text: &str) -> Result, ReservationError> { + match fromisoformat(&text.replace('Z', "+00:00")) { + None => Ok(None), + Some(IsoDateTime::Aware(micros)) => Ok(Some(Moment(i128::from(micros)))), + Some(IsoDateTime::Naive(_)) => Err(ReservationError::type_error( + "can't compare offset-naive and offset-aware datetimes", + )), + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const NOON: i128 = 1_763_035_200_000_000; + + #[rstest] + #[case::isoformat("2025-11-13T12:00:00+00:00", Some(NOON))] + #[case::zulu("2025-11-13T12:00:00Z", Some(NOON))] + #[case::micros("2025-11-13T12:00:00.000001+00:00", Some(NOON + 1))] + #[case::short_fraction("2025-11-13T12:00:00.5+00:00", Some(NOON + 500_000))] + #[case::long_fraction("2025-11-13T12:00:00.1234567+00:00", Some(NOON + 123_456))] + #[case::offset("2025-11-13T13:30:00+01:30", Some(NOON))] + #[case::negative_offset("2025-11-13T11:00:00-01:00", Some(NOON))] + #[case::space_separator("2025-11-13 12:00+00:00", Some(NOON))] + #[case::basic("20251113T120000+0000", Some(NOON))] + #[case::not_a_date("soon", None)] + #[case::bad_day("2025-02-30T12:00:00+00:00", None)] + #[case::bad_hour("2025-11-13T25:00:00+00:00", None)] + #[case::midnight_rollover("2025-11-13T24:00:00+00:00", Some(NOON + 43_200_000_000))] + #[case::trailing("2025-11-13T12:00:00+00:00 ", None)] + fn an_aware_reset_parses_as_python_fromisoformat_does( + #[case] text: &str, + #[case] expected: Option, + ) { + assert_eq!( + parse_reset(text).unwrap(), + expected.map(Moment::from_unix_micros) + ); + } + + #[rstest] + #[case::date_only("2025-11-13")] + #[case::no_offset("2025-11-13T12:00:00")] + fn a_naive_reset_raises_the_comparison_type_error(#[case] text: &str) { + assert_eq!( + parse_reset(text).unwrap_err(), + ReservationError::type_error("can't compare offset-naive and offset-aware datetimes") + ); + } +} diff --git a/litellm/rust/crates/callbacks-budgets/tests/admission.rs b/litellm/rust/crates/callbacks-budgets/tests/admission.rs new file mode 100644 index 00000000000..079ee1e475f --- /dev/null +++ b/litellm/rust/crates/callbacks-budgets/tests/admission.rs @@ -0,0 +1,230 @@ +use std::sync::Arc; + +use litellm_callbacks::{BudgetPlan, BudgetPlanCounter, BudgetReservation}; +use litellm_callbacks_budgets::{Admission, BudgetCounterSpend, Budgets, Settle, Settlement}; +use litellm_redis_store::{ + Number, + spend_counters::{SpendCounter, SpendCounters}, +}; +use rstest::rstest; +use serde_json::json; + +fn counter(key: &str, max_budget: f64, entity_type: &str, entity_id: &str) -> BudgetPlanCounter { + BudgetPlanCounter { + counter_key: Some(key.to_owned()), + max_budget: Some(max_budget), + fallback_spend: Some(0.0), + entity_type: Some(entity_type.to_owned()), + entity_id: Some(entity_id.to_owned()), + spend_log_entity_id: None, + } +} + +fn plan(counters: Vec, cost: f64) -> BudgetPlan { + BudgetPlan { + counters: Some(counters), + reservation_cost: Some(cost), + input_cost: Some(cost / 4.0), + input_tokens: Some(12), + throttle_counter: None, + fail_closed: Some(false), + status_code: Some(400), + reservation_id: Some("reservation-1".to_owned()), + ..BudgetPlan::default() + } +} + +fn store(seed: &[(&str, f64)]) -> Arc { + let store = Arc::new(SpendCounters::new(None)); + for (key, value) in seed { + store + .set_local( + &SpendCounter::parse(key).unwrap(), + Number::Float(*value), + None, + ) + .unwrap(); + } + store +} + +fn value(store: &SpendCounters, key: &str) -> Option { + store + .get_local(&SpendCounter::parse(key).unwrap()) + .unwrap() + .map(Number::as_f64) +} + +const KEY: &str = "spend:key:hashed"; + +fn observed(spends: &[(&str, f64)]) -> Vec { + spends + .iter() + .map(|(key, spend)| BudgetCounterSpend { + counter_key: Some((*key).to_owned()), + spend: Some(*spend), + }) + .collect() +} + +async fn admit(budgets: &Budgets, plan: &BudgetPlan, spends: &[(&str, f64)]) -> Admission { + budgets + .admit(plan.clone(), &observed(spends)) + .await + .unwrap() +} + +async fn held(budgets: &Budgets, plan: &BudgetPlan) -> BudgetReservation { + let Admission::Held { reservation } = admit(budgets, plan, &[]).await else { + panic!("expected a reservation"); + }; + reservation +} + +fn stamp(record: &mut BudgetReservation, adjustments: &[Option]) { + for (entry, target) in record.entries.iter_mut().flatten().zip(adjustments) { + if let (Some(entry), Some(target)) = (entry, target) { + entry.applied_adjustment = Some(*target); + } + } +} + +async fn settle( + budgets: &Budgets, + store: &SpendCounters, + record: &BudgetReservation, + actual: f64, +) -> Settlement { + let mut observed = Vec::new(); + for entry in record.entries.iter().flatten().flatten() { + let counter = SpendCounter::parse(entry.counter_key.as_deref().unwrap()).unwrap(); + observed.push( + store + .read(&counter) + .await + .unwrap() + .value() + .map(Number::as_f64), + ); + } + budgets + .settle(&Settle { + reservation: record, + window_resets: &[], + observed: &observed, + actual, + apply_consistent: true, + }) + .await + .unwrap() +} + +#[rstest] +#[tokio::test] +async fn an_admission_holds_and_returns_the_hold_as_the_reservation() { + let store = store(&[(KEY, 1.0)]); + let budgets = Budgets::new(Arc::clone(&store)); + let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0); + let mut record = held(&budgets, &plan).await; + assert_eq!(value(&store, KEY), Some(3.0)); + assert_eq!(record.reserved_cost, Some(2.0)); + assert_eq!(record.input_cost, Some(0.5)); + assert_eq!(record.input_tokens, Some(12)); + assert_eq!(record.finalized, None); + let settled = settle(&budgets, &store, &record, 0.5).await; + assert_eq!(settled.adjustments, vec![Some(-1.5)]); + assert!(settled.reseed.is_empty()); + assert_eq!(value(&store, KEY), Some(1.5)); + stamp(&mut record, &settled.adjustments); + let again = settle(&budgets, &store, &record, 0.5).await; + assert_eq!(again.adjustments, vec![None]); + assert_eq!(value(&store, KEY), Some(1.5)); +} + +#[rstest] +#[case::no_counters(BudgetPlan { counters: None, ..plan(vec![], 2.0) })] +#[case::no_status_code(BudgetPlan { status_code: None, ..plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0) })] +#[tokio::test] +async fn a_plan_missing_a_required_member_fails_without_holding(#[case] plan: BudgetPlan) { + let store = store(&[(KEY, 1.0)]); + let budgets = Budgets::new(Arc::clone(&store)); + assert!(budgets.admit(plan, &[]).await.is_err()); + assert_eq!(value(&store, KEY), Some(1.0)); +} + +#[rstest] +#[case::once(&[0.25])] +#[case::twice_on_the_same_cost(&[0.25, 0.25])] +#[case::grown_then_shrunk(&[0.75, 0.25])] +#[tokio::test] +async fn a_settlement_through_the_reservation_dict_lands_on_the_last_cost(#[case] costs: &[f64]) { + let store = store(&[(KEY, 1.0)]); + let budgets = Budgets::new(Arc::clone(&store)); + let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0); + let mut record = held(&budgets, &plan).await; + for cost in costs { + let settled = settle(&budgets, &store, &record, *cost).await; + stamp(&mut record, &settled.adjustments); + } + assert_eq!(value(&store, KEY), Some(1.0 + costs[costs.len() - 1])); + assert_eq!(record.finalized, None); +} + +#[rstest] +#[tokio::test] +async fn a_refusal_returns_the_typed_budget_error() { + let store = store(&[(KEY, 10.0)]); + let budgets = Budgets::new(Arc::clone(&store)); + let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0); + let Admission::Refused { kind, fields } = admit(&budgets, &plan, &[]).await else { + panic!("expected a refusal"); + }; + assert_eq!(kind, "budget_exceeded"); + assert_eq!( + serde_json::Value::Object(fields), + json!({ + "current_cost": 12.0, + "max_budget": 10.0, + "message": "Budget has been exceeded! Key=hashed Current cost: 12.0, Max budget: 10.0", + "entity_type": "key", + "entity_id": "hashed", + }) + ); + assert_eq!(value(&store, KEY), Some(10.0)); +} + +#[rstest] +#[case::observed(&[(KEY, 1.0)])] +#[case::not_observed(&[])] +#[tokio::test] +async fn a_counter_python_observed_is_charged_on_the_store(#[case] observed: &[(&str, f64)]) { + let store = store(&[(KEY, 1.0)]); + let budgets = Budgets::new(Arc::clone(&store)); + let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0); + let Admission::Held { .. } = admit(&budgets, &plan, observed).await else { + panic!("expected the reservation"); + }; + assert_eq!(value(&store, KEY), Some(3.0)); +} + +#[rstest] +#[tokio::test] +async fn an_observed_spend_with_no_room_charges_counters_one_at_a_time_and_refuses_the_full_one() { + const TEAM: &str = "spend:team:team-1"; + let store = store(&[(KEY, 10.0), (TEAM, 0.0)]); + let budgets = Budgets::new(Arc::clone(&store)); + let plan = plan( + vec![ + counter(TEAM, 10.0, "Team", "team-1"), + counter(KEY, 10.0, "Key", "hashed"), + ], + 2.0, + ); + let Admission::Refused { kind, .. } = admit(&budgets, &plan, &[(KEY, 10.0), (TEAM, 0.0)]).await + else { + panic!("expected the refusal"); + }; + assert_eq!(kind, "budget_exceeded"); + assert_eq!(value(&store, KEY), Some(10.0)); + assert_eq!(value(&store, TEAM), Some(0.0)); +} diff --git a/litellm/rust/crates/callbacks-legacy-python/python_contract.json b/litellm/rust/crates/callbacks-legacy-python/python_contract.json index 505651b5f16..2024400495e 100644 --- a/litellm/rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm/rust/crates/callbacks-legacy-python/python_contract.json @@ -159,6 +159,7 @@ "endpoint_type", "request_body", "chunks", - "error" + "error", + "head" ] } diff --git a/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs index 5f451f59c31..2b36f1ebff5 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs @@ -38,6 +38,7 @@ struct PassThroughStream { /// What the Messages stream iterator keeps for its end-of-stream billing. struct DeliveredStream { + head: Py, chunks: Py, first_chunk: Option>, } @@ -65,22 +66,14 @@ pub struct LegacyLogging { /// When the provider request went out, until its window is recorded (Messages only). api_started: Option, host_answered: bool, - /// Whether the call's Messages stream starts only on its first read, as Python's iterator - /// does. Only `azure_ai`: by the owner's ruling `anthropic` keeps main's billing of an - /// unread stream. + /// Whether the call's stream starts only on its first read, as Python's Messages iterator + /// does. stream_starts_on_first_read: bool, /// Set once the route sends through the host's own client, whose timed `post` already /// reports the exchange. host_client: Arc, } -/// The one provider whose unread Messages stream bills nothing, as Python's does. -const STREAM_STARTS_ON_FIRST_READ_PROVIDER: &str = "azure_ai"; - -/// The one provider whose exchange through the host's client is timed only by that client, as -/// Python's is. By the owner's ruling `anthropic` keeps main's second report. -const HOST_CLIENT_TIMED_PROVIDER: &str = "azure_ai"; - fn retry_reason(reason: &RetryReason) -> Value { match reason { RetryReason::ThinkingSignature => serde_json::json!({"kind": "thinking_signature"}), @@ -504,6 +497,7 @@ impl LegacyLogging { ), &stream.chunks, error, + &stream.head, ), ); match scheduled { @@ -675,8 +669,7 @@ impl LegacyLogging { )?; if self.operation == LoggingOperation::Messages && self.asynchronous - && !(context.custom_llm_provider == HOST_CLIENT_TIMED_PROVIDER - && self.host_client.load(Ordering::Acquire)) + && !self.host_client.load(Ordering::Acquire) && self.times_provider_exchange(py)? { self.api_started = Some(epoch_seconds()); @@ -729,8 +722,7 @@ impl LegacyLogging { ) -> PyResult> { use litellm_host::interceptors::{Cost, ResultSource}; - self.stream_starts_on_first_read = - facts.provider.provider == STREAM_STARTS_ON_FIRST_READ_PROVIDER; + self.stream_starts_on_first_read = self.operation == LoggingOperation::Messages; self.cache_key = match &facts.source { ResultSource::Provider => None, ResultSource::Cache { key } => Some(key.clone()), @@ -940,6 +932,7 @@ impl LegacyLogging { ), )?; self.stream = Some(DeliveredStream { + head: head.clone_ref(py), chunks: PyList::empty(py).unbind(), first_chunk: None, }); @@ -995,6 +988,7 @@ impl PythonOwned for LegacyLogging { visit.call(&self.response)?; visit.call(&self.error)?; if let Some(stream) = &self.stream { + visit.call(&stream.head)?; visit.call(&stream.chunks)?; visit.call(&stream.first_chunk)?; } diff --git a/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs index ab1a4ab9cdb..858d6573faf 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs @@ -107,7 +107,7 @@ FAKES = { 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk, stream_teardown: logger.record( 'stream_success', list(chunks) ), - 'stream_failure': lambda logger, endpoint_type, request_body, chunks, error: logger.record('stream_failure', error), + 'stream_failure': lambda logger, endpoint_type, request_body, chunks, error, head: logger.record('stream_failure', error), } assert FAKES.keys() == CONTRACT.keys(), sorted(FAKES.keys() ^ CONTRACT.keys()) for name, fake in FAKES.items(): diff --git a/litellm/rust/crates/callbacks-s3-log/src/key.rs b/litellm/rust/crates/callbacks-s3-log/src/key.rs index a0f3fc12b10..6f408e37c43 100644 --- a/litellm/rust/crates/callbacks-s3-log/src/key.rs +++ b/litellm/rust/crates/callbacks-s3-log/src/key.rs @@ -1,6 +1,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use litellm_callbacks::Moment; +use litellm_python_compat::datetime::UtcDateTime; use sha2::{Digest, Sha256}; use crate::settings::{Granularity, Settings}; @@ -58,16 +59,18 @@ impl Stamp { pub fn utc(time: SystemTime) -> Self { let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default(); let seconds = elapsed.as_secs(); - let (year, month, day) = civil_from_days((seconds / 86_400) as i64); - let of_day = seconds % 86_400; + let moment = UtcDateTime::from_timestamp( + i64::try_from(seconds).unwrap_or(i64::MAX), + elapsed.subsec_micros(), + ); Self { - year, - month, - day, - hour: (of_day / 3600) as u32, - minute: (of_day / 60 % 60) as u32, - second: (of_day % 60) as u32, - micros: elapsed.subsec_micros(), + year: moment.year, + month: moment.month, + day: moment.day, + hour: moment.hour, + minute: moment.minute, + second: moment.second, + micros: moment.microsecond, } } @@ -84,24 +87,6 @@ impl Stamp { } } -fn civil_from_days(days: i64) -> (i64, u32, u32) { - let shifted = days + 719_468; - let era = shifted.div_euclid(146_097); - let day_of_era = shifted.rem_euclid(146_097); - let year_of_era = - (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; - let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); - let month_index = (5 * day_of_year + 2) / 153; - let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32; - let month = if month_index < 10 { - month_index + 3 - } else { - month_index - 9 - } as u32; - let year = year_of_era + era * 400 + i64::from(month <= 2); - (year, month, day) -} - pub fn logging_id(start: &Stamp, id: &str) -> String { format!("time-{}_{id}", start.clock_micros()) } @@ -212,15 +197,7 @@ pub fn parent(key: &str) -> &str { } pub fn quote(key: &str) -> String { - let mut encoded = String::with_capacity(key.len()); - for byte in key.bytes() { - if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') { - encoded.push(byte as char); - } else { - encoded.push_str(&format!("%{byte:02X}")); - } - } - encoded + litellm_python_compat::urllib::quote(key, "/") } pub fn dns_suffix(region: Option<&str>) -> &'static str { diff --git a/litellm/rust/crates/callbacks-s3-log/src/lib.rs b/litellm/rust/crates/callbacks-s3-log/src/lib.rs index c8328c6ace6..6a4a0de1f73 100644 --- a/litellm/rust/crates/callbacks-s3-log/src/lib.rs +++ b/litellm/rust/crates/callbacks-s3-log/src/lib.rs @@ -1,6 +1,5 @@ mod key; mod limiter; -mod pyjson; mod settings; mod sink; mod upload; diff --git a/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs b/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs deleted file mode 100644 index c2f442c9af9..00000000000 --- a/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs +++ /dev/null @@ -1,214 +0,0 @@ -use serde_json::Value; - -const HEX: &[u8; 16] = b"0123456789abcdef"; - -pub fn write_str(out: &mut Vec, text: &str) { - out.push(b'"'); - let mut units = [0u16; 2]; - for character in text.chars() { - match character { - '"' => out.extend_from_slice(b"\\\""), - '\\' => out.extend_from_slice(b"\\\\"), - '\n' => out.extend_from_slice(b"\\n"), - '\r' => out.extend_from_slice(b"\\r"), - '\t' => out.extend_from_slice(b"\\t"), - '\u{8}' => out.extend_from_slice(b"\\b"), - '\u{c}' => out.extend_from_slice(b"\\f"), - ' '..='~' => out.push(character as u8), - _ => { - for unit in character.encode_utf16(&mut units) { - write_unit(out, *unit); - } - } - } - } - out.push(b'"'); -} - -fn write_unit(out: &mut Vec, unit: u16) { - out.extend_from_slice(b"\\u"); - for shift in [12, 8, 4, 0] { - out.push(HEX[usize::from((unit >> shift) & 0xf)]); - } -} - -pub fn float_repr(value: f64) -> String { - if value.is_nan() { - return "NaN".to_owned(); - } - if value.is_infinite() { - return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned(); - } - let shortest = format!("{value:e}"); - let length = shortest - .split('e') - .next() - .unwrap_or_default() - .chars() - .filter(char::is_ascii_digit) - .count(); - let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1)); - let (negative, scientific) = match scientific.strip_prefix('-') { - Some(rest) => (true, rest), - None => (false, scientific.as_str()), - }; - let (mantissa, exponent) = scientific.split_once('e').unwrap_or((scientific, "0")); - let exponent: i32 = exponent.parse().unwrap_or(0); - let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); - let mut text = String::with_capacity(digits.len() + 8); - if negative { - text.push('-'); - } - let point = exponent + 1; - if (-3..=16).contains(&point) { - let count = digits.len() as i32; - if point <= 0 { - text.push_str("0."); - text.extend(std::iter::repeat_n('0', (-point) as usize)); - text.push_str(&digits); - } else if point >= count { - text.push_str(&digits); - text.extend(std::iter::repeat_n('0', (point - count) as usize)); - text.push_str(".0"); - } else { - let (whole, fraction) = digits.split_at(point as usize); - text.push_str(whole); - text.push('.'); - text.push_str(fraction); - } - return text; - } - let (first, rest) = digits.split_at(1); - text.push_str(first); - if !rest.is_empty() { - text.push('.'); - text.push_str(rest); - } - text.push('e'); - text.push(if exponent < 0 { '-' } else { '+' }); - text.push_str(&format!("{:02}", exponent.unsigned_abs())); - text -} - -pub fn write_value(out: &mut Vec, value: &Value) { - match value { - Value::Null => out.extend_from_slice(b"null"), - Value::Bool(true) => out.extend_from_slice(b"true"), - Value::Bool(false) => out.extend_from_slice(b"false"), - Value::Number(number) => match (number.as_i64(), number.as_u64(), number.as_f64()) { - (Some(integer), _, _) => out.extend_from_slice(integer.to_string().as_bytes()), - (None, Some(integer), _) => out.extend_from_slice(integer.to_string().as_bytes()), - (None, None, Some(float)) => out.extend_from_slice(float_repr(float).as_bytes()), - (None, None, None) => out.extend_from_slice(number.to_string().as_bytes()), - }, - Value::String(text) => write_str(out, text), - Value::Array(items) => { - out.push(b'['); - for (index, item) in items.iter().enumerate() { - if index > 0 { - out.extend_from_slice(b", "); - } - write_value(out, item); - } - out.push(b']'); - } - Value::Object(entries) => { - out.push(b'{'); - for (index, (key, item)) in entries.iter().enumerate() { - if index > 0 { - out.extend_from_slice(b", "); - } - write_str(out, key); - out.extend_from_slice(b": "); - write_value(out, item); - } - out.push(b'}'); - } - } -} - -pub fn dumps(value: &Value) -> String { - let mut out = Vec::new(); - write_value(&mut out, value); - String::from_utf8(out).unwrap_or_default() -} - -pub fn py_str(value: &Value) -> String { - match value { - Value::Null => "None".to_owned(), - Value::Bool(true) => "True".to_owned(), - Value::Bool(false) => "False".to_owned(), - Value::String(text) => text.clone(), - Value::Number(number) => match number.as_f64() { - Some(float) if number.is_f64() => float_repr(float), - _ => number.to_string(), - }, - Value::Array(_) | Value::Object(_) => dumps(value), - } -} - -pub fn truthy(value: &Value) -> bool { - match value { - Value::Null => false, - Value::Bool(flag) => *flag, - Value::Number(number) => number.as_f64().is_some_and(|float| float != 0.0), - Value::String(text) => !text.is_empty(), - Value::Array(items) => !items.is_empty(), - Value::Object(entries) => !entries.is_empty(), - } -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - use serde_json::json; - - use super::*; - - #[rstest] - #[case(0.0, "0.0")] - #[case(-0.0, "-0.0")] - #[case(1.0, "1.0")] - #[case(0.1, "0.1")] - #[case(1.5e-5, "1.5e-05")] - #[case(1e-4, "0.0001")] - #[case(1e-5, "1e-05")] - #[case(1e15, "1000000000000000.0")] - #[case(1e16, "1e+16")] - #[case(1.2345678901234568e17, "1.2345678901234568e+17")] - #[case(123.456, "123.456")] - #[case(1e100, "1e+100")] - #[case(-2.5e-300, "-2.5e-300")] - #[case(f64::NAN, "NaN")] - #[case(f64::INFINITY, "Infinity")] - #[case(f64::NEG_INFINITY, "-Infinity")] - #[case(0.30000000000000004, "0.30000000000000004")] - #[case(0.07872390747070312, "0.07872390747070312")] - #[case(1794700894.4414062, "1794700894.4414062")] - fn floats_print_as_python_repr_prints_them(#[case] value: f64, #[case] expected: &str) { - assert_eq!(float_repr(value), expected); - } - - #[rstest] - #[case("plain", "\"plain\"")] - #[case("quote\" back\\", "\"quote\\\" back\\\\\"")] - #[case("\n\r\t\u{8}\u{c}\u{1}\u{7f}", "\"\\n\\r\\t\\b\\f\\u0001\\u007f\"")] - #[case("café", "\"caf\\u00e9\"")] - #[case("😀", "\"\\ud83d\\ude00\"")] - fn strings_are_escaped_as_ensure_ascii_escapes_them( - #[case] text: &str, - #[case] expected: &str, - ) { - let mut out = Vec::new(); - write_str(&mut out, text); - assert_eq!(String::from_utf8(out).unwrap(), expected); - } - - #[rstest] - fn containers_use_the_default_separators_and_keep_key_order() { - assert_eq!( - dumps(&json!({"b": [1, 2.5, null, true], "a": {"x": "y"}})), - "{\"b\": [1, 2.5, null, true], \"a\": {\"x\": \"y\"}}" - ); - } -} diff --git a/litellm/rust/crates/callbacks-s3-log/src/settings.rs b/litellm/rust/crates/callbacks-s3-log/src/settings.rs index c266f612c94..1dd09cf675c 100644 --- a/litellm/rust/crates/callbacks-s3-log/src/settings.rs +++ b/litellm/rust/crates/callbacks-s3-log/src/settings.rs @@ -5,28 +5,28 @@ use litellm_secret_value::SecretValue; use serde::Deserialize; use serde_json::{Map, Value}; -use crate::pyjson::{py_str, truthy}; +use litellm_python_compat::{BigInt, json::from_json, pydantic, repr}; + +fn python(value: &Value) -> litellm_python_compat::Value { + from_json(value.clone()) +} fn py_repr(value: &Value) -> String { - litellm_python_compat::repr::repr(&litellm_python_compat::json::from_json(value.clone())) + repr::repr(&python(value)) +} + +pub fn py_str(value: &Value) -> String { + repr::to_str(&python(value)) +} + +pub fn truthy(value: &Value) -> bool { + litellm_python_compat::truthy::truthy(&python(value)) } fn py_optional(value: Option) -> String { value.map_or_else(|| "None".to_owned(), |value| value.to_string()) } -fn py_type_name(value: &Value) -> &'static str { - match value { - Value::Null => "NoneType", - Value::Bool(_) => "bool", - Value::Number(number) if number.is_f64() => "float", - Value::Number(_) => "int", - Value::String(_) => "str", - Value::Array(_) => "list", - Value::Object(_) => "dict", - } -} - pub const PARTITION_GRANULARITY_ENV_VAR: &str = "S3_PARTITION_GRANULARITY"; pub const LOG_PROMPTS_ONLY_ENV_VAR: &str = "S3_LOG_PROMPTS_ONLY"; const DEFAULT_MAX_QUEUE_SIZE: usize = 50_000; @@ -122,71 +122,17 @@ fn is_blank(value: &Value) -> bool { } pub fn lax_bool(value: &Value) -> Option { - match value { - Value::Bool(flag) => Some(*flag), - Value::Number(number) => number - .as_f64() - .and_then(|float| { - [(0.0, false), (1.0, true)] - .into_iter() - .find(|(bound, _)| *bound == float) - }) - .map(|(_, flag)| flag), - Value::String(text) => match text.trim().to_ascii_lowercase().as_str() { - "0" | "off" | "f" | "false" | "n" | "no" => Some(false), - "1" | "on" | "t" | "true" | "y" | "yes" => Some(true), - _ => None, - }, - _ => None, - } + pydantic::lax_bool(&python(value)) } pub fn lax_int(value: &Value) -> Option { - match value { - Value::Bool(flag) => Some(i128::from(*flag)), - Value::Number(number) => { - if let Some(integer) = number.as_i64() { - return Some(i128::from(integer)); - } - if let Some(integer) = number.as_u64() { - return Some(i128::from(integer)); - } - let float = number.as_f64()?; - (float.is_finite() && float.fract() == 0.0 && float.abs() <= i64::MAX as f64) - .then_some(float as i128) - } - Value::String(text) => int_text(text.trim()), - _ => None, - } -} - -fn int_text(text: &str) -> Option { - let (whole, fraction) = text.split_once('.').unwrap_or((text, "")); - if !fraction.chars().all(|character| character == '0') { - return None; - } - if text.contains('.') && fraction.is_empty() { - return None; - } - let (sign, digits) = match whole.strip_prefix('-') { - Some(rest) => (-1, rest), - None => (1, whole.strip_prefix('+').unwrap_or(whole)), - }; - let well_formed = !digits.is_empty() - && !digits.starts_with('_') - && !digits.ends_with('_') - && !digits.contains("__") - && digits - .chars() - .all(|character| character.is_ascii_digit() || character == '_'); - if !well_formed { - return None; - } - digits - .replace('_', "") - .parse::() - .ok() - .map(|number| sign * number) + pydantic::lax_int(&python(value)).map(|number| { + i128::try_from(&number).unwrap_or(if number < BigInt::from(0) { + i128::MIN + } else { + i128::MAX + }) + }) } fn strip_text(value: &Value) -> Value { @@ -291,7 +237,7 @@ fn sse_value(name: &str, value: Option<&Value>) -> Option { Some(other) => { tracing::warn!( "s3 logging: ignoring {name} because it has invalid type {}; expected a string", - py_type_name(other) + python(other).type_name() ); None } @@ -390,6 +336,8 @@ impl Settings { web_identity_token: secret(params, "s3_aws_web_identity_token"), sts_endpoint: text(params, "s3_aws_sts_endpoint"), external_id: None, + session_tags: None, + session_policy: None, }, use_team_prefix: truthy_param("s3_use_team_prefix"), use_key_prefix: truthy_param("s3_use_key_prefix"), @@ -556,6 +504,16 @@ mod tests { assert_eq!(lax_bool(&strip_text(&value)), expected); } + #[rstest] + #[case(json!(["us-east-1"]), "['us-east-1']")] + #[case(json!({"a": true}), "{'a': True}")] + #[case(json!(null), "None")] + #[case(json!(1.5), "1.5")] + #[case(json!("us-east-1"), "us-east-1")] + fn values_print_as_python_str_prints_them(#[case] value: Value, #[case] expected: &str) { + assert_eq!(py_str(&value), expected); + } + #[rstest] fn defaults_match_the_python_constructor() { let settings = settings(json!({})); diff --git a/litellm/rust/crates/callbacks-s3-log/src/upload.rs b/litellm/rust/crates/callbacks-s3-log/src/upload.rs index b5e4f0aa39f..6ca69562e39 100644 --- a/litellm/rust/crates/callbacks-s3-log/src/upload.rs +++ b/litellm/rust/crates/callbacks-s3-log/src/upload.rs @@ -384,7 +384,7 @@ impl Uploader { .settings .region_name .as_ref() - .map(crate::pyjson::py_str) + .map(crate::settings::py_str) else { tracing::error!("Error downloading from S3: no s3_region_name to sign with"); return None; diff --git a/litellm/rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json b/litellm/rust/crates/db/.sqlx/query-805611b4198a8f11e66e29b77afa021f3a07befaedd77d4949e9401b120a5322.json similarity index 59% rename from litellm/rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json rename to litellm/rust/crates/db/.sqlx/query-805611b4198a8f11e66e29b77afa021f3a07befaedd77d4949e9401b120a5322.json index 7eef5278cb5..7fb7d2aed5f 100644 --- a/litellm/rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json +++ b/litellm/rust/crates/db/.sqlx/query-805611b4198a8f11e66e29b77afa021f3a07befaedd77d4949e9401b120a5322.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT set_config('lock_timeout', '5000ms', true)", + "query": "SELECT set_config('lock_timeout', $1::text, true)", "describe": { "columns": [ { @@ -11,11 +11,13 @@ } ], "parameters": { - "Left": [] + "Left": [ + "Text" + ] }, "nullable": [ null ] }, - "hash": "5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4" + "hash": "805611b4198a8f11e66e29b77afa021f3a07befaedd77d4949e9401b120a5322" } diff --git a/litellm/rust/crates/db/AGENTS.md b/litellm/rust/crates/db/AGENTS.md index 1e08317230e..c7ebedc6948 100644 --- a/litellm/rust/crates/db/AGENTS.md +++ b/litellm/rust/crates/db/AGENTS.md @@ -23,3 +23,54 @@ SQLX_OFFLINE=false cargo sqlx prepare ``` Tests use transactions and leave them uncommitted so fixture rows roll back when each test finishes + +# Boundary with the hosts + +The Python gateway and the Rust gateway both host this crate during the migration, so it takes only what both can hand it: + +- Pool size is a plain `max_connections`. How a worker's `connection_limit` is split between Prisma and SQLx is decided by the Python host (`litellm/proxy/db/connection_budget.py`) and disappears with Prisma. Never add budget or Prisma knowledge here +- Credentials arrive as a password (`set_writer_password`). Each process runs one token refresh loop owned by its host, so this crate never mints or refreshes a token on its own. When the Rust gateway needs IAM or Entra tokens, the minting logic moves here as a function the host's loop calls, and the Python loop switches to it, so the resolution logic exists once + +# Parity with the Python Prisma stack + +Python and SQLx run side by side until every repository operation is native, so the SQLx pool must honor every database setting Python honors. `Endpoint` is resolved in Python from the final `DATABASE_URL` (`litellm/proxy/db/native_database.py`); Rust never reads `DATABASE_*` env vars or parses a Prisma URL. When `litellm/proxy/db/` gains a setting or behavior, add it here + +| Python behavior | Where | Native status | +| --- | --- | --- | +| URL assembled from `DATABASE_*`, token minted, libpq TLS translated | `db_url_settings.py` | Inherited: the native endpoint reads the final URL | +| `host` Unix socket directory | URL param | `Endpoint::socket` | +| `sslmode`, `sslaccept=strict`, `sslcert` | `db_url_settings.py` | `strict` is `verify-full` with `sslcert` as the root; without it nothing is verified, so `sslcert` is dropped. `prefer` + `strict` asks the server once at startup and pins `verify-full` or `disable` | +| `sslidentity`, `sslpassword` | URL params | Python opens the PKCS#12 file and hands the client certificate and key over as PEM | +| `connection_limit` | `proxy_cli.py`, `connection_budget.py` | Split, not added: `ConnectionBudget.split` gives the Rust pool the hard-coded `RUST_RESERVED_CONNECTIONS` and Prisma the rest, so an upgrade never grows the budget. No setting names Rust, SQLx or Prisma. The reserve is 0 until a ported query needs the pool, so Prisma keeps its whole limit and no native pool opens. Once it is above 0, Prisma connections are tagged `application_name=litellm-prisma` and native ones `litellm-native` | +| `pool_timeout`, `max_idle_connection_lifetime` | `proxy_cli.py`, `db_url_settings.py` | `PoolLimits`, with Prisma's 10s default. `pool_timeout=0` waits forever | +| `socket_timeout` | `proxy_cli.py` | `PoolLimits::socket_timeout_seconds`, a deadline on each native operation | +| `connect_timeout` | `proxy_cli.py` | Not carried over: SQLx has no per-connect timeout, so a black-holed connect fails at `pool_timeout` | +| `options` (`statement_timeout`, `lock_timeout`, `idle_in_transaction_session_timeout`) | `proxy_cli.py` | `Endpoint::options`, sent as startup `-c` settings. Tokens that are not `-c name=value` are logged at boot and dropped | +| `schema` | URL param | `search_path` startup setting when not `public`; a custom schema behind PgBouncer is rejected by PgBouncer | +| `statement_cache_size`, `pgbouncer=true`, `DATABASE_DISABLE_PREPARED_STATEMENTS` | URL params, `db_url_settings.py` | `statement_cache_capacity`: `statement_cache_size` (Prisma's default 100), 0 under `pgbouncer=true`. Run every query through `Statements::new(executor, endpoint.names_statements())` so a 0 capacity sends unnamed statements like Prisma, until a sqlx release carries transact-rs/sqlx#4337 | +| `application_name` | URL param | A value the operator set is used on both pools; Prisma's `litellm-prisma` default becomes `litellm-native` | +| Rotating RDS IAM / Entra token | `token_auth.py`, `PrismaWrapper` | Python mints per side, `NativeDatabase.set_writer_password` / `set_reader_password` swap connect options | +| In-container PgBouncer | `pgbouncer.py` | Inherited: the pooled URL carries PgBouncer's credential | +| Startup connection budget | `postgres_connection_budget_message` | Counts the Rust writer and reader pools' shares | +| Read replica routing | `routing_prisma_wrapper.py`, `DATABASE_URL_READ_REPLICA` | `Pools`: callers pick `writer()`, `read()` or `read_after_write()`. `read()` prefers the reader and `read_after_write()` the writer (`WriterPinnedClient`); either falls back to the other side when its pool cannot connect and skips that side for 30s, so reads keep working on the reader while the writer is down. Writes have no fallback | +| `lock_timeout` on rollups | `rollup_lock_timeout.py` | `begin` takes the caller's `lock_timeout` as whole positive milliseconds, like `SPEND_ROLLUP_LOCK_TIMEOUT_MS`, so it can never send `0ms`, which Postgres reads as no timeout | +| Error classification | `exception_handler.py` | `Error::kind`, `is_unavailable`, `is_requeue_safe`. The bridge raises `NativeDatabaseFailure` with the kind and both answers, and `is_database_connection_error` and `db_write_failure_is_requeue_safe` return them instead of classifying the error again | +| Recreate the client after 25006 (writer became read-only) or a stale cached plan | `exception_handler.py`, `_query_first_with_cached_plan_fallback` | `Pools::recycle_after` drops every pooled connection opened before the error, so the next acquire reconnects and prepares fresh. Callers retry a `StalePlan` once | +| `ServiceTypes.DB` events per round trip | `db_span.py`, `log_db_metrics.py`, `prometheus_services.py`, `otel/model/spans.py` | Python host only. Wrap each native operation in `observe` with the call type and table its Prisma version uses, and the bridge forwards events to `ServiceLoggingEvents`. Nesting under an outer Python `db_span` is not detected. A Rust host has no sink yet and must emit the names below | +| Auth lookup gate, fail open on DB outage, reader choice per query | `db_lookup_gate.py`, `allow_requests_on_db_unavailable` | With each ported auth read | +| Connect at boot, so a database that cannot be reached is reported before traffic | `PrismaClient.connect` | The pool stays lazy and never fails boot. `Database.open` pings it once in the background and logs a warning when it cannot connect, so a mistranslated endpoint is not first found by a ported query | +| Health watchdog, engine reaper, drain tracker, engine recreation | `utils.py`, `query_engine_reaper.py`, `prisma_client.py` | Not needed: SQLx pools reconnect per connection and `close` waits for checked-out connections | +| Migrations, `check_migration`, views | `prisma_client.py`, `create_views.py` | Prisma keeps them until the SQLx cutover | + +## DB event names a Rust host sink must match + +Prometheus and OTel dashboards aggregate across the Python proxy and any Rust server, so a Rust sink emits the same names and labels Python does, pinned by inline cases on both sides + +- Prometheus + - `litellm_postgres_latency`, a histogram with label `postgres="postgres"` + - `litellm_postgres_total_requests` + - `litellm_postgres_failed_requests`, adding `error_class` (`NativeDatabaseError` for native failures) and `function_name` (the call type) +- OTel span name, from `service_span_name` + - `postgres.{verb} {table}` when the call type maps to a known Prisma helper and its table resolves + - `postgres.{verb}` for targetless verbs, for example `postgres.ping` for `health_check` + - `postgres {call_type}` otherwise diff --git a/litellm/rust/crates/db/Cargo.toml b/litellm/rust/crates/db/Cargo.toml index 4f8fe4e28e2..a9cebd9d5f5 100644 --- a/litellm/rust/crates/db/Cargo.toml +++ b/litellm/rust/crates/db/Cargo.toml @@ -4,16 +4,23 @@ version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true +include = ["Cargo.toml", "src/**", "queries/**", ".sqlx/**"] [dependencies] chrono.workspace = true +futures-util.workspace = true +litellm-circuit-breaker.workspace = true +litellm-error.workspace = true +litellm-secret-value.workspace = true macro_rules_attribute.workspace = true serde.workspace = true serde_json.workspace = true sqlx.workspace = true strum.workspace = true thiserror.workspace = true +tokio = { workspace = true, features = ["time"] } [dev-dependencies] rstest.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["test-util"] } +url.workspace = true diff --git a/litellm/rust/crates/db/queries/.gitattributes b/litellm/rust/crates/db/queries/.gitattributes new file mode 100644 index 00000000000..cc2d335b830 --- /dev/null +++ b/litellm/rust/crates/db/queries/.gitattributes @@ -0,0 +1 @@ +*.sql text eol=lf diff --git a/litellm/rust/crates/db/src/deadline.rs b/litellm/rust/crates/db/src/deadline.rs new file mode 100644 index 00000000000..d986daa108c --- /dev/null +++ b/litellm/rust/crates/db/src/deadline.rs @@ -0,0 +1,51 @@ +use std::{io, time::Duration}; + +use crate::Error; + +pub async fn within( + socket_timeout: Option, + operation: impl Future>, +) -> Result { + let Some(limit) = socket_timeout else { + return operation.await; + }; + tokio::time::timeout(limit, operation) + .await + .unwrap_or_else(|_| { + Err(Error::Query(sqlx::Error::Io( + io::ErrorKind::TimedOut.into(), + ))) + }) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::within; + use crate::Error; + + #[tokio::test(start_paused = true)] + async fn a_slow_operation_fails_like_a_socket_timeout() { + let result = within(Some(Duration::from_secs(5)), async { + tokio::time::sleep(Duration::from_secs(6)).await; + Ok::<_, Error>(()) + }) + .await; + + assert!( + matches!(result, Err(Error::Query(sqlx::Error::Io(error))) if error.kind() == std::io::ErrorKind::TimedOut) + ); + } + + #[tokio::test(start_paused = true)] + async fn no_timeout_waits_for_the_operation() { + let result = within(None, async { + tokio::time::sleep(Duration::from_secs(3600)).await; + Ok::<_, Error>(7) + }) + .await; + + assert_eq!(result.ok(), Some(7)); + } +} diff --git a/litellm/rust/crates/db/src/endpoint.rs b/litellm/rust/crates/db/src/endpoint.rs new file mode 100644 index 00000000000..4eba6a9942c --- /dev/null +++ b/litellm/rust/crates/db/src/endpoint.rs @@ -0,0 +1,337 @@ +use std::{path::PathBuf, time::Duration}; + +use litellm_secret_value::SecretValue; +use serde::Deserialize; +use sqlx::{ + PgPool, + postgres::{PgConnectOptions, PgPoolOptions, PgSslMode}, +}; + +use crate::Error; + +const WAIT_FOREVER: Duration = Duration::from_secs(100 * 365 * 24 * 60 * 60); + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct Endpoint { + pub host: String, + pub port: u16, + pub socket: Option, + pub username: String, + pub password: Option, + pub database: String, + pub ssl_mode: SslMode, + pub ssl_root_cert: Option, + pub ssl_client: Option, + pub application_name: Option, + pub options: Vec<(String, String)>, + pub statement_cache_capacity: usize, + pub pool: PoolLimits, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum SslMode { + Disable, + Prefer, + Require, + VerifyCa, + VerifyFull, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ClientIdentity { + pub cert_pem: String, + pub key_pem: SecretValue, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq)] +pub struct PoolLimits { + pub max_connections: u32, + pub acquire_timeout_seconds: Option, + pub idle_timeout_seconds: Option, + pub socket_timeout_seconds: Option, +} + +impl From for PgSslMode { + fn from(mode: SslMode) -> Self { + match mode { + SslMode::Disable => Self::Disable, + SslMode::Prefer => Self::Prefer, + SslMode::Require => Self::Require, + SslMode::VerifyCa => Self::VerifyCa, + SslMode::VerifyFull => Self::VerifyFull, + } + } +} + +impl Endpoint { + pub fn with_password(&self, password: SecretValue) -> Self { + Self { + password: Some(password), + ..self.clone() + } + } + + pub fn connect_options(&self) -> PgConnectOptions { + let options = PgConnectOptions::new_without_pgpass() + .host(&self.host) + .port(self.port) + .username(&self.username) + .database(&self.database) + .ssl_mode(self.ssl_mode.into()) + .options(self.options.iter().map(|(name, value)| (name, value))) + .statement_cache_capacity(self.statement_cache_capacity); + let options = match &self.socket { + Some(directory) => options.socket(directory), + None => options, + }; + let options = match &self.password { + Some(password) => options.password(password.expose()), + None => options, + }; + let options = match &self.application_name { + Some(name) => options.application_name(name), + None => options, + }; + let options = match &self.ssl_client { + Some(identity) => options + .ssl_client_cert_from_pem(&identity.cert_pem) + .ssl_client_key_from_pem(identity.key_pem.expose()), + None => options, + }; + match &self.ssl_root_cert { + Some(cert) => options.ssl_root_cert(cert), + None => options, + } + } + + pub fn pool_options(&self) -> Result { + let limits = self.pool; + Ok(PgPoolOptions::new() + .min_connections(0) + .max_connections(limits.max_connections) + .acquire_timeout(seconds(limits.acquire_timeout_seconds)?.unwrap_or(WAIT_FOREVER)) + .idle_timeout(seconds(limits.idle_timeout_seconds)?)) + } + + pub fn pool(&self) -> Result { + Ok(self + .pool_options()? + .connect_lazy_with(self.connect_options())) + } + + pub fn names_statements(&self) -> bool { + self.statement_cache_capacity > 0 + } + + pub fn socket_timeout(&self) -> Result, Error> { + seconds(self.pool.socket_timeout_seconds) + } +} + +fn seconds(value: Option) -> Result, Error> { + value + .map(Duration::try_from_secs_f64) + .transpose() + .map_err(Error::from) +} + +#[cfg(test)] +mod tests { + use std::{error::Error as StdError, path::PathBuf, time::Duration}; + + use litellm_secret_value::SecretValue; + use rstest::rstest; + use sqlx::postgres::PgSslMode; + + use super::{Endpoint, PoolLimits, SslMode, WAIT_FOREVER}; + use crate::{Error, test_support::database_endpoint}; + + const LIMITS: PoolLimits = PoolLimits { + max_connections: 7, + acquire_timeout_seconds: Some(2.5), + idle_timeout_seconds: Some(60.0), + socket_timeout_seconds: None, + }; + + fn endpoint() -> Endpoint { + Endpoint { + host: "db.internal".to_owned(), + port: 6543, + socket: None, + username: "litellm".to_owned(), + password: Some(SecretValue::new("first")), + database: "proxy".to_owned(), + ssl_mode: SslMode::VerifyFull, + ssl_root_cert: None, + ssl_client: None, + application_name: Some("litellm-native".to_owned()), + options: vec![("statement_timeout".to_owned(), "30000".to_owned())], + statement_cache_capacity: 100, + pool: LIMITS, + } + } + + #[test] + fn maps_endpoint_to_connect_options() { + let options = endpoint().connect_options(); + + assert_eq!(options.get_host(), "db.internal"); + assert_eq!(options.get_port(), 6543); + assert_eq!(options.get_socket(), None); + assert_eq!(options.get_username(), "litellm"); + assert_eq!(options.get_database(), Some("proxy")); + assert_eq!(options.get_application_name(), Some("litellm-native")); + assert!(matches!(options.get_ssl_mode(), PgSslMode::VerifyFull)); + assert_eq!(options.get_options(), Some("-c statement_timeout=30000")); + } + + #[test] + fn connects_through_a_unix_socket_directory() { + let options = Endpoint { + socket: Some(PathBuf::from("/cloudsql/project:region:instance")), + ..endpoint() + } + .connect_options(); + + assert_eq!( + options.get_socket(), + Some(&PathBuf::from("/cloudsql/project:region:instance")) + ); + } + + #[tokio::test] + async fn applies_pool_limits() -> Result<(), Error> { + let pool = endpoint().pool()?; + + assert_eq!(pool.options().get_max_connections(), 7); + assert_eq!(pool.options().get_min_connections(), 0); + assert_eq!( + pool.options().get_acquire_timeout(), + Duration::from_millis(2500) + ); + assert_eq!( + pool.options().get_idle_timeout(), + Some(Duration::from_secs(60)) + ); + assert_eq!(pool.size(), 0); + Ok(()) + } + + #[tokio::test] + async fn missing_acquire_timeout_waits_forever() -> Result<(), Error> { + let pool = Endpoint { + pool: PoolLimits { + acquire_timeout_seconds: None, + ..LIMITS + }, + ..endpoint() + } + .pool()?; + + assert_eq!(pool.options().get_acquire_timeout(), WAIT_FOREVER); + Ok(()) + } + + #[rstest] + #[case::negative(-1.0)] + #[case::not_a_number(f64::NAN)] + #[tokio::test] + async fn rejects_invalid_durations(#[case] seconds: f64) { + let error = Endpoint { + pool: PoolLimits { + acquire_timeout_seconds: Some(seconds), + ..LIMITS + }, + ..endpoint() + } + .pool() + .expect_err("an invalid duration must not build a pool"); + + assert!(matches!(error, Error::InvalidDuration(_))); + } + + #[test] + fn redacts_password() { + assert!(!format!("{:?}", endpoint()).contains("first")); + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn waits_past_an_exhausted_pool_without_a_timeout() -> Result<(), Box> { + let endpoint = database_endpoint()?; + let pool = Endpoint { + pool: PoolLimits { + max_connections: 1, + acquire_timeout_seconds: None, + ..endpoint.pool + }, + ..endpoint + } + .pool()?; + let held = pool.acquire().await?; + let waiter = tokio::spawn({ + let pool = pool.clone(); + async move { pool.acquire().await.map(drop) } + }); + + tokio::time::sleep(Duration::from_millis(300)).await; + assert!(!waiter.is_finished()); + drop(held); + + waiter.await??; + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn applies_session_options() -> Result<(), Box> { + let pool = Endpoint { + options: vec![ + ("statement_timeout".to_owned(), "1234".to_owned()), + ( + "search_path".to_owned(), + "\"Other Schema\", public".to_owned(), + ), + ], + application_name: Some("litellm-native".to_owned()), + ..database_endpoint()? + } + .pool()?; + + #[expect( + clippy::disallowed_methods, + reason = "Session settings are not part of the compile-checked schema" + )] + let (statement_timeout, search_path, application_name): (String, String, String) = + sqlx::query_as( + "SELECT current_setting('statement_timeout'), current_setting('search_path'), \ + current_setting('application_name')", + ) + .fetch_one(&pool) + .await?; + + assert_eq!(statement_timeout, "1234ms"); + assert_eq!(search_path, "\"Other Schema\", public"); + assert_eq!(application_name, "litellm-native"); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn rotated_password_reaches_new_connections() -> Result<(), Box> { + let endpoint = database_endpoint()?; + let pool = endpoint + .with_password(SecretValue::new("not-the-password")) + .pool()?; + #[expect(clippy::disallowed_methods, reason = "A probe query needs no schema")] + let probe = || sqlx::query("SELECT 1"); + let stale = probe().execute(&pool).await; + + pool.set_connect_options(endpoint.connect_options()); + probe().execute(&pool).await?; + + assert!(stale.is_err()); + Ok(()) + } +} diff --git a/litellm/rust/crates/db/src/error.rs b/litellm/rust/crates/db/src/error.rs index af0ee030896..22ddb643d9c 100644 --- a/litellm/rust/crates/db/src/error.rs +++ b/litellm/rust/crates/db/src/error.rs @@ -6,23 +6,80 @@ pub enum Error { InvalidSpendBatch { ids: usize, costs: usize }, #[error("invalid spend timestamp")] InvalidTimestamp(#[from] chrono::ParseError), + #[error("invalid pool duration")] + InvalidDuration(#[from] std::time::TryFromFloatSecsError), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr)] +#[strum(serialize_all = "snake_case")] +pub enum ErrorKind { + Unreachable, + Capacity, + Deadlock, + SerializationFailure, + LockTimeout, + ReadOnly, + StalePlan, + DataRejected, + Other, } impl Error { - pub fn is_retryable(&self) -> bool { + pub fn kind(&self) -> ErrorKind { match self { + Self::Query(sqlx::Error::Io(_)) => ErrorKind::Unreachable, + Self::Query(sqlx::Error::PoolTimedOut) => ErrorKind::Capacity, Self::Query(sqlx::Error::Database(error)) => { - matches!( - error.code().as_deref(), - Some("40P01" | "55P03" | "40001" | "53300") - ) + sqlstate_kind(error.code().as_deref(), error.message()) } - Self::Query(sqlx::Error::Io(_) | sqlx::Error::PoolTimedOut) => true, + _ => ErrorKind::Other, + } + } + + pub fn is_retryable(&self) -> bool { + matches!( + self.kind(), + ErrorKind::Unreachable + | ErrorKind::Capacity + | ErrorKind::Deadlock + | ErrorKind::SerializationFailure + | ErrorKind::LockTimeout + | ErrorKind::StalePlan + ) + } + + pub fn needs_fresh_connections(&self) -> bool { + matches!(self.kind(), ErrorKind::ReadOnly | ErrorKind::StalePlan) + } + + pub fn is_unavailable(&self) -> bool { + matches!(self.kind(), ErrorKind::Unreachable | ErrorKind::Capacity) + } + + pub fn is_requeue_safe(&self) -> bool { + match self { + Self::Query(sqlx::Error::PoolTimedOut | sqlx::Error::PoolClosed) => true, + Self::Query(sqlx::Error::Database(_)) => self.kind() != ErrorKind::DataRejected, _ => false, } } } +fn sqlstate_kind(code: Option<&str>, message: &str) -> ErrorKind { + match code { + Some("0A000") if message.contains("cached plan must not change result type") => { + ErrorKind::StalePlan + } + Some("53300") => ErrorKind::Capacity, + Some("40P01") => ErrorKind::Deadlock, + Some("40001") => ErrorKind::SerializationFailure, + Some("55P03") => ErrorKind::LockTimeout, + Some("25006") => ErrorKind::ReadOnly, + Some(code) if code.starts_with("22") || code.starts_with("23") => ErrorKind::DataRejected, + _ => ErrorKind::Other, + } +} + #[cfg(test)] mod tests { use std::error::Error as StdError; @@ -30,29 +87,59 @@ mod tests { use rstest::rstest; use sqlx::PgPool; - use super::Error; + use super::{Error, ErrorKind}; use crate::test_support::database_pool; #[rstest] - #[case::closed_pool(sqlx::Error::PoolClosed, false)] - #[case::pool_timeout(sqlx::Error::PoolTimedOut, true)] - #[case::io(sqlx::Error::Io(std::io::ErrorKind::ConnectionReset.into()), true)] - #[case::missing_row(sqlx::Error::RowNotFound, false)] - fn classifies_client_errors(#[case] error: sqlx::Error, #[case] retryable: bool) { - assert_eq!(Error::from(error).is_retryable(), retryable); + #[case::closed_pool(sqlx::Error::PoolClosed, ErrorKind::Other, false, false, true)] + #[case::pool_timeout(sqlx::Error::PoolTimedOut, ErrorKind::Capacity, true, true, true)] + #[case::io( + sqlx::Error::Io(std::io::ErrorKind::ConnectionReset.into()), + ErrorKind::Unreachable, + true, + true, + false + )] + #[case::missing_row(sqlx::Error::RowNotFound, ErrorKind::Other, false, false, false)] + fn classifies_client_errors( + #[case] error: sqlx::Error, + #[case] kind: ErrorKind, + #[case] retryable: bool, + #[case] unavailable: bool, + #[case] requeue_safe: bool, + ) { + let error = Error::from(error); + + assert_eq!(error.kind(), kind); + assert_eq!(error.is_retryable(), retryable); + assert_eq!(error.is_unavailable(), unavailable); + assert_eq!(error.is_requeue_safe(), requeue_safe); } #[rstest] - #[case::connection_capacity("53300", true)] - #[case::deadlock("40P01", true)] - #[case::lock_unavailable("55P03", true)] - #[case::serialization_failure("40001", true)] - #[case::unique_violation("23505", false)] + #[case::connection_capacity("53300", ErrorKind::Capacity, true, true, true)] + #[case::deadlock("40P01", ErrorKind::Deadlock, true, false, true)] + #[case::lock_unavailable("55P03", ErrorKind::LockTimeout, true, false, true)] + #[case::serialization_failure("40001", ErrorKind::SerializationFailure, true, false, true)] + #[case::read_only_after_failover("25006", ErrorKind::ReadOnly, false, false, true)] + #[case::unique_violation("23505", ErrorKind::DataRejected, false, false, false)] + #[case::numeric_overflow("22003", ErrorKind::DataRejected, false, false, false)] + #[case::undefined_table("42P01", ErrorKind::Other, false, false, true)] + #[case::feature_not_supported_without_a_stale_plan( + "0A000", + ErrorKind::Other, + false, + false, + true + )] #[tokio::test] #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] async fn classifies_server_errors( #[case] code: &str, + #[case] kind: ErrorKind, #[case] retryable: bool, + #[case] unavailable: bool, + #[case] requeue_safe: bool, #[future(awt)] database_pool: Result>, ) -> Result<(), Box> { let pool = database_pool?; @@ -72,7 +159,11 @@ mod tests { error.as_database_error().and_then(|error| error.code()), Some(code.into()) ); - assert_eq!(Error::from(error).is_retryable(), retryable); + let error = Error::from(error); + assert_eq!(error.kind(), kind); + assert_eq!(error.is_retryable(), retryable); + assert_eq!(error.is_unavailable(), unavailable); + assert_eq!(error.is_requeue_safe(), requeue_safe); tx.rollback().await?; Ok(()) } diff --git a/litellm/rust/crates/db/src/health.rs b/litellm/rust/crates/db/src/health.rs new file mode 100644 index 00000000000..bd734b267fb --- /dev/null +++ b/litellm/rust/crates/db/src/health.rs @@ -0,0 +1,29 @@ +use sqlx::{Connection, PgPool}; + +use crate::Error; + +pub async fn health_check(pool: &PgPool) -> Result<(), Error> { + pool.acquire().await?.ping().await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use rstest::rstest; + use sqlx::PgPool; + + use super::health_check; + use crate::test_support::database_pool; + + #[rstest] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn answers_when_the_database_is_up( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + health_check(&database_pool?).await?; + Ok(()) + } +} diff --git a/litellm/rust/crates/db/src/lib.rs b/litellm/rust/crates/db/src/lib.rs index 541f97c7db9..89d655a8502 100644 --- a/litellm/rust/crates/db/src/lib.rs +++ b/litellm/rust/crates/db/src/lib.rs @@ -7,20 +7,32 @@ macro_rules_attribute::attribute_alias! { } mod daily_spend; +mod deadline; mod end_user; +mod endpoint; mod error; mod failure; +mod health; +mod observe; +mod pools; mod spend; +mod statements; #[cfg(test)] mod test_support; pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; +pub use deadline::within; pub use end_user::{EndUser, RateLimits, find_end_user}; -pub use error::Error; +pub use endpoint::{ClientIdentity, Endpoint, PoolLimits, SslMode}; +pub use error::{Error, ErrorKind}; pub use failure::DbFailure; +pub use health::health_check; +pub use observe::{Call, Observer, Outcome, QueryEvent, observe}; +pub use pools::Pools; pub use spend::{ DailySpend, DailyTable, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, add_end_user_spend, add_key_spend, add_team_member_spend, add_team_spend, add_user_spend, begin, insert_spend_logs, lock_team, upsert_daily_end_user_spend, upsert_daily_spend, upsert_daily_team_spend, upsert_daily_user_spend, }; +pub use statements::Statements; diff --git a/litellm/rust/crates/db/src/observe.rs b/litellm/rust/crates/db/src/observe.rs new file mode 100644 index 00000000000..bc02a0d5f21 --- /dev/null +++ b/litellm/rust/crates/db/src/observe.rs @@ -0,0 +1,128 @@ +use std::time::{Duration, Instant, SystemTime}; + +use crate::Error; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Call { + pub call_type: &'static str, + pub table: Option<&'static str>, + pub operation: Option<&'static str>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + Succeeded, + Failed(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct QueryEvent { + pub call: Call, + pub started_at: SystemTime, + pub duration: Duration, + pub outcome: Outcome, +} + +pub trait Observer: Send + Sync { + fn record(&self, event: QueryEvent); +} + +pub async fn observe( + observer: &dyn Observer, + call: Call, + query: impl Future>, +) -> Result { + let started_at = SystemTime::now(); + let clock = Instant::now(); + let result = query.await; + let outcome = match &result { + Ok(_) => Some(Outcome::Succeeded), + Err(error @ Error::Query(_)) => Some(Outcome::Failed( + litellm_error::Report::new(error).to_string(), + )), + Err(_) => None, + }; + if let Some(outcome) = outcome { + observer.record(QueryEvent { + call, + started_at, + duration: clock.elapsed(), + outcome, + }); + } + result +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::{Call, Observer, Outcome, QueryEvent, observe}; + use crate::Error; + + const CALL: Call = Call { + call_type: "add_key_spend", + table: Some("LiteLLM_VerificationToken"), + operation: Some("update"), + }; + + #[derive(Default)] + struct Recording(Mutex>); + + impl Observer for Recording { + fn record(&self, event: QueryEvent) { + self.0.lock().expect("recording lock").push(event); + } + } + + impl Recording { + fn outcomes(&self) -> Vec<(Call, Outcome)> { + self.0 + .lock() + .expect("recording lock") + .iter() + .map(|event| (event.call, event.outcome.clone())) + .collect() + } + } + + #[tokio::test] + async fn records_a_successful_round_trip() { + let recording = Recording::default(); + + let value = observe(&recording, CALL, async { Ok::<_, Error>(7) }).await; + + assert_eq!(value.ok(), Some(7)); + assert_eq!(recording.outcomes(), vec![(CALL, Outcome::Succeeded)]); + } + + #[tokio::test] + async fn records_a_database_failure() { + let recording = Recording::default(); + + let result = observe(&recording, CALL, async { + Err::<(), _>(Error::from(sqlx::Error::PoolTimedOut)) + }) + .await; + + assert!(result.is_err()); + let outcomes = recording.outcomes(); + assert_eq!(outcomes.len(), 1); + assert!( + matches!(&outcomes[0], (call, Outcome::Failed(message)) if *call == CALL && message.contains("pool timed out")) + ); + } + + #[tokio::test] + async fn skips_failures_that_never_reached_the_database() { + let recording = Recording::default(); + + let result = observe(&recording, CALL, async { + Err::<(), _>(Error::InvalidSpendBatch { ids: 1, costs: 2 }) + }) + .await; + + assert!(result.is_err()); + assert!(recording.outcomes().is_empty()); + } +} diff --git a/litellm/rust/crates/db/src/pools.rs b/litellm/rust/crates/db/src/pools.rs new file mode 100644 index 00000000000..59bcd68e5c5 --- /dev/null +++ b/litellm/rust/crates/db/src/pools.rs @@ -0,0 +1,399 @@ +use std::{ + sync::{Arc, Mutex, PoisonError}, + time::{Duration, Instant}, +}; + +use litellm_circuit_breaker::{Breaker, Config, Failure}; +use sqlx::{PgPool, Postgres, pool::PoolConnection}; + +use crate::{Endpoint, Error}; + +const RECOVERY: Duration = Duration::from_secs(30); + +type Connection = PoolConnection; + +#[derive(Default)] +struct Recycled(Mutex>); + +impl Recycled { + fn mark(&self, now: Instant) { + *self.0.lock().unwrap_or_else(PoisonError::into_inner) = Some(now); + } + + fn predates(&self, now: Instant, age: Duration) -> bool { + let Some(recycled_at) = *self.0.lock().unwrap_or_else(PoisonError::into_inner) else { + return false; + }; + now.checked_sub(age) + .is_none_or(|opened_at| opened_at <= recycled_at) + } +} + +#[derive(Clone)] +struct Side { + pool: PgPool, + breaker: Breaker, + recycled: Arc, +} + +impl Side { + fn new(endpoint: &Endpoint) -> Result { + let recycled = Arc::new(Recycled::default()); + let pool = endpoint + .pool_options()? + .before_acquire({ + let recycled = Arc::clone(&recycled); + move |_, metadata| { + let fresh = !recycled.predates(Instant::now(), metadata.age); + Box::pin(async move { Ok(fresh) }) + } + }) + .connect_lazy_with(endpoint.connect_options()); + Ok(Self { + pool, + recycled, + breaker: Breaker::new( + Config { + failure_threshold: 1, + recovery_timeout: RECOVERY, + ..Config::default() + }, + Box::new(|_| {}), + ), + }) + } + + async fn acquire_unless_skipped(&self) -> Option { + let admission = self.breaker.admit(Instant::now()).ok()?; + match self.pool.acquire().await { + Ok(connection) => { + admission.succeeded(); + Some(connection) + } + Err(_) => { + admission.failed(Failure::Hard, Instant::now()); + None + } + } + } + + async fn acquire(&self) -> Result { + let result = self.pool.acquire().await; + if result.is_err() { + self.breaker.failed(Failure::Hard, Instant::now()); + } + Ok(result?) + } +} + +async fn acquire_preferring( + preferred: &Side, + fallback: Option<&Side>, +) -> Result { + let Some(fallback) = fallback else { + return preferred.acquire().await; + }; + match preferred.acquire_unless_skipped().await { + Some(connection) => Ok(connection), + None => fallback.acquire().await, + } +} + +#[derive(Clone)] +pub struct Pools { + writer: Side, + reader: Option, +} + +impl Pools { + pub fn new(writer: &Endpoint, reader: Option<&Endpoint>) -> Result { + Ok(Self { + writer: Side::new(writer)?, + reader: reader.map(Side::new).transpose()?, + }) + } + + pub fn writer(&self) -> &PgPool { + &self.writer.pool + } + + pub fn reader_pool(&self) -> Option<&PgPool> { + self.reader.as_ref().map(|reader| &reader.pool) + } + + pub async fn read(&self) -> Result { + match &self.reader { + Some(reader) => acquire_preferring(reader, Some(&self.writer)).await, + None => self.writer.acquire().await, + } + } + + pub async fn read_after_write(&self) -> Result { + acquire_preferring(&self.writer, self.reader.as_ref()).await + } + + pub fn recycle_after(&self, error: &Error) -> bool { + if !error.needs_fresh_connections() { + return false; + } + let now = Instant::now(); + self.writer.recycled.mark(now); + if let Some(reader) = &self.reader { + reader.recycled.mark(now); + } + true + } + + pub async fn close(&self) { + self.writer.pool.close().await; + if let Some(reader) = &self.reader { + reader.pool.close().await; + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + error::Error as StdError, + time::{Duration, Instant}, + }; + + use rstest::rstest; + use sqlx::{Connection as _, PgConnection}; + + use super::{Pools, Recycled}; + use crate::{Endpoint, Error, ErrorKind, PoolLimits, test_support::database_endpoint}; + + #[rstest] + #[case::never_recycled(None, 5, false)] + #[case::opened_before_the_recycle(Some(10), 30, true)] + #[case::opened_after_the_recycle(Some(10), 5, false)] + fn discards_connections_opened_before_a_recycle( + #[case] recycled_seconds_ago: Option, + #[case] age_seconds: u64, + #[case] stale: bool, + ) { + let now = Instant::now() + Duration::from_secs(60); + let recycled = Recycled::default(); + if let Some(ago) = recycled_seconds_ago { + recycled.mark(now - Duration::from_secs(ago)); + } + + assert_eq!( + recycled.predates(now, Duration::from_secs(age_seconds)), + stale + ); + } + + async fn backend_pid(pools: &Pools) -> Result { + #[expect(clippy::disallowed_methods, reason = "A probe query needs no schema")] + sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *pools.writer().acquire().await?) + .await + } + + async fn raised(pools: &Pools, code: &str) -> Result { + #[expect( + clippy::disallowed_methods, + reason = "Test query must raise the requested SQLSTATE" + )] + let error = sqlx::query(sqlx::AssertSqlSafe(format!( + "DO $$ BEGIN RAISE EXCEPTION USING ERRCODE = '{code}'; END $$" + ))) + .execute(pools.writer()) + .await + .expect_err("PostgreSQL must raise the requested error"); + Ok(Error::from(error)) + } + + #[rstest] + #[case::read_only_writer("25006", true)] + #[case::lock_timeout("55P03", false)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn recycles_pooled_connections_only_after_errors_that_poison_them( + #[case] code: &str, + #[case] recycled: bool, + ) -> Result<(), Box> { + let pools = Pools::new(&database_endpoint()?, None)?; + let before = backend_pid(&pools).await?; + + assert_eq!(pools.recycle_after(&raised(&pools, code).await?), recycled); + + assert_eq!(backend_pid(&pools).await? != before, recycled); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn a_schema_change_under_a_cached_plan_is_recovered_by_a_recycle() + -> Result<(), Box> { + let endpoint = database_endpoint()?; + let pools = Pools::new( + &Endpoint { + pool: PoolLimits { + max_connections: 1, + ..endpoint.pool + }, + ..endpoint.clone() + }, + None, + )?; + let table = format!("native_stale_plan_probe_{}", std::process::id()); + let select = format!("SELECT * FROM {table}"); + #[expect(clippy::disallowed_methods, reason = "A scratch table needs no schema")] + let run = |sql: &str| sqlx::query(sqlx::AssertSqlSafe(sql.to_owned())); + let mut admin = PgConnection::connect_with(&endpoint.connect_options()).await?; + run(&format!("CREATE TABLE {table} (id int)")) + .execute(&mut admin) + .await?; + + let outcome = async { + run(&select).fetch_all(pools.writer()).await?; + run(&format!("ALTER TABLE {table} ADD COLUMN extra int")) + .execute(&mut admin) + .await?; + let stale = Error::from( + run(&select) + .fetch_all(pools.writer()) + .await + .expect_err("the cached plan must be stale"), + ); + let recycled = pools.recycle_after(&stale); + run(&select).fetch_all(pools.writer()).await?; + Ok::<_, Box>((stale.kind(), recycled)) + } + .await; + run(&format!("DROP TABLE {table}")) + .execute(&mut admin) + .await?; + + assert_eq!(outcome?, (ErrorKind::StalePlan, true)); + Ok(()) + } + + fn read_only(endpoint: &Endpoint) -> Endpoint { + Endpoint { + options: vec![("default_transaction_read_only".to_owned(), "on".to_owned())], + ..endpoint.clone() + } + } + + fn unreachable(endpoint: &Endpoint) -> Endpoint { + Endpoint { + host: "127.0.0.1".to_owned(), + port: 1, + pool: PoolLimits { + acquire_timeout_seconds: Some(2.0), + ..endpoint.pool + }, + ..endpoint.clone() + } + } + + async fn is_read_only(connection: &mut PgConnection) -> Result { + #[expect( + clippy::disallowed_methods, + reason = "Session settings are not part of the compile-checked schema" + )] + let setting: String = sqlx::query_scalar("SHOW default_transaction_read_only") + .fetch_one(connection) + .await?; + Ok(setting == "on") + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn reads_go_to_the_reader_and_read_after_write_to_the_writer() + -> Result<(), Box> { + let writer = database_endpoint()?; + let pools = Pools::new(&writer, Some(&read_only(&writer)))?; + + assert!(is_read_only(&mut *pools.read().await?).await?); + assert!(!is_read_only(&mut *pools.read_after_write().await?).await?); + assert!(!is_read_only(&mut *pools.writer().acquire().await?).await?); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn writes_sent_to_the_reader_fail_read_only() -> Result<(), Box> { + let writer = database_endpoint()?; + let pools = Pools::new(&writer, Some(&read_only(&writer)))?; + let mut connection = pools.read().await?; + + #[expect(clippy::disallowed_methods, reason = "A scratch table needs no schema")] + let error = sqlx::query("CREATE TABLE native_reader_write_probe (id int)") + .execute(&mut *connection) + .await + .expect_err("a read-only session must reject writes"); + + assert_eq!( + error.as_database_error().and_then(|error| error.code()), + Some("25006".into()) + ); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn every_read_uses_the_writer_without_a_reader() -> Result<(), Box> { + let pools = Pools::new(&database_endpoint()?, None)?; + + assert!(pools.reader_pool().is_none()); + assert!(!is_read_only(&mut *pools.read().await?).await?); + assert!(!is_read_only(&mut *pools.read_after_write().await?).await?); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn unreachable_reader_falls_back_once_then_skips_it() -> Result<(), Box> { + let writer = database_endpoint()?; + let pools = Pools::new(&writer, Some(&unreachable(&writer)))?; + + assert!(!is_read_only(&mut *pools.read().await?).await?); + let started = Instant::now(); + assert!(!is_read_only(&mut *pools.read().await?).await?); + + assert!(started.elapsed().as_secs_f64() < 1.0); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn reads_stay_on_the_reader_while_the_writer_is_down() -> Result<(), Box> { + let reader = read_only(&database_endpoint()?); + let pools = Pools::new(&unreachable(&reader), Some(&reader))?; + + assert!(is_read_only(&mut *pools.read().await?).await?); + assert!(pools.writer().acquire().await.is_err()); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn read_after_write_yields_to_the_reader_while_the_writer_is_down() + -> Result<(), Box> { + let reader = read_only(&database_endpoint()?); + let pools = Pools::new(&unreachable(&reader), Some(&reader))?; + + assert!(is_read_only(&mut *pools.read_after_write().await?).await?); + let started = Instant::now(); + assert!(is_read_only(&mut *pools.read_after_write().await?).await?); + + assert!(started.elapsed().as_secs_f64() < 1.0); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn read_after_write_fails_when_the_only_database_is_down() -> Result<(), Box> + { + let pools = Pools::new(&unreachable(&database_endpoint()?), None)?; + + assert!(pools.read_after_write().await.is_err()); + Ok(()) + } +} diff --git a/litellm/rust/crates/db/src/spend/team_lock.rs b/litellm/rust/crates/db/src/spend/team_lock.rs index 7a6b9c7f7d0..5f958d6b9ce 100644 --- a/litellm/rust/crates/db/src/spend/team_lock.rs +++ b/litellm/rust/crates/db/src/spend/team_lock.rs @@ -1,28 +1,38 @@ +use std::num::NonZeroU32; + use sqlx::{PgPool, Postgres, Transaction}; -use crate::Error; +use crate::{Error, Statements}; -pub async fn begin(pool: &PgPool) -> Result, Error> { +pub async fn begin( + pool: &PgPool, + lock_timeout_ms: NonZeroU32, + named: bool, +) -> Result, Error> { let mut transaction = pool.begin().await?; - sqlx::query!("SELECT set_config('lock_timeout', '5000ms', true)") - .fetch_one(&mut *transaction) - .await?; + sqlx::query!( + "SELECT set_config('lock_timeout', $1::text, true)", + format!("{lock_timeout_ms}ms") + ) + .fetch_one(Statements::new(&mut *transaction, named)) + .await?; Ok(transaction) } pub async fn lock_team( transaction: &mut Transaction<'_, Postgres>, team_id: &str, + named: bool, ) -> Result<(), Error> { sqlx::query_file!("queries/spend/lock_team.sql", team_id) - .fetch_one(&mut **transaction) + .fetch_one(Statements::new(&mut **transaction, named)) .await?; Ok(()) } #[cfg(test)] mod tests { - use std::error::Error as StdError; + use std::{error::Error as StdError, num::NonZeroU32}; use super::{begin, lock_team}; use crate::test_support::database_pool; @@ -42,7 +52,12 @@ mod tests { .max_connections(2) .connect_with(database_pool.connect_options().as_ref().clone()) .await?; - let mut first = begin(&pool).await?; + let mut first = begin( + &pool, + NonZeroU32::new(1500).ok_or("1500 is not zero")?, + true, + ) + .await?; let mut second = pool.begin().await?; #[expect( clippy::disallowed_methods, @@ -51,7 +66,7 @@ mod tests { let original_timeout: String = sqlx::query_scalar("SELECT current_setting('lock_timeout')") .fetch_one(&mut *second) .await?; - lock_team(&mut first, team_id).await?; + lock_team(&mut first, team_id, true).await?; #[expect( clippy::disallowed_methods, reason = "Test assertions are outside the compile-checked query set" @@ -60,7 +75,7 @@ mod tests { sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')") .fetch_one(&mut *first) .await?; - assert_eq!(timeout, "5s"); + assert_eq!(timeout, "1500ms"); #[expect( clippy::disallowed_methods, reason = "Test assertions are outside the compile-checked query set" diff --git a/litellm/rust/crates/db/src/spend/team_member_spend.rs b/litellm/rust/crates/db/src/spend/team_member_spend.rs index 965a39ada4f..9b4317fd137 100644 --- a/litellm/rust/crates/db/src/spend/team_member_spend.rs +++ b/litellm/rust/crates/db/src/spend/team_member_spend.rs @@ -1,7 +1,7 @@ use sqlx::{Postgres, Transaction}; use super::lock_team; -use crate::Error; +use crate::{Error, Statements}; #[macro_rules_attribute::apply(crate::db_input)] pub struct TeamMemberSpend { @@ -13,6 +13,7 @@ pub struct TeamMemberSpend { pub async fn add_team_member_spend( transaction: &mut Transaction<'_, Postgres>, members: &[TeamMemberSpend], + named: bool, ) -> Result<(), Error> { if members.is_empty() { return Ok(()); @@ -27,7 +28,7 @@ pub async fn add_team_member_spend( .collect(); teams.dedup(); for team in teams { - lock_team(transaction, team).await?; + lock_team(transaction, team, named).await?; } let rows = serde_json::Value::Array( members @@ -42,14 +43,14 @@ pub async fn add_team_member_spend( .collect(), ); sqlx::query_file!("queries/spend/upsert_team_member_spend.sql", rows) - .execute(&mut **transaction) + .execute(Statements::new(&mut **transaction, named)) .await?; Ok(()) } #[cfg(test)] mod tests { - use std::error::Error as StdError; + use std::{error::Error as StdError, num::NonZeroU32}; use super::super::begin; use super::{TeamMemberSpend, add_team_member_spend}; @@ -90,7 +91,12 @@ mod tests { #[future(awt)] database_pool: Result>, ) -> Result<(), Box> { let pool = database_pool?; - let mut tx = begin(&pool).await?; + let mut tx = begin( + &pool, + NonZeroU32::new(5000).ok_or("5000 is not zero")?, + true, + ) + .await?; #[expect( clippy::disallowed_methods, reason = "Fixture writes are outside the compile-checked query set" @@ -113,10 +119,10 @@ mod tests { ) .execute(&mut *tx) .await?; - add_team_member_spend(&mut tx, &[]).await?; - add_team_member_spend(&mut tx, &members(0.0)).await?; - add_team_member_spend(&mut tx, &members(cost)).await?; - add_team_member_spend(&mut tx, &members(cost)).await?; + add_team_member_spend(&mut tx, &[], true).await?; + add_team_member_spend(&mut tx, &members(0.0), true).await?; + add_team_member_spend(&mut tx, &members(cost), true).await?; + add_team_member_spend(&mut tx, &members(cost), true).await?; #[expect( clippy::disallowed_methods, reason = "Test assertions are outside the compile-checked query set" diff --git a/litellm/rust/crates/db/src/statements.rs b/litellm/rust/crates/db/src/statements.rs new file mode 100644 index 00000000000..a3da065fe81 --- /dev/null +++ b/litellm/rust/crates/db/src/statements.rs @@ -0,0 +1,152 @@ +use futures_util::{future::BoxFuture, stream::BoxStream}; +use sqlx::{ + Describe, Either, Execute, Executor, Postgres, SqlStr, + error::BoxDynError, + postgres::{PgArguments, PgQueryResult, PgRow, PgStatement, PgTypeInfo}, +}; + +// TODO(transact-rs/sqlx#4337): sqlx-postgres 0.9.0 still names every statement when +// statement_cache_capacity is 0, leaving one open per query on the server and colliding +// as "sqlx_s_1 already exists" behind transaction poolers. #4337 makes capacity 0 send +// unnamed statements on its own; delete this wrapper once a sqlx release carries it. +#[derive(Debug)] +pub struct Statements { + executor: E, + named: bool, +} + +impl Statements { + pub fn new(executor: E, named: bool) -> Self { + Self { executor, named } + } +} + +struct Naming { + query: Q, + named: bool, +} + +impl<'q, Q: Execute<'q, Postgres>> Execute<'q, Postgres> for Naming { + fn sql(self) -> SqlStr { + self.query.sql() + } + + fn statement(&self) -> Option<&PgStatement> { + self.query.statement() + } + + fn take_arguments(&mut self) -> Result, BoxDynError> { + self.query.take_arguments() + } + + fn persistent(&self) -> bool { + self.named && self.query.persistent() + } +} + +impl<'c, E: Executor<'c, Database = Postgres>> Executor<'c> for Statements { + type Database = Postgres; + + fn fetch_many<'e, 'q: 'e, Q>( + self, + query: Q, + ) -> BoxStream<'e, Result, sqlx::Error>> + where + 'c: 'e, + Q: 'q + Execute<'q, Postgres>, + { + self.executor.fetch_many(Naming { + query, + named: self.named, + }) + } + + fn fetch_optional<'e, 'q: 'e, Q>( + self, + query: Q, + ) -> BoxFuture<'e, Result, sqlx::Error>> + where + 'c: 'e, + Q: 'q + Execute<'q, Postgres>, + { + self.executor.fetch_optional(Naming { + query, + named: self.named, + }) + } + + fn prepare_with<'e>( + self, + sql: SqlStr, + parameters: &'e [PgTypeInfo], + ) -> BoxFuture<'e, Result> + where + 'c: 'e, + { + self.executor.prepare_with(sql, parameters) + } + + fn describe<'e>(self, sql: SqlStr) -> BoxFuture<'e, Result, sqlx::Error>> + where + 'c: 'e, + { + self.executor.describe(sql) + } +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use rstest::rstest; + use sqlx::{AssertSqlSafe, PgConnection}; + + use super::Statements; + use crate::{Endpoint, test_support::database_endpoint}; + + async fn named_statements( + connection: &mut PgConnection, + named: bool, + ) -> Result { + #[expect( + clippy::disallowed_methods, + reason = "pg_prepared_statements is not part of the compile-checked schema" + )] + sqlx::query_scalar("SELECT count(*) FROM pg_prepared_statements WHERE NOT from_sql") + .fetch_one(Statements::new(connection, named)) + .await + } + + #[rstest] + #[case::cache_off(0, 0)] + #[case::cache_on(100, 4)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn the_endpoint_cache_decides_whether_statements_stay_named( + #[case] statement_cache_capacity: usize, + #[case] expected: i64, + ) -> Result<(), Box> { + let endpoint = Endpoint { + statement_cache_capacity, + ..database_endpoint()? + }; + let pool = endpoint.pool()?; + let mut connection = pool.acquire().await?; + + for query in ["SELECT 1", "SELECT 2", "SELECT 3"] { + #[expect(clippy::disallowed_methods, reason = "probe queries need no schema")] + sqlx::query(AssertSqlSafe(query)) + .execute(Statements::new( + &mut *connection, + endpoint.names_statements(), + )) + .await?; + } + + assert_eq!( + named_statements(&mut connection, endpoint.names_statements()).await?, + expected + ); + Ok(()) + } +} diff --git a/litellm/rust/crates/db/src/test_support.rs b/litellm/rust/crates/db/src/test_support.rs index 992be4a4447..40446ca53dc 100644 --- a/litellm/rust/crates/db/src/test_support.rs +++ b/litellm/rust/crates/db/src/test_support.rs @@ -1,10 +1,40 @@ -use std::error::Error as StdError; +use std::{error::Error as StdError, str::FromStr}; +use litellm_secret_value::SecretValue; use rstest::fixture; -use sqlx::PgPool; +use sqlx::{PgPool, postgres::PgConnectOptions}; + +use crate::{Endpoint, PoolLimits, SslMode}; #[fixture] pub async fn database_pool() -> Result> { let database_url = std::env::var("DATABASE_URL")?; Ok(PgPool::connect(&database_url).await?) } + +pub fn database_endpoint() -> Result> { + let database_url = std::env::var("DATABASE_URL")?; + let options = PgConnectOptions::from_str(&database_url)?; + Ok(Endpoint { + host: options.get_host().to_owned(), + port: options.get_port(), + socket: None, + username: options.get_username().to_owned(), + password: url::Url::parse(&database_url)? + .password() + .map(SecretValue::new), + database: options.get_database().unwrap_or("postgres").to_owned(), + ssl_mode: SslMode::Prefer, + ssl_root_cert: None, + ssl_client: None, + application_name: None, + options: Vec::new(), + statement_cache_capacity: 100, + pool: PoolLimits { + max_connections: 4, + acquire_timeout_seconds: Some(10.0), + idle_timeout_seconds: None, + socket_timeout_seconds: None, + }, + }) +} diff --git a/litellm/rust/crates/gateway-inference/Cargo.toml b/litellm/rust/crates/gateway-inference/Cargo.toml index 69ca561e4b5..271b668e4f3 100644 --- a/litellm/rust/crates/gateway-inference/Cargo.toml +++ b/litellm/rust/crates/gateway-inference/Cargo.toml @@ -32,6 +32,7 @@ serde_json.workspace = true thiserror.workspace = true [dev-dependencies] +litellm-core-utils.workspace = true litellm-cache-memory.workspace = true futures-util.workspace = true tokio = { workspace = true, features = ["io-util"] } diff --git a/litellm/rust/crates/gateway-inference/src/error.rs b/litellm/rust/crates/gateway-inference/src/error.rs index 190561079f6..d05ecb2ff68 100644 --- a/litellm/rust/crates/gateway-inference/src/error.rs +++ b/litellm/rust/crates/gateway-inference/src/error.rs @@ -324,8 +324,8 @@ mod tests { #[rstest] #[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)] #[case::declined( - Error::Route(InferenceError::from(litellm_http::Error::computed_header( - "authorization" + Error::Route(InferenceError::Declined(litellm_core_utils::decline::Decline::new( + "unrecognized request parameter" ))), StatusCode::NOT_IMPLEMENTED )] diff --git a/litellm/rust/crates/gateway-inference/src/messages.rs b/litellm/rust/crates/gateway-inference/src/messages.rs index 79b787a3c53..56e283d63b9 100644 --- a/litellm/rust/crates/gateway-inference/src/messages.rs +++ b/litellm/rust/crates/gateway-inference/src/messages.rs @@ -108,7 +108,7 @@ fn project( .chain(tools) .collect(); Ok(MessagesCall { - body: messages_body(body, deployment.custom_llm_provider.as_deref())?, + body: messages_body(body)?, api_key: deployment.api_key.clone(), api_base: deployment.api_base.clone(), custom_llm_provider: deployment.custom_llm_provider.clone(), diff --git a/litellm/rust/crates/host-python/AGENTS.md b/litellm/rust/crates/host-python/AGENTS.md index 02b8d4f2dcb..1f7dbca461a 100644 --- a/litellm/rust/crates/host-python/AGENTS.md +++ b/litellm/rust/crates/host-python/AGENTS.md @@ -16,15 +16,15 @@ Boundary tests exercise behavior with a supplied lifecycle binding without impor `HookChain` composes Python runtime hooks in order. Each argument, wire-request and response transformation feeds its result to the next hook. After all argument transformations, the driver calls `arguments_prepared` on every hook in order. Retained callback views must adopt that dictionary before later policy hooks can mutate or reject it. SDK policy is supplied by bridge composition as a hook, never a separate driver phase or parameter. Hooks implement only the stages they need. Default stages preserve the supplied values -Terminal notifications share the selected response or exception. An ordinary notification error is reported as unraisable and does not skip the next hook or replace the selected outcome. Preparation, interception and transformation errors stop the chain. Cancellation stops all further hook dispatch. Suspensions stay inline in the existing driver, and the chain traverses retained event values for GC +Terminal notifications share the selected response or exception. An ordinary notification error is reported as unraisable and does not skip the next hook or replace the selected outcome. Preparation, interception and transformation errors stop the chain. Cancellation stops all further hook dispatch, except on a stream whose head asks for a detached drain: the first cancellation during a read keeps reading to the end, delivers every chunk and the success, then raises the cancellation, and a second cancellation stops it. Suspensions stay inline in the existing driver, and the chain traverses retained event values for GC ## Existing runtime invariants - Keep this crate the CPython runtime adapter and nothing more: Serde marshalling, interpreter detachment, tokio/asyncio glue, the `Execution` handle, the call driver and the `PythonBinding`, `PythonHostCalls` and `PythonOwned` traits, and the `PythonRuntime` specialization of `host::hooks::CallHooks` - No LiteLLM domain dependencies beyond `litellm-host`: no route types, no `Logging` policy, no public API registration, no cdylib build features - `PythonCallHooks` only constrains the shared call-stage interface to `PythonRuntime` and Python ownership. It must not redeclare the stages - - `PythonCallEvent` is a specialization of the shared `CallEvent`, never a separately defined lifecycle. The driver emits `Succeeded` or `Failed` exactly once and never dispatches callbacks after a cancellation. Which Python objects consume those events is the legacy adapter's business - - `CallOptions` can publish snapshots independently of callback delivery. Terminal snapshots follow completed hook dispatch, and cancellation never calls a Python callback. For Python-driven calls, leave the machine's observation publisher unset so the driver is the sole publisher of intercepted provider-response snapshots + - `PythonCallEvent` is a specialization of the shared `CallEvent`, never a separately defined lifecycle. The driver emits `Succeeded` or `Failed` exactly once and never dispatches callbacks after a cancellation, apart from the detached drain above. Which Python objects consume those events is the legacy adapter's business + - `CallOptions` can publish snapshots independently of callback delivery. Terminal snapshots follow completed hook dispatch, and cancellation never calls a Python callback outside a detached drain. For Python-driven calls, leave the machine's observation publisher unset so the driver is the sole publisher of intercepted provider-response snapshots - `PythonBinding::decode_request` receives the keyword view returned by `prepare_arguments` and updated by `arguments_prepared`, not the caller's dict. A binding that decodes from it inherits all composed argument rewrites - A native failure, including one a host op returns as `InvokeError::Native`, is classified exactly once through the binding's `map_error`. A Python exception raised inside the call, and a failure in `prepare_arguments` or `transform_response`, is raised as is - A failing `map_error` is raised with the native error's text as its `__context__`, never swallowed diff --git a/litellm/rust/crates/host-python/src/driver.rs b/litellm/rust/crates/host-python/src/driver.rs index ba0c174c875..541e6e41b0a 100644 --- a/litellm/rust/crates/host-python/src/driver.rs +++ b/litellm/rust/crates/host-python/src/driver.rs @@ -14,7 +14,7 @@ use litellm_host::{ lifecycle::{CallEvent, ExecutionEvent, FailureOrigin, Timing, epoch_seconds}, machine::{HostFailure, Machine, MachineStep}, observation::ObservationSender, - protocol::{HostRequest, InterceptRequest, Protocol, Reply, StreamDelivery}, + protocol::{HostRequest, InterceptRequest, Protocol, Reply, StreamDelivery, UnobservedTail}, }; use crate::PythonHostCalls; @@ -133,6 +133,9 @@ where native: NativeMachine, start: Option, M>>, streaming: bool, + detached: bool, + drains: bool, + unobserved_tail: Option, } impl CallBody @@ -238,6 +241,13 @@ where NativePoll::Ready(result) => result, NativePoll::Suspend(awaitable) => match self.caller.await_python(awaitable).await { Ok(_) => self.native.take_result()?, + Err(error) if self.drains_after(&error) => { + self.detached = true; + Python::attach(|py| { + self.parts().interrupted = Some(error.into_value(py)); + }); + continue; + } Err(error) if self.parts().interrupted.is_none() => { interruption = Some(self.interrupt(error)); continue; @@ -260,6 +270,14 @@ where } } + fn drains_after(&self, error: &PyErr) -> bool { + self.drains + && self.streaming + && !self.detached + && self.parts().interrupted.is_none() + && Python::attach(|py| is_cancellation(py, error)) + } + fn interrupt(&self, error: PyErr) -> HostFailure> { Python::attach(|py| { let cancelled = is_cancellation(py, &error); @@ -320,6 +338,8 @@ where } HostRequest::Stream(StreamDelivery::Open(head, reply)) => { self.streaming = true; + self.drains = ProtocolOf::::detached_drain(&head).is_some(); + self.unobserved_tail = ProtocolOf::::unobserved_tail(&head); let head = { let mut parts = self.parts(); let Parts { binding, hooks, .. } = &mut *parts; @@ -333,16 +353,28 @@ where Ok(()) } HostRequest::Stream(StreamDelivery::Chunk(chunk, reply)) => { + let observed = !self + .unobserved_tail + .as_ref() + .is_some_and(UnobservedTail::begun); let chunk = { let mut parts = self.parts(); let Parts { binding, hooks, .. } = &mut *parts; Python::attach(|py| { let chunk = binding.encode_chunk(py, chunk)?; - hooks.on_stream_chunk(py, &chunk)?; + if observed { + hooks.on_stream_chunk(py, &chunk)?; + } Ok::<_, PyErr>(chunk) })? }; - reply.send(self.caller.yield_chunk(chunk).await); + if self.detached { + reply.send(std::ops::ControlFlow::Break(())); + return Ok(()); + } + let flow = self.caller.yield_chunk(chunk).await; + self.detached = flow.is_break(); + reply.send(flow); Ok(()) } } @@ -384,7 +416,14 @@ where HostedCompletion::Complete(response) => response, HostedCompletion::StreamEnded | HostedCompletion::Detached => { let none = Python::attach(|py| py.None()); - return self.succeeded(none).await; + let succeeded = self.succeeded(none).await; + let cancelled = self.parts().interrupted.take(); + return match cancelled { + Some(cancelled) => Err(Python::attach(|py| { + PyErr::from_value(cancelled.into_bound(py).into_any()) + })), + None => succeeded, + }; } }; let encoded = { @@ -645,6 +684,9 @@ where native: NativeMachine::new(asynchronous), start: Some(Box::new(start)), streaming: false, + detached: false, + drains: false, + unobserved_tail: None, } .run() } diff --git a/litellm/rust/crates/host-python/src/native.rs b/litellm/rust/crates/host-python/src/native.rs index 4e6b59813f9..13168c10e2e 100644 --- a/litellm/rust/crates/host-python/src/native.rs +++ b/litellm/rust/crates/host-python/src/native.rs @@ -67,13 +67,14 @@ where let state = Arc::clone(self.state.as_ref().ok_or(Error::MissingState)?); let future = async move { let mut state = state.lock().await; - let result = match interruption { - Some(failure) => state + let result = match (interruption, state.result.take()) { + (Some(failure), _) => state .machine .interrupt(failure) .await .map(MachineStep::Complete), - None => state.machine.resume().await, + (None, Some(settled)) => settled, + (None, None) => state.machine.resume().await, }; state.result = Some(result); Ok(()) diff --git a/litellm/rust/crates/host-python/src/runtime.rs b/litellm/rust/crates/host-python/src/runtime.rs index 2a8fc598fc4..bc16faae0da 100644 --- a/litellm/rust/crates/host-python/src/runtime.rs +++ b/litellm/rust/crates/host-python/src/runtime.rs @@ -8,6 +8,7 @@ use crate::fork_gate::{ForkGate, Refused, RuntimeAlreadyStarted}; use crate::{Error, Pythonized, panic_to_pyerr, release_gil}; use futures_util::FutureExt; use litellm_error::Report; +use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; use serde::Serialize; @@ -95,18 +96,32 @@ where pyo3_async_runtimes::tokio::future_into_py(py, future) } +fn loop_future(py: Python<'_>) -> PyResult> { + py.import("asyncio")? + .call_method0("get_running_loop")? + .call_method0("create_future") +} + pub fn ready_future<'py>( py: Python<'py>, value: &Bound<'py, PyAny>, ) -> PyResult> { - let future = py - .import("asyncio")? - .call_method0("get_running_loop")? - .call_method0("create_future")?; + let future = loop_future(py)?; future.call_method1("set_result", (value,))?; Ok(future) } +fn settle<'py>( + future: Bound<'py, PyAny>, + result: PyResult>, +) -> PyResult> { + match result { + Ok(value) => future.call_method1("set_result", (value,))?, + Err(error) => future.call_method1("set_exception", (error.into_value(future.py()),))?, + }; + Ok(future) +} + pub fn run_sync( py: Python<'_>, future: F, @@ -181,7 +196,18 @@ where T: for<'py> IntoPyObject<'py> + Send + 'static, F: Future> + Send + 'static, { - future_into_py(py, async move { catch_future_panic(future).await? }) + let settled = loop_future(py)?; + let mut future = Box::pin(async move { Ok(catch_future_panic(future).await.flatten()) }); + // If the future is already done (e.g. in-memory read), just poll once to + // return it instead of relying on a separate thread to wakeup. This saves + // ~100 microseconds per call + match poll_async_value(py, future.as_mut())? { + Poll::Ready(result) => settle( + settled, + result.and_then(|value| value.into_bound_py_any(py)), + ), + Poll::Pending => future_into_py(py, async move { future.await? }), + } } pub fn poll_async_value(py: Python<'_>, future: Pin<&mut F>) -> PyResult> @@ -520,6 +546,83 @@ asyncio.run(exercise()) }); } + #[pyfunction] + fn inline_value(py: Python<'_>, mode: String) -> PyResult> { + let caller = thread::current().id(); + run_async_value(py, async move { + match mode.as_str() { + "error" => Err(PyLookupError::new_err("inline failure")), + "pending" => { + tokio::task::yield_now().await; + Ok(thread::current().id() == caller) + } + _ => Ok(thread::current().id() == caller), + } + }) + } + + #[rstest] + #[case::ready("ready", "done and await future is True")] + #[case::error("error", "done and await future is LookupError")] + #[case::pending("pending", "not done and await future is False")] + fn an_async_value_settles_inline_only_when_its_first_poll_completes( + #[from(initialized_python)] python: &InitializedPython, + #[case] mode: &str, + #[case] expected: &str, + ) { + python.attach(|py| { + let locals = PyDict::new(py); + locals + .set_item("inline_value", wrap_pyfunction!(inline_value, py).unwrap()) + .unwrap(); + locals.set_item("mode", mode).unwrap(); + py.run( + c" +import asyncio + +async def exercise(): + future = inline_value(mode) + assert future.get_loop() is asyncio.get_running_loop() + state = 'done' if future.done() else 'not done' + try: + value = repr(await asyncio.wait_for(future, 2)) + except LookupError: + value = 'LookupError' + return f'{state} and await future is {value}' + +result = asyncio.run(exercise()) +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let result: String = locals + .get_item("result") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(result, expected); + }); + } + + #[rstest] + fn an_async_value_without_a_running_loop_fails_before_running_the_future( + #[from(initialized_python)] python: &InitializedPython, + ) { + python.attach(|py| { + let ran = Arc::new(AtomicBool::new(false)); + let flag = Arc::clone(&ran); + let error = run_async_value(py, async move { + flag.store(true, Ordering::SeqCst); + Ok(()) + }) + .unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(!ran.load(Ordering::SeqCst)); + }); + } + #[rstest] fn sync_runner_polls_future_on_the_caller_thread( #[from(initialized_python)] python: &InitializedPython, diff --git a/litellm/rust/crates/host-python/tests/driver.rs b/litellm/rust/crates/host-python/tests/driver.rs index 157cfc92148..1f555bcb165 100644 --- a/litellm/rust/crates/host-python/tests/driver.rs +++ b/litellm/rust/crates/host-python/tests/driver.rs @@ -23,8 +23,9 @@ use pyo3::{ use rstest::rstest; use support::{ HookScript, Interpreter, Log, MachineFactory, OpScript, Synthetic, binding, call_options, - failing_machine, handed, hooks, interpreter, lifecycle_module, open_stream, read_all, - run_composed, run_scripted, settled, success_machine, + draining_streaming_machine, failing_machine, handed, hooks, interpreter, lifecycle_module, + open_stream, open_stream_from, read_all, run_composed, run_scripted, settled, success_machine, + tailed_streaming_machine, }; fn observed(capacity: usize) -> (CallOptions, impl FnMut() -> Vec) { @@ -403,6 +404,151 @@ fn a_stream_carries_its_head_before_the_first_chunk( }); } +#[rstest] +fn chunks_on_the_unobserved_tail_reach_the_caller_without_the_stream_hooks( + #[from(interpreter)] _python: Interpreter, + #[values(false, true)] asynchronous: bool, +) { + Python::attach(|py| { + let log = Log::default(); + let stream = open_stream_from( + py, + tailed_streaming_machine(), + hooks(&log, HookScript::Plain), + call_options(asynchronous), + ); + assert_eq!( + read_all(py, &stream, asynchronous), + ["first", "second", "trailer"] + ); + assert_eq!( + log.entries(), + [ + "started", + "begin", + "opened", + "delivered", + "delivered", + "succeeded:None", + "adapter.close" + ] + ); + }); +} + +#[rstest] +fn a_draining_stream_reports_every_chunk_to_the_stream_hooks_after_the_caller_closes( + #[from(interpreter)] _python: Interpreter, + #[values(false, true)] asynchronous: bool, +) { + Python::attach(|py| { + let log = Log::default(); + let stream = open_stream_from( + py, + draining_streaming_machine(true, std::time::Duration::ZERO), + hooks(&log, HookScript::Plain), + call_options(asynchronous), + ); + if asynchronous { + for method in ["__anext__", "aclose"] { + let stop = stream + .call_method0(method) + .unwrap() + .call_method1("send", (py.None(),)) + .unwrap_err(); + assert!(stop.is_instance_of::(py)); + } + } else { + stream.call_method0("__next__").unwrap(); + stream.call_method0("close").unwrap(); + } + assert_eq!( + log.entries(), + [ + "started", + "begin", + "opened", + "delivered", + "delivered", + "delivered", + "succeeded:None", + "adapter.close" + ] + ); + }); +} + +#[rstest] +#[case::draining( + true, + "cancelled", + &["started", "begin", "opened", "delivered", "delivered", "delivered", "succeeded:None", "adapter.close"] +)] +#[case::not_draining(false, "cancelled", &["started", "begin", "opened", "delivered", "adapter.close"])] +fn a_caller_cancelled_mid_read_leaves_only_a_draining_stream_reading_on( + #[from(interpreter)] _python: Interpreter, + #[case] drain: bool, + #[case] outcome: &str, + #[case] logged: &[&str], +) { + Python::attach(|py| { + let log = Log::default(); + let opened = handed( + py, + run_call( + py, + move |_, _, request| { + Ok(draining_streaming_machine( + drain, + std::time::Duration::from_millis(200), + )(request)) + }, + support::StreamingBinding, + hooks(&log, HookScript::Plain), + PyDict::new(py).unbind(), + call_options(true), + ), + true, + ) + .unwrap(); + let locals = PyDict::new(py); + locals.set_item("opened", opened).unwrap(); + py.run( + c" +import asyncio + +async def read_then_cancel(): + stream = await opened + await stream.__anext__() + reading = asyncio.ensure_future(stream.__anext__()) + await asyncio.sleep(0.05) + reading.cancel() + try: + await reading + except asyncio.CancelledError: + outcome = 'cancelled' + else: + outcome = 'read' + await stream.aclose() + return outcome + +outcome = asyncio.run(read_then_cancel()) +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let actual: String = locals + .get_item("outcome") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(actual, outcome); + assert_eq!(log.entries(), logged); + }); +} + struct Classification { provider_fails: bool, op: OpScript, diff --git a/litellm/rust/crates/host-python/tests/support/mod.rs b/litellm/rust/crates/host-python/tests/support/mod.rs index 25b68b782b2..8b9b98d656d 100644 --- a/litellm/rust/crates/host-python/tests/support/mod.rs +++ b/litellm/rust/crates/host-python/tests/support/mod.rs @@ -2,13 +2,15 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use futures_util::StreamExt; + use litellm_host::{ call::{CallOutput, HostedMachine, hosted_call}, hooks::CallHooks, interceptors::{Interceptors, RawResponse, RequestContext, WireRequest}, lifecycle::{ExecutionEvent, Timing}, machine::{CallMachine, MachineFault}, - protocol::{Protocol, Reply}, + protocol::{DetachedDrain, Protocol, Reply, UnobservedTail}, }; use litellm_host_python::{ CallOptions, HookStep, InvokeError, PythonBinding, PythonCallEvent, PythonCallHooks, @@ -559,7 +561,21 @@ impl Protocol for Streaming { type Request = (); type HostCall = std::convert::Infallible; type Chunk = &'static str; - type StreamHead = Vec<(&'static str, &'static str)>; + type StreamHead = StreamingHead; + + fn detached_drain(head: &StreamingHead) -> Option { + head.drain + } + + fn unobserved_tail(head: &StreamingHead) -> Option { + head.tail.clone() + } +} + +pub struct StreamingHead { + headers: Vec<(&'static str, &'static str)>, + tail: Option, + drain: Option, } pub struct StreamingBinding; @@ -576,13 +592,9 @@ impl PythonBinding for StreamingBinding { Ok(()) } - fn encode_stream_head( - &mut self, - py: Python<'_>, - head: Vec<(&'static str, &'static str)>, - ) -> PyResult> { + fn encode_stream_head(&mut self, py: Python<'_>, head: StreamingHead) -> PyResult> { let headers = PyDict::new(py); - for (name, value) in head { + for (name, value) in head.headers { headers.set_item(name, value)?; } let hidden = PyDict::new(py); @@ -629,25 +641,94 @@ pub fn streaming_machine() -> impl FnOnce(()) -> HostedMachine + Send move |()| { hosted_call((), None, |(), _, _, _observations| async { Ok(CallOutput::Stream { - head: vec![("request-id", "req_1")], + head: StreamingHead { + headers: vec![("request-id", "req_1")], + tail: None, + drain: None, + }, chunks: Box::pin(futures_util::stream::iter([Ok("first"), Ok("second")])), }) }) } } +/// Streams `first` and `second`, then `trailer` on its unobserved tail. +pub fn tailed_streaming_machine() -> impl FnOnce(()) -> HostedMachine + Send + Sync { + move |()| { + hosted_call((), None, |(), _, _, _observations| async { + let tail = UnobservedTail::default(); + let begun = tail.clone(); + let trailer = futures_util::stream::once(async move { + begun.begin(); + Ok("trailer") + }); + Ok(CallOutput::Stream { + head: StreamingHead { + headers: Vec::new(), + tail: Some(tail), + drain: None, + }, + chunks: Box::pin(futures_util::StreamExt::chain( + futures_util::stream::iter([Ok("first"), Ok("second")]), + trailer, + )), + }) + }) + } +} + +pub fn draining_streaming_machine( + drain: bool, + delay: std::time::Duration, +) -> impl FnOnce(()) -> HostedMachine + Send + Sync { + move |()| { + hosted_call((), None, move |(), _, _, _observations| async move { + let second = futures_util::stream::once(async move { + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + Ok("second") + }); + Ok(CallOutput::Stream { + head: StreamingHead { + headers: Vec::new(), + tail: None, + drain: drain.then_some(DetachedDrain { + timeout: std::time::Duration::from_secs(5), + limit: 4, + }), + }, + chunks: Box::pin( + futures_util::stream::iter([Ok("first")]) + .chain(second) + .chain(futures_util::stream::iter([Ok("third")])), + ), + }) + }) + } +} + /// Starts a streaming call and returns the caller's stream object. pub fn open_stream( py: Python<'_>, hooks: SyntheticHooks, options: CallOptions, +) -> Bound<'_, PyAny> { + open_stream_from(py, streaming_machine(), hooks, options) +} + +pub fn open_stream_from( + py: Python<'_>, + machine: impl FnOnce(()) -> HostedMachine + Send + Sync + 'static, + hooks: SyntheticHooks, + options: CallOptions, ) -> Bound<'_, PyAny> { let asynchronous = options.asynchronous; let handed = handed( py, run_call( py, - |_, _, request| Ok(streaming_machine()(request)), + move |_, _, request| Ok(machine(request)), StreamingBinding, hooks, PyDict::new(py).unbind(), diff --git a/litellm/rust/crates/host/Cargo.toml b/litellm/rust/crates/host/Cargo.toml index 73392827f31..284c1c5f5f0 100644 --- a/litellm/rust/crates/host/Cargo.toml +++ b/litellm/rust/crates/host/Cargo.toml @@ -13,7 +13,7 @@ litellm-coroutine.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true -tokio = { workspace = true, features = ["sync"] } +tokio = { workspace = true, features = ["sync", "time"] } [dev-dependencies] rstest.workspace = true diff --git a/litellm/rust/crates/host/src/call.rs b/litellm/rust/crates/host/src/call.rs index d8187a216e5..b965a6b6d78 100644 --- a/litellm/rust/crates/host/src/call.rs +++ b/litellm/rust/crates/host/src/call.rs @@ -1,13 +1,58 @@ use crate::observation::ObservationSender; -use std::future::Future; +use std::{ + future::Future, + sync::atomic::{AtomicUsize, Ordering}, +}; use futures_util::{TryStreamExt, stream::BoxStream}; use crate::{ - machine::{CallMachine, ChannelInterceptors, HostServices, MachineFault}, - protocol::Protocol, + machine::{CallMachine, ChannelInterceptors, HostServices, MachineFault, StreamSender}, + protocol::{DetachedDrain, Protocol}, }; +static DETACHED_DRAINS: AtomicUsize = AtomicUsize::new(0); + +struct DrainSlot; + +impl DrainSlot { + fn claim(limit: usize) -> Option { + DETACHED_DRAINS + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |running| { + (running < limit).then_some(running + 1) + }) + .ok() + .map(|_| Self) + } +} + +impl Drop for DrainSlot { + fn drop(&mut self) { + DETACHED_DRAINS.fetch_sub(1, Ordering::AcqRel); + } +} + +async fn drain_detached

( + stream: &StreamSender

, + mut chunks: BoxStream<'static, Result>, + drain: DetachedDrain, +) where + P: Protocol, + P::Error: From, +{ + let Some(_slot) = DrainSlot::claim(drain.limit) else { + return; + }; + let delivered = async { + while let Ok(Some(chunk)) = chunks.try_next().await { + if stream.send_chunk(chunk).await.is_err() { + return; + } + } + }; + let _ = tokio::time::timeout(drain.timeout, delivered).await; +} + pub enum CallOutput { Complete(Response), Stream { @@ -61,11 +106,15 @@ where match execute(request, host.services, host.interceptors, host.observers).await? { CallOutput::Complete(response) => Ok(HostedCompletion::Complete(response)), CallOutput::Stream { head, mut chunks } => { + let drain = P::detached_drain(&head); if host.stream.open_stream(head).await?.is_break() { return Ok(HostedCompletion::Detached); } while let Some(chunk) = chunks.try_next().await? { if host.stream.send_chunk(chunk).await?.is_break() { + if let Some(drain) = drain { + drain_detached(&host.stream, chunks, drain).await; + } return Ok(HostedCompletion::Detached); } } diff --git a/litellm/rust/crates/host/src/protocol.rs b/litellm/rust/crates/host/src/protocol.rs index 5ec566b6a65..8a9bf712543 100644 --- a/litellm/rust/crates/host/src/protocol.rs +++ b/litellm/rust/crates/host/src/protocol.rs @@ -1,4 +1,10 @@ -use std::ops::ControlFlow; +use std::{ + ops::ControlFlow, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; pub use litellm_coroutine::{Abandoned, Answer, Reply, reply}; @@ -11,6 +17,37 @@ pub trait Protocol: Send + Sync + 'static { type HostCall: Send + 'static; type Chunk: Send + 'static; type StreamHead: Send + 'static; + + fn detached_drain(_head: &Self::StreamHead) -> Option { + None + } + + fn unobserved_tail(_head: &Self::StreamHead) -> Option { + None + } +} + +/// Where a stream's chunks stop reaching the stream hooks: once the route begins the tail, the +/// chunks it delivers go to the caller alone, as a frame Python's relay adds after billing does. +#[derive(Clone, Debug, Default)] +pub struct UnobservedTail(Arc); + +impl UnobservedTail { + pub fn begin(&self) { + self.0.store(true, Ordering::Release); + } + + pub fn begun(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + +/// A stream whose upstream is read to its end after the caller detaches, so the provider's +/// terminal usage still reaches the stream hooks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DetachedDrain { + pub timeout: std::time::Duration, + pub limit: usize, } pub enum HostRequest { diff --git a/litellm/rust/crates/host/tests/call.rs b/litellm/rust/crates/host/tests/call.rs index a8ba71e7b07..454c3c9272c 100644 --- a/litellm/rust/crates/host/tests/call.rs +++ b/litellm/rust/crates/host/tests/call.rs @@ -6,6 +6,7 @@ use std::{ Arc, Mutex, atomic::{AtomicUsize, Ordering}, }, + time::Duration, }; use futures_util::{StreamExt, stream}; @@ -13,7 +14,7 @@ use litellm_host::{ call::{CallOutput, HostedCompletion, hosted_call}, lifecycle::{CallEvent, CallObserver, observe_call, observe_unary}, machine::{Machine, MachineFault, MachineStep}, - protocol::{HostRequest, Protocol}, + protocol::{DetachedDrain, HostRequest, Protocol}, }; use rstest::{fixture, rstest}; @@ -94,6 +95,86 @@ async fn delivery_obeys_demand_and_distinguishes_detachment( assert_eq!(delivered, (0..expected_polls).collect::>()); } +struct DrainingProtocol; + +impl Protocol for DrainingProtocol { + type Response = &'static str; + type Error = TestError; + type Request = usize; + type HostCall = Infallible; + type Chunk = usize; + type StreamHead = Option; + + fn detached_drain(head: &Option) -> Option { + *head + } +} + +#[rstest] +#[case::draining( + Some(DetachedDrain { timeout: Duration::from_secs(5), limit: 4 }), + vec![0, 1, 2] +)] +#[case::no_drain(None, vec![0])] +#[case::no_free_slot(Some(DetachedDrain { timeout: Duration::from_secs(5), limit: 0 }), vec![0])] +#[tokio::test] +async fn a_draining_stream_delivers_the_rest_after_the_caller_detaches( + #[case] drain: Option, + #[case] expected: Vec, +) { + let mut machine = + hosted_call::(3, None, move |count, _, _, _| async move { + Ok(CallOutput::Stream { + head: drain, + chunks: stream::iter((0..count).map(Ok)).boxed(), + }) + }); + let mut delivered = Vec::new(); + let completed = loop { + match machine.resume().await.unwrap() { + MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Open(_, reply))) => { + reply.send(ControlFlow::Continue(())); + } + MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Chunk(chunk, reply))) => { + delivered.push(chunk); + reply.send(ControlFlow::Break(())); + } + MachineStep::Complete(result) => break result, + _ => panic!("unexpected operation"), + } + }; + assert_eq!(completed, HostedCompletion::Detached); + assert_eq!(delivered, expected); +} + +#[tokio::test] +async fn a_detached_drain_stops_at_its_timeout() { + let mut machine = + hosted_call::(0, None, move |_, _, _, _| async move { + let pending = stream::iter([Ok(0)]).chain(stream::pending()); + Ok(CallOutput::Stream { + head: Some(DetachedDrain { + timeout: Duration::from_millis(50), + limit: 4, + }), + chunks: pending.boxed(), + }) + }); + let completed = loop { + match machine.resume().await.unwrap() { + MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Open(_, reply))) => { + reply.send(ControlFlow::Continue(())); + } + MachineStep::Suspended(HostRequest::Stream(StreamDelivery::Chunk(_, reply))) => { + reply.send(ControlFlow::Break(())); + } + MachineStep::Complete(result) => break result, + _ => panic!("unexpected operation"), + } + }; + assert_eq!(completed, HostedCompletion::Detached); +} + #[derive(Default)] struct Observer(Observations); struct Observations { diff --git a/litellm/rust/crates/http/src/error.rs b/litellm/rust/crates/http/src/error.rs index d9fdfb5486f..ab3c0a768fd 100644 --- a/litellm/rust/crates/http/src/error.rs +++ b/litellm/rust/crates/http/src/error.rs @@ -52,8 +52,6 @@ pub(crate) enum Repr { RequestUrl(#[source] reqwest::Error), #[error("invalid request URL")] ParseUrl(#[source] url::ParseError), - #[error("request forwards the header {0:?}, which the signer computes")] - ComputedHeader(String), #[error("request signing failed: a streaming body cannot be signed")] UnsignableBody, #[error("request forwards a header that cannot be signed")] @@ -247,10 +245,6 @@ impl Error { } } - pub fn computed_header(name: impl Into) -> Self { - Repr::ComputedHeader(name.into()).into() - } - pub fn request_body(error: serde_json::Error) -> Self { Repr::RequestBody(error).into() } @@ -292,7 +286,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -318,7 +311,6 @@ impl Error { Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableHeader(_) | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) @@ -364,7 +356,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -392,51 +383,10 @@ impl Error { } } - pub fn is_computed_header(&self) -> bool { - match &*self.0 { - Repr::ComputedHeader(_) => true, - Repr::Read { .. } - | Repr::InvalidPem { .. } - | Repr::Client(_) - | Repr::TlsConfig(_) - | Repr::RequestBody(_) - | Repr::Request(_) - | Repr::RequestUrl(_) - | Repr::ParseUrl(_) - | Repr::UnsignableBody - | Repr::UnsignableHeader(_) - | Repr::Signature(_) - | Repr::Connect(_) - | Repr::Network(_) - | Repr::BodyCut { .. } - | Repr::Timeout(_) - | Repr::ReadTimeout { .. } - | Repr::Status { .. } - | Repr::WebSocketUrl(_) - | Repr::WebSocketScheme(_) - | Repr::WebSocketSchemeRewrite(_) - | Repr::Handshake(_) - | Repr::HandshakeTimeout(_) - | Repr::WebSocket(_) - | Repr::WebSocketClosed - | Repr::BlockedUrl - | Repr::Resolve(_) - | Repr::DownloadDisabled - | Repr::DownloadTooLarge - | Repr::TooManyRedirects - | Repr::MissingRedirectLocation - | Repr::InvalidRedirect(_) - | Repr::DownloadStatus(_) - | Repr::Download(_) - | Repr::DownloadTimeout(_) => false, - } - } - pub fn is_connect(&self) -> bool { match &*self.0 { Repr::Connect(_) => true, - Repr::ComputedHeader(_) - | Repr::Read { .. } + Repr::Read { .. } | Repr::InvalidPem { .. } | Repr::Client(_) | Repr::TlsConfig(_) @@ -486,7 +436,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -523,7 +472,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -564,7 +512,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) @@ -600,7 +547,6 @@ impl Error { | Repr::Request(_) | Repr::RequestUrl(_) | Repr::ParseUrl(_) - | Repr::ComputedHeader(_) | Repr::UnsignableBody | Repr::UnsignableHeader(_) | Repr::Signature(_) diff --git a/litellm/rust/crates/http/src/outbound.rs b/litellm/rust/crates/http/src/outbound.rs index 49ff7fa6df0..7636eff0f76 100644 --- a/litellm/rust/crates/http/src/outbound.rs +++ b/litellm/rust/crates/http/src/outbound.rs @@ -19,6 +19,10 @@ use crate::{Error, error::Repr}; /// Returns the headers to add to the request; it never sees a mutable request. pub trait RequestSigner: Send + Sync { fn sign(&self, request: &Request) -> Result; + + fn owned_headers(&self) -> &'static [&'static str] { + &[] + } } pub fn json( @@ -65,7 +69,11 @@ pub fn from_http>(request: http::Request) -> Result Result { let signature = signer.sign(&request)?; - request.headers_mut().extend(signature); + let headers = request.headers_mut(); + for name in signer.owned_headers() { + headers.remove(*name); + } + headers.extend(signature); Ok(request) } @@ -159,7 +167,7 @@ mod tests { struct Refuses; impl RequestSigner for Refuses { fn sign(&self, _request: &Request) -> Result { - Err(Error::computed_header("authorization")) + Err(Error::unsignable_body()) } } @@ -168,7 +176,44 @@ mod tests { &Refuses, ) .unwrap_err(); - assert!(error.is_computed_header()); + assert!(!error.dispatched()); + } + + #[test] + fn a_header_the_signer_owns_is_dropped_even_when_the_signer_omits_it() { + struct OwnsDate; + impl RequestSigner for OwnsDate { + fn sign(&self, _request: &Request) -> Result { + Ok(HeaderMap::from_iter([( + reqwest::header::AUTHORIZATION, + HeaderValue::from_static("signed"), + )])) + } + + fn owned_headers(&self) -> &'static [&'static str] { + &["date", "authorization"] + } + } + + let request = signed( + json( + "https://u.test/", + &[ + ("Date".into(), "caller".into()), + ("Authorization".into(), "caller".into()), + ("x-kept".into(), "1".into()), + ], + &json!({}), + None, + ) + .unwrap(), + &OwnsDate, + ) + .unwrap(); + + assert!(!request.headers().contains_key("date")); + assert_eq!(request.headers()["authorization"], "signed"); + assert_eq!(request.headers()["x-kept"], "1"); } #[rstest] diff --git a/litellm/rust/crates/inference-chat/tests/request.rs b/litellm/rust/crates/inference-chat/tests/request.rs index 121f7347c24..d9efbc68419 100644 --- a/litellm/rust/crates/inference-chat/tests/request.rs +++ b/litellm/rust/crates/inference-chat/tests/request.rs @@ -328,13 +328,23 @@ async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() { } #[rstest] -#[case::authorization("Authorization")] -#[case::amz_date("x-amz-date")] -#[case::security_token("x-amz-security-token")] -#[case::date("Date")] +#[case::authorization("Authorization", Some("forged"), false, None)] +#[case::amz_date("x-amz-date", None, false, Some("x-amz-date"))] +#[case::security_token( + "x-amz-security-token", + None, + true, + Some("x-amz-date;x-amz-security-token") +)] +#[case::date("Date", None, false, Some("content-type;date;host"))] #[tokio::test] -async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) { - let upstream = wiremock::MockServer::start().await; +async fn a_forwarded_header_the_signer_computes_is_sent_as_python_sends_it( + #[case] forwarded: &str, + #[case] authorization: Option<&str>, + #[case] kept: bool, + #[case] signed_headers: Option<&str>, +) { + let upstream = upstream([bedrock_response()]).await; let base = upstream.uri(); let call = ChatCompletionsRequest { api_key: None, @@ -351,11 +361,27 @@ async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) }), ) }; - let error = complete(call) - .await - .expect_err("conflicting signing headers fail"); - assert!(error.declined().is_some(), "{error:?}"); - assert!(received(&upstream).await.is_empty()); + complete(call).await.expect("call succeeds"); + + let sent = only_request(&upstream).await; + let signature = sent.header("authorization").expect("an authorization"); + match authorization { + Some(expected) => assert_eq!(signature, expected), + None => { + assert!(signature.starts_with("AWS4-HMAC-SHA256")); + assert_eq!(sent.header(forwarded) == Some("forged"), kept); + assert!(signed_headers.is_some_and(|names| signature.contains(names))); + } + } + let dated_by_date = forwarded == "Date"; + assert_eq!( + ( + sent.header("x-amz-date") + .is_some_and(|date| date != "forged"), + sent.header("date").is_some_and(|date| date != "forged"), + ), + (!dated_by_date, dated_by_date) + ); } #[rstest] diff --git a/litellm/rust/crates/inference-messages/Cargo.toml b/litellm/rust/crates/inference-messages/Cargo.toml index 0ca371c3ce7..ed84b0f864d 100644 --- a/litellm/rust/crates/inference-messages/Cargo.toml +++ b/litellm/rust/crates/inference-messages/Cargo.toml @@ -20,6 +20,7 @@ litellm-http.workspace = true litellm-inference.workspace = true litellm-llms.workspace = true litellm-llms-types.workspace = true +litellm-python-compat.workspace = true litellm-router-types.workspace = true litellm-secrets.workspace = true litellm-tracing.workspace = true diff --git a/litellm/rust/crates/inference-messages/src/common_utils.rs b/litellm/rust/crates/inference-messages/src/common_utils.rs index d0f109c1e99..4f86ff42f06 100644 --- a/litellm/rust/crates/inference-messages/src/common_utils.rs +++ b/litellm/rust/crates/inference-messages/src/common_utils.rs @@ -52,11 +52,15 @@ pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option &TENCENT_MESSAGES_CONFIG, LlmProviders::Anthropic => &ANTHROPIC_MESSAGES_CONFIG, LlmProviders::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG, - LlmProviders::Bedrock if model.starts_with("claude_platform/") => { + LlmProviders::Bedrock if has_route_prefix(model, "claude_platform/") => { &BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG } - LlmProviders::Bedrock if model.starts_with("mantle/") => &BEDROCK_MANTLE_MESSAGES_CONFIG, - LlmProviders::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG, + LlmProviders::Bedrock if has_route_prefix(model, "converse/") => return None, + LlmProviders::Bedrock if has_route_prefix(model, "mantle/") => { + &BEDROCK_MANTLE_MESSAGES_CONFIG + } + LlmProviders::Bedrock if model.contains("claude") => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG, + LlmProviders::Bedrock => return None, LlmProviders::BedrockMantle => &BEDROCK_MANTLE_MESSAGES_CONFIG, LlmProviders::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG, LlmProviders::Edenai => &EDENAI_MESSAGES_CONFIG, @@ -70,6 +74,10 @@ pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option bool { + model.starts_with(prefix) || model.contains(&format!("/{prefix}")) +} + pub(super) fn string_headers( extra_headers: Option>, ) -> Result, Error> { @@ -155,6 +163,72 @@ mod tests { assert!(messages_provider(provider, model).is_none()); } + #[rstest] + #[case::invoke_claude("anthropic.claude-sonnet-4-5-20250929-v1:0", Some("bedrock"))] + #[case::invoke_prefix("invoke/anthropic.claude-3-5-sonnet", Some("bedrock"))] + #[case::cross_region_claude("us.anthropic.claude-opus-4-5", Some("bedrock"))] + #[case::claude_arn( + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5", + Some("bedrock") + )] + #[case::converse_claude("converse/anthropic.claude-sonnet-4-5", None)] + #[case::nested_converse("us-west-2/converse/anthropic.claude-sonnet-4-5", None)] + #[case::non_claude("amazon.nova-pro-v1:0", None)] + #[case::non_claude_invoke("invoke/meta.llama3-70b-instruct-v1:0", None)] + #[case::capitalized_claude_is_not_claude("anthropic.Claude-x", None)] + #[case::mantle("mantle/anthropic.claude-sonnet-5", Some("bedrock"))] + #[case::claude_platform("claude_platform/claude-sonnet-4-5", Some("bedrock"))] + #[case::bedrock_mantle_provider_prefix_is_not_the_mantle_route( + "bedrock_mantle/openai.gpt-5", + None + )] + fn bedrock_models_route_as_python_selects_their_messages_config( + #[case] model: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!( + messages_provider(LlmProviders::Bedrock, model).map(|provider| provider.as_str()), + expected + ); + } + + #[rstest] + #[case::invoke("anthropic.claude-3", false, false)] + #[case::mantle("mantle/anthropic.claude-sonnet-5", true, false)] + #[case::claude_platform("claude_platform/claude-sonnet-4-5", false, true)] + fn bedrock_route_prefixes_pick_their_config( + #[case] model: &str, + #[case] mantle: bool, + #[case] claude_platform: bool, + ) { + use litellm_llms::{ + base_llm::messages::transformation::BaseMessagesConfig, + bedrock::claude_platform::messages::transformation::BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG, + bedrock_mantle::messages::transformation::BEDROCK_MANTLE_MESSAGES_CONFIG, + }; + let params = litellm_router_types::LitellmParams::default(); + let region = |key: &str| { + key.starts_with("AWS_REGION") + .then(|| "us-east-1".to_owned()) + }; + let url = |config: &dyn BaseMessagesConfig| { + format!( + "{:?}", + config.get_complete_url(None, model, ¶ms, false, ®ion) + ) + }; + let routed = url(messages_provider(LlmProviders::Bedrock, model) + .unwrap() + .config()); + assert_eq!( + ( + routed == url(&BEDROCK_MANTLE_MESSAGES_CONFIG), + routed == url(&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG), + ), + (mantle, claude_platform) + ); + } + #[rstest] #[case::int(json!(12345), "int")] #[case::float(json!(1.5), "float")] diff --git a/litellm/rust/crates/inference-messages/src/handler.rs b/litellm/rust/crates/inference-messages/src/handler.rs index bc71053a453..e1bc2322560 100644 --- a/litellm/rust/crates/inference-messages/src/handler.rs +++ b/litellm/rust/crates/inference-messages/src/handler.rs @@ -9,15 +9,18 @@ use litellm_core_utils::{ dot_notation_indexing::delete_nested_value, get_provider_specific_headers::get_provider_specific_headers, settings::Lookup, }; -use litellm_host::interceptors::{ - Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext, - ResponseHead, ResultSource, WireRequest, +use litellm_host::{ + interceptors::{ + Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext, + ResponseHead, ResultSource, WireRequest, + }, + protocol::UnobservedTail, }; use litellm_llms::base_llm::{ auth::{Authenticated, AzureAdTokenHosts, ValidatedEnvironment, resolve_auth}, messages::{ context::MessagesTransformContext, - streaming::{self, ByteStream, StreamDecoder}, + streaming::{ByteStream, StreamDecoder}, transformation::{BaseMessagesConfig, FederatedEnvironment}, }, }; @@ -53,12 +56,38 @@ struct ProviderCall { stream: bool, whole_sse_frames: bool, joined_reads: bool, + detached_drain: Option, transform_context: MessagesTransformContext, + native_output_fallback: Option, ascii_response_headers: bool, unencodable_request_header: Option>, ascii_request_headers: bool, } +struct NativeOutputFallback { + input: MessagesRequest, + context: MessagesTransformContext, + headers: Vec<(String, String)>, + tool_custom_field: litellm_llms::base_llm::messages::tools::ToolCustomField, + drops_unsupported_params: bool, + drop_params: bool, +} + +impl NativeOutputFallback { + fn body(&self, config: &dyn BaseMessagesConfig) -> Result { + let rebuilt = config.request_body( + self.tool_custom_field.apply( + config.transform_anthropic_messages_request(self.input.clone(), &self.context)?, + self.drops_unsupported_params, + ), + &self.headers, + self.drop_params, + )?; + let (rebuilt, _) = config.finalize_request(rebuilt, self.headers.clone(), &self.context)?; + Ok(config.wire_body(serde_json::to_value(&rebuilt).map_err(serialize_failure)?)) + } +} + impl ProviderCall { fn cacheable(&self) -> bool { self.signer.is_none() @@ -290,7 +319,7 @@ async fn provider_call( )?; let transform_context = MessagesTransformContext { drops_unsupported_params: shaping.settings.drops_unsupported_params, - logs_dropped_params: config.logs_dropped_params(), + global_drop_params: shaping.settings.global_drop_params, ..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params) }; let dropped_keys = top_level_paths(&shaping.settings.additional_drop_params); @@ -358,6 +387,9 @@ async fn provider_call( let tool_custom_field = config.tool_custom_field(&trimmed.model); let requested_output_format = trimmed.params.output_format.clone(); + let fallback = config + .native_output_fallback(&trimmed, &transform_context) + .map(|fallback_context| (trimmed.clone(), fallback_context)); let shaped = config.request_body( tool_custom_field.apply( config.transform_anthropic_messages_request(trimmed, &transform_context)?, @@ -374,17 +406,26 @@ async fn provider_call( &env_lookup, )?; let body = config.request_for_url(shaped, &url); - let environment = ValidatedEnvironment { - headers: config.headers_for_url( - config.request_headers( - with_python_default_headers(validated.headers, config.default_headers()), - &with_requested_output_format(&body, requested_output_format), - ), - &url, + let headers = config.headers_for_url( + config.request_headers( + with_python_default_headers(validated.headers, config.default_headers()), + &with_requested_output_format(&body, requested_output_format), ), + &url, + ); + let native_output_fallback = fallback.map(|(input, context)| NativeOutputFallback { + input, + context, + headers: headers.clone(), + tool_custom_field, + drops_unsupported_params: shaping.settings.drops_unsupported_params, + drop_params: shaping.settings.drop_params, + }); + let (body, headers) = config.finalize_request(body, headers, &transform_context)?; + let environment = ValidatedEnvironment { + headers, auth: validated.auth, }; - let body = config.finalize_request(body, &environment.headers, &transform_context); let request_context = RequestContext { model: body.model.clone(), @@ -393,6 +434,11 @@ async fn provider_call( secret_fields: Vec::new(), api_key: api_key.map(SecretValue::new), }; + let serialized = serde_json::to_value(&body).map_err(serialize_failure)?; + let serialized_stream = serialized.get("stream").is_some(); + let wire_body = config.wire_body(serialized); + let stream_moved_to_url = + body.params.streams() && serialized_stream && wire_body.get("stream").is_none(); let authenticated = resolve_auth(credentials.auth, environment, &|key| { std::env::var(key).ok() }) @@ -407,15 +453,15 @@ async fn provider_call( WireRequest { url, headers: joined_header_values(authenticated.headers), - body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?), + body: wire_body, }, request_context, ) .await?; - let stream = wire - .body - .get("stream") - .is_some_and(litellm_llms_types::recognized::python_truthy); + let stream = wire.body.get("stream").map_or( + stream_moved_to_url, + litellm_llms_types::recognized::python_truthy, + ); Ok(ProviderCall { identity, wire, @@ -425,7 +471,9 @@ async fn provider_call( stream, whole_sse_frames: shaping.settings.whole_sse_frames, joined_reads: shaping.settings.joined_reads, + detached_drain: shaping.settings.detached_drain(), transform_context, + native_output_fallback, ascii_response_headers: shaping.settings.ascii_response_headers, unencodable_request_header: shaping.settings.unencodable_request_header, ascii_request_headers: shaping.settings.ascii_request_headers, @@ -499,6 +547,13 @@ fn requested_optional_params( if provider.accepts_service_tier() { return Ok(request); } + if request.params.service_tier.is_some() { + litellm_tracing::warn!( + "Dropping `service_tier` for model={} on provider={}. Only the Anthropic API routes accept it.", + request.model, + provider.as_str() + ); + } Ok(MessagesRequest { params: MessagesOptionalParams { service_tier: None, @@ -561,6 +616,7 @@ async fn call_provider( stream, whole_sse_frames, joined_reads, + detached_drain, .. } = request; let provider_name = provider.as_str(); @@ -582,6 +638,7 @@ async fn call_provider( timeout, joined_reads, ); + let streamed = draining_after_detach(streamed, config.stream_decoder(), detached_drain); return Ok(if whole_sse_frames { framed(streamed) } else { @@ -617,6 +674,7 @@ async fn hosted_response( identity, provider, whole_sse_frames, + detached_drain, .. } = request; let provider_name = provider.as_str(); @@ -634,6 +692,7 @@ async fn hosted_response( .await?; let streamed = hosted_streaming_response(headers, chunks, config.stream_decoder(), provider_name); + let streamed = draining_after_detach(streamed, config.stream_decoder(), detached_drain); return Ok(if whole_sse_frames { framed(streamed) } else { @@ -707,6 +766,7 @@ async fn send_recovering( let config = request.provider.config(); let mut body = std::borrow::Cow::Borrowed(&request.wire.body); let mut attempt = 1; + let mut recovered: Vec<(u16, String)> = Vec::new(); loop { log_request_body(request.provider.as_str(), request.stream, &body); if let Some(value) = &request.unencodable_request_header { @@ -780,6 +840,20 @@ async fn send_recovering( } hosted @ Upstream::Hosted(_) => return Ok((hosted, sent_body)), }; + if let Some(fallback) = request.native_output_fallback.as_ref().filter(|_| { + config.retries_without_native_output( + failure.status, + &failure.text, + &body, + &request.identity.model, + ) + }) { + let rebuilt = fallback.body(config)?; + let replayed = replayed_recoveries(config, rebuilt, &recovered, request); + body = std::borrow::Cow::Owned(replayed.body.clone()); + context.provider_request_retried(replayed).await?; + continue; + } let retried = if attempt == MESSAGES_HTTP_ERROR_ATTEMPTS { None } else { @@ -794,6 +868,7 @@ async fn send_recovering( return Err(failure.into()); }; attempt += 1; + recovered.push((failure.status, failure.text.clone())); tracing::debug!( "Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt {attempt}/{MESSAGES_HTTP_ERROR_ATTEMPTS})." ); @@ -802,6 +877,36 @@ async fn send_recovering( } } +fn replayed_recoveries( + config: &dyn BaseMessagesConfig, + rebuilt: Value, + recovered: &[(u16, String)], + request: &ProviderCall, +) -> litellm_host::interceptors::RetriedRequest { + recovered.iter().fold( + litellm_host::interceptors::RetriedRequest { + reasons: Vec::new(), + body: rebuilt, + }, + |replayed, (status, text)| match config.retry_after_http_error( + *status, + text, + &replayed.body, + &request.transform_context, + ) { + Some(retried) => litellm_host::interceptors::RetriedRequest { + reasons: replayed + .reasons + .into_iter() + .chain(retried.reasons) + .collect(), + body: retried.body, + }, + None => replayed, + }, + ) +} + fn serialize_failure(err: serde_json::Error) -> Error { Error::invalid_request(litellm_llms::ErrorDetail::failed( "Anthropic messages request serialization", @@ -973,16 +1078,23 @@ fn streaming_response( _ => chunks.boxed(), } .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); - let chunks = match decoder { + let unobserved_tail = decoder.map(|_| UnobservedTail::default()); + let chunks = match decoder.zip(unobserved_tail.clone()) { None => chunks.map_err(Error::from).boxed(), - Some(decode) => decoded_chunks( + Some((decode, tail)) => decoded_chunks( chunks.map_err(std::io::Error::other).boxed(), decode, + headers.clone(), provider, + tail, ), }; MessagesCallResponse::Stream { - head: super::route::MessagesStreamHead { headers }, + head: super::route::MessagesStreamHead { + headers, + detached_drain: None, + unobserved_tail, + }, chunks, } } @@ -994,20 +1106,44 @@ fn hosted_streaming_response( provider: &'static str, ) -> MessagesCallResponse { let chunks = chunks.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); - let chunks = match decoder { + let unobserved_tail = decoder.map(|_| UnobservedTail::default()); + let chunks = match decoder.zip(unobserved_tail.clone()) { None => chunks.boxed(), - Some(decode) => decoded_chunks( + Some((decode, tail)) => decoded_chunks( chunks.map_err(std::io::Error::other).boxed(), decode, + headers.clone(), provider, + tail, ), }; MessagesCallResponse::Stream { - head: super::route::MessagesStreamHead { headers }, + head: super::route::MessagesStreamHead { + headers, + detached_drain: None, + unobserved_tail, + }, chunks, } } +fn draining_after_detach( + response: MessagesCallResponse, + decoder: Option, + detached_drain: Option, +) -> MessagesCallResponse { + match response { + MessagesCallResponse::Stream { head, chunks } => MessagesCallResponse::Stream { + head: super::route::MessagesStreamHead { + detached_drain: decoder.and(detached_drain), + ..head + }, + chunks, + }, + complete => complete, + } +} + fn framed(response: MessagesCallResponse) -> MessagesCallResponse { let MessagesCallResponse::Stream { head, chunks } = response else { return response; @@ -1057,17 +1193,14 @@ fn complete_frames_end(pending: &[u8]) -> usize { fn decoded_chunks( bytes: ByteStream, decode: StreamDecoder, + headers: Vec<(String, String)>, provider: &'static str, + tail: UnobservedTail, ) -> BoxStream<'static, Result> { - futures_util::stream::try_unfold(decode(bytes), move |mut events| async move { - let Some(event) = events.try_next().await? else { - return Ok(None); - }; - let chunk = streaming::encode(&event)?; - log_chunk(provider, "client_response", &chunk); - Ok(Some((chunk, events))) - }) - .boxed() + let frames = decode(bytes, headers).map_err(Error::from).boxed(); + super::relay::relayed(frames, tail) + .inspect_ok(move |chunk| log_chunk(provider, "client_response", chunk)) + .boxed() } fn log_request_body(provider: &str, stream: bool, body: &serde_json::Value) { @@ -1113,6 +1246,7 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) { #[cfg(test)] mod tests { + use litellm_llms::base_llm::messages::streaming; use litellm_llms_types::headers::ProviderSpecificHeaders; use rstest::{fixture, rstest}; use serde_json::{Map, Value, json}; @@ -1850,8 +1984,8 @@ mod tests { #[rstest] #[case::event( - "data: {\"type\":\"ping\"}\n\n", - Some("event: ping\ndata: {\"type\":\"ping\"}\n\n") + "data: {\"type\":\"message_stop\"}\n\n", + Some("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n") )] #[rstest::rstest] #[case::invalid_event("data: invalid\n\ndata: {\"type\":\"ping\"}\n\n", None)] @@ -1872,7 +2006,11 @@ mod tests { .unwrap(); let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response( response, - Some(streaming::event_stream), + Some(|bytes, _| { + streaming::event_stream(bytes) + .and_then(|event| futures_util::future::ready(streaming::encode(&event))) + .boxed() + }), "test", Duration::from_secs(5), false, @@ -1888,6 +2026,78 @@ mod tests { assert!(chunks.next().await.is_none()); } + async fn upstream_stream(body: &'static str) -> reqwest::Response { + let upstream = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(200).set_body_raw(body, "text/event-stream")) + .mount(&upstream) + .await; + litellm_http::Client::for_test() + .get(upstream.uri()) + .send() + .await + .unwrap() + } + + fn reencoding(bytes: ByteStream, _: Vec<(String, String)>) -> streaming::FrameStream { + streaming::event_stream(bytes) + .and_then(|event| futures_util::future::ready(streaming::encode(&event))) + .boxed() + } + + #[rstest] + #[case::decoded_and_cut_short( + Some(reencoding as StreamDecoder), + "data: {\"type\":\"ping\"}\n\n", + true + )] + #[case::decoded_and_stopped( + Some(reencoding as StreamDecoder), + "data: {\"type\":\"ping\"}\n\ndata: {\"type\":\"message_stop\"}\n\n", + false + )] + #[case::passed_through_and_cut_short(None, "data: {\"type\":\"ping\"}\n\n", false)] + #[tokio::test] + async fn a_decoded_stream_cut_short_ends_with_the_incomplete_stream_error_on_its_tail( + #[case] decoder: Option, + #[case] body: &'static str, + #[case] incomplete: bool, + ) { + let relays = decoder.is_some(); + let MessagesCallResponse::Stream { head, mut chunks } = streaming_response( + upstream_stream(body).await, + decoder, + "test", + Duration::from_secs(5), + false, + ) else { + panic!("a streaming response returns chunks"); + }; + let tail = head.unobserved_tail; + assert_eq!(tail.is_some(), relays); + let mut delivered = Vec::new(); + let mut on_tail = Vec::new(); + while let Some(chunk) = chunks.try_next().await.unwrap() { + match tail.as_ref().is_some_and(UnobservedTail::begun) { + true => on_tail.push(chunk), + false => delivered.push(chunk), + } + } + let error_event = Bytes::from(format!( + "event: error\ndata: {{\"type\": \"error\", \"error\": {{\"type\": \"api_error\", \"message\": \"{}\"}}}}\n\n", + super::super::relay::INCOMPLETE_STREAM_ERROR_MESSAGE + )); + assert!(!delivered.contains(&error_event)); + assert_eq!( + on_tail, + if incomplete { + vec![error_event] + } else { + Vec::new() + } + ); + } + #[rstest] #[case::no_delimiter(b"data: a".as_slice(), 0)] #[case::lf(b"data: a\n\ndata: b", 9)] diff --git a/litellm/rust/crates/inference-messages/src/lib.rs b/litellm/rust/crates/inference-messages/src/lib.rs index a932fcb3d2d..5a1a725a772 100644 --- a/litellm/rust/crates/inference-messages/src/lib.rs +++ b/litellm/rust/crates/inference-messages/src/lib.rs @@ -5,6 +5,7 @@ mod handler; mod host_answer; mod pre_request; mod reads; +mod relay; pub mod route; mod safeguards; mod transport; diff --git a/litellm/rust/crates/inference-messages/src/relay.rs b/litellm/rust/crates/inference-messages/src/relay.rs new file mode 100644 index 00000000000..1b5022e95d9 --- /dev/null +++ b/litellm/rust/crates/inference-messages/src/relay.rs @@ -0,0 +1,202 @@ +use bytes::Bytes; +use futures_util::{StreamExt, TryStreamExt, stream::BoxStream}; +use litellm_host::protocol::UnobservedTail; +use litellm_python_compat::{Value, json}; + +use super::Error; + +pub(crate) const INCOMPLETE_STREAM_ERROR_MESSAGE: &str = "Provider stream ended before emitting a message_stop event; the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated."; + +type Frames = BoxStream<'static, Result>; + +/// `_incomplete_stream_error_sse_event`. +fn incomplete_stream_error_event() -> Bytes { + Bytes::from(format!( + "event: error\ndata: {{\"type\": \"error\", \"error\": {{\"type\": \"api_error\", \"message\": \"{INCOMPLETE_STREAM_ERROR_MESSAGE}\"}}}}\n\n" + )) +} + +/// `bytes.splitlines()`. +fn lines(chunk: &[u8]) -> impl Iterator { + let mut rest = Some(chunk); + std::iter::from_fn(move || { + let current = rest.filter(|current| !current.is_empty())?; + match current + .iter() + .position(|byte| matches!(byte, b'\n' | b'\r')) + { + Some(at) => { + let skip = if current[at..].starts_with(b"\r\n") { + 2 + } else { + 1 + }; + rest = Some(¤t[at + skip..]); + Some(¤t[..at]) + } + None => { + rest = None; + Some(current) + } + } + }) +} + +fn field<'a>(fields: &'a [(Value, Value)], name: &str) -> Option<&'a Value> { + fields + .iter() + .find(|(key, _)| matches!(key, Value::Str(key) if key == name)) + .map(|(_, value)| value) +} + +/// `_decoded_sse_data_line`. +fn decoded_data_line(line: &[u8]) -> Option { + let data = line.strip_prefix(b"data:")?.trim_ascii(); + json::loads(std::str::from_utf8(data).ok()?).ok() +} + +/// `is_message_stop_chunk` for a chunk of SSE bytes. +fn is_message_stop_chunk(chunk: &[u8]) -> bool { + lines(chunk).any(|line| line == b"event: message_stop") +} + +/// `is_provider_error_chunk` for a chunk of SSE bytes. +fn is_provider_error_chunk(chunk: &[u8]) -> bool { + lines(chunk) + .filter_map(decoded_data_line) + .find_map(|candidate| match candidate { + Value::Dict(fields) + if matches!(field(&fields, "type"), Some(Value::Str(kind)) if kind == "error") => + { + Some(fields) + } + _ => None, + }) + .is_some_and(|fields| matches!(field(&fields, "error"), Some(Value::Dict(_)))) +} + +/// `_is_terminal_stream_chunk`. +fn is_terminal_stream_chunk(chunk: &[u8]) -> bool { + is_message_stop_chunk(chunk) || is_provider_error_chunk(chunk) +} + +struct Relay { + frames: Option, + terminal: bool, + tail: UnobservedTail, +} + +/// A decoded provider stream as `BaseAnthropicMessagesStreamingIterator.async_sse_wrapper` +/// relays it: one that ends without a `message_stop` or provider error event closes with an +/// `api_error` event. Python adds that event after billing, so it goes out on `tail`, which the +/// stream hooks never observe. +pub(crate) fn relayed(frames: Frames, tail: UnobservedTail) -> Frames { + let relay = Relay { + frames: Some(frames), + terminal: false, + tail, + }; + futures_util::stream::try_unfold(relay, |mut relay| async move { + let Some(frames) = relay.frames.as_mut() else { + return Ok(None); + }; + if let Some(frame) = frames.try_next().await? { + relay.terminal |= is_terminal_stream_chunk(&frame); + return Ok(Some((frame, relay))); + } + relay.frames = None; + if relay.terminal { + return Ok(None); + } + relay.tail.begin(); + Ok(Some((incomplete_stream_error_event(), relay))) + }) + .boxed() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const START: &str = "event: message_start\ndata: {\"type\": \"message_start\"}\n\n"; + const STOP: &str = "event: message_stop\ndata: {\"type\": \"message_stop\"}\n\n"; + const PROVIDER_ERROR: &str = "event: error\ndata: {\"type\": \"error\", \"error\": {\"type\": \"overloaded_error\"}}\n\n"; + const UNSHAPED_ERROR: &str = + "event: error\ndata: {\"type\": \"error\", \"error\": \"overloaded\"}\n\n"; + + fn frames(items: Vec>) -> Frames { + futures_util::stream::iter(items.into_iter().map(|item| item.map(Bytes::from))).boxed() + } + + async fn relayed_frames( + items: Vec>, + ) -> (Vec>, UnobservedTail) { + let tail = UnobservedTail::default(); + let relayed = relayed(frames(items), tail.clone()).collect().await; + (relayed, tail) + } + + #[rstest] + #[case::message_stop(vec![START, STOP])] + #[case::provider_error(vec![START, PROVIDER_ERROR])] + #[case::stop_then_more(vec![START, STOP, START])] + #[tokio::test] + async fn a_terminated_stream_is_relayed_as_it_is(#[case] sent: Vec<&'static str>) { + let (relayed, tail) = relayed_frames(sent.iter().copied().map(Ok).collect()).await; + let relayed: Vec = relayed.into_iter().map(Result::unwrap).collect(); + assert_eq!( + relayed, + sent.into_iter().map(Bytes::from).collect::>() + ); + assert!(!tail.begun()); + } + + #[rstest] + #[case::empty(vec![])] + #[case::cut_short(vec![START])] + #[case::error_without_an_error_object(vec![START, UNSHAPED_ERROR])] + #[tokio::test] + async fn an_unterminated_stream_ends_with_the_incomplete_stream_error_on_the_tail( + #[case] sent: Vec<&'static str>, + ) { + let (relayed, tail) = relayed_frames(sent.iter().copied().map(Ok).collect()).await; + let mut relayed: Vec = relayed.into_iter().map(Result::unwrap).collect(); + assert_eq!( + relayed.pop().unwrap(), + Bytes::from(format!( + "event: error\ndata: {{\"type\": \"error\", \"error\": {{\"type\": \"api_error\", \"message\": \"{INCOMPLETE_STREAM_ERROR_MESSAGE}\"}}}}\n\n" + )) + ); + assert_eq!( + relayed, + sent.into_iter().map(Bytes::from).collect::>() + ); + assert!(tail.begun()); + } + + #[tokio::test] + async fn a_failed_stream_ends_with_its_failure() { + let failure = Error::from(litellm_http::Error::upstream(500, "cut".to_owned())); + let (relayed, tail) = relayed_frames(vec![Ok(START), Err(failure)]).await; + assert_eq!(relayed.len(), 2); + assert_eq!(relayed[0].as_ref().unwrap(), START.as_bytes()); + assert!(relayed[1].is_err()); + assert!(!tail.begun()); + } + + #[rstest] + #[case::crlf("event: message_stop\r\ndata: {}\r\n\r\n", true)] + #[case::cr("event: ping\revent: message_stop\r", true)] + #[case::unspaced_event("event:message_stop\n\n", false)] + #[case::error_data_padded("data: {\"type\": \"error\", \"error\": {}} \n\n", true)] + #[case::first_error_payload_decides( + "data: {\"type\": \"error\", \"error\": 1}\ndata: {\"type\": \"error\", \"error\": {}}\n\n", + false + )] + #[case::undecodable_data("data: {\n\n", false)] + fn terminal_chunks_are_python_s(#[case] chunk: &str, #[case] terminal: bool) { + assert_eq!(is_terminal_stream_chunk(chunk.as_bytes()), terminal); + } +} diff --git a/litellm/rust/crates/inference-messages/src/route.rs b/litellm/rust/crates/inference-messages/src/route.rs index fa850b23bf2..054bd0585ab 100644 --- a/litellm/rust/crates/inference-messages/src/route.rs +++ b/litellm/rust/crates/inference-messages/src/route.rs @@ -13,6 +13,8 @@ pub type MessagesOutput = HostedCompletion; /// The upstream response as the caller sees it at stream hand-off, before any chunk. pub struct MessagesStreamHead { pub headers: Vec<(String, String)>, + pub detached_drain: Option, + pub unobserved_tail: Option, } pub struct Messages; @@ -24,6 +26,16 @@ impl Protocol for Messages { type HostCall = Infallible; type Chunk = Bytes; type StreamHead = MessagesStreamHead; + + fn detached_drain(head: &MessagesStreamHead) -> Option { + head.detached_drain + } + + fn unobserved_tail( + head: &MessagesStreamHead, + ) -> Option { + head.unobserved_tail.clone() + } } pub type MessagesMachine = HostedMachine; @@ -66,6 +78,8 @@ impl litellm_inference::caching::StreamCachable for Messages { Some(litellm_host::call::CallOutput::Stream { head: MessagesStreamHead { headers: Vec::new(), + detached_drain: None, + unobserved_tail: None, }, chunks: Box::pin(futures_util::stream::iter([Ok(data)])), }) diff --git a/litellm/rust/crates/inference-messages/src/safeguards.rs b/litellm/rust/crates/inference-messages/src/safeguards.rs index 78e4a752536..4511fa7a241 100644 --- a/litellm/rust/crates/inference-messages/src/safeguards.rs +++ b/litellm/rust/crates/inference-messages/src/safeguards.rs @@ -168,6 +168,8 @@ mod tests { MessagesCallResponse::Stream { head: MessagesStreamHead { headers: Vec::new(), + detached_drain: None, + unobserved_tail: None, }, chunks: futures_util::stream::iter(chunks).boxed(), } diff --git a/litellm/rust/crates/inference-messages/src/types.rs b/litellm/rust/crates/inference-messages/src/types.rs index f265a9edc2b..a5b392a2c72 100644 --- a/litellm/rust/crates/inference-messages/src/types.rs +++ b/litellm/rust/crates/inference-messages/src/types.rs @@ -1,15 +1,13 @@ use std::time::Duration; use bytes::Bytes; -use litellm_core_utils::get_llm_provider_logic::LlmProviders; use litellm_host::call::CallOutput; -use litellm_inference::provider::resolve_llm_provider; use litellm_llms::{ ErrorDetail, base_llm::messages::context::{MessagesModelCapabilities, MessagesShapeOptions}, }; use litellm_llms_types::{ - formats::messages::{MessagesRequest, MessagesResponse, SystemPrompt}, + formats::messages::{MessagesRequest, MessagesResponse}, headers::ProviderSpecificHeaders, }; use litellm_router_types::LitellmParams; @@ -31,32 +29,10 @@ pub struct MessagesCall { pub shaping: MessagesShaping, } -/// The request body. `custom_llm_provider` is the call's, as the provider is resolved from it -/// and the model. -pub fn messages_body( - body: Map, - custom_llm_provider: Option<&str>, -) -> Result { - if !reads_unparsed_system(&body, custom_llm_provider) - && let Some(system) = body.get(SYSTEM).filter(|system| !system.is_null()) - && let Err(err) = SystemPrompt::deserialize(system) - { - return Err(invalid_request(err)); - } +pub fn messages_body(body: Map) -> Result { serde_json::from_value(Value::Object(body)).map_err(invalid_request) } -const SYSTEM: &str = "system"; - -/// Only `azure_ai` keeps a system prompt Rust could not type as sent, as Python does. By the -/// owner's ruling every other provider, `anthropic` included, keeps main's request error for it. -fn reads_unparsed_system(body: &Map, custom_llm_provider: Option<&str>) -> bool { - body.get("model") - .and_then(Value::as_str) - .and_then(|model| resolve_llm_provider(model, custom_llm_provider, "messages").ok()) - .is_some_and(|resolved| resolved.provider == LlmProviders::AzureAi) -} - /// The caller's litellm params, projected by a host from the keys [`LitellmParams::fields`] /// names. A key present with a value of the wrong type is a request error, as it is for /// Python's `GenericLiteLLMParams(**kwargs)`. @@ -110,6 +86,8 @@ pub struct MessagesSettings { #[serde(default)] pub drops_unsupported_params: bool, #[serde(default)] + pub global_drop_params: bool, + #[serde(default)] pub intercepted: bool, #[serde(default)] pub mcp_gateway: bool, @@ -126,6 +104,10 @@ pub struct MessagesSettings { /// settings dict. #[serde(skip)] pub module_api_key: Option, + #[serde(default)] + pub detached_drain_timeout: Option, + #[serde(default)] + pub detached_drain_limit: Option, } impl MessagesSettings { @@ -133,10 +115,18 @@ impl MessagesSettings { MessagesShapeOptions { reasoning_auto_summary: self.reasoning_auto_summary, presanitized: self.presanitized, - ..MessagesShapeOptions::default() } } + /// Python's Bedrock Messages relay keeps reading the provider stream after its client leaves, + /// for at most this long and in at most this many calls at once. + pub fn detached_drain(&self) -> Option { + Some(litellm_host::protocol::DetachedDrain { + timeout: Duration::try_from_secs_f64(self.detached_drain_timeout?).ok()?, + limit: self.detached_drain_limit?, + }) + } + pub fn request_timeout(&self) -> Option { self.request_timeout .and_then(|secs| Duration::try_from_secs_f64(secs).ok()) @@ -225,6 +215,7 @@ mod tests { supports_sampling_params: false, supports_speed: true, supports_mid_conversation_system: false, + bedrock: Default::default(), effort_tiers: SupportedEffortTiers { minimal: false, low: true, diff --git a/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs b/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs index 5207f4e9d58..c19ffc5a47c 100644 --- a/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs +++ b/litellm/rust/crates/inference-messages/tests/bedrock_native/main.rs @@ -227,6 +227,57 @@ async fn managed_aws_credentials_sign_the_exact_native_request( assert_eq!(request.json()["model"], "claude-test"); } +#[rstest] +#[tokio::test] +async fn bedrock_invoke_keeps_the_region_path_and_signs_caller_headers_as_python_does( + call: MessagesCall, +) { + let upstream = upstream([message_response()]).await; + let secrets = Arc::new(RecordingSecrets::new([("AWS_REGION_NAME", "us-east-1")])); + run_with( + secrets, + MessagesCall { + api_base: Some(upstream.uri()), + litellm_params: litellm_router_types::LitellmParams { + aws: litellm_auth::AwsParams { + aws_access_key_id: Some("test-access".into()), + aws_secret_access_key: Some("test-secret".into()), + ..Default::default() + }, + ..Default::default() + }, + extra_headers: Some(serde_json::Map::from_iter([ + ("x-amz-date".to_string(), json!("20200101T000000Z")), + ("x-amz-security-token".to_string(), json!("caller-token")), + ("date".to_string(), json!("Wed, 01 Jan 2020 00:00:00 GMT")), + ])), + ..with_model( + call, + "bedrock/invoke/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0", + ) + }, + ) + .await + .unwrap(); + let request = only_request(&upstream).await; + let auth = request.header("authorization").unwrap(); + assert_eq!( + request.url.path(), + "/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke" + ); + assert!(auth.starts_with("AWS4-HMAC-SHA256 Credential=test-access/")); + assert!(auth.contains( + "/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;date;host;x-amz-security-token, " + )); + assert_eq!(request.header("x-amz-security-token"), Some("caller-token")); + assert_eq!(request.header("x-amz-date"), None); + assert!( + request + .header("date") + .is_some_and(|date| date.ends_with(" -0000") && !date.contains("2020")) + ); +} + #[rstest] #[tokio::test] async fn private_mantle_endpoint_is_used_and_signed(call: MessagesCall) { diff --git a/litellm/rust/crates/inference-messages/tests/messages/request.rs b/litellm/rust/crates/inference-messages/tests/messages/request.rs index 4c025c9ad51..72447a17019 100644 --- a/litellm/rust/crates/inference-messages/tests/messages/request.rs +++ b/litellm/rust/crates/inference-messages/tests/messages/request.rs @@ -920,13 +920,12 @@ impl litellm_tracing::Sink for Warnings { } #[rstest] -#[case::anthropic_logs_as_main("anthropic", false)] -#[case::azure_ai_logs_python_s_warning("azure_ai", true)] +#[case::anthropic("anthropic")] +#[case::azure_ai("azure_ai")] #[tokio::test] -async fn a_speed_dropped_under_drop_params_is_logged_only_on_azure_ai( +async fn a_speed_dropped_under_drop_params_logs_python_s_warning( call: MessagesCall, #[case] provider: &str, - #[case] logged: bool, ) { let upstream = upstream([message_response()]).await; let warnings = Warnings::default(); @@ -956,11 +955,7 @@ async fn a_speed_dropped_under_drop_params_is_logged_only_on_azure_ai( .filter(|message| message.starts_with("Dropping unsupported `speed`")) .cloned() .collect(); - assert_eq!( - speed_warnings.len(), - usize::from(logged), - "{speed_warnings:?}" - ); + assert_eq!(speed_warnings.len(), 1, "{speed_warnings:?}"); } #[rstest] diff --git a/litellm/rust/crates/inference-messages/tests/messages/response.rs b/litellm/rust/crates/inference-messages/tests/messages/response.rs index 365d6ed711c..0f4809b72fc 100644 --- a/litellm/rust/crates/inference-messages/tests/messages/response.rs +++ b/litellm/rust/crates/inference-messages/tests/messages/response.rs @@ -483,7 +483,7 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes #[case::mistyped_param(json!({"model": MODEL, "messages": "hello", "max_tokens": 16}))] #[case::missing_messages(json!({"model": MODEL, "max_tokens": 16}))] fn a_body_that_does_not_parse_is_an_invalid_request(#[case] raw: Value) { - let error = messages_body(object(raw), None).expect_err("the body is rejected"); + let error = messages_body(object(raw)).expect_err("the body is rejected"); assert!(error.is_request(), "{error:?}"); assert!( @@ -502,38 +502,27 @@ const UNPARSED_SYSTEMS: [&str; 4] = [ ]; #[rstest] -fn a_system_prompt_that_does_not_parse_is_main_s_request_error_except_on_azure_ai( +fn a_system_prompt_that_does_not_parse_is_kept_as_python_keeps_it( #[values(0, 1, 2, 3)] system: usize, #[values( - ("anthropic/claude-sonnet-4-5", None), - ("claude-sonnet-4-5", Some("anthropic")), - ("vertex_ai/claude-sonnet-4-5", None), - ("azure_ai/claude-sonnet-4-5", None), - ("claude-sonnet-4-5", Some("azure_ai")) + "anthropic/claude-sonnet-4-5", + "vertex_ai/claude-sonnet-4-5", + "azure_ai/claude-sonnet-4-5" )] - call: (&str, Option<&str>), + model: &str, ) { - let (model, custom_llm_provider) = call; let system: Value = serde_json::from_str(UNPARSED_SYSTEMS[system]).unwrap(); let body = object(json!({"model": model, "messages": [], "max_tokens": 16, "system": system})); - let parsed = messages_body(body, custom_llm_provider); - let azure_ai = model.starts_with("azure_ai/") || custom_llm_provider == Some("azure_ai"); - match parsed { + match messages_body(body) { Err(error) => { - assert!( - !azure_ai || system.is_number() || system.is_boolean(), - "{error:?}" - ); + assert!(system.is_number() || system.is_boolean(), "{error:?}"); let text = litellm_error::Report::new(&error).to_string(); - let expected = if azure_ai { - "system must be a string or a list of content blocks" - } else { - "invalid Anthropic messages request: data did not match any variant of untagged enum SystemPrompt" - }; - assert!(text.contains(expected), "{text}"); + assert!( + text.contains("system must be a string or a list of content blocks"), + "{text}" + ); } Ok(request) => { - assert!(azure_ai, "{request:?}"); assert!( request .params diff --git a/litellm/rust/crates/inference/src/error.rs b/litellm/rust/crates/inference/src/error.rs index 5f6e1b3f70d..0f5eacf26b5 100644 --- a/litellm/rust/crates/inference/src/error.rs +++ b/litellm/rust/crates/inference/src/error.rs @@ -215,11 +215,6 @@ impl Error { impl From for Error { fn from(error: litellm_http::Error) -> Self { - // Python drops the caller's copy and prefers a forwarded Authorization - // over the signature, so leave the request to it. - if error.is_computed_header() { - return Self::Declined(Decline::new("request forwards a header AWS SigV4 computes")); - } Self::Http(error) } } @@ -330,16 +325,6 @@ mod tests { assert!(error.is_missing_credential()); } - #[rstest] - #[case::direct(litellm_http::Error::computed_header("authorization").into())] - #[case::through_a_transformation( - LlmError::Http(litellm_http::Error::computed_header("authorization")).into() - )] - fn a_header_the_signer_computes_is_left_to_the_host_on_every_route(#[case] error: Error) { - assert!(error.declined().is_some(), "{error:?}"); - assert!(!error.dispatched()); - } - #[test] fn a_transformation_decline_survives_into_the_route_error() { let decline = Decline::new("unrecognized request parameter"); diff --git a/litellm/rust/crates/llms-types/src/formats/messages/streaming.rs b/litellm/rust/crates/llms-types/src/formats/messages/streaming.rs index b2f70ee3753..58f2db287aa 100644 --- a/litellm/rust/crates/llms-types/src/formats/messages/streaming.rs +++ b/litellm/rust/crates/llms-types/src/formats/messages/streaming.rs @@ -137,6 +137,8 @@ pub enum MessagesStreamEvent { MessageStop { #[serde(default, skip_serializing_if = "Option::is_none")] usage: Option, + #[serde(flatten)] + extra: Map, }, Ping, Error { diff --git a/litellm/rust/crates/llms/Cargo.toml b/litellm/rust/crates/llms/Cargo.toml index f779d4edf66..f537a26bd73 100644 --- a/litellm/rust/crates/llms/Cargo.toml +++ b/litellm/rust/crates/llms/Cargo.toml @@ -27,6 +27,7 @@ litellm-model-catalog.workspace = true litellm-tracing.workspace = true base64.workspace = true bytes.workspace = true +crc32fast = "1.5" data-url = "0.3.2" fancy-regex.workspace = true futures-util.workspace = true @@ -39,6 +40,7 @@ strum.workspace = true thiserror.workspace = true time.workspace = true tokio = { workspace = true, features = ["sync"] } +tokio-util = { version = "0.7", features = ["codec"] } url.workspace = true [dev-dependencies] diff --git a/litellm/rust/crates/llms/src/anthropic/beta_gated_fields.rs b/litellm/rust/crates/llms/src/anthropic/beta_gated_fields.rs index 4445fb49a2b..b06e78f4cf0 100644 --- a/litellm/rust/crates/llms/src/anthropic/beta_gated_fields.rs +++ b/litellm/rust/crates/llms/src/anthropic/beta_gated_fields.rs @@ -1,8 +1,13 @@ use std::collections::BTreeSet; +use std::fmt::Display; + use litellm_llms_types::{ - formats::messages::{ContentBlock, ContentBlockType, Message, MessageContent}, - providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet}, + formats::messages::{ + ContentBlock, ContentBlockType, Message, MessageContent, MessagesOptionalParams, + MessagesRequest, ThinkingConfig, + }, + providers::anthropic::{AnthropicBeta, BetaSet}, recognized::Recognized, }; use serde_json::{Map, Value}; @@ -16,15 +21,20 @@ const CONTENT: &str = "content"; const TYPE: &str = "type"; const OUTPUT_CONFIG: &str = "output_config"; const TOOL_CHANGE_BLOCK_TYPES: [&str; 2] = ["tool_addition", "tool_removal"]; +const BLOCK_BINDING: &str = "block_binding"; +const DISPLAY: &str = "display"; +const DISPLAY_UPDATES: &str = "updates"; -/// The mid-conversation system fields of Python's `BETA_GATED_FIELDS`, named as Python's warning -/// names them and declared in the order it sorts those names. +/// The fields of Python's `BETA_GATED_FIELDS`, named as Python's warning names them and declared +/// in the order it sorts those names. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] enum GatedField { Emptied, ClearAt, ToolChanges, OutputConfig, + BlockBinding, + DisplayUpdates, } impl GatedField { @@ -34,6 +44,8 @@ impl GatedField { Self::ClearAt => "messages[].system.clear_at", Self::ToolChanges => "messages[].system.content[].tool_addition|tool_removal", Self::OutputConfig => "messages[].system.output_config", + Self::BlockBinding => "thinking.block_binding", + Self::DisplayUpdates => "thinking.display=updates", } } @@ -49,6 +61,8 @@ impl GatedField { AnthropicBeta::MidConversationOutputConfig20260701, AnthropicBeta::PerTurnControl20260701, ], + Self::BlockBinding => &[AnthropicBeta::ThinkingBindingControls20260801], + Self::DisplayUpdates => &[AnthropicBeta::ThinkingDisplayUpdates20260818], } } @@ -82,11 +96,98 @@ pub fn strip_beta_gated_system_messages( active: &BetaSet, drops_unsupported_params: bool, model: &str, - provider: BetaProvider, + provider: impl Display, ) -> Vec> { if !drops_unsupported_params { return messages; } + let (messages, gated) = stripped_system_messages(messages, active); + log_beta_gated_strips(&gated, model, provider); + messages +} + +/// Python's `strip_beta_gated_native_request` and `log_beta_gated_strips`, run once the final +/// `anthropic-beta` values are known: under drop_params the `thinking` fields and the system +/// message fields no beta in `active` unlocks are dropped, and one warning names them all. +pub fn strip_beta_gated_request( + request: MessagesRequest, + active: &BetaSet, + drops_unsupported_params: bool, + provider: impl Display, +) -> MessagesRequest { + if !drops_unsupported_params { + return request; + } + let (thinking, thinking_gated) = stripped_thinking(request.params.thinking, active); + let (messages, messages_gated) = stripped_system_messages(request.messages, active); + log_beta_gated_strips( + &thinking_gated.union(&messages_gated).copied().collect(), + &request.model, + provider, + ); + MessagesRequest { + messages, + params: MessagesOptionalParams { + thinking, + ..request.params + }, + ..request + } +} + +fn log_beta_gated_strips(gated: &BTreeSet, model: &str, provider: impl Display) { + if gated.is_empty() { + return; + } + litellm_tracing::warn!( + "Dropped {} for model={model} provider={provider} because drop_params is on and the provider would reject them on this request, either since this provider leg does not send that anthropic-beta (see anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user messages.", + gated + .iter() + .copied() + .map(GatedField::described) + .collect::>() + .join(", ") + ); +} + +/// Python's `strip_beta_gated_thinking`, which reads `thinking` as any mapping. +fn stripped_thinking( + thinking: Option>, + active: &BetaSet, +) -> (Option>, BTreeSet) { + let Some(Value::Object(fields)) = thinking.as_ref().and_then(|t| serde_json::to_value(t).ok()) + else { + return (thinking, BTreeSet::new()); + }; + let binding_gated = + fields.contains_key(BLOCK_BINDING) && GatedField::BlockBinding.gated(active); + let display_gated = fields.get(DISPLAY).and_then(Value::as_str) == Some(DISPLAY_UPDATES) + && GatedField::DisplayUpdates.gated(active); + if !binding_gated && !display_gated { + return (thinking, BTreeSet::new()); + } + let kept: Map = fields + .into_iter() + .filter(|(key, _)| { + !((binding_gated && key == BLOCK_BINDING) || (display_gated && key == DISPLAY)) + }) + .collect(); + let gated = [ + (binding_gated, GatedField::BlockBinding), + (display_gated, GatedField::DisplayUpdates), + ] + .into_iter() + .filter_map(|(gated, field)| gated.then_some(field)) + .collect(); + let thinking = serde_json::from_value(Value::Object(kept.clone())) + .unwrap_or(Recognized::Unrecognized(Value::Object(kept))); + (Some(thinking), gated) +} + +fn stripped_system_messages( + messages: Vec>, + active: &BetaSet, +) -> (Vec>, BTreeSet) { let stripped: Vec<(Option>, Vec)> = messages .iter() .cloned() @@ -112,24 +213,16 @@ pub fn strip_beta_gated_system_messages( .flat_map(|(_, gated)| gated.iter().copied()) .collect(); if gated.is_empty() { - return messages; + return (messages, gated); } let kept: Vec> = stripped .into_iter() .filter_map(|(message, _)| message) .collect(); if ends_on_assistant(&kept) && !ends_on_assistant(&messages) { - return messages; + return (messages, BTreeSet::new()); } - litellm_tracing::warn!( - "Dropped {} for model={model} provider={provider} because drop_params is on and the provider would reject them on this request, either since this provider leg does not send that anthropic-beta (see anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user messages.", - gated - .into_iter() - .map(GatedField::described) - .collect::>() - .join(", ") - ); - kept + (kept, gated) } fn strip_system_message(message: Message, active: &BetaSet) -> (Option, Vec) { @@ -263,14 +356,14 @@ fn ends_on_assistant(messages: &[Recognized]) -> bool { #[cfg(test)] mod tests { use litellm_llms_types::{ - formats::messages::Message, + formats::messages::{Message, MessagesRequest}, providers::anthropic::{BetaProvider, BetaSet}, recognized::Recognized, }; use rstest::rstest; use serde_json::{Value, json}; - use super::strip_beta_gated_system_messages; + use super::{strip_beta_gated_request, strip_beta_gated_system_messages}; fn strip( messages: Vec>, @@ -445,6 +538,99 @@ mod tests { assert_eq!(strip(sent.clone(), &BetaSet::default(), true), sent); } + fn thinking_request(thinking: Value, system: Value) -> MessagesRequest { + serde_json::from_value(json!({ + "model": "test-model", + "max_tokens": 16, + "messages": conversation(system), + "thinking": thinking, + })) + .unwrap() + } + + #[rstest] + #[case::block_binding_without_its_beta( + json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}), + "", + json!({"type": "adaptive"}) + )] + #[case::block_binding_with_its_beta( + json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}), + "thinking-binding-controls-2026-08-01", + json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}) + )] + #[case::display_updates_without_its_beta( + json!({"type": "adaptive", "display": "updates"}), + "", + json!({"type": "adaptive"}) + )] + #[case::display_updates_with_its_beta( + json!({"type": "adaptive", "display": "updates"}), + "thinking-display-updates-2026-08-18", + json!({"type": "adaptive", "display": "updates"}) + )] + #[case::another_display( + json!({"type": "adaptive", "display": "summarized"}), + "", + json!({"type": "adaptive", "display": "summarized"}) + )] + #[case::both_on_enabled_thinking( + json!({"type": "enabled", "budget_tokens": 1024, "display": "updates", "block_binding": {}}), + "", + json!({"type": "enabled", "budget_tokens": 1024}) + )] + fn gated_thinking_fields_are_dropped( + #[case] thinking: Value, + #[case] active: &str, + #[case] expected: Value, + ) { + let stripped = strip_beta_gated_request( + thinking_request(thinking, json!({"role": "user", "content": "more"})), + &betas(active), + true, + "test", + ); + assert_eq!( + serde_json::to_value(stripped.params.thinking).unwrap(), + expected + ); + } + + #[rstest] + fn the_request_strip_drops_gated_thinking_and_system_fields_together() { + let stripped = strip_beta_gated_request( + thinking_request( + json!({"type": "adaptive", "display": "updates"}), + json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}), + ), + &BetaSet::default(), + true, + "test", + ); + assert_eq!( + ( + serde_json::to_value(stripped.params.thinking).unwrap(), + stripped.messages + ), + ( + json!({"type": "adaptive"}), + messages(conversation(json!({"role": "system", "content": "note"}))) + ) + ); + } + + #[rstest] + fn without_drop_params_the_request_is_sent_as_is() { + let sent = thinking_request( + json!({"type": "adaptive", "display": "updates", "block_binding": {}}), + json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}), + ); + assert_eq!( + strip_beta_gated_request(sent.clone(), &BetaSet::default(), false, "test"), + sent + ); + } + #[rstest] fn non_system_messages_keep_their_fields() { let sent = messages(json!([ diff --git a/litellm/rust/crates/llms/src/anthropic/beta_headers.rs b/litellm/rust/crates/llms/src/anthropic/beta_headers.rs index 4a3575c9bbc..6f78a71950c 100644 --- a/litellm/rust/crates/llms/src/anthropic/beta_headers.rs +++ b/litellm/rust/crates/llms/src/anthropic/beta_headers.rs @@ -16,20 +16,38 @@ pub enum BetaPolicy { impl BetaPolicy { pub fn apply(self, headers: Headers) -> Headers { + self.apply_keeping(headers, &BetaSet::default()) + } + + /// Python's `update_headers_with_filtered_beta` with `keep`: a beta the routed model + /// accepts is sent as the caller sent it, after the betas the filter kept. + pub fn apply_keeping(self, headers: Headers, keep: &BetaSet) -> Headers { let provider = match self { Self::Forward => return headers, Self::Drop => return without_headers(headers, &[BETA_HEADER]), Self::Filter(provider) => provider, }; - let accepted: BetaSet = existing_betas(&headers) + let sent = existing_betas(&headers); + let accepted: BetaSet = sent .iter() + .filter(|beta| !keep.contains(beta)) .filter_map(|beta| beta.on(provider)) .collect(); + let kept: Vec<&AnthropicBeta> = sent + .iter() + .filter(|beta| keep.contains(beta) && !accepted.contains(beta)) + .collect(); let headers = without_headers(headers, &[BETA_HEADER]); - if accepted.is_empty() { + if accepted.is_empty() && kept.is_empty() { return headers; } - with_header(headers, BETA_HEADER, accepted.to_string()) + let value = accepted + .iter() + .chain(kept) + .map(AnthropicBeta::as_str) + .collect::>() + .join(","); + with_header(headers, BETA_HEADER, value) } } @@ -46,7 +64,7 @@ pub fn accepted_or_unlisted(betas: BetaSet, provider: BetaProvider) -> BetaSet { #[cfg(test)] mod tests { use crate::anthropic::beta_headers::BetaPolicy; - use litellm_llms_types::providers::anthropic::BetaProvider; + use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet}; use rstest::rstest; fn headers(pairs: &[(&str, &str)]) -> Vec<(String, String)> { @@ -125,6 +143,37 @@ mod tests { ); } + #[rstest] + #[case::kept_after_the_filtered_betas( + "per-turn-control-2026-07-01,advanced-tool-use-2025-11-20,example-beta-2099-01-01", + Some("tool-search-tool-2025-10-19,per-turn-control-2026-07-01") + )] + #[case::kept_alone( + "per-turn-control-2026-07-01,example-beta-2099-01-01", + Some("per-turn-control-2026-07-01") + )] + #[case::not_sent("advanced-tool-use-2025-11-20", Some("tool-search-tool-2025-10-19"))] + #[case::nothing_survives("example-beta-2099-01-01", None)] + fn filter_sends_a_kept_beta_as_sent_after_the_filtered_ones( + #[case] sent: &str, + #[case] expected: Option<&str>, + ) { + let keep: BetaSet = "per-turn-control-2026-07-01" + .parse() + .unwrap_or_else(|never| match never {}); + assert_eq!( + BetaPolicy::Filter(BetaProvider::Bedrock).apply_keeping( + headers(&[("x-api-key", "k"), ("anthropic-beta", sent)]), + &keep + ), + [("x-api-key", "k")] + .into_iter() + .chain(expected.map(|beta| ("anthropic-beta", beta))) + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>() + ); + } + #[rstest] #[case::anthropic( BetaProvider::Anthropic, diff --git a/litellm/rust/crates/llms/src/anthropic/common_utils.rs b/litellm/rust/crates/llms/src/anthropic/common_utils.rs index 38113e2fce6..86f649eac2b 100644 --- a/litellm/rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm/rust/crates/llms/src/anthropic/common_utils.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::LazyLock; -use crate::base_llm::messages::context::{MessagesModelCapabilities, UnparsedMessages}; +use crate::base_llm::messages::context::MessagesModelCapabilities; use fancy_regex::Regex; use litellm_auth::CredentialPlacement; use litellm_secret_value::SecretValue; @@ -322,26 +322,19 @@ pub fn has_advisor_tool(tools: Option<&[Recognized]>) -> bool { pub fn requires_native_compaction_beta( compaction: Option<&Value>, messages: &[Recognized], - unparsed: UnparsedMessages, ) -> bool { - compaction.is_some() - || messages - .iter() - .any(|message| carries_signed_compaction_block(message, unparsed)) + compaction.is_some() || messages.iter().any(carries_signed_compaction_block) } -/// Python's `carries_signed_compaction_block` reads any mapping, so where unparsed messages are -/// read a message that did not parse is scanned through its raw content too. -fn carries_signed_compaction_block( - message: &Recognized, - unparsed: UnparsedMessages, -) -> bool { +/// Python's `carries_signed_compaction_block` reads any mapping, so a message that did not parse +/// is scanned through its raw content too. +fn carries_signed_compaction_block(message: &Recognized) -> bool { match message { Recognized::Known(message) => message.blocks().iter().any(|block| { block.is_type(ContentBlockType::Compaction) && block.signature.as_deref().is_some_and(|s| !s.is_empty()) }), - Recognized::Unrecognized(Value::Object(fields)) if unparsed.read() => matches!( + Recognized::Unrecognized(Value::Object(fields)) => matches!( fields.get("content"), Some(Value::Array(blocks)) if blocks.iter().any(|block| { block.get("type").and_then(Value::as_str) == Some("compaction") @@ -2881,33 +2874,19 @@ mod tests { #[case] messages: Value, #[case] expected: bool, ) { - for unparsed in [UnparsedMessages::Read, UnparsedMessages::Skipped] { - assert_eq!( - requires_native_compaction_beta( - compaction.as_ref(), - &history(messages.clone()), - unparsed - ), - expected - ); - } + assert_eq!( + requires_native_compaction_beta(compaction.as_ref(), &history(messages)), + expected + ); } - #[rstest] - #[case::read(UnparsedMessages::Read, true)] - #[case::skipped(UnparsedMessages::Skipped, false)] - fn a_signed_compaction_block_in_an_unparsed_message_counts_only_where_unparsed_messages_are_read( - #[case] unparsed: UnparsedMessages, - #[case] expected: bool, - ) { + #[test] + fn a_signed_compaction_block_in_an_unparsed_message_counts() { let messages = history(json!([{"role": "assistant", "content": [ {"type": "text", "text": 5}, {"type": "compaction", "content": "c", "signature": "s"} ]}])); - assert_eq!( - requires_native_compaction_beta(None, &messages, unparsed), - expected - ); + assert!(requires_native_compaction_beta(None, &messages)); } #[rstest] @@ -3083,6 +3062,7 @@ mod tests { supports_speed: false, supports_mid_conversation_system: false, effort_tiers: tiers(false, false, false, false, false, false), + bedrock: Default::default(), } ); assert_eq!( diff --git a/litellm/rust/crates/llms/src/anthropic/messages/handler.rs b/litellm/rust/crates/llms/src/anthropic/messages/handler.rs index 2709e729d05..4bd598e8a18 100644 --- a/litellm/rust/crates/llms/src/anthropic/messages/handler.rs +++ b/litellm/rust/crates/llms/src/anthropic/messages/handler.rs @@ -3,7 +3,7 @@ use litellm_llms_types::{ AdaptiveThinking, EnabledThinking, Message, MessagesOptionalParams, MessagesRequest, ThinkingConfig, ThinkingDisplay, }, - recognized::{Recognized, map_known_runs}, + recognized::Recognized, }; use serde_json::{Value, json}; @@ -15,30 +15,25 @@ use crate::{ strip_empty_raw_content_blocks, strip_provider_specific_fields, strip_raw_provider_specific_fields, }, - base_llm::messages::context::{MessagesShapeOptions, UnparsedMessages}, + base_llm::messages::context::MessagesShapeOptions, }; pub fn shape_anthropic_messages_request( request: MessagesRequest, options: MessagesShapeOptions, ) -> Result { - let messages = match (options.presanitized, options.unparsed) { - (true, UnparsedMessages::Read) => map_messages( + let messages = if options.presanitized { + map_messages( request.messages, strip_provider_specific_fields, strip_raw_provider_specific_fields, - ), - (false, UnparsedMessages::Read) => map_messages( + ) + } else { + map_messages( request.messages, sanitize_anthropic_messages, sanitize_raw_anthropic_message, - ), - (true, UnparsedMessages::Skipped) => { - map_known_runs(request.messages, strip_provider_specific_fields) - } - (false, UnparsedMessages::Skipped) => { - map_known_runs(request.messages, sanitize_anthropic_messages) - } + ) }; Ok(MessagesRequest { messages, @@ -334,7 +329,6 @@ mod tests { MessagesShapeOptions { reasoning_auto_summary: true, presanitized: false, - unparsed: UnparsedMessages::Skipped, }, ) .unwrap(); @@ -355,7 +349,6 @@ mod tests { #[rstest] #[case::sanitized( false, - UnparsedMessages::Read, json!([ {"role": "user", "content": "hi"}, {"role": "assistant", "content": [ @@ -365,7 +358,6 @@ mod tests { )] #[case::presanitized( true, - UnparsedMessages::Read, json!([ {"role": "user", "content": "hi"}, {"role": "user", "content": [{"type": "text", "text": 5}]}, @@ -375,33 +367,8 @@ mod tests { ]} ]) )] - #[case::skipped( - false, - UnparsedMessages::Skipped, - json!([ - {"role": "user", "content": "hi"}, - {"role": "user", "content": [{"type": "text", "text": 5}]}, - {"role": "assistant", "content": [ - {"type": "text", "text": 5}, - {"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}, "provider_specific_fields": {"x": 1}} - ]} - ]) - )] - #[case::skipped_presanitized( - true, - UnparsedMessages::Skipped, - json!([ - {"role": "user", "content": "hi"}, - {"role": "user", "content": [{"type": "text", "text": 5}]}, - {"role": "assistant", "content": [ - {"type": "text", "text": 5}, - {"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}, "provider_specific_fields": {"x": 1}} - ]} - ]) - )] - fn shaping_rewrites_a_message_that_did_not_parse_only_where_unparsed_messages_are_read( + fn shaping_rewrites_a_message_that_did_not_parse( #[case] presanitized: bool, - #[case] unparsed: UnparsedMessages, #[case] expected: Value, ) { let shaped = shape_anthropic_messages_request( @@ -416,7 +383,6 @@ mod tests { MessagesShapeOptions { reasoning_auto_summary: false, presanitized, - unparsed, }, ) .unwrap(); @@ -434,7 +400,6 @@ mod tests { MessagesShapeOptions { reasoning_auto_summary: false, presanitized: true, - unparsed: UnparsedMessages::Skipped, }, ) .unwrap(); diff --git a/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs index 20310544245..9380cd126a9 100644 --- a/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs @@ -17,9 +17,11 @@ use super::{ scalar::{is_one, python_str, python_value}, thinking::{drop_unsupported_effort, translate_thinking}, }; -use crate::anthropic::beta_headers::accepted_or_unlisted; +use crate::anthropic::{ + beta_gated_fields::strip_beta_gated_request, beta_headers::accepted_or_unlisted, +}; use crate::base_llm::messages::{ - context::{MessagesShapeOptions, MessagesTransformContext, UnparsedMessages}, + context::{MessagesShapeOptions, MessagesTransformContext}, tools::ToolCustomField, }; use crate::{ @@ -32,7 +34,7 @@ use crate::{ is_anthropic_invalid_thinking_block_error, is_anthropic_thinking_binding_mismatch_error, is_claude_strict_tools_rejection, is_tool_search_used, map_messages, merge_beta_headers, minted_oauth_environment, optionally_handle_anthropic_oauth, - requires_native_compaction_beta, strip_advisor_blocks, strip_encrypted_reasoning_blocks, + requires_native_compaction_beta, strip_advisor_blocks, strip_encrypted_reasoning_blocks_restoring_redacted_data, strip_raw_advisor_blocks, strip_raw_encrypted_reasoning_blocks, strip_thinking_blocks_from_anthropic_messages_request, strip_unsignable_thinking_blocks, @@ -133,6 +135,23 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { ) } + /// Python's transform strips the thinking and system fields the final header does not + /// unlock. + fn finalize_request( + &self, + request: MessagesRequest, + headers: Headers, + context: &MessagesTransformContext, + ) -> Result<(MessagesRequest, Headers), Error> { + let stripped = strip_beta_gated_request( + request, + &existing_betas(&headers), + context.drops_unsupported_params, + BetaProvider::Anthropic, + ); + Ok((stripped, headers)) + } + fn retry_after_http_error( &self, status: u16, @@ -324,19 +343,16 @@ pub(crate) fn transform_messages_request( .context_management .clone() .map(map_openai_context_management_to_anthropic); - let reads_unparsed = context.unparsed.read(); let messages = if has_advisor_tool(request.params.tools.as_deref()) || continues_message_thread(&request.params) { request.messages - } else if reads_unparsed { + } else { map_messages( request.messages, strip_advisor_blocks, strip_raw_advisor_blocks, ) - } else { - map_known_runs(request.messages, strip_advisor_blocks) }; let messages = if context.drops_unsupported_params { drop_non_string_tool_result_texts(messages) @@ -344,15 +360,11 @@ pub(crate) fn transform_messages_request( messages }; Ok(MessagesRequest { - messages: if reads_unparsed { - map_messages( - messages, - strip_encrypted_reasoning_blocks_restoring_redacted_data, - strip_raw_encrypted_reasoning_blocks, - ) - } else { - map_known_runs(messages, strip_encrypted_reasoning_blocks) - }, + messages: map_messages( + messages, + strip_encrypted_reasoning_blocks_restoring_redacted_data, + strip_raw_encrypted_reasoning_blocks, + ), params: MessagesOptionalParams { context_management, extra: without_null_safeguards(request.params.extra), @@ -451,21 +463,17 @@ pub(crate) fn update_headers_with_anthropic_beta( headers: Headers, request: &MessagesRequest, ) -> Headers { - merge_beta_headers(headers, feature_betas(request, UnparsedMessages::Skipped)) + merge_beta_headers(headers, feature_betas(request)) } -/// `unparsed` says whether a beta trigger in a message that did not parse counts, as it does -/// for Python, which reads any mapping. pub(crate) fn update_headers_with_safeguards_beta( headers: Headers, request: &MessagesRequest, provider: BetaProvider, - unparsed: UnparsedMessages, ) -> Headers { merge_beta_headers( headers, - feature_betas(request, unparsed) - .union(on_provider(mapped_host_betas(&request.params), provider)), + feature_betas(request).union(on_provider(mapped_host_betas(&request.params), provider)), ) } @@ -473,7 +481,7 @@ pub(crate) fn update_headers_with_safeguards_beta( /// through that column. pub(crate) fn provider_feature_betas(request: &MessagesRequest, provider: BetaProvider) -> BetaSet { on_provider( - feature_betas(request, UnparsedMessages::Skipped).union(mapped_host_betas(&request.params)), + feature_betas(request).union(mapped_host_betas(&request.params)), provider, ) } @@ -506,7 +514,7 @@ fn is_threaded(params: &MessagesOptionalParams) -> bool { .is_some_and(|thread| !thread.is_null()) } -fn feature_betas(request: &MessagesRequest, unparsed: UnparsedMessages) -> BetaSet { +fn feature_betas(request: &MessagesRequest) -> BetaSet { let params = &request.params; let tools = params.tools.as_deref(); let scanned_messages: &[Recognized] = if continues_message_thread(params) { @@ -515,19 +523,19 @@ fn feature_betas(request: &MessagesRequest, unparsed: UnparsedMessages) -> BetaS &request.messages }; [ - requires_native_compaction_beta(params.compaction.as_ref(), scanned_messages, unparsed) + requires_native_compaction_beta(params.compaction.as_ref(), scanned_messages) .then_some(AnthropicBeta::Compact20260904), uses_structured_output(params).then_some(AnthropicBeta::StructuredOutputs20251113), (params.speed == Some(Recognized::Known(Speed::Fast))) .then_some(AnthropicBeta::FastMode20260201), - messages_carry_output_config(scanned_messages, unparsed) + messages_carry_output_config(scanned_messages) .then_some(AnthropicBeta::PerTurnControl20260701), is_threaded(params).then_some(AnthropicBeta::MessageThreads20260812), uses_thinking_display_updates(params) .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818), uses_thinking_block_binding(params) .then_some(AnthropicBeta::ThinkingBindingControls20260801), - messages_carry_tool_changes(scanned_messages, unparsed) + messages_carry_tool_changes(scanned_messages) .then_some(AnthropicBeta::MidConversationToolChanges20260701), has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301), is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120), @@ -573,17 +581,12 @@ fn carries_safeguards(params: &MessagesOptionalParams) -> bool { .is_some_and(|value| !value.is_null()) } -/// Python's `messages_carry_output_config` reads any mapping, so where unparsed messages are read -/// a message that did not parse, such as a system turn sent without content, counts too. -fn messages_carry_output_config( - messages: &[Recognized], - unparsed: UnparsedMessages, -) -> bool { +/// Python's `messages_carry_output_config` reads any mapping, so a message that did not parse, +/// such as a system turn sent without content, counts too. +fn messages_carry_output_config(messages: &[Recognized]) -> bool { messages.iter().any(|message| match message { Recognized::Known(message) => message.extra.contains_key(OUTPUT_CONFIG), - Recognized::Unrecognized(Value::Object(fields)) => { - unparsed.read() && fields.contains_key(OUTPUT_CONFIG) - } + Recognized::Unrecognized(Value::Object(fields)) => fields.contains_key(OUTPUT_CONFIG), Recognized::Unrecognized(_) => false, }) } @@ -641,19 +644,15 @@ fn is_raw_tool_reference_change(block: &Value) -> bool { == Some("tool_reference") } -/// `AnthropicModelInfo.is_mid_conversation_tool_change_used` reads any mapping, so where unparsed -/// messages are read a system message that did not parse is scanned through its raw content too. -fn messages_carry_tool_changes( - messages: &[Recognized], - unparsed: UnparsedMessages, -) -> bool { +/// `AnthropicModelInfo.is_mid_conversation_tool_change_used` reads any mapping, so a system +/// message that did not parse is scanned through its raw content too. +fn messages_carry_tool_changes(messages: &[Recognized]) -> bool { messages.iter().any(|message| match message { Recognized::Known(message) => { message.role == "system" && message.blocks().iter().any(is_tool_reference_change) } Recognized::Unrecognized(Value::Object(fields)) => { - unparsed.read() - && fields.get("role").and_then(Value::as_str) == Some("system") + fields.get("role").and_then(Value::as_str) == Some("system") && matches!( fields.get("content"), Some(Value::Array(blocks)) if blocks.iter().any(is_raw_tool_reference_change) @@ -694,11 +693,9 @@ pub fn drop_unsupported_params( let speed = match ¶ms.speed { Some(speed) if !capabilities.supports_speed => { reject("speed", speed_repr(speed), "")?; - if context.logs_dropped_params { - litellm_tracing::warn!( - "Dropping unsupported `speed` for model={model} (drop_params=True). Fast mode is only supported on select Opus models." - ); - } + litellm_tracing::warn!( + "Dropping unsupported `speed` for model={model} (drop_params=True). Fast mode is only supported on select Opus models." + ); None } _ => params.speed.clone(), @@ -1415,8 +1412,7 @@ mod tests { }, drop_params: true, drops_unsupported_params: false, - unparsed: UnparsedMessages::Skipped, - logs_dropped_params: false, + global_drop_params: false, } ); return; @@ -1805,10 +1801,7 @@ mod tests { &["message-threads-2026-08-12", "per-turn-control-2026-07-01"] )] fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) { - assert_eq!( - feature_betas(&request(fields), UnparsedMessages::Read), - betas(expected) - ); + assert_eq!(feature_betas(&request(fields)), betas(expected)); } #[rstest] @@ -1827,21 +1820,16 @@ mod tests { {"role": "user", "content": "Use the ping tool"}, {"role": "system", "content": [{"type": "text", "text": 5}, {"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, ]}))] - fn a_beta_trigger_in_an_unparsed_message_counts_only_where_unparsed_messages_are_read( - #[case] fields: Value, - ) { + fn a_beta_trigger_in_an_unparsed_message_counts_on_every_provider(#[case] fields: Value) { let request = request(fields); + assert_ne!(feature_betas(&request), BetaSet::default()); assert_eq!( - feature_betas(&request, UnparsedMessages::Skipped), - BetaSet::default() + existing_betas(&update_headers_with_anthropic_beta(Vec::new(), &request)), + feature_betas(&request) ); assert_eq!( provider_feature_betas(&request, BetaProvider::Anthropic), - BetaSet::default() - ); - assert_ne!( - feature_betas(&request, UnparsedMessages::Read), - BetaSet::default() + feature_betas(&request) ); } diff --git a/litellm/rust/crates/llms/src/anthropic/model_capabilities.rs b/litellm/rust/crates/llms/src/anthropic/model_capabilities.rs index 1f31473db45..7aff234f778 100644 --- a/litellm/rust/crates/llms/src/anthropic/model_capabilities.rs +++ b/litellm/rust/crates/llms/src/anthropic/model_capabilities.rs @@ -1,19 +1,32 @@ use std::sync::LazyLock; -use crate::base_llm::messages::context::{MessagesModelCapabilities, SupportedEffortTiers}; +use crate::base_llm::messages::context::{ + BedrockCapabilities, MessagesModelCapabilities, NativeOutputSupport, SupportedEffortTiers, +}; use crate::bedrock::common_utils::{ - bedrock_base_model, is_bedrock_application_inference_profile_arn, without_routing_prefixes, + bedrock_base_model, is_bedrock_application_inference_profile_arn, strip_bedrock_routing_prefix, }; use fancy_regex::Regex; use litellm_auth_aws::is_bedrock_region; use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; +use litellm_llms_types::formats::messages::EffortLevel; use litellm_model_catalog::{Catalog, wire::Fields}; +use litellm_python_compat::{json::from_json, truthy::truthy}; +use serde::Deserialize; +use serde_json::Value; const SAMPLING_RESTRICTED_NAMES: [&str; 9] = [ "fable", "opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7", "opus-4-8", "opus_4_8", "opus-4.8", "opus_4.8", ]; const SAMPLING_TOLERANT_PROVIDERS: [&str; 1] = ["openrouter"]; +const BEDROCK: &str = "bedrock"; +const NATIVE_OUTPUT: &str = "supports_native_structured_output"; +const BEDROCK_NATIVE_OUTPUT: &str = "supports_bedrock_native_structured_output"; +const NATIVE_OUTPUT_BY_DEFAULT: &str = "supports_bedrock_native_structured_output_by_default"; +const OUTPUT_CONFIG: &str = "supports_output_config"; +const PER_TURN_CONTROL: &str = "supports_per_turn_control"; +const PROMPT_CACHE_TTL_PRICE: &str = "cache_creation_input_token_cost_above_1hr"; const BACKSTOP_PREFIXES: [&str; 4] = [ "bedrock/converse/", "bedrock/invoke/", @@ -45,7 +58,8 @@ pub fn anthropic_model_capabilities( .map_or((model, "anthropic"), |resolved| { (resolved.model, resolved.custom_llm_provider) }); - let (model, name) = match base_model.map(without_routing_prefixes) { + let requested = model; + let (model, name) = match base_model.map(strip_bedrock_routing_prefix) { Some(base) if is_bedrock(provider) && is_bedrock_application_inference_profile_arn(resolved) => { @@ -53,18 +67,7 @@ pub fn anthropic_model_capabilities( } _ => (model, resolved), }; - let probe = Probe { - catalog, - declared, - fallbacks: backstop_candidates(model), - info: catalog.model_info( - &candidate_names(name, provider) - .each_ref() - .map(String::as_str), - provider, - ), - bare: catalog.lookup(name).map(|found| found.entry.fields()), - }; + let probe = Probe::new(catalog, model, name, provider, declared); let thinking_always_on = probe.supports("thinking_always_on"); MessagesModelCapabilities { supports_reasoning: probe.supports("supports_reasoning"), @@ -98,9 +101,73 @@ pub fn anthropic_model_capabilities( xhigh: probe.supports("supports_xhigh_reasoning_effort"), max: probe.supports("supports_max_reasoning_effort"), }, + bedrock: if provider == BEDROCK { + bedrock_model_capabilities(catalog, requested, declared) + } else { + BedrockCapabilities::default() + }, } } +pub fn bedrock_model_capabilities( + catalog: &Catalog, + model: &str, + declared: &dyn Fn(&str) -> Option, +) -> BedrockCapabilities { + let model = bedrock_model(model); + let probe = Probe::new(catalog, model, bedrock_model(model), BEDROCK, &|_| None); + let mapped = probe.factory(NATIVE_OUTPUT); + let opted_out = declared(BEDROCK_NATIVE_OUTPUT).map_or_else( + || mapped_native_output_flag(catalog, model) == Some(false), + |opted_in| !opted_in, + ); + BedrockCapabilities { + output_config_effort_ceiling: probe + .info + .as_ref() + .and_then(|info| info.get("bedrock_output_config_effort_ceiling")) + .and_then(|ceiling| EffortLevel::deserialize(ceiling).ok()), + supports_output_config: probe.factory(OUTPUT_CONFIG), + supports_prompt_cache_ttl: [model, bedrock_base_model(model)].iter().any(|name| { + catalog + .get(name) + .and_then(|fields| fields.get(PROMPT_CACHE_TTL_PRICE)) + .is_some_and(|price| !price.is_null()) + }), + supports_tool_search: probe.supports("supports_tool_search"), + supports_per_turn_control: declared(PER_TURN_CONTROL) + .unwrap_or_else(|| probe.supports(PER_TURN_CONTROL)), + native_output: NativeOutputSupport { + enabled: declared(NATIVE_OUTPUT) + .unwrap_or_else(|| declared(BEDROCK_NATIVE_OUTPUT) != Some(false) && mapped), + opt_in: declared(BEDROCK_NATIVE_OUTPUT) + .unwrap_or_else(|| probe.factory(BEDROCK_NATIVE_OUTPUT)), + mapped, + replaces_unforceable_tool: backstop( + catalog, + &probe.fallbacks, + "supports_forced_tool_use", + ) == Some(false) + && !opted_out, + }, + } +} + +fn bedrock_model(model: &str) -> &str { + get_custom_llm_provider(model, Some(BEDROCK)).map_or(model, |found| found.model) +} + +fn mapped_native_output_flag(catalog: &Catalog, model: &str) -> Option { + let routed = strip_bedrock_routing_prefix(model); + [ + format!("{BEDROCK}/{routed}"), + routed.to_owned(), + without_region(routed).to_owned(), + ] + .iter() + .find_map(|name| flag(catalog.get(name), BEDROCK_NATIVE_OUTPUT)) +} + struct Probe<'a> { catalog: &'a Catalog, declared: &'a dyn Fn(&str) -> Option, @@ -109,7 +176,28 @@ struct Probe<'a> { bare: Option<&'a Fields>, } -impl Probe<'_> { +impl<'a> Probe<'a> { + fn new( + catalog: &'a Catalog, + model: &str, + name: &str, + provider: &str, + declared: &'a dyn Fn(&str) -> Option, + ) -> Self { + Self { + catalog, + declared, + fallbacks: backstop_candidates(model), + info: catalog.model_info( + &candidate_names(name, provider) + .each_ref() + .map(String::as_str), + provider, + ), + bare: catalog.lookup(name).map(|found| found.entry.fields()), + } + } + fn supports(&self, capability: &str) -> bool { if let Some(supported) = (self.declared)(capability) { return supported; @@ -117,10 +205,35 @@ impl Probe<'_> { if let Some(supported) = flag(self.info.as_ref(), capability) { return supported; } - if self.info.is_some() && flag(self.bare, capability) == Some(true) { - return true; + self.bare_supports(capability) + || backstop(self.catalog, &self.fallbacks, capability) == Some(true) + } + + fn factory(&self, capability: &str) -> bool { + match self.info.as_ref() { + Some(info) => { + model_info_flag(info, capability).unwrap_or_else(|| self.bare_supports(capability)) + } + None => false, } - backstop(self.catalog, &self.fallbacks, capability) == Some(true) + } + + fn bare_supports(&self, capability: &str) -> bool { + self.info.is_some() && flag(self.bare, capability) == Some(true) + } +} + +fn model_info_flag(info: &Fields, capability: &str) -> Option { + match capability { + NATIVE_OUTPUT + if info + .get(NATIVE_OUTPUT_BY_DEFAULT) + .is_some_and(|value| truthy(&from_json(value.clone()))) => + { + Some(info.get(BEDROCK_NATIVE_OUTPUT) != Some(&Value::Bool(false))) + } + OUTPUT_CONFIG => None, + _ => flag(Some(info), capability), } } @@ -170,7 +283,7 @@ fn split_model<'a>(model: &'a str, provider: &str) -> &'a str { if !is_bedrock(provider) { return model; } - let routed = without_routing_prefixes(model); + let routed = strip_bedrock_routing_prefix(model); let region_free = without_region(routed); if !region_free.starts_with("arn:") { return routed; @@ -239,7 +352,9 @@ mod tests { use rstest::{fixture, rstest}; use serde_json::Value; - use super::anthropic_model_capabilities; + use super::{anthropic_model_capabilities, bedrock_model_capabilities}; + use crate::base_llm::messages::context::{BedrockCapabilities, NativeOutputSupport}; + use litellm_llms_types::formats::messages::EffortLevel; const CATALOG: &str = r#"{ "claude-test-5": { @@ -460,6 +575,137 @@ mod tests { ); } + #[fixture] + fn bundled() -> Catalog { + Catalog::parse(BUNDLED_REGISTRY).unwrap().value + } + + type Expected = (Option, [bool; 8]); + + fn projected(capabilities: BedrockCapabilities) -> Expected { + let BedrockCapabilities { + output_config_effort_ceiling, + supports_output_config, + supports_prompt_cache_ttl, + supports_tool_search, + supports_per_turn_control, + native_output: + NativeOutputSupport { + enabled, + opt_in, + mapped, + replaces_unforceable_tool, + }, + } = capabilities; + ( + output_config_effort_ceiling, + [ + supports_output_config, + supports_prompt_cache_ttl, + supports_tool_search, + supports_per_turn_control, + enabled, + opt_in, + mapped, + replaces_unforceable_tool, + ], + ) + } + + const XHIGH: Option = Some(EffortLevel::Xhigh); + const PROFILE: &str = "bedrock/arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-opus-4-8"; + const FOUNDATION: &str = "bedrock/invoke/arn:aws:bedrock:us-east-1:123456789012:foundation-model/anthropic.claude-opus-4-8"; + const APPLICATION: &str = + "bedrock/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123"; + + #[rstest] + #[case::us("bedrock/us.anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, true, true, true, false]))] + #[case::eu("bedrock/eu.anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, true, true, true, false]))] + #[case::global("bedrock/global.anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, true, true, true, false]))] + #[case::invoke_prefix("bedrock/invoke/us.anthropic.claude-opus-4-8", (XHIGH, [false, true, true, true, true, true, true, false]))] + #[case::region_path("bedrock/us-east-1/anthropic.claude-opus-4-8", (XHIGH, [false, true, true, true, false, true, false, false]))] + #[case::inference_profile_arn(PROFILE, (XHIGH, [false, true, true, true, true, true, true, false]))] + #[case::invoke_foundation_arn(FOUNDATION, (XHIGH, [false, true, true, true, false, true, false, false]))] + #[case::unprefixed("anthropic.claude-opus-4-8", (XHIGH, [true, true, true, true, false, true, false, false]))] + #[case::other_casing("bedrock/ANTHROPIC.CLAUDE-OPUS-4-8", (XHIGH, [true, false, true, true, false, true, false, false]))] + #[case::sonnet_4_5("bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", (None, [false, true, true, true, true, false, true, false]))] + #[case::converse_prefix("bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0", (None, [false, true, true, true, true, false, true, false]))] + #[case::throughput_suffix("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0:1000k", (None, [false, true, true, true, true, false, true, false]))] + #[case::claude_3_5("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", (None, [false; 8]))] + #[case::opus_4_7_without_version("bedrock/global.anthropic.claude-opus-4-7", (XHIGH, [true, true, true, true, false, false, false, false]))] + #[case::opus_4_7_invoke("bedrock/invoke/global.anthropic.claude-opus-4-7-v1:0", (None, [false, false, true, true, false, false, false, false]))] + #[case::unforceable_tool("bedrock/anthropic.claude-fable-5-1", (XHIGH, [true, true, true, true, false, false, false, true]))] + #[case::native_by_default("bedrock/us.anthropic.claude-mythos-5-2", (XHIGH, [true, true, true, true, true, false, true, false]))] + #[case::opt_in_only("bedrock/anthropic.claude-sonnet-5", (XHIGH, [true, true, true, true, false, true, false, false]))] + #[case::unmapped_claude_version("bedrock/anthropic.claude-opus-9-9-v1:0", (None, [false, false, true, true, false, false, false, false]))] + #[case::unmapped_claude("bedrock/anthropic.claude-newthing", (None, [false; 8]))] + #[case::application_profile(APPLICATION, (None, [false; 8]))] + #[case::other_family("bedrock/amazon.nova-pro-v1:0", (None, [false; 8]))] + #[case::unknown("bedrock/unknown-model", (None, [false; 8]))] + fn bedrock_capabilities_match_python( + bundled: Catalog, + #[case] model: &str, + #[case] expected: Expected, + ) { + assert_eq!( + projected(bedrock_model_capabilities(&bundled, model, &|_| None)), + expected + ); + } + + #[rstest] + #[case::native_on("bedrock/us.anthropic.claude-opus-4-8", &[("supports_native_structured_output", true)], (XHIGH, [true, true, true, true, true, true, true, false]))] + #[case::native_off("bedrock/us.anthropic.claude-opus-4-8", &[("supports_native_structured_output", false)], (XHIGH, [true, true, true, true, false, true, true, false]))] + #[case::bedrock_native_off("bedrock/us.anthropic.claude-opus-4-8", &[("supports_bedrock_native_structured_output", false)], (XHIGH, [true, true, true, true, false, false, true, false]))] + #[case::per_turn_off("bedrock/us.anthropic.claude-opus-4-8", &[("supports_per_turn_control", false)], (XHIGH, [true, true, true, false, true, true, true, false]))] + #[case::native_off_opt_in( + "bedrock/us.anthropic.claude-opus-4-8", + &[("supports_native_structured_output", false), ("supports_bedrock_native_structured_output", true)], + (XHIGH, [true, true, true, true, false, true, true, false]) + )] + #[case::unforceable_native_on("bedrock/anthropic.claude-fable-5-1", &[("supports_native_structured_output", true)], (XHIGH, [true, true, true, true, true, false, false, true]))] + #[case::unforceable_opt_in("bedrock/anthropic.claude-fable-5-1", &[("supports_bedrock_native_structured_output", true)], (XHIGH, [true, true, true, true, false, true, false, true]))] + #[case::unforceable_opted_out("bedrock/anthropic.claude-fable-5-1", &[("supports_bedrock_native_structured_output", false)], (XHIGH, [true, true, true, true, false, false, false, false]))] + #[case::bedrock_native_on("bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", &[("supports_bedrock_native_structured_output", true)], (None, [false, true, true, true, true, true, true, false]))] + #[case::per_turn_off_haiku("bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", &[("supports_per_turn_control", false)], (None, [false, true, true, false, true, false, true, false]))] + fn deployment_model_info_overrides_the_catalog_as_python_does( + bundled: Catalog, + #[case] model: &str, + #[case] declared: &[(&str, bool)], + #[case] expected: Expected, + ) { + let lookup = |capability: &str| { + declared + .iter() + .find(|(name, _)| *name == capability) + .map(|(_, supported)| *supported) + }; + + assert_eq!( + projected(bedrock_model_capabilities(&bundled, model, &lookup)), + expected + ); + } + + #[rstest] + #[case::bedrock("bedrock/us.anthropic.claude-opus-4-8", None, true)] + #[case::configured_bedrock("us.anthropic.claude-opus-4-8", Some("bedrock"), true)] + #[case::anthropic("anthropic/claude-opus-4-8", None, false)] + fn only_bedrock_deployments_carry_bedrock_capabilities( + bundled: Catalog, + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, + #[case] carried: bool, + ) { + let capabilities = + anthropic_model_capabilities(&bundled, model, custom_llm_provider, None, &|_| None); + + assert_eq!( + capabilities.bedrock != BedrockCapabilities::default(), + carried + ); + } + #[rstest] fn every_bundled_anthropic_row_shapes_with_its_own_flags() { let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value; diff --git a/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs index 77b4c0b9d78..f78f5929959 100644 --- a/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -16,7 +16,7 @@ use serde_json::Value; use crate::{ Error, anthropic::{ - beta_gated_fields::strip_beta_gated_system_messages, + beta_gated_fields::strip_beta_gated_request, beta_headers::BetaPolicy, common_utils::{ API_KEY_PLACEMENT, existing_betas, map_messages, strip_raw_unsignable_thinking_blocks, @@ -39,7 +39,7 @@ use crate::{ base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, messages::{ - context::{MessagesShapeOptions, MessagesTransformContext, UnparsedMessages}, + context::{MessagesShapeOptions, MessagesTransformContext}, normalization::{normalize_system_role_messages, strip_cache_control_scope}, transformation::BaseMessagesConfig, }, @@ -64,10 +64,6 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { } } - fn logs_dropped_params(&self) -> bool { - true - } - fn retry_after_http_error( &self, status: u16, @@ -83,7 +79,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { request: MessagesRequest, options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, options.reading_unparsed_messages()) + shape_anthropic_messages_request(request, options) } fn request_for_url(&self, request: MessagesRequest, _url: &str) -> MessagesRequest { @@ -116,12 +112,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { request: MessagesRequest, context: &MessagesTransformContext, ) -> Result { - transform_messages_request( - request, - &context - .keeping_display_updates() - .reading_unparsed_messages(), - ) + transform_messages_request(request, &context.keeping_display_updates()) } fn secret_names(&self) -> &'static [&'static str] { @@ -230,30 +221,25 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { headers, request, BetaProvider::AzureAi, - UnparsedMessages::Read, )) } - /// Python's transform strips the system fields the final header does not unlock, then - /// normalizes system roles and drops `cache_control.scope`, all after the betas were chosen - /// from the request as sent. + /// Python's transform strips the thinking and system fields the final header does not + /// unlock, then normalizes system roles and drops `cache_control.scope`, all after the betas + /// were chosen from the request as sent. fn finalize_request( &self, request: MessagesRequest, - headers: &Headers, + headers: Headers, context: &MessagesTransformContext, - ) -> MessagesRequest { - let stripped = MessagesRequest { - messages: strip_beta_gated_system_messages( - request.messages, - &existing_betas(headers), - context.drops_unsupported_params, - &request.model, - BetaProvider::AzureAi, - ), - ..request - }; - strip_cache_control_scope(normalize_system_role_messages( + ) -> Result<(MessagesRequest, Headers), Error> { + let stripped = strip_beta_gated_request( + request, + &existing_betas(&headers), + context.drops_unsupported_params, + BetaProvider::AzureAi, + ); + let normalized = strip_cache_control_scope(normalize_system_role_messages( stripped, context .thinking @@ -261,7 +247,8 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { .supports_mid_conversation_system, context.drops_unsupported_params, BetaProvider::AzureAi, - )) + )); + Ok((normalized, headers)) } } @@ -488,7 +475,12 @@ mod tests { .iter() .map(|(name, value)| (name.to_string(), value.to_string())) .collect(); - to_value(AZURE_ANTHROPIC_MESSAGES_CONFIG.finalize_request(request, &headers, context)) + to_value( + AZURE_ANTHROPIC_MESSAGES_CONFIG + .finalize_request(request, headers, context) + .expect("finalized request") + .0, + ) } #[rstest] diff --git a/litellm/rust/crates/llms/src/base_llm/auth.rs b/litellm/rust/crates/llms/src/base_llm/auth.rs index 1d9444a2ac4..7ec1d2fbf9c 100644 --- a/litellm/rust/crates/llms/src/base_llm/auth.rs +++ b/litellm/rust/crates/llms/src/base_llm/auth.rs @@ -9,7 +9,7 @@ use litellm_auth::{AuthServices, CredentialPlacement, TokenProviderHandle}; use litellm_auth_azure::AzureAuthInputs; use litellm_secret_value::SecretValue; -use litellm_auth_aws::{AwsCredentialSource, SigV4Signer}; +use litellm_auth_aws::{AwsCredentialSource, CallerSigningHeaders, SigV4Signer}; use litellm_auth_gcp::VertexConfig; use litellm_http::request::with_header; @@ -57,6 +57,7 @@ pub enum AuthScheme { region: String, service: &'static str, credentials: Box, + caller_headers: CallerSigningHeaders, }, } @@ -191,11 +192,13 @@ pub async fn resolve_auth( region, service, credentials, + caller_headers, } => Ok(Authenticated { headers, signer: Some( SigV4Signer::resolve(&services.aws, region, service, *credentials, env_lookup) - .await?, + .await? + .with_caller_headers(caller_headers), ), }), } @@ -445,6 +448,7 @@ mod tests { None, "test", ))), + caller_headers: CallerSigningHeaders::Dropped, }, ) .await; diff --git a/litellm/rust/crates/llms/src/base_llm/messages/context.rs b/litellm/rust/crates/llms/src/base_llm/messages/context.rs index 9b65620bf7e..b3dd3b53aeb 100644 --- a/litellm/rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm/rust/crates/llms/src/base_llm/messages/context.rs @@ -1,4 +1,5 @@ use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_llms_types::formats::messages::EffortLevel; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -23,39 +24,10 @@ impl SupportedEffortTiers { } } -/// What the shared Anthropic shaping does with a message, or a beta trigger in one, that Rust -/// could not type. `Read` reads it as Python does, off any mapping. `Skipped` leaves it as sent -/// and finds no beta in it, as main's native route does. Only `azure_ai` reads them: by the -/// owner's ruling `anthropic` keeps main's behaviour. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum UnparsedMessages { - #[default] - Skipped, - Read, -} - -impl UnparsedMessages { - pub fn read(self) -> bool { - self == Self::Read - } -} - #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct MessagesShapeOptions { pub reasoning_auto_summary: bool, pub presanitized: bool, - pub unparsed: UnparsedMessages, -} - -impl MessagesShapeOptions { - /// The same options for a provider whose shaping reads unparsed messages. - #[must_use] - pub fn reading_unparsed_messages(self) -> Self { - Self { - unparsed: UnparsedMessages::Read, - ..self - } - } } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -80,6 +52,46 @@ pub struct MessagesModelCapabilities { pub supports_mid_conversation_system: bool, #[serde(default)] pub effort_tiers: SupportedEffortTiers, + #[serde(default)] + pub bedrock: BedrockCapabilities, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct BedrockCapabilities { + #[serde(default)] + pub output_config_effort_ceiling: Option, + #[serde(default)] + pub supports_output_config: bool, + #[serde(default)] + pub supports_prompt_cache_ttl: bool, + #[serde(default)] + pub supports_tool_search: bool, + #[serde(default)] + pub supports_per_turn_control: bool, + #[serde(default)] + pub native_output: NativeOutputSupport, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct NativeOutputSupport { + #[serde(default)] + pub enabled: bool, + #[serde(default)] + pub opt_in: bool, + #[serde(default)] + pub mapped: bool, + #[serde(default)] + pub replaces_unforceable_tool: bool, +} + +impl NativeOutputSupport { + pub fn is_opt_in(self) -> bool { + !self.enabled && (self.opt_in || self.replaces_unforceable_tool) + } + + pub fn retries(self) -> bool { + self.enabled || self.is_opt_in() + } } fn default_true() -> bool { @@ -99,6 +111,7 @@ impl Default for MessagesModelCapabilities { supports_speed: false, supports_mid_conversation_system: false, effort_tiers: SupportedEffortTiers::default(), + bedrock: BedrockCapabilities::default(), } } } @@ -111,6 +124,7 @@ pub struct ThinkingBudgets { pub high: u64, pub xhigh: u64, pub max: u64, + pub bedrock_min: u64, } impl Default for ThinkingBudgets { @@ -122,6 +136,7 @@ impl Default for ThinkingBudgets { high: 4096, xhigh: 8192, max: 16384, + bedrock_min: 1024, } } } @@ -140,6 +155,9 @@ impl ThinkingBudgets { high: tier("HIGH", defaults.high), xhigh: tier("XHIGH", defaults.xhigh), max: tier("MAX", defaults.max), + bedrock_min: env + .parsed::("BEDROCK_MIN_THINKING_BUDGET_TOKENS") + .unwrap_or(defaults.bedrock_min), } } } @@ -159,9 +177,7 @@ pub struct MessagesTransformContext { pub thinking: ThinkingContext, pub drop_params: bool, pub drops_unsupported_params: bool, - pub unparsed: UnparsedMessages, - /// Logs Python's warning for a param `drop_params` drops because the model lacks it. - pub logs_dropped_params: bool, + pub global_drop_params: bool, } impl MessagesTransformContext { pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self { @@ -182,8 +198,7 @@ impl MessagesTransformContext { }, drop_params, drops_unsupported_params: false, - unparsed: UnparsedMessages::Skipped, - logs_dropped_params: false, + global_drop_params: false, } } @@ -198,13 +213,4 @@ impl MessagesTransformContext { ..*self } } - - /// The same context for a provider whose transform reads unparsed messages. - #[must_use] - pub fn reading_unparsed_messages(&self) -> Self { - Self { - unparsed: UnparsedMessages::Read, - ..*self - } - } } diff --git a/litellm/rust/crates/llms/src/base_llm/messages/streaming.rs b/litellm/rust/crates/llms/src/base_llm/messages/streaming.rs index 8c301e67de3..42cafb23805 100644 --- a/litellm/rust/crates/llms/src/base_llm/messages/streaming.rs +++ b/litellm/rust/crates/llms/src/base_llm/messages/streaming.rs @@ -5,7 +5,8 @@ pub use crate::base_llm::base_model_iterator::ByteStream; use crate::{Error, base_llm::sse}; pub type EventStream = sse::EventStream; -pub type StreamDecoder = fn(ByteStream) -> EventStream; +pub type FrameStream = futures_util::stream::BoxStream<'static, Result>; +pub type StreamDecoder = fn(ByteStream, Vec<(String, String)>) -> FrameStream; fn no_control_frames(_: &litellm_framer::sse::SseEvent) -> bool { false @@ -36,7 +37,7 @@ mod tests { MessagesContentBlockDelta, MessagesStreamUsage, }; use rstest::rstest; - use serde_json::json; + use serde_json::{Map, json}; use super::*; @@ -61,6 +62,7 @@ mod tests { output_tokens: Some(7), ..MessagesStreamUsage::default() }), + extra: Map::new(), }, ] } @@ -88,7 +90,10 @@ mod tests { text: "hello".into(), }, }, - MessagesStreamEvent::MessageStop { usage: None }, + MessagesStreamEvent::MessageStop { + usage: None, + extra: Map::new(), + }, ]; let streamed = event_stream(in_pieces(wire.as_bytes(), size)) .try_collect::>() @@ -138,7 +143,11 @@ mod tests { #[test] fn an_event_is_named_by_its_type() { - let encoded = encode(&MessagesStreamEvent::MessageStop { usage: None }).unwrap(); + let encoded = encode(&MessagesStreamEvent::MessageStop { + usage: None, + extra: Map::new(), + }) + .unwrap(); assert_eq!( encoded, Bytes::from(format!( diff --git a/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs index c29fb7e869e..3965f6826c7 100644 --- a/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs @@ -49,15 +49,28 @@ pub trait BaseMessagesConfig: Sync { Ok(request) } - fn tool_custom_field(&self, _model: &str) -> ToolCustomField { - ToolCustomField::Keep + fn native_output_fallback( + &self, + _request: &MessagesRequest, + _context: &MessagesTransformContext, + ) -> Option { + None } - /// Whether a param `drop_params` drops because the model lacks it logs Python's warning. - fn logs_dropped_params(&self) -> bool { + fn retries_without_native_output( + &self, + _status: u16, + _error_text: &str, + _request_body: &Value, + _model: &str, + ) -> bool { false } + fn tool_custom_field(&self, _model: &str) -> ToolCustomField { + ToolCustomField::Keep + } + fn transform_anthropic_messages_response( &self, _model: &str, @@ -153,14 +166,15 @@ pub trait BaseMessagesConfig: Sync { } /// The request steps Python runs once the final `anthropic-beta` header is known, which - /// read that header or must not change which betas the request asks for. + /// read that header, must not change which betas the request asks for, or rewrite the + /// headers that go out with the body. fn finalize_request( &self, request: MessagesRequest, - _headers: &Headers, + headers: Headers, _context: &MessagesTransformContext, - ) -> MessagesRequest { - request + ) -> Result<(MessagesRequest, Headers), Error> { + Ok((request, headers)) } /// The JSON that goes on the wire, for a host whose body differs from the typed request: @@ -253,7 +267,6 @@ mod tests { MessagesShapeOptions { reasoning_auto_summary, presanitized, - ..MessagesShapeOptions::default() } ), Ok(request) diff --git a/litellm/rust/crates/llms/src/bedrock/audio_transcription/mod.rs b/litellm/rust/crates/llms/src/bedrock/audio_transcription/mod.rs index 1c976c1f73d..d3a14b98a26 100644 --- a/litellm/rust/crates/llms/src/bedrock/audio_transcription/mod.rs +++ b/litellm/rust/crates/llms/src/bedrock/audio_transcription/mod.rs @@ -1,8 +1,7 @@ use litellm_auth::AwsParams; use litellm_auth_aws::{ - AwsCredentialSource, bedrock_model_id_and_region, - constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, - resolve_bedrock_region, + AwsCredentialSource, CallerSigningHeaders, bedrock_model_id_and_region, + bedrock_runtime_endpoint, constants::BEDROCK_SERVICE, resolve_bedrock_region, }; use litellm_core_utils::core_helpers::json_type_name; use litellm_llms_types::{ @@ -161,7 +160,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { .map(str::trim) .filter(|value| !value.is_empty()) .map(str::to_string) - .unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", ®ion)); + .unwrap_or_else(|| bedrock_runtime_endpoint(®ion)); Ok(format!( "{}/model/{model_id}/converse", endpoint.trim_end_matches('/') @@ -187,6 +186,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { region: resolve_bedrock_region(model_region.as_deref(), ¶ms, env_lookup), service: BEDROCK_SERVICE, credentials: Box::new(AwsCredentialSource::from_params(¶ms, env_lookup)), + caller_headers: CallerSigningHeaders::Dropped, }, }) } diff --git a/litellm/rust/crates/llms/src/bedrock/chat/AGENTS.md b/litellm/rust/crates/llms/src/bedrock/chat/AGENTS.md index 8627e3f73ef..605c2b33698 100644 --- a/litellm/rust/crates/llms/src/bedrock/chat/AGENTS.md +++ b/litellm/rust/crates/llms/src/bedrock/chat/AGENTS.md @@ -2,4 +2,5 @@ - Chat Completions over Converse. Converse wire docs live in `llms-types/src/providers/bedrock/AGENTS.md` - `topK` is not forwarded, because Python's placement depends on the model catalog this crate cannot see -- `invoke_handler.rs` decodes InvokeModel event streams for Claude, reusing `anthropic/chat` response handling +- `invoke_handler.rs` ports Python's `AWSEventStreamDecoder`: it decodes InvokeModel event streams the way botocore's `EventStreamBuffer` does +- `invoke_transformations/base_invoke_transformation.rs` ports the InvokeModel helpers of Python's `base_invoke_transformation.py` diff --git a/litellm/rust/crates/llms/src/bedrock/chat/converse_transformation.rs b/litellm/rust/crates/llms/src/bedrock/chat/converse_transformation.rs index e02e7b9a155..cdc3581a4ce 100644 --- a/litellm/rust/crates/llms/src/bedrock/chat/converse_transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -3,8 +3,9 @@ use litellm_auth::CredentialPlacement; use litellm_secret_value::SecretValue; use litellm_auth_aws::{ - AwsCredentialSource, bedrock_model_id_and_region, - constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, + AwsCredentialSource, CallerSigningHeaders, bedrock_model_id_and_region, + bedrock_runtime_endpoint, + constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_SERVICE}, resolve_bedrock_region, }; use litellm_core_utils::decline::Decline; @@ -128,7 +129,7 @@ impl BaseConfig for AmazonConverseConfig { .map(str::trim) .filter(|value| !value.is_empty()) .map(str::to_string) - .unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", ®ion)); + .unwrap_or_else(|| bedrock_runtime_endpoint(®ion)); let endpoint = endpoint.trim_end_matches('/'); // A host that already built the full Converse URL (LiteLLM's Python // path encodes the model id itself) passes it through untouched, the @@ -253,6 +254,7 @@ impl BaseConfig for AmazonConverseConfig { region: resolve_bedrock_region(model_region.as_deref(), ¶ms, env_lookup), service: BEDROCK_SERVICE, credentials: Box::new(AwsCredentialSource::from_params(¶ms, env_lookup)), + caller_headers: CallerSigningHeaders::Botocore, }, }) } @@ -582,16 +584,25 @@ mod tests { ); } - #[test] - fn builds_the_converse_url_from_the_region_in_the_model_id() { + #[rstest] + #[case::commercial( + "us-east-1/anthropic.claude-v2", + "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse" + )] + #[case::china_partition( + "cn-north-1/anthropic.claude-v2", + "https://bedrock-runtime.cn-north-1.amazonaws.com.cn/model/anthropic.claude-v2/converse" + )] + fn builds_the_converse_url_from_the_region_in_the_model_id( + #[case] model: &str, + #[case] expected: &str, + ) { let config = &BEDROCK_CHAT_COMPLETIONS_CONFIG; assert_eq!( config - .get_complete_url(None, "us-east-1/anthropic.claude-v2", &Map::new(), &|_| { - None - }) + .get_complete_url(None, model, &Map::new(), &|_| None) .expect("url builds"), - "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse" + expected ); } diff --git a/litellm/rust/crates/llms/src/bedrock/chat/invoke_handler.rs b/litellm/rust/crates/llms/src/bedrock/chat/invoke_handler.rs index 15a6ff38822..626c6faceaf 100644 --- a/litellm/rust/crates/llms/src/bedrock/chat/invoke_handler.rs +++ b/litellm/rust/crates/llms/src/bedrock/chat/invoke_handler.rs @@ -1,132 +1,685 @@ +use std::sync::{Arc, Mutex, PoisonError}; + use base64::Engine; -use bytes::Buf; -use futures_util::{Stream, StreamExt}; +use bytes::BytesMut; +use futures_util::{StreamExt, stream}; use litellm_framer::{ EventStreamError, - aws_event_stream::{AwsEventStreamCodec, Message}, + aws_event_stream::{AwsEventStreamCodec, HeaderValue, Message}, frames, }; -use litellm_llms_types::formats::messages::streaming::MessagesStreamEvent; -use serde::Deserialize; +use litellm_python_compat::{Value as PythonValue, json as python_json, repr}; use serde_json::Value; +use tokio_util::codec::Decoder; -use crate::{ - Error, - base_llm::{ - base_model_iterator::tunnelled_http, - messages::streaming::{ByteStream, EventStream}, - }, -}; +use crate::{Error, ErrorDetail, base_llm::base_model_iterator::tunnelled_http}; -#[derive(Deserialize)] -struct InvokeChunkPayload { - bytes: String, +const STREAM_HEAD_BYTES: usize = 200; +const PRELUDE_BYTES: usize = 12; +const MAX_HEADERS_BYTES: u32 = 128 * 1024; +const MAX_PAYLOAD_BYTES: i64 = 24 * 1024 * 1024; + +const PROVIDER_REQUEST_ID_HEADERS: [&str; 7] = [ + "x-amzn-requestid", + "x-request-id", + "request-id", + "x-ms-request-id", + "apim-request-id", + "x-goog-request-id", + "cf-ray", +]; + +const STREAM_EVENT_STATUSES: [(&str, Option); 13] = [ + ("chunk", None), + ("messageStart", None), + ("contentBlockStart", None), + ("contentBlockDelta", None), + ("contentBlockStop", None), + ("messageStop", None), + ("metadata", None), + ("internalServerException", Some(500)), + ("modelStreamErrorException", Some(424)), + ("validationException", Some(400)), + ("throttlingException", Some(429)), + ("modelTimeoutException", Some(408)), + ("serviceUnavailableException", Some(503)), +]; + +const RESPONSE_STREAM_EXCEPTION_STATUSES: [(&str, u16); 6] = [ + ("internalServerException", 500), + ("modelStreamErrorException", 424), + ("validationException", 400), + ("throttlingException", 429), + ("modelTimeoutException", 408), + ("serviceUnavailableException", 503), +]; + +/// An exception Python's Bedrock stream decoder raises as it is, unmapped: botocore's +/// `EventStreamBuffer` parser errors, `json.loads` failing on an event payload, and the +/// chunk parser's `pop` on a payload that is not a JSON object. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum BedrockStreamRaise { + #[error("Checksum mismatch: expected 0x{expected:08x}, calculated 0x{calculated:08x}")] + ChecksumMismatch { expected: u32, calculated: u32 }, + #[error("Header length of {0} exceeded the maximum of {MAX_HEADERS_BYTES}")] + InvalidHeadersLength(u32), + #[error("Payload length of {0} exceeded the maximum of {MAX_PAYLOAD_BYTES}")] + InvalidPayloadLength(i64), + #[error("Duplicate header present: \"{0}\"")] + DuplicateHeader(String), + #[error("Bedrock event payload is not JSON: {0}")] + InvalidJson(String), + #[error("Bedrock chunk is not a JSON object: {0}")] + NotAnObject(String), } -pub fn decode_invoke_chunk(message: Message) -> Result { - let payload: InvokeChunkPayload = - serde_json::from_slice(message.payload()).map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid("Bedrock event payload", error)) - })?; - let chunk = base64::engine::general_purpose::STANDARD - .decode(payload.bytes) - .map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid( - "Bedrock event payload base64", - error, - )) - })?; - serde_json::from_slice(&chunk).map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid("Anthropic stream event", error)) +pub(crate) fn raised(raise: BedrockStreamRaise) -> Error { + Error::InvalidResponse(ErrorDetail::failed("Bedrock event stream", raise)) +} + +/// Port of Python's `_EventStreamTally`. +#[derive(Default)] +struct EventStreamTally { + bytes_received: usize, + bytes_decoded: usize, + events: usize, + recognized_events: usize, + unrecognized_event_types: std::collections::BTreeSet, + unrecognized_head: Vec, + head: Vec, +} + +type SharedEventStreamTally = Arc>; + +fn with_tally( + tally: &SharedEventStreamTally, + update: impl FnOnce(&mut EventStreamTally) -> T, +) -> T { + update(&mut tally.lock().unwrap_or_else(PoisonError::into_inner)) +} + +#[derive(Debug, thiserror::Error)] +enum FrameError { + #[error(transparent)] + Raise(#[from] BedrockStreamRaise), + #[error(transparent)] + Framing(#[from] EventStreamError), +} + +fn be_u32(bytes: &[u8], at: usize) -> u32 { + u32::from_be_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]]) +} + +fn checked(data: &[u8], expected: u32, seed: u32) -> Result<(), BedrockStreamRaise> { + let mut hasher = crc32fast::Hasher::new_with_initial(seed); + hasher.update(data); + let calculated = hasher.finalize(); + if calculated == expected { + return Ok(()); + } + Err(BedrockStreamRaise::ChecksumMismatch { + expected, + calculated, }) } -pub fn invoke_chunk_stream(input: S) -> impl Stream> + Send -where - S: Stream> + Send, - B: Buf + Send, - E: std::error::Error + Send + Sync + 'static, -{ - frames(input, AwsEventStreamCodec).map(|message| decode_invoke_chunk(message.map_err(framing)?)) +/// Validates each frame in the order botocore's `EventStreamBuffer` does, so a body that is not +/// an event stream fails with the parser error Python raises. +struct BotocoreCodec { + tally: SharedEventStreamTally, } -fn framing(error: EventStreamError) -> Error { +impl BotocoreCodec { + fn prelude(src: &[u8]) -> Result { + let total = be_u32(src, 0); + let headers = be_u32(src, 4); + checked(&src[..PRELUDE_BYTES - 4], be_u32(src, 8), 0)?; + if headers > MAX_HEADERS_BYTES { + return Err(BedrockStreamRaise::InvalidHeadersLength(headers)); + } + let payload = i64::from(total) - i64::from(headers) - 16; + if payload > MAX_PAYLOAD_BYTES { + return Err(BedrockStreamRaise::InvalidPayloadLength(payload)); + } + Ok(total as usize) + } + + fn unique_headers(message: &Message) -> Result<(), BedrockStreamRaise> { + let mut seen = std::collections::HashSet::new(); + message + .headers() + .iter() + .map(|header| header.name().as_str()) + .find(|name| !seen.insert(*name)) + .map_or(Ok(()), |name| { + Err(BedrockStreamRaise::DuplicateHeader(name.to_string())) + }) + } +} + +impl Decoder for BotocoreCodec { + type Item = Message; + type Error = FrameError; + + fn decode(&mut self, src: &mut BytesMut) -> Result, FrameError> { + if src.len() < PRELUDE_BYTES { + return Ok(None); + } + let length = Self::prelude(src)?; + if src.len() < length { + return Ok(None); + } + if length >= 16 { + checked( + &src[PRELUDE_BYTES - 4..length - 4], + be_u32(src, length - 4), + be_u32(src, 8), + )?; + } + let Some(message) = AwsEventStreamCodec.decode(src)? else { + return Ok(None); + }; + Self::unique_headers(&message)?; + with_tally(&self.tally, |tally| tally.bytes_decoded += length); + Ok(Some(message)) + } + + fn decode_eof(&mut self, src: &mut BytesMut) -> Result, FrameError> { + self.decode(src) + } +} + +impl From for FrameError { + fn from(error: std::io::Error) -> Self { + Self::Framing(EventStreamError::Body(error)) + } +} + +fn header_text<'a>(message: &'a Message, name: &str) -> Option<&'a str> { + message + .headers() + .iter() + .find(|header| header.name().as_str() == name) + .and_then(|header| match header.value() { + HeaderValue::String(value) => Some(value.as_str()), + _ => None, + }) +} + +fn event_status(event_type: &str) -> Option> { + STREAM_EVENT_STATUSES + .iter() + .find(|(known, _)| *known == event_type) + .map(|(_, status)| *status) +} + +fn python_bytes_repr(bytes: &[u8]) -> String { + repr::repr(&PythonValue::Bytes(bytes.to_vec())) +} + +fn python_optional_str_repr(value: Option<&str>) -> String { + repr::repr(&value.map_or(PythonValue::None, |text| PythonValue::Str(text.to_string()))) +} + +fn response_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) +} + +fn request_id_headers(headers: &[(String, String)]) -> Vec<(String, String)> { + headers + .iter() + .filter(|(name, value)| { + PROVIDER_REQUEST_ID_HEADERS.contains(&name.to_lowercase().as_str()) && value.is_ascii() + }) + .cloned() + .collect() +} + +fn bedrock_error(status: u16, message: String, headers: &[(String, String)]) -> Error { + Error::Http(litellm_http::Error::upstream_with_headers( + status, + message, + request_id_headers(headers), + )) +} + +/// Port of Python's `_EventStreamTally.add_event`. +fn add_event(tally: &SharedEventStreamTally, message: &Message) { + let message_type = header_text(message, ":message-type"); + let event_type = header_text(message, ":event-type"); + with_tally(tally, |tally| { + tally.events += 1; + if message_type != Some("event") || event_type.and_then(event_status).is_some() { + tally.recognized_events += 1; + return; + } + tally + .unrecognized_event_types + .insert(event_type.unwrap_or("").to_string()); + if tally.unrecognized_head.is_empty() { + let payload = message.payload(); + tally.unrecognized_head = payload[..payload.len().min(STREAM_HEAD_BYTES)].to_vec(); + } + }); +} + +/// Port of Python's `_EventStreamTally.undecoded_stream_error`. +fn undecoded_stream_error(tally: &EventStreamTally, headers: &[(String, String)]) -> Option { + let undecoded = tally.bytes_received.saturating_sub(tally.bytes_decoded); + if tally.recognized_events > 0 && undecoded == 0 { + return None; + } + let detail = format!( + "content-type={}, x-amzn-requestid={}, {} bytes received", + python_optional_str_repr(response_header(headers, "content-type")), + python_optional_str_repr(response_header(headers, "x-amzn-requestid")), + tally.bytes_received + ); + let message = if tally.events == 0 { + format!( + "Bedrock answered the stream with HTTP 200 but its body decoded to no events ({detail}, first bytes={})", + python_bytes_repr(&tally.head) + ) + } else if tally.recognized_events == 0 { + let event_types = PythonValue::List( + tally + .unrecognized_event_types + .iter() + .cloned() + .map(PythonValue::Str) + .collect(), + ); + format!( + "Bedrock answered the stream with HTTP 200 but none of its {} events carried a known event type (event types={}, {detail}, first payload={})", + tally.events, + repr::repr(&event_types), + python_bytes_repr(&tally.unrecognized_head) + ) + } else { + format!( + "Bedrock stream ended with {undecoded} undecoded bytes after {} events ({detail})", + tally.events + ) + }; + Some(bedrock_error(502, message, headers)) +} + +fn framing(error: FrameError) -> Error { + let error = match error { + FrameError::Raise(raise) => return raised(raise), + FrameError::Framing(error) => error, + }; match &error { EventStreamError::Body(body) if let Some(http) = tunnelled_http(body) => { Error::Http(http.clone()) } - EventStreamError::Body(_) - | EventStreamError::InvalidLength(_) - | EventStreamError::Truncated - | EventStreamError::Malformed(_) => { - Error::InvalidResponse(crate::ErrorDetail::failed("stream framing", error)) - } + _ => Error::InvalidResponse(ErrorDetail::failed("stream framing", error)), } } -pub fn decode_invoke_anthropic_chunk(chunk: Value) -> Result { - serde_json::from_value(chunk).map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid("Anthropic stream event", error)) - }) +fn invalid_payload(error: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::InvalidResponse(ErrorDetail::invalid("Bedrock event payload", error)) } -pub fn invoke_anthropic_event_stream(bytes: ByteStream) -> EventStream { - Box::pin(invoke_chunk_stream(bytes).map(|chunk| decode_invoke_anthropic_chunk(chunk?))) +/// Port of Python's `bedrock_event_chunk_text`. +fn bedrock_event_chunk_text(payload: &[u8]) -> Result>, Error> { + let parsed: Value = serde_json::from_slice(payload).map_err(invalid_payload)?; + let Some(chunk) = parsed.as_object().filter(|chunk| !chunk.is_empty()) else { + return Ok(None); + }; + let encoded = chunk + .get("bytes") + .and_then(Value::as_str) + .ok_or(Error::InvalidResponse(ErrorDetail::MissingField { + subject: "Bedrock chunk event", + field: "bytes", + }))?; + base64::engine::general_purpose::STANDARD + .decode(encoded) + .map(Some) + .map_err(invalid_payload) +} + +/// Port of Python's `AWSEventStreamDecoder._parse_message_from_response`, parsed with `json.loads`. +fn parse_message_from_response( + message: &Message, + headers: &[(String, String)], +) -> Result, Error> { + let body = String::from_utf8_lossy(message.payload()); + if header_text(message, ":message-type") != Some("event") { + let error = match header_text(message, ":exception-type") { + Some(exception_type) => { + let status = RESPONSE_STREAM_EXCEPTION_STATUSES + .iter() + .find(|(known, _)| *known == exception_type) + .map_or(400, |(_, status)| *status); + bedrock_error(status, format!("{exception_type} {body}"), headers) + } + None => bedrock_error(400, body.into_owned(), headers), + }; + return Err(error); + } + let event_type = header_text(message, ":event-type"); + if let Some((event_type, status)) = + event_type.and_then(|event_type| Some((event_type, event_status(event_type).flatten()?))) + { + return Err(bedrock_error( + status, + format!("{event_type} {body}"), + headers, + )); + } + let text = if event_type == Some("chunk") { + bedrock_event_chunk_text(message.payload())? + } else { + Some(message.payload().to_vec()).filter(|payload| !payload.is_empty()) + }; + text.filter(|text| !text.is_empty()) + .map(|text| { + let text = String::from_utf8_lossy(&text); + python_json::loads(&text) + .map_err(|_| raised(BedrockStreamRaise::InvalidJson(text.into_owned()))) + }) + .transpose() +} + +/// Port of Python's `AWSEventStreamDecoder.aiter_bytes`, yielding the decoded JSON of each event. +pub fn aiter_bytes( + bytes: crate::base_llm::messages::streaming::ByteStream, + headers: Vec<(String, String)>, +) -> impl futures_util::Stream> + Send { + let tally = SharedEventStreamTally::default(); + let counted = { + let tally = tally.clone(); + bytes.inspect(move |chunk| { + if let Ok(chunk) = chunk { + with_tally(&tally, |tally| { + tally.bytes_received += chunk.len(); + let room = STREAM_HEAD_BYTES.saturating_sub(tally.head.len()); + tally + .head + .extend_from_slice(&chunk[..chunk.len().min(room)]); + }); + } + }) + }; + let headers = Arc::new(headers); + let events = { + let tally = tally.clone(); + let headers = headers.clone(); + frames( + counted, + BotocoreCodec { + tally: tally.clone(), + }, + ) + .map(move |message| { + let message = message.map_err(framing)?; + add_event(&tally, &message); + parse_message_from_response(&message, &headers) + }) + }; + let end = stream::once(async move { + with_tally(&tally, |tally| undecoded_stream_error(tally, &headers)).map_or(Ok(None), Err) + }); + events + .chain(end) + .scan(false, |failed, item| { + if *failed { + return futures_util::future::ready(None); + } + *failed = item.is_err(); + futures_util::future::ready(Some(item)) + }) + .filter_map(|item| futures_util::future::ready(item.transpose())) } #[cfg(test)] mod tests { use aws_smithy_eventstream::frame::write_message_to; - use aws_smithy_types::event_stream::{Header, HeaderValue, Message}; + use aws_smithy_types::event_stream::{ + Header, HeaderValue as WireHeaderValue, Message as WireMessage, + }; use base64::engine::general_purpose::STANDARD; use bytes::Bytes; - use futures_util::TryStreamExt; - use litellm_llms_types::formats::messages::streaming::MessagesContentBlockDelta; + use futures_util::StreamExt; + use rstest::rstest; + use serde_json::json; use super::*; - use crate::base_llm::messages::streaming; - fn in_pieces(wire: &[u8]) -> ByteStream { - let pieces: Vec = wire.chunks(3).map(Bytes::copy_from_slice).collect(); - futures_util::stream::iter(pieces.into_iter().map(Ok)).boxed() - } - - const TEXT_DELTA: &str = - r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}"#; - - fn aws_wire(chunk: &str) -> Vec { - let payload = serde_json::json!({"bytes": STANDARD.encode(chunk)}); - let message = Message::new(Bytes::from(serde_json::to_vec(&payload).unwrap())).add_header( - Header::new(":event-type", HeaderValue::String("chunk".into())), + fn frame(headers: &[(&str, &str)], payload: &[u8]) -> Vec { + let message = headers.iter().fold( + WireMessage::new(Bytes::copy_from_slice(payload)), + |message, (name, value)| { + message.add_header(Header::new( + name.to_string(), + WireHeaderValue::String(value.to_string().into()), + )) + }, ); let mut wire = Vec::new(); write_message_to(&message, &mut wire).unwrap(); wire } + fn chunk(event: &Value) -> Vec { + let payload = json!({"bytes": STANDARD.encode(event.to_string())}); + frame( + &[(":event-type", "chunk"), (":message-type", "event")], + payload.to_string().as_bytes(), + ) + } + + fn response_headers() -> Vec<(String, String)> { + vec![ + ( + "content-type".into(), + "application/vnd.amazon.eventstream".into(), + ), + ("x-amzn-requestid".into(), "rid-1".into()), + ("x-other".into(), "dropped".into()), + ] + } + + async fn decoded(wire: Vec) -> (Vec, Option) { + let bytes = futures_util::stream::iter( + wire.chunks(5) + .map(|piece| Ok(Bytes::copy_from_slice(piece))) + .collect::>(), + ) + .boxed(); + let items: Vec> = + aiter_bytes(bytes, response_headers()).collect().await; + let (events, errors): (Vec<_>, Vec<_>) = items.into_iter().partition(Result::is_ok); + ( + events.into_iter().filter_map(Result::ok).collect(), + errors.into_iter().find_map(Result::err), + ) + } + + fn upstream(error: Option) -> (u16, String, Vec<(String, String)>) { + let Some(Error::Http(error)) = error else { + panic!("expected an upstream error, got {error:?}"); + }; + let (status, body) = error.upstream_response().unwrap(); + (status, body.to_string(), error.upstream_headers().to_vec()) + } + #[tokio::test] - async fn aws_and_sse_framing_decode_to_the_same_anthropic_events() { - let aws = aws_wire(TEXT_DELTA); - let sse = format!("event: content_block_delta\ndata: {TEXT_DELTA}\n\n"); - - let from_aws = invoke_anthropic_event_stream(in_pieces(&aws)) - .try_collect::>() - .await - .unwrap(); - let from_sse = streaming::event_stream(in_pieces(sse.as_bytes())) - .try_collect::>() - .await - .unwrap(); - + async fn chunk_events_decode_to_anthropic_events() { + let event = json!({"type": "message_stop"}); + let (events, error) = decoded(chunk(&event)).await; assert_eq!( - from_aws, - vec![MessagesStreamEvent::ContentBlockDelta { - index: 0, - delta: MessagesContentBlockDelta::TextDelta { - text: "hello".into(), - }, - }] + (events, error.is_none()), + (vec![python_json::loads(&event.to_string()).unwrap()], true) ); - assert_eq!(from_aws, from_sse); + } + + #[rstest] + #[case::throttling("throttlingException", 429)] + #[case::model_stream_error("modelStreamErrorException", 424)] + #[case::validation("validationException", 400)] + #[case::unmodeled("somethingNewException", 400)] + #[tokio::test] + async fn exception_frames_raise_their_modeled_status( + #[case] exception_type: &str, + #[case] status: u16, + ) { + let wire = [ + chunk(&json!({"type": "ping"})), + frame( + &[ + (":message-type", "exception"), + (":exception-type", exception_type), + ], + br#"{"message":"slow down"}"#, + ), + ] + .concat(); + let (events, error) = decoded(wire).await; + assert_eq!(events.len(), 1); + assert_eq!( + upstream(error), + ( + status, + format!(r#"{exception_type} {{"message":"slow down"}}"#), + vec![("x-amzn-requestid".to_string(), "rid-1".to_string())] + ) + ); + } + + #[tokio::test] + async fn an_error_event_type_raises_its_status() { + let wire = frame( + &[ + (":message-type", "event"), + (":event-type", "internalServerException"), + ], + b"boom", + ); + assert_eq!(upstream(decoded(wire).await.1).0, 500); + } + + #[tokio::test] + async fn an_empty_body_is_a_502() { + let (_, body, _) = upstream(decoded(Vec::new()).await.1); + assert_eq!( + body, + "Bedrock answered the stream with HTTP 200 but its body decoded to no events (content-type='application/vnd.amazon.eventstream', x-amzn-requestid='rid-1', 0 bytes received, first bytes=b'')" + ); + } + + #[tokio::test] + async fn unknown_event_types_are_a_502() { + let wire = frame( + &[(":message-type", "event"), (":event-type", "mystery")], + b"{}", + ); + let (status, body, _) = upstream(decoded(wire.clone()).await.1); + assert_eq!( + (status, body), + ( + 502, + format!( + "Bedrock answered the stream with HTTP 200 but none of its 1 events carried a known event type (event types=['mystery'], content-type='application/vnd.amazon.eventstream', x-amzn-requestid='rid-1', {} bytes received, first payload=b'{{}}')", + wire.len() + ) + ) + ); + } + + #[tokio::test] + async fn trailing_undecoded_bytes_are_a_502() { + let complete = chunk(&json!({"type": "ping"})); + let truncated = chunk(&json!({"type": "message_stop"})); + let wire = [complete, truncated[..20].to_vec()].concat(); + let (events, error) = decoded(wire).await; + let (status, body, _) = upstream(error); + assert_eq!(events.len(), 1); + assert_eq!(status, 502); + assert!( + body.starts_with("Bedrock stream ended with 20 undecoded bytes after 1 events ("), + "{body}" + ); + } + + #[tokio::test] + async fn decoding_stops_at_the_first_error() { + let wire = [ + frame( + &[ + (":message-type", "exception"), + (":exception-type", "throttlingException"), + ], + b"x", + ), + chunk(&json!({"type": "ping"})), + ] + .concat(); + let bytes = futures_util::stream::iter([Ok(Bytes::from(wire))]).boxed(); + let items: Vec> = aiter_bytes(bytes, Vec::new()).collect().await; + assert_eq!(items.len(), 1); + assert!(items[0].is_err()); + } + + fn raise_of(error: Option) -> BedrockStreamRaise { + litellm_error::find::(&error.expect("an error")) + .cloned() + .expect("a botocore parser error") + } + + #[tokio::test] + async fn a_json_body_fails_the_prelude_checksum_as_botocore_does() { + let body = br#"{"id":"msg_1","type":"message","content":[]}"#; + assert_eq!( + raise_of(decoded(body.to_vec()).await.1), + BedrockStreamRaise::ChecksumMismatch { + expected: be_u32(body, 8), + calculated: crc32fast::hash(&body[..8]), + } + ); + } + + fn with_prelude(total: u32, headers: u32) -> Vec { + let mut prelude = [total.to_be_bytes(), headers.to_be_bytes()].concat(); + let crc = crc32fast::hash(&prelude); + prelude.extend_from_slice(&crc.to_be_bytes()); + prelude + } + + #[rstest] + #[case::headers_too_long( + with_prelude(200_000, 140_000), + BedrockStreamRaise::InvalidHeadersLength(140_000) + )] + #[case::payload_too_long( + with_prelude(30_000_000, 0), + BedrockStreamRaise::InvalidPayloadLength(30_000_000 - 16) + )] + #[case::duplicate_header( + frame(&[(":message-type", "event"), (":message-type", "event")], b"{}"), + BedrockStreamRaise::DuplicateHeader(":message-type".into()) + )] + #[tokio::test] + async fn a_malformed_frame_raises_the_botocore_parser_error( + #[case] wire: Vec, + #[case] expected: BedrockStreamRaise, + ) { + assert_eq!(raise_of(decoded(wire).await.1), expected); + } + + #[tokio::test] + async fn a_corrupt_message_checksum_raises_a_checksum_mismatch() { + let mut wire = chunk(&json!({"type": "ping"})); + let last = wire.len() - 1; + wire[last] ^= 0xff; + assert!(matches!( + raise_of(decoded(wire).await.1), + BedrockStreamRaise::ChecksumMismatch { .. } + )); } } diff --git a/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/base_invoke_transformation.rs b/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/base_invoke_transformation.rs new file mode 100644 index 00000000000..8a221943b1f --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/base_invoke_transformation.rs @@ -0,0 +1,186 @@ +use litellm_python_compat::{json as python_json, repr}; +use serde_json::{Map, Value}; + +use crate::Error; + +pub const GUARDRAIL_CONFIG_PARAM: &str = "guardrailConfig"; + +pub(crate) const GUARDRAIL_IDENTIFIER_HEADER: &str = "X-Amzn-Bedrock-GuardrailIdentifier"; +pub(crate) const GUARDRAIL_VERSION_HEADER: &str = "X-Amzn-Bedrock-GuardrailVersion"; +const GUARDRAIL_TRACE_HEADER: &str = "X-Amzn-Bedrock-Trace"; +const GUARDRAIL_CONFIG_EXPECTED_FORMAT: &str = "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"; +const GUARDRAIL_TRACE_VALUES: [&str; 3] = ["enabled", "disabled", "enabled_full"]; + +fn python_repr(value: &Value) -> String { + repr::to_str(&python_json::from_json(value.clone())) +} + +fn python_type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) => "dict", + } +} + +fn validation_error(location: &str, message: &str, kind: &str, input: &Value) -> String { + let location = if location.is_empty() { + String::new() + } else { + format!("\n{location}") + }; + format!( + "1 validation error for GuardrailConfigBlock{location}\n {message} [type={kind}, input_value={}, input_type={}]", + python_repr(input), + python_type_name(input) + ) +} + +fn guardrail_violation(config: &Map) -> Option { + ["guardrailIdentifier", "guardrailVersion"] + .into_iter() + .find_map(|field| { + config + .get(field) + .filter(|value| !value.is_string()) + .map(|value| { + validation_error( + field, + "Input should be a valid string", + "string_type", + value, + ) + }) + }) + .or_else(|| { + config + .get("trace") + .filter(|value| { + !value + .as_str() + .is_some_and(|trace| GUARDRAIL_TRACE_VALUES.contains(&trace)) + }) + .map(|value| { + validation_error( + "trace", + "Input should be 'enabled', 'disabled' or 'enabled_full'", + "literal_error", + value, + ) + }) + }) +} + +fn guardrail_error(message: String) -> Error { + Error::Http(litellm_http::Error::upstream(400, message)) +} + +/// Port of Python's `bedrock_invoke_guardrail_headers`. +pub(crate) fn bedrock_invoke_guardrail_headers( + raw: &Value, +) -> Result, Error> { + let Value::Object(config) = raw else { + return Err(guardrail_error(format!( + "Invalid guardrailConfig={}. Expected format: {GUARDRAIL_CONFIG_EXPECTED_FORMAT}. Error: {}", + python_repr(raw), + validation_error("", "Input should be a valid dictionary", "dict_type", raw) + ))); + }; + if let Some(violation) = guardrail_violation(config) { + return Err(guardrail_error(format!( + "Invalid guardrailConfig={}. Expected format: {GUARDRAIL_CONFIG_EXPECTED_FORMAT}. Error: {violation}", + python_repr(raw) + ))); + } + let Some(identifier) = config.get("guardrailIdentifier").and_then(Value::as_str) else { + return Err(guardrail_error(format!( + "guardrailConfig={} is missing 'guardrailIdentifier'. Expected format: {GUARDRAIL_CONFIG_EXPECTED_FORMAT}", + python_repr(raw) + ))); + }; + Ok([ + (GUARDRAIL_IDENTIFIER_HEADER, Some(identifier.to_string())), + ( + GUARDRAIL_VERSION_HEADER, + config + .get("guardrailVersion") + .and_then(Value::as_str) + .map(str::to_string), + ), + ( + GUARDRAIL_TRACE_HEADER, + config + .get("trace") + .and_then(Value::as_str) + .map(str::to_uppercase), + ), + ] + .into_iter() + .filter_map(|(name, value)| Some((name.to_string(), value?))) + .collect()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn rejection(error: Error) -> (u16, String) { + let Error::Http(error) = error else { + panic!("expected a 400, got {error:?}"); + }; + let (status, body) = error.upstream_response().unwrap(); + (status, body.to_string()) + } + + #[rstest] + #[case::enabled("enabled", "ENABLED")] + #[case::disabled("disabled", "DISABLED")] + #[case::enabled_full("enabled_full", "ENABLED_FULL")] + fn guardrail_config_becomes_invoke_headers(#[case] trace: &str, #[case] expected: &str) { + assert_eq!( + bedrock_invoke_guardrail_headers( + &json!({"guardrailIdentifier": "gid", "guardrailVersion": "2", "trace": trace}) + ) + .unwrap(), + vec![ + (GUARDRAIL_IDENTIFIER_HEADER.to_string(), "gid".to_string()), + (GUARDRAIL_VERSION_HEADER.to_string(), "2".to_string()), + (GUARDRAIL_TRACE_HEADER.to_string(), expected.to_string()), + ] + ); + } + + #[test] + fn a_guardrail_config_without_an_identifier_is_a_400() { + assert_eq!( + rejection(bedrock_invoke_guardrail_headers(&json!({"guardrailVersion": "2"})).unwrap_err()), + ( + 400, + "guardrailConfig={'guardrailVersion': '2'} is missing 'guardrailIdentifier'. Expected format: {'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}".to_string() + ) + ); + } + + #[rstest] + #[case::bad_trace(json!({"guardrailIdentifier": "gid", "trace": "loud"}))] + #[case::not_a_dict(json!("gid"))] + #[case::non_string_version(json!({"guardrailIdentifier": "gid", "guardrailVersion": 2}))] + fn an_invalid_guardrail_config_is_a_400(#[case] raw: Value) { + let (status, body) = rejection(bedrock_invoke_guardrail_headers(&raw).unwrap_err()); + assert_eq!(status, 400); + assert!( + body.starts_with(&format!( + "Invalid guardrailConfig={}. Expected format: ", + python_repr(&raw) + )), + "{body}" + ); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/mod.rs b/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/mod.rs new file mode 100644 index 00000000000..418ceff691e --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/chat/invoke_transformations/mod.rs @@ -0,0 +1 @@ +pub mod base_invoke_transformation; diff --git a/litellm/rust/crates/llms/src/bedrock/chat/mod.rs b/litellm/rust/crates/llms/src/bedrock/chat/mod.rs index a46514aa697..0dd26c8cfc5 100644 --- a/litellm/rust/crates/llms/src/bedrock/chat/mod.rs +++ b/litellm/rust/crates/llms/src/bedrock/chat/mod.rs @@ -1,2 +1,3 @@ pub mod converse_transformation; pub mod invoke_handler; +pub mod invoke_transformations; diff --git a/litellm/rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs b/litellm/rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs index 52c93d097f5..e4deb3adb86 100644 --- a/litellm/rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs @@ -5,7 +5,7 @@ use std::sync::LazyLock; use litellm_auth::{AwsParams, CredentialPlacement, ParamKind, ParamSpec}; use litellm_secret_value::SecretValue; -use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region}; +use litellm_auth_aws::{AwsCredentialSource, CallerSigningHeaders, resolve_aws_region}; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{header_value, with_header, without_headers}; use litellm_llms_types::{ @@ -24,7 +24,7 @@ use crate::{ base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, messages::{ - context::{MessagesShapeOptions, MessagesTransformContext, UnparsedMessages}, + context::{MessagesShapeOptions, MessagesTransformContext}, transformation::BaseMessagesConfig, }, }, @@ -123,6 +123,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { region: platform_region(base.as_deref(), params, env)?, service: SERVICE, credentials: Box::new(AwsCredentialSource::from_params(¶ms.aws, env)), + caller_headers: CallerSigningHeaders::Botocore, }, }) } @@ -196,12 +197,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()), None => headers, }; - update_headers_with_safeguards_beta( - headers, - request, - BetaProvider::Anthropic, - UnparsedMessages::Skipped, - ) + update_headers_with_safeguards_beta(headers, request, BetaProvider::Anthropic) } fn wire_body(&self, body: Value) -> Value { diff --git a/litellm/rust/crates/llms/src/bedrock/common_utils.rs b/litellm/rust/crates/llms/src/bedrock/common_utils.rs index 05f6871529a..295641c5b20 100644 --- a/litellm/rust/crates/llms/src/bedrock/common_utils.rs +++ b/litellm/rust/crates/llms/src/bedrock/common_utils.rs @@ -1,7 +1,10 @@ use litellm_auth_aws::is_bedrock_region; +use serde_json::{Map, Value, json}; use crate::{Error, ErrorDetail}; +pub mod structured_output; + const ROUTE_PREFIXES: [&str; 3] = ["bedrock/converse/", "bedrock/", "converse/"]; const ROUTING_PREFIXES: [&str; 8] = [ "bedrock/", @@ -27,7 +30,7 @@ pub fn bedrock_base_model(model: &str) -> &str { if unrouted.starts_with("nova/") { return "amazon.nova-custom"; } - let named = without_throughput_suffix(model_name_from_arn(without_routing_prefixes(model))); + let named = without_throughput_suffix(model_name_from_arn(strip_bedrock_routing_prefix(model))); if let Some((region, rest)) = named.split_once('.') && CROSS_REGION_INFERENCE_PREFIXES.contains(®ion) { @@ -43,7 +46,8 @@ pub fn is_bedrock_application_inference_profile_arn(model: &str) -> bool { model.contains(":application-inference-profile/") } -pub fn without_routing_prefixes(model: &str) -> &str { +/// Port of Python's `strip_bedrock_routing_prefix`. +pub fn strip_bedrock_routing_prefix(model: &str) -> &str { ROUTING_PREFIXES.into_iter().fold(model, |model, prefix| { model.strip_prefix(prefix).unwrap_or(model) }) @@ -83,6 +87,181 @@ fn is_digits(value: &str) -> bool { !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) } +const UNNAMED_TOOL_PREFIX: &str = "litellm_unnamed_tool_"; + +pub(crate) fn is_int(value: &Value) -> bool { + value.is_i64() || value.is_u64() +} + +fn effort_rank(effort: &str) -> Option { + match effort { + "low" => Some(0), + "medium" => Some(1), + "high" => Some(2), + "max" => Some(3), + "xhigh" => Some(4), + _ => None, + } +} + +pub(crate) fn clamp_effort(effort: &str, ceiling: Option<&str>) -> Option { + let ceiling = ceiling?; + (effort_rank(effort)? > effort_rank(ceiling)?).then(|| ceiling.to_string()) +} + +/// Port of Python's `normalize_bedrock_opus_output_config_effort`. +pub(crate) fn normalize_bedrock_opus_output_config_effort( + output_config: Option<&mut Value>, + ceiling: Option<&str>, +) { + let Some(Value::Object(output_config)) = output_config else { + return; + }; + if let Some(clamped) = output_config + .get("effort") + .and_then(Value::as_str) + .and_then(|effort| clamp_effort(effort, ceiling)) + { + output_config.insert("effort".into(), Value::String(clamped)); + } +} + +/// Port of Python's `strip_unsupported_bedrock_invoke_output_config_keys`. +pub(crate) fn strip_unsupported_bedrock_invoke_output_config_keys( + model: &str, + request_body: &mut Map, + accepts_output_config: bool, +) { + let Some(Value::Object(output_config)) = request_body.get("output_config") else { + return; + }; + if output_config.keys().all(|key| key == "format") || accepts_output_config { + return; + } + litellm_tracing::warn!( + "Bedrock Invoke: stripping unsupported `output_config` keys for model={model}: neither `supports_output_config` nor any `supports_*_reasoning_effort` flag is set in model_prices_and_context_window.json. Add the capability flag to the model JSON entry if this model accepts `output_config`." + ); + match output_config + .get("format") + .filter(|format| !format.is_null()) + .cloned() + { + Some(format) => { + request_body.insert("output_config".into(), json!({"format": format})); + } + None => { + request_body.shift_remove("output_config"); + } + } +} + +fn tools_mut(request_body: &mut Map) -> Option<&mut Vec> { + match request_body.get_mut("tools") { + Some(Value::Array(tools)) => Some(tools), + _ => None, + } +} + +/// Port of Python's `_normalize_custom_field_on_tool`. +fn normalize_custom_field_on_tool(tool: &mut Value) { + let Value::Object(tool) = tool else { + return; + }; + if let Some(Value::Object(custom)) = tool.shift_remove("custom") + && !tool.contains_key("defer_loading") + && let Some(deferred @ Value::Bool(_)) = custom.get("defer_loading") + { + tool.insert("defer_loading".into(), deferred.clone()); + } +} + +/// Port of Python's `normalize_custom_field_on_tools`. +pub(crate) fn normalize_custom_field_on_tools(request_body: &mut Map) { + if let Some(tools) = tools_mut(request_body) { + tools.iter_mut().for_each(normalize_custom_field_on_tool); + } +} + +/// Port of Python's `normalize_json_schema_custom_types_to_object`. +fn normalize_json_schema_custom_types_to_object(schema: &mut Value) { + let Value::Object(node) = schema else { + return; + }; + if node.get("type").and_then(Value::as_str) == Some("custom") { + node.insert("type".into(), Value::String("object".into())); + } + for key in ["items", "additionalProperties"] { + if let Some(child) = node.get_mut(key) { + normalize_json_schema_custom_types_to_object(child); + } + } + if let Some(Value::Object(properties)) = node.get_mut("properties") { + properties + .values_mut() + .for_each(normalize_json_schema_custom_types_to_object); + } + for combiner in ["allOf", "anyOf", "oneOf"] { + if let Some(Value::Array(branches)) = node.get_mut(combiner) { + branches + .iter_mut() + .for_each(normalize_json_schema_custom_types_to_object); + } + } +} + +/// Port of Python's `normalize_tool_input_schema_types_for_bedrock_invoke`. +pub(crate) fn normalize_tool_input_schema_types_for_bedrock_invoke( + request_body: &mut Map, +) { + let Some(tools) = tools_mut(request_body) else { + return; + }; + tools + .iter_mut() + .filter_map(|tool| tool.get_mut("input_schema")) + .filter(|input_schema| input_schema.is_object()) + .for_each(normalize_json_schema_custom_types_to_object); +} + +/// Port of Python's `ensure_bedrock_anthropic_messages_tool_names`. +pub(crate) fn ensure_bedrock_anthropic_messages_tool_names(request_body: &mut Map) { + let Some(tools) = tools_mut(request_body) else { + return; + }; + for (index, tool) in tools.iter_mut().enumerate() { + let Value::Object(tool) = tool else { + continue; + }; + let unnamed = match tool.get("name") { + None | Some(Value::Null) => true, + Some(Value::String(name)) => name.trim().is_empty(), + Some(_) => false, + }; + if unnamed { + tool.insert( + "name".into(), + Value::String(format!("{UNNAMED_TOOL_PREFIX}{index}")), + ); + } + } +} + +/// Port of Python's `tools_without_eager_input_streaming`. +pub(crate) fn tools_without_eager_input_streaming(request_body: &mut Map) { + let Some(tools) = tools_mut(request_body) else { + return; + }; + if !tools.iter().all(Value::is_object) { + return; + } + tools + .iter_mut() + .filter_map(Value::as_object_mut) + .for_each(|tool| { + tool.shift_remove("eager_input_streaming"); + }); +} + pub(crate) fn validated_region(region: String) -> Result { if !region.is_empty() && region @@ -101,8 +280,42 @@ pub(crate) fn validated_region(region: String) -> Result { #[cfg(test)] mod tests { use rstest::rstest; + use serde_json::{Value, json}; - use super::bedrock_base_model; + use super::*; + use crate::{ + anthropic::common_utils::supports_effort_param, + base_llm::messages::context::{ + BedrockCapabilities, MessagesModelCapabilities, SupportedEffortTiers, + }, + }; + + fn adaptive() -> MessagesModelCapabilities { + MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { + low: true, + medium: true, + high: true, + max: true, + xhigh: true, + ..SupportedEffortTiers::default() + }, + ..MessagesModelCapabilities::default() + } + } + + fn with_bedrock( + capabilities: MessagesModelCapabilities, + bedrock: BedrockCapabilities, + ) -> MessagesModelCapabilities { + MessagesModelCapabilities { + bedrock, + ..capabilities + } + } #[rstest] #[case::cross_region("us.anthropic.claude-x-v1:0", "anthropic.claude-x-v1:0")] @@ -132,4 +345,66 @@ mod tests { fn base_model_matches_the_python_resolution(#[case] model: &str, #[case] expected: &str) { assert_eq!(bedrock_base_model(model), expected); } + + #[rstest] + #[case::hoists_bool_defer_loading( + json!([{"name": "Read", "input_schema": {"type": "object"}, "custom": {"defer_loading": true}}, {"name": "Write", "input_schema": {"type": "object"}}]), + json!([{"name": "Read", "input_schema": {"type": "object"}, "defer_loading": true}, {"name": "Write", "input_schema": {"type": "object"}}]), + )] + #[case::keeps_an_existing_defer_loading( + json!([{"name": "Read", "defer_loading": false, "custom": {"defer_loading": true}}]), + json!([{"name": "Read", "defer_loading": false}]), + )] + #[case::drops_a_non_bool_defer_loading( + json!([{"name": "Read", "custom": {"defer_loading": "yes", "other": 1}}]), + json!([{"name": "Read"}]), + )] + #[case::custom_schema_types_become_object( + json!([{"name": "t", "input_schema": {"type": "custom", "properties": {"a": {"type": "custom", "items": {"type": "custom"}}}, "anyOf": [{"type": "custom"}]}}]), + json!([{"name": "t", "input_schema": {"type": "object", "properties": {"a": {"type": "object", "items": {"type": "object"}}}, "anyOf": [{"type": "object"}]}}]), + )] + #[case::unnamed_tools_get_their_index( + json!([{"name": "first"}, {"input_schema": {"type": "object"}}, {"name": " "}, {"name": null}]), + json!([{"name": "first"}, {"input_schema": {"type": "object"}, "name": "litellm_unnamed_tool_1"}, {"name": "litellm_unnamed_tool_2"}, {"name": "litellm_unnamed_tool_3"}]), + )] + fn tools_are_normalized_for_bedrock(#[case] tools: Value, #[case] expected: Value) { + let mut request = json!({"tools": tools}).as_object().unwrap().clone(); + normalize_custom_field_on_tools(&mut request); + normalize_tool_input_schema_types_for_bedrock_invoke(&mut request); + ensure_bedrock_anthropic_messages_tool_names(&mut request); + assert_eq!(request["tools"], expected); + } + + #[rstest] + #[case::stripped_without_support( + MessagesModelCapabilities::default(), + json!({"effort": "high"}), + None, + )] + #[case::kept_with_output_config_support( + adaptive(), + json!({"effort": "high"}), + Some(json!({"effort": "high"})), + )] + #[case::bedrock_flag_keeps_it( + with_bedrock(MessagesModelCapabilities::default(), BedrockCapabilities { supports_output_config: true, ..BedrockCapabilities::default() }), + json!({"effort": "high"}), + Some(json!({"effort": "high"})), + )] + fn output_config_keys_follow_model_support( + #[case] capabilities: MessagesModelCapabilities, + #[case] output_config: Value, + #[case] expected: Option, + ) { + let mut request = json!({"output_config": output_config}) + .as_object() + .unwrap() + .clone(); + strip_unsupported_bedrock_invoke_output_config_keys( + "model", + &mut request, + capabilities.bedrock.supports_output_config || supports_effort_param(&capabilities), + ); + assert_eq!(request.get("output_config").cloned(), expected); + } } diff --git a/litellm/rust/crates/llms/src/bedrock/common_utils/structured_output.rs b/litellm/rust/crates/llms/src/bedrock/common_utils/structured_output.rs new file mode 100644 index 00000000000..9056daa6da2 --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/common_utils/structured_output.rs @@ -0,0 +1,904 @@ +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use litellm_python_compat::{json as python_json, repr}; +use serde_json::{Map, Value, json}; + +use crate::{Error, ErrorDetail, base_llm::messages::context::NativeOutputSupport}; + +use litellm_llms_types::recognized::python_truthy; + +use super::is_int; + +type Body = Map; + +const NATIVE_OUTPUT_MAX_PROPERTIES: usize = 64; +const NATIVE_OUTPUT_MAX_UNION_PARAMETERS: usize = 16; +const NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: usize = 24; +const NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: u128 = 2048; +const MAX_SCHEMA_NESTING: usize = 1024; +const NATIVE_OUTPUT_STRING_FORMATS: [&str; 10] = [ + "date-time", + "time", + "date", + "duration", + "email", + "hostname", + "uri", + "ipv4", + "ipv6", + "uuid", +]; +const SUBSCHEMA_KEYWORDS: [&str; 11] = [ + "additionalItems", + "additionalProperties", + "contains", + "else", + "if", + "items", + "not", + "propertyNames", + "then", + "unevaluatedItems", + "unevaluatedProperties", +]; +const SUBSCHEMA_LIST_KEYWORDS: [&str; 5] = ["allOf", "anyOf", "items", "oneOf", "prefixItems"]; +const SUBSCHEMA_MAP_KEYWORDS: [&str; 5] = [ + "$defs", + "definitions", + "dependentSchemas", + "patternProperties", + "properties", +]; +const CONSTRAINT_LABELS: [(&str, &str); 25] = [ + ("minItems", "minimum number of items: {}"), + ("maxItems", "maximum number of items: {}"), + ("uniqueItems", "all array items must be unique"), + ("contains", "array must contain an item matching: {}"), + ("minContains", "minimum number of matching items: {}"), + ("maxContains", "maximum number of matching items: {}"), + ("prefixItems", "leading items must match, in order: {}"), + ("minimum", "minimum value: {}"), + ("maximum", "maximum value: {}"), + ("exclusiveMinimum", "exclusive minimum value: {}"), + ("exclusiveMaximum", "exclusive maximum value: {}"), + ("multipleOf", "must be a multiple of {}"), + ("minLength", "minimum length: {}"), + ("maxLength", "maximum length: {}"), + ("minProperties", "minimum number of properties: {}"), + ("maxProperties", "maximum number of properties: {}"), + ( + "patternProperties", + "properties whose names match each pattern must satisfy: {}", + ), + ("propertyNames", "property names must satisfy: {}"), + ("dependentRequired", "dependent required properties: {}"), + ("dependentSchemas", "dependent schemas: {}"), + ( + "unevaluatedProperties", + "unevaluated properties must satisfy: {}", + ), + ("if", "conditional (if): {}"), + ("then", "conditional (then): {}"), + ("else", "conditional (else): {}"), + ("not", "must not match: {}"), +]; + +type Schema = Map; + +fn subschemas(node: &Schema) -> impl Iterator { + node.iter().flat_map(|(key, value)| { + let key = key.as_str(); + let children: Vec<&Schema> = match value { + Value::Object(entries) if SUBSCHEMA_MAP_KEYWORDS.contains(&key) => { + entries.values().filter_map(Value::as_object).collect() + } + Value::Array(entries) if SUBSCHEMA_LIST_KEYWORDS.contains(&key) => { + entries.iter().filter_map(Value::as_object).collect() + } + Value::Object(child) if SUBSCHEMA_KEYWORDS.contains(&key) => vec![child], + _ => Vec::new(), + }; + children + }) +} + +fn schema_nodes(schema: &Schema) -> Vec<&Schema> { + let mut nodes = Vec::new(); + let mut frontier = vec![schema]; + let mut depth = 0; + while !frontier.is_empty() && depth < MAX_SCHEMA_NESTING { + let next: Vec<&Schema> = frontier.iter().flat_map(|node| subschemas(node)).collect(); + nodes.extend(frontier); + frontier = next; + depth += 1; + } + nodes +} + +fn schema_at<'a>(node: &'a Schema, key: &str) -> Option<&'a Schema> { + node.get(key).and_then(Value::as_object) +} + +fn list_at<'a>(node: &'a Schema, key: &str) -> &'a [Value] { + match node.get(key) { + Some(Value::Array(values)) => values, + _ => &[], + } +} + +fn schema_fragment_ref(reference: &str) -> String { + if reference.starts_with("/$defs/") || reference.starts_with("/definitions/") { + format!("#{reference}") + } else { + reference.to_string() + } +} + +fn refs_in(schema: &Schema) -> BTreeSet { + schema_nodes(schema) + .into_iter() + .filter_map(|node| node.get("$ref").and_then(Value::as_str)) + .map(schema_fragment_ref) + .collect() +} + +fn root_definitions(schema: &Schema) -> Vec<(String, &Schema)> { + ["$defs", "definitions"] + .into_iter() + .filter_map(|defs_key| schema_at(schema, defs_key).map(|defs| (defs_key, defs))) + .flat_map(|(defs_key, defs)| { + defs.iter().filter_map(move |(name, body)| { + body.as_object() + .map(|body| (format!("#/{defs_key}/{name}"), body)) + }) + }) + .collect() +} + +fn has_cycle(graph: &BTreeMap>) -> bool { + let mut unresolved: BTreeMap<&str, usize> = BTreeMap::new(); + let mut dependents: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + graph.iter().for_each(|(node, edges)| { + let known = edges.iter().filter(|edge| graph.contains_key(*edge)); + unresolved.insert(node, known.clone().count()); + known.for_each(|edge| dependents.entry(edge).or_default().push(node)); + }); + let mut ready: VecDeque<&str> = unresolved + .iter() + .filter(|(_, count)| **count == 0) + .map(|(node, _)| *node) + .collect(); + let mut resolved = 0; + while let Some(node) = ready.pop_front() { + resolved += 1; + dependents + .get(node) + .into_iter() + .flatten() + .for_each(|dependent| { + if let Some(count) = unresolved.get_mut(dependent) { + *count -= 1; + if *count == 0 { + ready.push_back(*dependent); + } + } + }); + } + resolved != graph.len() +} + +fn schema_has_unresolvable_ref(schema: &Schema) -> bool { + let definitions: BTreeMap = root_definitions(schema).into_iter().collect(); + if !refs_in(schema) + .iter() + .all(|reference| definitions.contains_key(reference)) + { + return true; + } + has_cycle( + &definitions + .iter() + .map(|(pointer, body)| (pointer.clone(), refs_in(body))) + .collect(), + ) +} + +fn is_object_schema(node: &Schema) -> bool { + match node.get("type") { + Some(Value::String(kind)) => kind == "object", + Some(Value::Array(kinds)) => kinds.iter().any(|kind| kind.as_str() == Some("object")), + _ => false, + } +} + +fn is_open_object(node: &Schema) -> bool { + let additional = node.get("additionalProperties"); + if additional == Some(&Value::Bool(true)) + || additional.is_some_and(Value::is_object) + || node.contains_key("patternProperties") + { + return true; + } + is_object_schema(node) + && !node.contains_key("properties") + && additional != Some(&Value::Bool(false)) +} + +fn uses_regex_lookaround(pattern: &str) -> bool { + ["(?=", "(?!", "(?<=", "(? bool { + match node.get("type") { + None => true, + Some(Value::String(kind)) => kind == "string", + Some(_) => list_at(node, "type") + .iter() + .any(|kind| kind.as_str() == Some("string")), + } +} + +fn optional_property_names(node: &Schema) -> BTreeSet<&str> { + let Some(properties) = schema_at(node, "properties") else { + return BTreeSet::new(); + }; + let required: BTreeSet<&str> = list_at(node, "required") + .iter() + .filter_map(Value::as_str) + .collect(); + properties + .keys() + .map(String::as_str) + .filter(|name| !required.contains(name)) + .collect() +} + +fn is_union(node: &Schema) -> bool { + node.get("anyOf").is_some_and(Value::is_array) + || node.get("oneOf").is_some_and(Value::is_array) + || list_at(node, "type").len() > 1 +} + +fn grammar_branches(node: &Schema) -> u128 { + let Some(properties) = schema_at(node, "properties") else { + return 0; + }; + let optional = optional_property_names(node).len(); + let required = (properties.len() - optional.min(properties.len())) as u128 + 1; + u32::try_from(optional) + .ok() + .and_then(|shift| 1u128.checked_shl(shift)) + .and_then(|branches| branches.checked_mul(required)) + .unwrap_or(u128::MAX) +} + +/// Port of Python's `schema_property_count`. +pub(crate) fn schema_property_count(schema: &Schema) -> usize { + schema_nodes(schema) + .into_iter() + .filter_map(|node| schema_at(node, "properties")) + .map(Map::len) + .sum() +} + +fn native_output_rejects(schema: &Schema) -> bool { + let nodes = schema_nodes(schema); + let string_formats_supported = nodes + .iter() + .filter(|node| may_be_string(node)) + .filter_map(|node| node.get("format").and_then(Value::as_str)) + .all(|format| NATIVE_OUTPUT_STRING_FORMATS.contains(&format)); + let union_properties = nodes + .iter() + .filter_map(|node| schema_at(node, "properties")) + .flat_map(|properties| properties.values().filter_map(Value::as_object)) + .filter(|property| is_union(property)) + .count(); + let optional_properties: usize = nodes + .iter() + .map(|node| optional_property_names(node).len()) + .sum(); + let grammar_branches = nodes + .iter() + .map(|node| grammar_branches(node)) + .fold(0u128, u128::saturating_add); + schema_has_unresolvable_ref(schema) + || nodes.iter().any(|node| is_open_object(node)) + || nodes.iter().any(|node| { + node.get("pattern") + .and_then(Value::as_str) + .is_some_and(uses_regex_lookaround) + }) + || !string_formats_supported + || union_properties > NATIVE_OUTPUT_MAX_UNION_PARAMETERS + || optional_properties > NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS + || grammar_branches > NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES +} + +fn enum_conflicts_with_declared_type(schema: &Schema) -> bool { + let Some(Value::Array(values)) = schema.get("enum") else { + return false; + }; + let declared = match schema.get("type") { + None | Some(Value::Null) => return false, + Some(Value::Array(_)) => return true, + Some(Value::String(kind)) => kind.as_str(), + Some(_) => return false, + }; + let check: fn(&Value) -> bool = match declared { + "null" => Value::is_null, + "boolean" => Value::is_boolean, + "integer" => is_int, + "number" => Value::is_number, + "string" => Value::is_string, + "array" => Value::is_array, + "object" => Value::is_object, + _ => return false, + }; + !values.iter().all(check) +} + +fn note_value(value: &Value) -> Result { + let python = python_json::from_json(value.clone()); + match value { + Value::Object(_) | Value::Array(_) => python_json::dumps(&python).map_err(|error| { + Error::InvalidRequest(ErrorDetail::failed( + "output schema note serialization", + error, + )) + }), + _ => Ok(repr::to_str(&python)), + } +} + +fn described_with_note(description: Option<&Value>, note: String) -> Result { + match description { + Some(Value::String(text)) if !text.is_empty() => { + Ok(Value::String(format!("{text} {note}"))) + } + Some(other) if python_truthy(other) => { + Err(Error::Declined(litellm_core_utils::decline::Decline::new( + "a non-string output schema description next to a dropped constraint", + ))) + } + _ => Ok(Value::String(note)), + } +} + +fn filtered_subschema(value: &Value) -> Result { + match value { + Value::Object(schema) => filter_output_schema(schema).map(Value::Object), + other => Ok(other.clone()), + } +} + +fn filtered_subschemas(value: &Value) -> Result { + match value { + Value::Object(entries) => entries + .iter() + .map(|(name, schema)| Ok((name.clone(), filtered_subschema(schema)?))) + .collect::, Error>>() + .map(Value::Object), + other => Ok(other.clone()), + } +} + +fn filtered_branches(branches: &[Value]) -> Result, Error> { + branches.iter().map(filtered_subschema).collect() +} + +fn extended_any_of(result: &Schema, branches: &[Value]) -> Result { + let filtered = filtered_branches(branches)?; + match result.get("anyOf") { + Some(Value::Array(existing)) => Ok(Value::Array( + existing.iter().cloned().chain(filtered).collect(), + )), + Some(_) => Err(Error::Declined(litellm_core_utils::decline::Decline::new( + "an output schema whose anyOf cannot be extended", + ))), + None => Ok(Value::Array(filtered)), + } +} + +pub(crate) fn filter_output_schema(schema: &Schema) -> Result { + let enforced_min_items = schema + .get("minItems") + .filter(|value| is_int(value)) + .and_then(Value::as_i64) + .is_some_and(|count| count == 0 || count == 1); + let unsupported = |field: &str| { + CONSTRAINT_LABELS.iter().any(|(label, _)| *label == field) + && !(field == "minItems" && enforced_min_items) + }; + let notes = CONSTRAINT_LABELS + .iter() + .filter(|(field, _)| unsupported(field)) + .filter_map(|(field, label)| schema.get(*field).map(|value| (label, value))) + .filter(|(_, value)| **value != Value::Bool(false)) + .map(|(label, value)| Ok(label.replacen("{}", ¬e_value(value)?, 1))) + .collect::, Error>>()?; + let mut result = Map::new(); + if !notes.is_empty() { + let note = format!("Note: {}.", notes.join(", ")); + result.insert( + "description".into(), + described_with_note(schema.get("description"), note)?, + ); + } + let drops_conflicting_type = enum_conflicts_with_declared_type(schema); + for (key, value) in schema { + let key = key.as_str(); + if unsupported(key) + || (key == "type" && drops_conflicting_type) + || (key == "description" && result.contains_key("description")) + { + continue; + } + let filtered = match (key, value) { + ("properties" | "$defs" | "definitions", _) => filtered_subschemas(value)?, + ("items", _) => filtered_subschema(value)?, + ("anyOf" | "oneOf", Value::Array(branches)) => { + let extended = extended_any_of(&result, branches)?; + result.insert("anyOf".into(), extended); + continue; + } + ("allOf", Value::Array(branches)) => Value::Array(filtered_branches(branches)?), + ("$ref", Value::String(reference)) => Value::String(schema_fragment_ref(reference)), + _ => value.clone(), + }; + result.insert(key.to_string(), filtered); + } + if is_object_schema(&result) { + result.insert("additionalProperties".into(), Value::Bool(false)); + } + Ok(result) +} + +/// Port of Python's `AnthropicConfig.native_output_schema`. +pub(crate) fn native_output_schema(schema: &Schema) -> Result, Error> { + if native_output_rejects(schema) { + return Ok(None); + } + filter_output_schema(schema).map(Some) +} + +/// Port of Python's `bedrock_native_output_format`, with `bedrock_native_output_schema` and +/// `bedrock_cleaned_output_schema` inlined. +pub(crate) fn bedrock_native_output_format( + output_format: &Schema, + native: NativeOutputSupport, +) -> Result, Error> { + let Some(Value::Object(schema)) = output_format.get("schema") else { + return Ok((native.enabled || native.opt_in).then(|| output_format.clone())); + }; + let cleaned = if !native.mapped && schema_property_count(schema) > NATIVE_OUTPUT_MAX_PROPERTIES + { + None + } else { + native_output_schema(schema)? + }; + let native_schema = if native.enabled { + Some(cleaned.unwrap_or_else(|| schema.clone())) + } else if native.opt_in { + cleaned + } else { + None + }; + Ok(native_schema.map(|native_schema| { + let mut format = output_format.clone(); + format.insert("schema".into(), Value::Object(native_schema)); + format + })) +} + +/// Port of Python's `pop_bedrock_invoke_output_config_format`. +fn pop_bedrock_invoke_output_config_format(body: &mut Body) -> Option { + let Some(Value::Object(output_config)) = body.get_mut("output_config") else { + return None; + }; + let format = output_config.shift_remove("format"); + if output_config.is_empty() { + body.shift_remove("output_config"); + } + match format { + Some(Value::Object(format)) => Some(format), + _ => None, + } +} + +/// Port of Python's `convert_bedrock_invoke_output_format_to_inline_schema`. +fn convert_bedrock_invoke_output_format_to_inline_schema( + output_format: &Schema, + body: &mut Body, +) -> Result<(), Error> { + let Some(schema) = output_format + .get("schema") + .filter(|schema| python_truthy(schema)) + else { + return Ok(()); + }; + let text = python_json::dumps(&python_json::from_json(schema.clone())).map_err(|error| { + Error::InvalidRequest(ErrorDetail::failed("output schema serialization", error)) + })?; + let schema_block = json!({"type": "text", "text": text}); + let Some(Value::Array(messages)) = body.get_mut("messages") else { + return Ok(()); + }; + let Some(last_user) = messages + .iter_mut() + .rev() + .find(|message| message.get("role").and_then(Value::as_str) == Some("user")) + .and_then(Value::as_object_mut) + else { + return Ok(()); + }; + let content = match last_user.get("content") { + None => Value::Array(vec![schema_block]), + Some(Value::String(text)) => json!([{"type": "text", "text": text}, schema_block]), + Some(Value::Array(blocks)) => { + Value::Array(blocks.iter().cloned().chain([schema_block]).collect()) + } + Some(_) => return Ok(()), + }; + last_user.insert("content".into(), content); + Ok(()) +} + +/// Port of Python's `apply_bedrock_invoke_structured_output`. +pub(crate) fn apply_bedrock_invoke_structured_output( + body: &mut Body, + model: &str, + native: NativeOutputSupport, +) -> Result<(), Error> { + let legacy = body.shift_remove("output_format"); + let output_config_format = pop_bedrock_invoke_output_config_format(body); + let schema_format = match legacy { + Some(Value::Object(legacy)) => Some(legacy), + _ => output_config_format, + }; + let Some(schema_format) = schema_format else { + return Ok(()); + }; + if let Some(native_format) = bedrock_native_output_format(&schema_format, native)? { + match body.get_mut("output_config") { + Some(Value::Object(output_config)) => { + output_config.insert("format".into(), Value::Object(native_format)); + } + _ => { + body.insert("output_config".into(), json!({"format": native_format})); + } + } + return Ok(()); + } + litellm_tracing::warn!( + "Bedrock Invoke: model={model} does not take native structured output for this schema per model_prices_and_context_window.json, so the JSON schema was inlined into the last user message and is NOT enforced by the model." + ); + convert_bedrock_invoke_output_format_to_inline_schema(&schema_format, body) +} + +/// Port of Python's `requests_output_config_format`. +fn requests_output_config_format(params: &Map) -> bool { + matches!(params.get("output_config"), Some(Value::Object(output_config)) if output_config.contains_key("format")) +} + +pub(crate) fn requested_output_format(params: &Map) -> Option<&Schema> { + match params.get("output_format") { + Some(Value::Object(legacy)) => Some(legacy), + _ => params + .get("output_config") + .and_then(Value::as_object) + .and_then(|output_config| output_config.get("format")) + .and_then(Value::as_object), + } +} + +/// Port of Python's `bedrock_native_output_retry_requested`. +pub(crate) fn bedrock_native_output_retry_requested( + params: &Map, + native: NativeOutputSupport, +) -> bool { + (params.get("output_format").is_some_and(Value::is_object) + || requests_output_config_format(params)) + && native.retries() +} + +pub(crate) fn sent_output_config_format(body: &Value) -> bool { + body.as_object().is_some_and(requests_output_config_format) +} + +pub(crate) const NATIVE_OUTPUT_UNSUPPORTED: &str = + "output_config.format: Extra inputs are not permitted"; + +const NATIVE_OUTPUT_REJECTIONS: [&str; 4] = [ + NATIVE_OUTPUT_UNSUPPORTED, + "compilation exceeded limits: Schema is too complex", + "The compiled grammar is too large", + "output_config.format.schema:", +]; + +/// Port of Python's `is_bedrock_native_output_rejection`. +pub(crate) fn is_bedrock_native_output_rejection(status: u16, error_text: &str) -> bool { + status == 400 + && NATIVE_OUTPUT_REJECTIONS + .iter() + .any(|rejection| error_text.contains(rejection)) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn schema(value: Value) -> Schema { + value.as_object().unwrap().clone() + } + + fn legacy() -> NativeOutputSupport { + NativeOutputSupport { + enabled: true, + mapped: true, + ..NativeOutputSupport::default() + } + } + + fn bedrock_opt_in() -> NativeOutputSupport { + NativeOutputSupport { + opt_in: true, + ..NativeOutputSupport::default() + } + } + + fn tree() -> Value { + json!({ + "type": "object", + "properties": {"root": {"$ref": "#/$defs/Node"}}, + "$defs": {"Node": {"type": "object", "properties": {"children": {"type": "array", "items": {"$ref": "#/$defs/Node"}}}}}, + }) + } + + fn open_object() -> Value { + json!({ + "type": "object", + "properties": {"counts": {"type": "object", "additionalProperties": {"type": "integer"}}}, + }) + } + + fn many_properties(count: usize) -> Value { + let properties: Map = (0..count) + .map(|index| (format!("p{index}"), json!({"type": "string"}))) + .collect(); + let required: Vec = (0..count).map(|index| json!(format!("p{index}"))).collect(); + json!({"type": "object", "properties": properties, "required": required, "additionalProperties": false}) + } + + fn sent(native: NativeOutputSupport, raw: Value) -> Body { + let mut body = schema(json!({ + "messages": [{"role": "user", "content": "Hello"}], + "output_config": {"format": {"type": "json_schema", "schema": raw}}, + })); + apply_bedrock_invoke_structured_output(&mut body, "model", native).unwrap(); + body + } + + #[test] + fn native_output_drops_schema_keywords_bedrock_rejects() { + let body = sent( + bedrock_opt_in(), + json!({ + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "ids": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + }, + "required": ["count", "ids"], + }), + ); + assert_eq!( + body["output_config"]["format"]["schema"], + json!({ + "type": "object", + "properties": { + "count": {"description": "Note: minimum value: 1.", "type": "integer"}, + "ids": {"description": "Note: all array items must be unique.", "type": "array", "items": {"type": "string"}}, + }, + "required": ["count", "ids"], + "additionalProperties": false, + }) + ); + } + + #[rstest] + #[case::recursive_on_a_bedrock_profile(bedrock_opt_in(), tree(), false)] + #[case::open_object_on_a_bedrock_profile(bedrock_opt_in(), open_object(), false)] + #[case::recursive_on_a_legacy_profile(legacy(), tree(), true)] + #[case::open_object_on_a_legacy_profile(legacy(), open_object(), true)] + fn schemas_native_output_cannot_hold( + #[case] native: NativeOutputSupport, + #[case] raw: Value, + #[case] forwarded: bool, + ) { + let body = sent(native, raw.clone()); + if forwarded { + assert_eq!(body["output_config"]["format"]["schema"], raw); + } else { + assert_eq!(body.get("output_config"), None); + let text = body["messages"][0]["content"][1]["text"].as_str().unwrap(); + assert_eq!(serde_json::from_str::(text).unwrap(), raw); + } + } + + #[test] + fn shared_defs_reach_native_output_once() { + let party = json!({"$ref": "#/$defs/Party"}); + let body = sent( + legacy(), + json!({ + "type": "object", + "properties": {"buyer": party, "seller": party}, + "required": ["buyer", "seller"], + "additionalProperties": false, + "$defs": {"Party": {"type": "object", "properties": {"email": {"anyOf": [{"type": "string"}, {"type": "null"}]}}, "required": ["email"], "additionalProperties": false}}, + }), + ); + let sent_schema = &body["output_config"]["format"]["schema"]; + assert_eq!(sent_schema["properties"]["buyer"], party); + assert_eq!(sent_schema.to_string().matches("\"anyOf\"").count(), 1); + } + + #[rstest] + #[case::unmapped_over_the_limit(NativeOutputSupport { opt_in: true, ..NativeOutputSupport::default() }, 65, false)] + #[case::unmapped_at_the_limit(NativeOutputSupport { opt_in: true, ..NativeOutputSupport::default() }, 24, true)] + #[case::mapped_over_the_limit(NativeOutputSupport { opt_in: true, mapped: true, ..NativeOutputSupport::default() }, 65, true)] + fn property_count_rule_applies_to_unmapped_models( + #[case] native: NativeOutputSupport, + #[case] properties: usize, + #[case] native_sent: bool, + ) { + assert_eq!( + sent(native, many_properties(properties)).contains_key("output_config"), + native_sent + ); + } + + #[test] + fn legacy_output_format_wins() { + let mut body = schema(json!({ + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + "output_format": {"type": "json_schema", "schema": {"type": "object", "properties": {"legacy": {"type": "string"}}}}, + "output_config": {"effort": "high", "format": {"type": "json_schema", "schema": {"type": "object", "properties": {"new": {"type": "string"}}}}}, + })); + apply_bedrock_invoke_structured_output(&mut body, "model", NativeOutputSupport::default()) + .unwrap(); + assert_eq!(body["output_config"], json!({"effort": "high"})); + assert_eq!( + body["messages"][0]["content"][1]["text"], + json!(r#"{"type": "object", "properties": {"legacy": {"type": "string"}}}"#) + ); + } + + #[rstest] + #[case::one_of_becomes_any_of( + json!({"oneOf": [{"type": "string"}, {"type": "integer", "maximum": 3}]}), + json!({"anyOf": [{"type": "string"}, {"description": "Note: maximum value: 3.", "type": "integer"}]}), + )] + #[case::small_min_items_is_kept( + json!({"type": "array", "minItems": 1, "maxItems": 4}), + json!({"description": "Note: maximum number of items: 4.", "type": "array", "minItems": 1}), + )] + #[case::false_constraints_add_no_note( + json!({"type": "array", "uniqueItems": false}), + json!({"type": "array"}), + )] + #[case::conflicting_enum_type_is_dropped( + json!({"type": "string", "enum": [1, 2]}), + json!({"enum": [1, 2]}), + )] + #[case::sub_schema_constraints_note_their_json( + json!({"type": "array", "contains": {"type": "string"}, "description": "Items"}), + json!({"description": "Items Note: array must contain an item matching: {\"type\": \"string\"}.", "type": "array"}), + )] + #[case::ref_becomes_a_fragment( + json!({"$ref": "/$defs/Node"}), + json!({"$ref": "#/$defs/Node"}), + )] + fn output_schema_filter_matches_the_anthropic_sdk(#[case] raw: Value, #[case] expected: Value) { + assert_eq!( + Value::Object(filter_output_schema(&schema(raw)).unwrap()), + expected + ); + } + + #[rstest] + #[case::lookaround(json!({"type": "string", "pattern": "^(?!x)"}), true)] + #[case::unsupported_format(json!({"type": "string", "format": "regex"}), true)] + #[case::supported_format(json!({"type": "string", "format": "email"}), false)] + #[case::unresolvable_ref(json!({"$ref": "#/$defs/Missing"}), true)] + #[case::too_many_optional(json!({"type": "object", "properties": (0..25).map(|i| (format!("p{i}"), json!({"type": "string"}))).collect::>(), "additionalProperties": false}), true)] + fn native_output_rejects_what_the_grammar_cannot_compile( + #[case] raw: Value, + #[case] rejected: bool, + ) { + assert_eq!(native_output_rejects(&schema(raw)), rejected); + } + + fn graph(edges: Vec<(&str, Vec<&str>)>) -> BTreeMap> { + edges + .into_iter() + .map(|(node, refs)| { + ( + node.to_string(), + refs.into_iter().map(str::to_string).collect(), + ) + }) + .collect() + } + + #[rstest] + #[case::none(vec![], false)] + #[case::chain(vec![("a", vec!["b"]), ("b", vec!["c"]), ("c", vec![])], false)] + #[case::diamond(vec![("a", vec!["b", "c"]), ("b", vec!["d"]), ("c", vec!["d"]), ("d", vec![])], false)] + #[case::unknown_edges_are_ignored(vec![("a", vec!["missing"])], false)] + #[case::self_reference(vec![("a", vec!["a"])], true)] + #[case::two_node_cycle(vec![("a", vec!["b"]), ("b", vec!["a"])], true)] + #[case::cycle_behind_a_resolved_node(vec![("a", vec![]), ("b", vec!["a", "c"]), ("c", vec!["b"])], true)] + fn has_cycle_finds_only_reference_loops( + #[case] edges: Vec<(&str, Vec<&str>)>, + #[case] expected: bool, + ) { + assert_eq!(has_cycle(&graph(edges)), expected); + } + + #[test] + fn has_cycle_stays_linear_on_a_long_reference_chain() { + let size = 50_000; + let chain: BTreeMap> = (0..size) + .map(|i| { + ( + format!("n{i}"), + (i + 1 < size) + .then(|| format!("n{}", i + 1)) + .into_iter() + .collect(), + ) + }) + .collect(); + let started = std::time::Instant::now(); + assert!(!has_cycle(&chain)); + assert!(started.elapsed() < std::time::Duration::from_secs(5)); + } + + #[rstest] + #[case::extra_inputs(400, "output_config.format: Extra inputs are not permitted", true)] + #[case::too_complex(400, "compilation exceeded limits: Schema is too complex", true)] + #[case::grammar_too_large(400, "The compiled grammar is too large", true)] + #[case::schema_error(400, "output_config.format.schema: bad", true)] + #[case::other_400(400, "messages: bad", false)] + #[case::not_a_400(500, "output_config.format: Extra inputs are not permitted", false)] + fn native_output_rejections_are_recognized( + #[case] status: u16, + #[case] text: &str, + #[case] expected: bool, + ) { + assert_eq!(is_bedrock_native_output_rejection(status, text), expected); + } + + #[rstest] + #[case::enabled(legacy(), true)] + #[case::opt_in(bedrock_opt_in(), true)] + #[case::replaces_unforceable_tool(NativeOutputSupport { replaces_unforceable_tool: true, ..NativeOutputSupport::default() }, true)] + #[case::unsupported(NativeOutputSupport::default(), false)] + fn native_output_retry_needs_native_support( + #[case] native: NativeOutputSupport, + #[case] expected: bool, + ) { + let params = schema(json!({"output_config": {"format": {"type": "json_schema"}}})); + assert_eq!( + bedrock_native_output_retry_requested(¶ms, native), + expected + ); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/messages/AGENTS.md b/litellm/rust/crates/llms/src/bedrock/messages/AGENTS.md index 48c222a1cdc..e82ac89380c 100644 --- a/litellm/rust/crates/llms/src/bedrock/messages/AGENTS.md +++ b/litellm/rust/crates/llms/src/bedrock/messages/AGENTS.md @@ -1,6 +1,7 @@ # rules -- Claude on Bedrock InvokeModel: reuses `anthropic/messages` shaping, moves the model into the URL, and decodes the AWS event stream +- Claude on Bedrock InvokeModel: reuses `anthropic/messages` shaping, moves the model into the URL, and decodes the AWS event stream through `chat/invoke_handler.rs` +- `invoke_transformations/` holds only what Python's `AmazonAnthropicClaudeMessagesConfig` holds, under its method names. Helpers Python keeps in `bedrock/common_utils.py` and `bedrock/request_metadata.py` live in the matching Rust modules - Invocation metrics in the response map onto Messages usage here # references diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs index fa037974342..b0cba0c9d3a 100644 --- a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs @@ -1,57 +1,63 @@ use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; -use litellm_core_utils::decline::Decline; use litellm_host::interceptors::RetriedRequest; -use std::convert::Infallible; use crate::{ anthropic::messages::handler::shape_anthropic_messages_request, - base_llm::messages::context::{MessagesShapeOptions, MessagesTransformContext}, + base_llm::messages::context::{ + MessagesShapeOptions, MessagesTransformContext, NativeOutputSupport, + }, }; -use futures_util::StreamExt; -use litellm_auth::AwsParams; use litellm_auth::CredentialPlacement; use litellm_secret_value::SecretValue; use litellm_auth_aws::{ - AwsCredentialSource, bedrock_model_id_and_region, + AwsCredentialSource, CallerSigningHeaders, bedrock_runtime_endpoint, constants::{ AWS_BEARER_TOKEN_BEDROCK, AWS_BEDROCK_RUNTIME_ENDPOINT, AWS_DEFAULT_REGION, AWS_REGION, - AWS_REGION_NAME, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE, + AWS_REGION_NAME, BEDROCK_SERVICE, }, resolve_bedrock_region, }; -use litellm_llms_types::formats::messages::{ - MessagesOptionalParams, MessagesRequest, - streaming::{MessagesStreamEvent, MessagesStreamUsage}, -}; -use litellm_llms_types::providers::bedrock::{ - INVOCATION_METRICS_KEY, INVOKE_PATH, INVOKE_STREAM_PATH, -}; +use litellm_llms_types::formats::messages::{MessagesOptionalParams, MessagesRequest}; +use litellm_llms_types::providers::bedrock::{INVOKE_PATH, INVOKE_STREAM_PATH}; use litellm_router_types::LitellmParams; -use serde_json::{Map, Value}; +use serde_json::Value; use crate::{ Error, base_llm::{ auth::AuthScheme, - base_model_iterator::{StreamError, StreamTransformer, transform_stream}, messages::{ - streaming::{ByteStream, EventStream, StreamDecoder}, + streaming::StreamDecoder, transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment}, }, }, - bedrock::chat::invoke_handler::{decode_invoke_anthropic_chunk, invoke_chunk_stream}, }; -const METRICS_USAGE_KEYS: [(&str, &str); 4] = [ - ("input_tokens", "inputTokenCount"), - ("output_tokens", "outputTokenCount"), - ("cache_read_input_tokens", "cacheReadInputTokenCount"), - ("cache_creation_input_tokens", "cacheWriteInputTokenCount"), -]; +use crate::bedrock::{ + chat::invoke_transformations::base_invoke_transformation::{ + GUARDRAIL_CONFIG_PARAM, bedrock_invoke_guardrail_headers, + }, + common_utils::{ + strip_bedrock_routing_prefix, + structured_output::{ + NATIVE_OUTPUT_UNSUPPORTED, bedrock_native_output_format, + bedrock_native_output_retry_requested, is_bedrock_native_output_rejection, + requested_output_format, sent_output_config_format, + }, + validated_region, + }, + request_metadata::{bedrock_request_metadata_headers, merge_bedrock_invoke_headers}, +}; + +use super::{request, stream}; const INVOKE_MODEL_PREFIX: &str = "invoke/"; +const MODEL_ID_PARAM: &str = "model_id"; + +const MODEL_SPEC_PREFIXES: [&str; 2] = ["nova-2/", "nova/"]; + const SECRET_NAMES: &[&str] = &[ AWS_BEARER_TOKEN_BEDROCK, AWS_BEDROCK_RUNTIME_ENDPOINT, @@ -76,16 +82,89 @@ fn bearer_token( .filter(|token| !token.is_empty()) } +/// Port of Python's `BaseAWSLLM.encode_model_id`. +pub fn encode_model_id(text: &str) -> String { + text.bytes() + .map(|byte| match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' => { + char::from(byte).to_string() + } + other => format!("%{other:02X}"), + }) + .collect() +} + +/// Port of Python's `BaseAWSLLM.get_bedrock_model_id` for Claude. +fn get_bedrock_model_id(model: &str, model_id: Option<&str>) -> String { + if let Some(model_id) = model_id { + return encode_model_id(model_id).replacen(INVOKE_MODEL_PREFIX, "", 1); + } + let stripped = strip_bedrock_routing_prefix(model); + if stripped.starts_with("arn:") { + return encode_model_id(stripped); + } + let model_id = stripped.replacen(INVOKE_MODEL_PREFIX, "", 1); + match MODEL_SPEC_PREFIXES + .iter() + .find(|spec| model_id.contains(**spec)) + { + Some(spec) => encode_model_id(&model_id.replace(spec, "")), + None => model_id, + } +} + +fn is_arn_partition_char(character: char) -> bool { + character.is_ascii_lowercase() || character.is_ascii_digit() || character == '-' +} + +/// Port of Python's `contains_bedrock_arn`. +fn contains_bedrock_arn(model: &str) -> bool { + model.match_indices("arn:aws").any(|(index, prefix)| { + let rest = &model[index + prefix.len()..]; + let after_partition = match rest.strip_prefix('-') { + Some(partition) => { + let trimmed = partition.trim_start_matches(is_arn_partition_char); + if trimmed.len() == partition.len() { + return false; + } + trimmed + } + None => rest, + }; + after_partition.starts_with(":bedrock") + }) +} + +/// Port of Python's `BaseAWSLLM._get_aws_region_from_model_arn`. +fn get_aws_region_from_model_arn(model: &str) -> Option { + contains_bedrock_arn(model) + .then(|| model.split(':').nth(3)) + .flatten() + .filter(|region| !region.is_empty() && region.chars().all(is_arn_partition_char)) + .map(str::to_string) +} + +/// Port of Python's `AmazonInvokeConfig.get_complete_url`. fn invoke_url( api_base: Option<&str>, model: &str, - aws: &AwsParams, + litellm_params: &LitellmParams, stream: bool, env_lookup: &dyn Fn(&str) -> Option, -) -> String { - let (model_id, model_region) = - bedrock_model_id_and_region(model.strip_prefix(INVOKE_MODEL_PREFIX).unwrap_or(model)); - let region = resolve_bedrock_region(model_region.as_deref(), aws, env_lookup); +) -> Result { + let aws = &litellm_params.aws; + let model_id = get_bedrock_model_id( + model, + litellm_params + .extra + .get(MODEL_ID_PARAM) + .and_then(Value::as_str), + ); + let region = resolve_bedrock_region( + get_aws_region_from_model_arn(model).as_deref(), + aws, + env_lookup, + ); let path = if stream { INVOKE_STREAM_PATH } else { @@ -97,11 +176,41 @@ fn invoke_url( .filter(|value| !value.is_empty()) .map(str::to_string) }; - let endpoint = configured(api_base) + let endpoint = match configured(api_base) .or_else(|| configured(aws.aws_bedrock_runtime_endpoint.as_deref())) .or_else(|| env_lookup(AWS_BEDROCK_RUNTIME_ENDPOINT)) - .unwrap_or_else(|| BEDROCK_RUNTIME_ENDPOINT_TEMPLATE.replace("{region}", ®ion)); - format!("{}/model/{model_id}/{path}", endpoint.trim_end_matches('/')) + { + Some(endpoint) => endpoint, + None => bedrock_runtime_endpoint(&validated_region(region)?), + }; + Ok(format!( + "{}/model/{model_id}/{path}", + endpoint.trim_end_matches('/') + )) +} + +/// The header half of Python's `validate_anthropic_messages_environment`. +fn invoke_headers(headers: Headers, litellm_params: &LitellmParams) -> Result { + let guardrail = match litellm_params.extra.get(GUARDRAIL_CONFIG_PARAM) { + None | Some(Value::Null) => Vec::new(), + Some(raw) => bedrock_invoke_guardrail_headers(raw)?, + }; + Ok(merge_bedrock_invoke_headers( + headers, + guardrail, + bedrock_request_metadata_headers(&litellm_params.extra), + )) +} + +fn without_native_output(context: &MessagesTransformContext) -> MessagesTransformContext { + let mut fallback = *context; + fallback.thinking.capabilities.bedrock.native_output = NativeOutputSupport { + enabled: false, + opt_in: false, + replaces_unforceable_tool: false, + ..fallback.thinking.capabilities.bedrock.native_output + }; + fallback } impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { @@ -131,23 +240,72 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { stream: bool, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { - Ok(invoke_url( - api_base, - model, - &litellm_params.aws, - stream, - env_lookup, - )) + invoke_url(api_base, model, litellm_params, stream, env_lookup) } - fn transform_anthropic_messages_request( + /// Python's `transform_anthropic_messages_request`, which picks the betas from the caller's + /// header and the request and sends them in the body. + fn finalize_request( &self, - _request: MessagesRequest, - _context: &MessagesTransformContext, - ) -> Result { - Err(Error::Declined(Decline::new( - "Bedrock invoke messages request shaping", - ))) + request: MessagesRequest, + headers: Headers, + context: &MessagesTransformContext, + ) -> Result<(MessagesRequest, Headers), Error> { + request::transform_anthropic_messages_request(request, context, headers) + } + + /// Port of Python's `native_output_fallback_litellm_params`. + fn native_output_fallback( + &self, + request: &MessagesRequest, + context: &MessagesTransformContext, + ) -> Option { + let native = context.thinking.capabilities.bedrock.native_output; + let Ok(Value::Object(params)) = serde_json::to_value(&request.params) else { + return None; + }; + if !bedrock_native_output_retry_requested(¶ms, native) { + return None; + } + let output_format = requested_output_format(¶ms)?; + bedrock_native_output_format(output_format, native) + .ok() + .flatten() + .map(|_| without_native_output(context)) + } + + /// Port of Python's `should_retry_anthropic_messages_without_native_output`. + fn retries_without_native_output( + &self, + status: u16, + error_text: &str, + request_body: &Value, + model: &str, + ) -> bool { + let retries = sent_output_config_format(request_body) + && is_bedrock_native_output_rejection(status, error_text); + if retries { + let opt_out_advice = if error_text.contains(NATIVE_OUTPUT_UNSUPPORTED) { + " Set `supports_bedrock_native_structured_output: false` in this deployment's model_info to skip the rejected call." + } else { + "" + }; + litellm_tracing::warn!( + "Bedrock rejected native structured output for model={model}, so litellm sent the request again without it.{opt_out_advice}" + ); + } + retries + } + + fn wire_body(&self, body: Value) -> Value { + match body { + Value::Object(mut fields) => { + fields.shift_remove("model"); + fields.shift_remove("stream"); + Value::Object(fields) + } + other => other, + } } fn secret_names(&self) -> &'static [&'static str] { @@ -165,6 +323,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { litellm_params: &LitellmParams, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { + let headers = invoke_headers(headers, litellm_params)?; if let Some(token) = bearer_token(api_key, env_lookup) { return Ok(ValidatedEnvironment { headers, @@ -174,15 +333,18 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { }, }); } - let (_, model_region) = - bedrock_model_id_and_region(model.strip_prefix(INVOKE_MODEL_PREFIX).unwrap_or(model)); let params = &litellm_params.aws; Ok(ValidatedEnvironment { headers, auth: AuthScheme::AwsSigV4 { - region: resolve_bedrock_region(model_region.as_deref(), params, env_lookup), + region: resolve_bedrock_region( + get_aws_region_from_model_arn(model).as_deref(), + params, + env_lookup, + ), service: BEDROCK_SERVICE, credentials: Box::new(AwsCredentialSource::from_params(params, env_lookup)), + caller_headers: CallerSigningHeaders::Botocore, }, }) } @@ -191,343 +353,20 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { &[("content-type", "application/json")] } + /// Port of Python's `get_async_streaming_response_iterator`. fn stream_decoder(&self) -> Option { - Some(bedrock_anthropic_messages_event_stream) - } -} - -fn with_invocation_usage(chunk: Value) -> Value { - match chunk { - Value::Object(fields) => Value::Object(with_metrics_usage(fields)), - other => other, - } -} - -fn with_metrics_usage(mut fields: Map) -> Map { - let Some(Value::Object(metrics)) = fields.remove(INVOCATION_METRICS_KEY) else { - return fields; - }; - if metrics.is_empty() { - return fields; - } - let preserved = match fields.remove("usage") { - Some(Value::Object(usage)) => usage, - _ => Map::new(), - }; - let usage: Map = METRICS_USAGE_KEYS - .iter() - .filter_map(|(anthropic, metric)| { - Some((anthropic.to_string(), metrics.get(*metric)?.clone())) - }) - .chain(preserved) - .collect(); - fields.insert("usage".to_string(), Value::Object(usage)); - fields -} - -pub fn bedrock_anthropic_messages_event_stream(bytes: ByteStream) -> EventStream { - let events = invoke_chunk_stream(bytes) - .map(|chunk| decode_invoke_anthropic_chunk(with_invocation_usage(chunk?))); - Box::pin( - transform_stream(events, MessageStopUsagePromoter::default()) - .map(|item| item.map_err(StreamError::into_decode)), - ) -} - -#[derive(Default)] -pub struct MessageStopUsagePromoter { - pending_delta: Option, - start_usage: Option, -} - -fn promoted_usage( - delta: Option, - stop: Option<&MessagesStreamUsage>, - start: Option<&MessagesStreamUsage>, -) -> Option { - let delta = delta.unwrap_or_default(); - let merged = MessagesStreamUsage { - input_tokens: stop - .and_then(|stop| stop.input_tokens) - .or(delta.input_tokens), - cache_creation_input_tokens: stop - .and_then(|stop| stop.cache_creation_input_tokens) - .or(delta.cache_creation_input_tokens) - .or_else(|| start.and_then(|start| start.cache_creation_input_tokens)), - cache_read_input_tokens: stop - .and_then(|stop| stop.cache_read_input_tokens) - .or(delta.cache_read_input_tokens) - .or_else(|| start.and_then(|start| start.cache_read_input_tokens)), - extra: delta - .extra - .into_iter() - .chain( - start - .and_then(|start| start.extra.get_key_value("cache_creation")) - .map(|(key, value)| (key.clone(), value.clone())), - ) - .fold(Map::new(), |mut extra, (key, value)| { - extra.entry(key).or_insert(value); - extra - }), - ..delta - }; - (merged != MessagesStreamUsage::default()).then_some(merged) -} - -fn promoted( - event: MessagesStreamEvent, - stop: Option<&MessagesStreamUsage>, - start: Option<&MessagesStreamUsage>, -) -> MessagesStreamEvent { - match event { - MessagesStreamEvent::MessageDelta { - delta, - usage, - context_management, - } => MessagesStreamEvent::MessageDelta { - delta, - usage: promoted_usage(usage, stop, start), - context_management, - }, - other => other, - } -} - -impl StreamTransformer for MessageStopUsagePromoter { - type Input = MessagesStreamEvent; - type Output = MessagesStreamEvent; - type Error = Infallible; - - fn transform( - &mut self, - input: MessagesStreamEvent, - ) -> Result, Infallible> { - let pending = self.pending_delta.take(); - match input { - MessagesStreamEvent::MessageDelta { .. } => { - self.pending_delta = Some(input); - Ok(pending.into_iter().collect()) - } - MessagesStreamEvent::MessageStop { usage } => Ok(pending - .map(|delta| promoted(delta, usage.as_ref(), self.start_usage.as_ref())) - .into_iter() - .chain([MessagesStreamEvent::MessageStop { usage }]) - .collect()), - MessagesStreamEvent::MessageStart { message } => { - self.start_usage = Some(message.usage.clone()); - Ok(pending - .into_iter() - .chain([MessagesStreamEvent::MessageStart { message }]) - .collect()) - } - other => Ok(pending.into_iter().chain([other]).collect()), - } - } - - fn finish(&mut self) -> Result, Infallible> { - Ok(self - .pending_delta - .take() - .map(|delta| promoted(delta, None, self.start_usage.as_ref())) - .into_iter() - .collect()) + Some(stream::get_async_streaming_response_iterator) } } #[cfg(test)] mod tests { - use aws_smithy_eventstream::frame::write_message_to; - use aws_smithy_types::event_stream::{Header, HeaderValue, Message}; - use base64::{Engine, engine::general_purpose::STANDARD}; - use bytes::Bytes; - use futures_util::TryStreamExt; use rstest::rstest; - use serde_json::json; + use litellm_auth::AwsParams; use litellm_auth_aws::constants::DEFAULT_BEDROCK_REGION; use super::*; - use crate::base_llm::messages::streaming; - - fn event(value: Value) -> MessagesStreamEvent { - serde_json::from_value(value).unwrap() - } - - fn message_start(usage: Value) -> MessagesStreamEvent { - event(json!({ - "type": "message_start", - "message": { - "id": "msg_1", "type": "message", "role": "assistant", "model": "m", - "content": [], "stop_reason": null, "stop_sequence": null, "usage": usage - } - })) - } - - fn message_delta(usage: Value) -> MessagesStreamEvent { - event(json!({ - "type": "message_delta", - "delta": {"stop_reason": "end_turn"}, - "usage": usage - })) - } - - fn message_stop(usage: Option) -> MessagesStreamEvent { - match usage { - Some(usage) => event(json!({"type": "message_stop", "usage": usage})), - None => event(json!({"type": "message_stop"})), - } - } - - fn promote(events: Vec) -> Vec { - let mut promoter = MessageStopUsagePromoter::default(); - let mut output: Vec<_> = events - .into_iter() - .flat_map(|event| promoter.transform(event).unwrap()) - .collect(); - output.extend(promoter.finish().unwrap()); - output - } - - #[rstest] - #[case::cache_fields_on_message_stop( - json!({"input_tokens": 10, "output_tokens": 0}), - json!({"output_tokens": 5}), - Some(json!({"input_tokens": 3, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20})), - json!({"input_tokens": 3, "output_tokens": 5, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20}), - )] - #[case::cache_only_on_message_start( - json!({"input_tokens": 10, "output_tokens": 0, "cache_read_input_tokens": 80, "cache_creation_input_tokens": 4, "cache_creation": {"ephemeral_5m_input_tokens": 4}}), - json!({"output_tokens": 5}), - Some(json!({"input_tokens": 10})), - json!({"input_tokens": 10, "output_tokens": 5, "cache_read_input_tokens": 80, "cache_creation_input_tokens": 4, "cache_creation": {"ephemeral_5m_input_tokens": 4}}), - )] - #[case::message_stop_wins_over_message_start( - json!({"input_tokens": 10, "cache_read_input_tokens": 80}), - json!({"output_tokens": 5}), - Some(json!({"cache_read_input_tokens": 100})), - json!({"output_tokens": 5, "cache_read_input_tokens": 100}), - )] - #[case::delta_cache_fields_are_kept( - json!({"input_tokens": 10, "cache_read_input_tokens": 80}), - json!({"output_tokens": 5, "cache_read_input_tokens": 7}), - None, - json!({"output_tokens": 5, "cache_read_input_tokens": 7}), - )] - fn message_delta_usage_is_completed_from_stop_then_start( - #[case] start: Value, - #[case] delta: Value, - #[case] stop: Option, - #[case] expected: Value, - ) { - let output = promote(vec![ - message_start(start), - message_delta(delta), - message_stop(stop.clone()), - ]); - - assert_eq!(output.len(), 3); - assert_eq!(output[1], message_delta(expected)); - assert_eq!(output[2], message_stop(stop)); - } - - #[test] - fn a_delta_is_flushed_with_start_usage_when_the_stream_ends_without_a_stop() { - let output = promote(vec![ - message_start(json!({"input_tokens": 10, "cache_read_input_tokens": 80})), - message_delta(json!({"output_tokens": 5})), - ]); - - assert_eq!( - output[1], - message_delta(json!({"output_tokens": 5, "cache_read_input_tokens": 80})) - ); - } - - #[test] - fn events_around_the_delta_keep_their_order() { - let ping = event(json!({"type": "ping"})); - let output = promote(vec![ - message_delta(json!({"output_tokens": 5})), - ping.clone(), - message_stop(None), - ]); - - assert_eq!( - output, - vec![ - message_delta(json!({"output_tokens": 5})), - ping, - message_stop(None) - ] - ); - } - - #[rstest] - #[case::metrics_fill_missing_usage( - json!({"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "outputTokenCount": 9}}), - json!({"type": "message_stop", "usage": {"input_tokens": 3, "output_tokens": 9}}), - )] - #[case::the_chunks_own_usage_wins( - json!({"type": "message_stop", "usage": {"input_tokens": 1}, "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "cacheReadInputTokenCount": 40}}), - json!({"type": "message_stop", "usage": {"cache_read_input_tokens": 40, "input_tokens": 1}}), - )] - #[case::no_metrics_leaves_the_chunk( - json!({"type": "message_stop"}), - json!({"type": "message_stop"}), - )] - #[case::empty_metrics_are_dropped( - json!({"type": "message_stop", "amazon-bedrock-invocationMetrics": {}}), - json!({"type": "message_stop"}), - )] - fn invocation_metrics_become_anthropic_usage(#[case] chunk: Value, #[case] expected: Value) { - assert_eq!(with_invocation_usage(chunk), expected); - } - - fn aws_frame(chunk: &Value) -> Vec { - let payload = json!({"bytes": STANDARD.encode(chunk.to_string())}); - let message = Message::new(Bytes::from(serde_json::to_vec(&payload).unwrap())).add_header( - Header::new(":event-type", HeaderValue::String("chunk".into())), - ); - let mut wire = Vec::new(); - write_message_to(&message, &mut wire).unwrap(); - wire - } - - #[tokio::test] - async fn bedrock_stream_yields_the_sse_an_anthropic_client_reads() { - let chunks = [ - json!({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 5}}), - json!({"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "cacheReadInputTokenCount": 40}}), - ]; - let wire: Vec = chunks.iter().flat_map(aws_frame).collect(); - let bytes: ByteStream = futures_util::stream::iter( - wire.chunks(7) - .map(|chunk| Ok(Bytes::copy_from_slice(chunk))) - .collect::>(), - ) - .boxed(); - - let sse = bedrock_anthropic_messages_event_stream(bytes) - .map_ok(|event| streaming::encode(&event).unwrap()) - .try_collect::>() - .await - .unwrap() - .concat(); - - let expected: Vec = [ - message_delta( - json!({"output_tokens": 5, "cache_read_input_tokens": 40, "input_tokens": 3}), - ), - message_stop(Some( - json!({"input_tokens": 3, "cache_read_input_tokens": 40}), - )), - ] - .iter() - .flat_map(|event| streaming::encode(event).unwrap()) - .collect(); - assert_eq!(sse, expected); - } fn in_region(region: &str) -> LitellmParams { LitellmParams { @@ -619,6 +458,37 @@ mod tests { ); } + #[rstest] + #[case::host_injection("evil.example.com/x?")] + #[case::uppercase("US-WEST-2")] + #[case::port("us-west-2:8443")] + fn a_region_outside_the_aws_alphabet_never_builds_a_url(#[case] region: &str) { + let url = AmazonAnthropicClaudeMessagesConfig.get_complete_url( + None, + "anthropic.claude-3", + &in_region(region), + false, + &|_| None, + ); + assert!(matches!(url, Err(Error::InvalidRequest(_))), "{url:?}"); + } + + #[test] + fn a_configured_endpoint_does_not_read_the_region() { + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .get_complete_url( + Some("https://base.test"), + "anthropic.claude-3", + &in_region("evil.example.com/x?"), + false, + &|_| None, + ) + .unwrap(), + "https://base.test/model/anthropic.claude-3/invoke" + ); + } + #[test] fn sigv4_scope_and_credentials_come_from_the_litellm_params() { let litellm_params = LitellmParams { @@ -702,4 +572,276 @@ mod tests { (other, _) => panic!("unexpected auth {other:?}"), } } + + const ARN: &str = "arn:aws:bedrock:us-east-1:086734376398:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0"; + const ENCODED_ARN: &str = "arn%3Aaws%3Abedrock%3Aus-east-1%3A086734376398%3Ainference-profile%2Fglobal.anthropic.claude-sonnet-4-5-20250929-v1%3A0"; + + #[rstest] + #[case::bedrock_prefixed_arn( + "bedrock/arn:aws:bedrock:us-east-1:086734376398:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0", + None, + ENCODED_ARN + )] + #[case::compound_prefixed_arn( + "bedrock/invoke/arn:aws:bedrock:us-east-1:086734376398:inference-profile/global.anthropic.claude-sonnet-4-5-20250929-v1:0", + None, + ENCODED_ARN + )] + #[case::bare_arn(ARN, None, ENCODED_ARN)] + #[case::regular_id( + "anthropic.claude-3-sonnet-20240229-v1:0", + None, + "anthropic.claude-3-sonnet-20240229-v1:0" + )] + #[case::invoke_prefix( + "invoke/anthropic.claude-3-sonnet-20240229-v1:0", + None, + "anthropic.claude-3-sonnet-20240229-v1:0" + )] + #[case::model_id_override( + "anthropic.claude-3", + Some("arn:aws:bedrock:us-east-1:1:application-inference-profile/abc"), + "arn%3Aaws%3Abedrock%3Aus-east-1%3A1%3Aapplication-inference-profile%2Fabc" + )] + fn invoke_model_ids_match_python( + #[case] model: &str, + #[case] model_id: Option<&str>, + #[case] expected: &str, + ) { + assert_eq!(get_bedrock_model_id(model, model_id), expected); + } + + #[rstest] + #[case::region_segment_stays_in_the_path( + "bedrock/invoke/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", + "us-west-2" + )] + #[case::region_segment_does_not_pick_the_region( + "invoke/eu-central-1/anthropic.claude-3-5-sonnet-20240620-v1:0", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/eu-central-1/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", + "us-west-2" + )] + #[case::params_region_hosts_a_region_segment( + "invoke/us-west-2/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + Some("us-east-1"), + "https://bedrock-runtime.us-east-1.amazonaws.com/model/us-west-2/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke", + "us-east-1" + )] + #[case::arn_after_a_region_segment_picks_the_region( + "invoke/us-west-2/arn:aws:bedrock:eu-west-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + None, + "https://bedrock-runtime.eu-west-1.amazonaws.com/model/us-west-2/arn:aws:bedrock:eu-west-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke", + "eu-west-1" + )] + #[case::arn_after_any_segment_picks_the_region( + "invoke/foo/arn:aws:bedrock:eu-west-1:1:x", + None, + "https://bedrock-runtime.eu-west-1.amazonaws.com/model/foo/arn:aws:bedrock:eu-west-1:1:x/invoke", + "eu-west-1" + )] + #[case::partitioned_arn( + "invoke/arn:aws-us-gov:bedrock:us-gov-west-1:1:inference-profile/x", + None, + "https://bedrock-runtime.us-gov-west-1.amazonaws.com/model/arn%3Aaws-us-gov%3Abedrock%3Aus-gov-west-1%3A1%3Ainference-profile%2Fx/invoke", + "us-gov-west-1" + )] + #[case::empty_partition_is_not_an_arn( + "invoke/arn:aws-:bedrock:eu-west-1:1:x", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws-%3Abedrock%3Aeu-west-1%3A1%3Ax/invoke", + "us-west-2" + )] + #[case::uppercase_arn_region_is_ignored( + "invoke/arn:aws:bedrock:EU:1:x", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3AEU%3A1%3Ax/invoke", + "us-west-2" + )] + #[case::nova_spec_is_removed_and_the_id_encoded( + "invoke/us-west-2/nova/anthropic.claude-3", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2%2Fanthropic.claude-3/invoke", + "us-west-2" + )] + #[case::one_inner_invoke_segment_is_removed( + "invoke/us-west-2/invoke/anthropic.claude-3", + None, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us-west-2/anthropic.claude-3/invoke", + "us-west-2" + )] + fn url_and_signing_region_match_python_for_every_model_shape( + #[case] model: &str, + #[case] region_param: Option<&str>, + #[case] expected_url: &str, + #[case] expected_region: &str, + ) { + let litellm_params = region_param.map_or_else(LitellmParams::default, in_region); + let url = AmazonAnthropicClaudeMessagesConfig + .get_complete_url(None, model, &litellm_params, false, &|_| None) + .unwrap(); + let validated = AmazonAnthropicClaudeMessagesConfig + .validate_environment( + Vec::new(), + None, + model, + &MessagesOptionalParams::default(), + &litellm_params, + &|_| None, + ) + .unwrap(); + let AuthScheme::AwsSigV4 { + region, + caller_headers, + .. + } = validated.auth + else { + panic!("expected SigV4, got {:?}", validated.auth); + }; + assert_eq!( + (url.as_str(), region.as_str(), caller_headers), + ( + expected_url, + expected_region, + CallerSigningHeaders::Botocore + ) + ); + } + + #[test] + fn an_arn_url_is_percent_encoded_in_the_arn_region() { + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .get_complete_url( + None, + &format!("bedrock/{ARN}"), + &LitellmParams::default(), + true, + &|_| None + ) + .unwrap(), + format!( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/{ENCODED_ARN}/invoke-with-response-stream" + ) + ); + } + + #[test] + fn the_model_id_param_overrides_the_url_model() { + let litellm_params: LitellmParams = serde_json::from_value(serde_json::json!({ + "model": "bedrock/anthropic.claude-3", + "aws_region_name": "us-west-2", + "model_id": "my/profile", + })) + .unwrap(); + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .get_complete_url(None, "anthropic.claude-3", &litellm_params, false, &|_| { + None + }) + .unwrap(), + "https://bedrock-runtime.us-west-2.amazonaws.com/model/my%2Fprofile/invoke" + ); + } + + #[test] + fn guardrail_and_metadata_headers_are_added_for_both_auth_schemes() { + let litellm_params: LitellmParams = serde_json::from_value(serde_json::json!({ + "model": "bedrock/anthropic.claude-3", + "guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "2", "trace": "enabled"}, + })) + .unwrap(); + let validated = AmazonAnthropicClaudeMessagesConfig + .validate_environment( + Vec::new(), + Some("token"), + "anthropic.claude-3", + &MessagesOptionalParams::default(), + &litellm_params, + &|_| None, + ) + .unwrap(); + assert_eq!( + validated.headers, + vec![ + ( + "X-Amzn-Bedrock-GuardrailIdentifier".to_string(), + "gid".to_string() + ), + ( + "X-Amzn-Bedrock-GuardrailVersion".to_string(), + "2".to_string() + ), + ("X-Amzn-Bedrock-Trace".to_string(), "ENABLED".to_string()), + ] + ); + } + + fn native_context(native_output: NativeOutputSupport) -> MessagesTransformContext { + let mut context = MessagesTransformContext::default(); + context.thinking.capabilities.bedrock.native_output = native_output; + context + } + + fn structured_request() -> MessagesRequest { + serde_json::from_value(serde_json::json!({ + "model": "anthropic.claude-sonnet-5", + "max_tokens": 10, + "messages": [{"role": "user", "content": "hi"}], + "output_config": {"format": {"type": "json_schema", "schema": {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}}}, + })) + .unwrap() + } + + #[rstest] + #[case::native_enabled(NativeOutputSupport { enabled: true, mapped: true, ..NativeOutputSupport::default() }, true)] + #[case::native_opt_in(NativeOutputSupport { opt_in: true, ..NativeOutputSupport::default() }, true)] + #[case::inline_only(NativeOutputSupport::default(), false)] + fn a_native_output_request_carries_an_inline_fallback( + #[case] native_output: NativeOutputSupport, + #[case] expected: bool, + ) { + let fallback = AmazonAnthropicClaudeMessagesConfig + .native_output_fallback(&structured_request(), &native_context(native_output)); + assert_eq!(fallback.is_some(), expected); + if let Some(fallback) = fallback { + let (rebuilt, _) = AmazonAnthropicClaudeMessagesConfig + .finalize_request(structured_request(), Vec::new(), &fallback) + .unwrap(); + let body = serde_json::to_value(rebuilt).unwrap(); + assert_eq!(body.get("output_config"), None); + assert_eq!( + body["messages"][0]["content"][1]["type"], + serde_json::json!("text") + ); + } + } + + #[rstest] + #[case::rejected_native_output(400, "output_config.format: Extra inputs are not permitted", serde_json::json!({"output_config": {"format": {}}}), true)] + #[case::inline_body_is_not_retried(400, "output_config.format: Extra inputs are not permitted", serde_json::json!({"messages": []}), false)] + #[case::other_error(429, "Too many requests", serde_json::json!({"output_config": {"format": {}}}), false)] + fn native_output_rejections_retry_without_it( + #[case] status: u16, + #[case] text: &str, + #[case] body: Value, + #[case] expected: bool, + ) { + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .retries_without_native_output(status, text, &body, "m"), + expected + ); + } + + #[test] + fn the_wire_body_drops_model_and_stream() { + assert_eq!( + AmazonAnthropicClaudeMessagesConfig + .wire_body(serde_json::json!({"model": "m", "stream": true, "max_tokens": 1})), + serde_json::json!({"max_tokens": 1}) + ); + } } diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/betas.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/betas.rs new file mode 100644 index 00000000000..8014c99d673 --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/betas.rs @@ -0,0 +1,702 @@ +use std::collections::BTreeSet; + +use litellm_llms_types::{ + providers::anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet}, + recognized::python_truthy, +}; +use serde_json::Value; + +use crate::{ + Error, ErrorDetail, + anthropic::beta_headers::BetaPolicy, + base_llm::{auth::Headers, messages::context::MessagesModelCapabilities}, +}; + +use super::request::Body; + +const PER_TURN_CONTROL: &str = "per-turn-control-2026-07-01"; +const DANGEROUS_TOOL_USE: &str = "dangerous-tool-use-2026-09-03"; +const TOOL_SEARCH_TOOL: &str = "tool-search-tool-2025-10-19"; +const ADVANCED_TOOL_USE: &str = "advanced-tool-use-2025-11-20"; +const TOOL_EXAMPLES: &str = "tool-examples-2025-10-29"; +const COMPACT_2026_09_04: &str = "compact-2026-09-04"; +const THINKING_BINDING_CONTROLS: &str = "thinking-binding-controls-2026-08-01"; +const THINKING_DISPLAY_UPDATES: &str = "thinking-display-updates-2026-08-18"; +const MID_CONVERSATION_OUTPUT_CONFIG: &str = "mid-conversation-output-config-2026-07-01"; +const MID_CONVERSATION_TOOL_CHANGES: &str = "mid-conversation-tool-changes-2026-07-01"; +const PROGRAMMATIC_CALLER: &str = "code_execution_20250825"; +const TOOL_SEARCH_TYPES: [&str; 2] = [ + "tool_search_tool_regex_20251119", + "tool_search_tool_bm25_20251119", +]; +const CONTEXT_MANAGEMENT_EDIT_BETAS: [(&str, &str); 2] = [ + ("compact_20260112", "compact-2026-01-12"), + ("clear_tool_uses_20250919", "context-management-2025-06-27"), +]; +const COMPUTER_TOOL_BETAS: [(&str, &str); 3] = [ + ("computer_20251124", "computer-use-2025-11-24"), + ("computer_20250124", "computer-use-2025-01-24"), + ("computer_20241022", "computer-use-2024-10-22"), +]; +const OPUS_4_5_EFFORT_PATTERNS: [&str; 2] = ["opus-4-5", "opus_4_5"]; + +fn on_bedrock(beta: &str) -> Option { + beta.parse::() + .unwrap_or_else(|never| match never {}) + .on(BetaProvider::Bedrock) + .map(|beta| beta.as_str().to_string()) +} + +/// Port of Python's `model_specific_anthropic_betas`. +fn model_specific_anthropic_betas( + capabilities: &MessagesModelCapabilities, +) -> BTreeSet<&'static str> { + if capabilities.bedrock.supports_per_turn_control { + BTreeSet::from([PER_TURN_CONTROL]) + } else { + BTreeSet::new() + } +} + +fn header_value<'a>(headers: &'a Headers, name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) +} + +fn user_betas(headers: &Headers) -> BTreeSet { + let Some(header) = header_value(headers, BETA_HEADER).filter(|value| !value.is_empty()) else { + return BTreeSet::new(); + }; + if header.starts_with('[') + && header.ends_with(']') + && let Ok(Value::Array(values)) = serde_json::from_str::(header) + { + return values + .iter() + .map(|value| match value { + Value::String(text) => text.trim().to_string(), + other => other.to_string(), + }) + .filter(|value| !value.is_empty()) + .collect(); + } + header + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .collect() +} + +fn tools(value: Option<&Value>) -> &[Value] { + match value { + Some(Value::Array(tools)) => tools, + _ => &[], + } +} + +fn tool_type(tool: &Value) -> Option<&str> { + tool.get("type").and_then(Value::as_str) +} + +fn lists_programmatic_caller(value: Option<&Value>) -> bool { + matches!(value, Some(Value::Array(callers)) if callers.iter().any(|caller| caller.as_str() == Some(PROGRAMMATIC_CALLER))) +} + +fn programmatic_tool_calling_used(tools: &[Value]) -> bool { + tools.iter().any(|tool| { + lists_programmatic_caller(tool.get("allowed_callers")) + || lists_programmatic_caller( + tool.get("function") + .and_then(|function| function.get("allowed_callers")), + ) + }) +} + +fn non_empty_list(value: Option<&Value>) -> bool { + matches!(value, Some(Value::Array(items)) if !items.is_empty()) +} + +fn input_examples_used(tools: &[Value]) -> bool { + tools.iter().any(|tool| { + non_empty_list(tool.get("input_examples")) + || non_empty_list( + tool.get("function") + .and_then(|function| function.get("input_examples")), + ) + }) +} + +fn eager_input_streaming_flag(tool: &Value) -> Result, Error> { + let Value::Object(tool) = tool else { + return Ok(None); + }; + let strict_bool = |value: Option<&Value>| match value { + None | Some(Value::Null) => Ok(None), + Some(Value::Bool(flag)) => Ok(Some(*flag)), + Some(_) => Err(Error::InvalidRequest(ErrorDetail::Expected { + subject: "eager_input_streaming", + expected: "a boolean", + })), + }; + let top_level = strict_bool(tool.get("eager_input_streaming"))?; + let function = match tool.get("function") { + None | Some(Value::Null) => None, + Some(Value::Object(function)) => strict_bool(function.get("eager_input_streaming"))?, + Some(_) => { + return Err(Error::InvalidRequest(ErrorDetail::Expected { + subject: "eager_input_streaming", + expected: "a boolean", + })); + } + }; + Ok(top_level.or(function)) +} + +fn eager_input_streaming_used(tools: &[Value]) -> Result { + tools.iter().try_fold(false, |used, tool| { + Ok(eager_input_streaming_flag(tool)? == Some(true) || used) + }) +} + +fn effort_used(params: &Body, model: &str, capabilities: &MessagesModelCapabilities) -> bool { + if params.is_empty() || capabilities.supports_adaptive_thinking { + return false; + } + let lowered = model.to_lowercase(); + let opus_4_5_effort = OPUS_4_5_EFFORT_PATTERNS + .iter() + .any(|pattern| lowered.contains(pattern)) + && params + .get("reasoning_effort") + .and_then(Value::as_str) + .is_some_and(|effort| !effort.is_empty()); + let output_config_effort = params + .get("output_config") + .and_then(|output_config| output_config.get("effort")) + .and_then(Value::as_str) + .is_some_and(|effort| !effort.is_empty()); + opus_4_5_effort || output_config_effort +} + +fn system_messages(messages: &[Value]) -> impl Iterator { + messages + .iter() + .filter(|message| message.get("role").and_then(Value::as_str) == Some("system")) +} + +fn mid_conversation_tool_change_used(messages: &[Value]) -> bool { + system_messages(messages).any(|message| { + tools(message.get("content")).iter().any(|block| { + matches!(tool_type(block), Some("tool_addition" | "tool_removal")) + && block.get("tool").and_then(tool_type) == Some("tool_reference") + }) + }) +} + +fn thinking_betas(thinking: Option<&Value>) -> impl Iterator { + let display = super::request::is_thinking_display_updates_used(thinking) + .then_some(THINKING_DISPLAY_UPDATES); + let binding = + matches!(thinking, Some(Value::Object(thinking)) if thinking.contains_key("block_binding")) + .then_some(THINKING_BINDING_CONTROLS); + display.into_iter().chain(binding) +} + +fn carries_signed_compaction_block(messages: &[Value]) -> bool { + messages + .iter() + .flat_map(|message| tools(message.get("content"))) + .any(|block| { + tool_type(block) == Some("compaction") + && block + .get("signature") + .and_then(Value::as_str) + .is_some_and(|signature| !signature.is_empty()) + }) +} + +/// Port of Python's `_filter_context_management_for_bedrock_invoke`. +fn filter_context_management_for_bedrock_invoke(body: &mut Body, betas: &mut BTreeSet) { + let Some(Value::Object(context_management)) = body.get("context_management") else { + return; + }; + let Some(Value::Array(edits)) = context_management.get("edits") else { + body.shift_remove("context_management"); + return; + }; + let retained: Vec<(Value, &'static str)> = edits + .iter() + .filter_map(|edit| { + let edit_type = edit.as_object()?.get("type")?.as_str()?; + CONTEXT_MANAGEMENT_EDIT_BETAS + .iter() + .find(|(supported, _)| *supported == edit_type) + .map(|(_, beta)| (edit.clone(), *beta)) + }) + .collect(); + if retained.is_empty() { + body.shift_remove("context_management"); + return; + } + let mut filtered = context_management.clone(); + betas.extend(retained.iter().map(|(_, beta)| (*beta).to_string())); + filtered.insert( + "edits".into(), + Value::Array(retained.into_iter().map(|(edit, _)| edit).collect()), + ); + body.insert("context_management".into(), Value::Object(filtered)); +} + +/// Port of the handler's `update_headers_with_filtered_beta`, keeping +/// `model_specific_anthropic_betas`. +fn update_headers_with_filtered_beta(headers: Headers, keep: &BTreeSet<&'static str>) -> Headers { + BetaPolicy::Filter(BetaProvider::Bedrock).apply_keeping( + headers, + &keep + .iter() + .map(|beta| { + beta.parse::() + .unwrap_or_else(|never| match never {}) + }) + .collect::(), + ) +} + +/// Port of Python's `AnthropicModelInfo.get_anthropic_beta_list` for `custom_llm_provider="bedrock"`. +fn get_anthropic_beta_list( + body: &Body, + params: &Body, + model: &str, + capabilities: &MessagesModelCapabilities, + outgoing_tools: &[Value], + messages: &[Value], +) -> BTreeSet { + let mut betas = BTreeSet::new(); + if effort_used(params, model, capabilities) { + betas.insert("effort-2025-11-24".into()); + } + if let Some(beta) = outgoing_tools + .iter() + .find_map(|tool| tool_type(tool).filter(|kind| kind.starts_with("computer_"))) + .and_then(|kind| COMPUTER_TOOL_BETAS.iter().find(|(tool, _)| *tool == kind)) + .map(|(_, beta)| *beta) + { + betas.insert(beta.into()); + } + if params.get("mcp_servers").is_some_and(python_truthy) { + betas.insert("mcp-client-2025-04-04".into()); + } + if messages.iter().any(|message| { + message + .as_object() + .is_some_and(|message| message.contains_key("output_config")) + }) { + betas.insert(MID_CONVERSATION_OUTPUT_CONFIG.into()); + } + betas.extend(thinking_betas(body.get("thinking")).map(str::to_string)); + if mid_conversation_tool_change_used(messages) { + betas.insert(MID_CONVERSATION_TOOL_CHANGES.into()); + } + betas +} + +/// Port of Python's `_add_compaction_beta_for_bedrock_invoke`. +fn add_compaction_beta_for_bedrock_invoke( + body: &Body, + messages: &[Value], + betas: &mut BTreeSet, +) { + if body.get("compaction").is_some_and(Value::is_object) + || carries_signed_compaction_block(messages) + { + betas.insert(COMPACT_2026_09_04.into()); + } +} + +/// Port of Python's `_get_tool_search_beta_header_for_bedrock`. +fn get_tool_search_beta_header_for_bedrock( + capabilities: &MessagesModelCapabilities, + tool_search_used: bool, + programmatic_tool_calling_used: bool, + input_examples_used: bool, + betas: &mut BTreeSet, +) { + if tool_search_used && !(programmatic_tool_calling_used || input_examples_used) { + betas.remove(ADVANCED_TOOL_USE); + if capabilities.bedrock.supports_tool_search { + betas.insert(TOOL_SEARCH_TOOL.into()); + } + } +} + +/// Port of Python's `_get_bedrock_invoke_anthropic_beta_headers`. +pub(super) fn get_bedrock_invoke_anthropic_beta_headers( + body: &mut Body, + params: &Body, + headers: Headers, + model: &str, + capabilities: &MessagesModelCapabilities, + injected_thinking: bool, +) -> Result<(Vec, Headers), Error> { + let keep = model_specific_anthropic_betas(capabilities); + let headers = update_headers_with_filtered_beta(headers, &keep); + let user = user_betas(&headers); + let mut betas = user.clone(); + let outgoing_tools = tools(body.get("tools")).to_vec(); + let messages = tools(body.get("messages")).to_vec(); + betas.extend(get_anthropic_beta_list( + body, + params, + model, + capabilities, + &outgoing_tools, + &messages, + )); + if injected_thinking { + betas.insert("interleaved-thinking-2025-05-14".into()); + } + if eager_input_streaming_used(&outgoing_tools)? { + betas.insert("fine-grained-tool-streaming-2025-05-14".into()); + } + if params + .get("safeguards") + .is_some_and(|safeguards| !safeguards.is_null()) + { + betas.insert(DANGEROUS_TOOL_USE.into()); + } else { + betas.remove(DANGEROUS_TOOL_USE); + } + filter_context_management_for_bedrock_invoke(body, &mut betas); + add_compaction_beta_for_bedrock_invoke(body, &messages, &mut betas); + let tool_search_used = outgoing_tools + .iter() + .any(|tool| tool_type(tool).is_some_and(|kind| TOOL_SEARCH_TYPES.contains(&kind))); + get_tool_search_beta_header_for_bedrock( + capabilities, + tool_search_used, + programmatic_tool_calling_used(&outgoing_tools), + input_examples_used(&outgoing_tools), + &mut betas, + ); + if betas.contains(TOOL_SEARCH_TOOL) { + betas.insert(TOOL_EXAMPLES.into()); + } + let model_supported = user.iter().filter(|beta| keep.contains(beta.as_str())); + let filtered: BTreeSet = betas + .iter() + .filter_map(|beta| on_bedrock(beta)) + .chain(model_supported.cloned()) + .collect(); + let dropped: Vec<&String> = user + .iter() + .filter(|beta| !keep.contains(beta.as_str()) && on_bedrock(beta).is_none()) + .collect(); + if !dropped.is_empty() { + litellm_tracing::warn!( + "Bedrock Invoke: dropping unsupported anthropic-beta values from client headers: {dropped:?}. Bedrock has no mapping entry for these; forwarding them would cause a 400." + ); + } + Ok((filtered.into_iter().collect(), headers)) +} + +#[cfg(test)] +mod tests { + use litellm_llms_types::formats::messages::MessagesRequest; + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::{ + base_llm::messages::{ + context::{BedrockCapabilities, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, + bedrock::messages::invoke_transformations::anthropic_claude_transformation::AmazonAnthropicClaudeMessagesConfig, + }; + + fn sent( + params: Value, + capabilities: MessagesModelCapabilities, + header: Option<&str>, + drop_params: bool, + ) -> (Value, Headers) { + let mut request = json!({ + "model": "anthropic.claude-3-haiku-20240307-v1:0", + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + "max_tokens": 128, + }); + request + .as_object_mut() + .unwrap() + .extend(params.as_object().unwrap().clone()); + let request: MessagesRequest = serde_json::from_value(request).unwrap(); + let context = MessagesTransformContext { + drops_unsupported_params: drop_params, + ..MessagesTransformContext::new(capabilities, drop_params) + }; + let headers = header + .map(|value| vec![("anthropic-beta".to_string(), value.to_string())]) + .unwrap_or_default(); + let (request, headers) = AmazonAnthropicClaudeMessagesConfig + .finalize_request(request, headers, &context) + .unwrap(); + (serde_json::to_value(request).unwrap(), headers) + } + + fn betas( + params: Value, + capabilities: MessagesModelCapabilities, + header: Option<&str>, + ) -> Value { + sent(params, capabilities, header, false) + .0 + .get("anthropic_beta") + .cloned() + .unwrap_or(Value::Null) + } + + fn bedrock(bedrock: BedrockCapabilities) -> MessagesModelCapabilities { + MessagesModelCapabilities { + bedrock, + ..MessagesModelCapabilities::default() + } + } + + #[rstest] + #[case::unlisted_beta_dropped("advisor-tool-2026-03-01,context-1m-2025-08-07", json!(["context-1m-2025-08-07"]))] + #[case::aliased_beta_renamed("advanced-tool-use-2025-11-20", json!(["tool-search-tool-2025-10-19"]))] + #[case::unknown_beta_dropped("not-a-real-beta-2099-01-01", Value::Null)] + #[case::dangerous_tool_use_dropped_without_safeguards( + "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14", + Value::Null + )] + fn user_betas_go_through_the_bedrock_column(#[case] header: &str, #[case] expected: Value) { + assert_eq!( + betas( + json!({}), + MessagesModelCapabilities::default(), + Some(header) + ), + expected + ); + } + + #[rstest] + #[case::client_sends_beta("dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14")] + #[case::client_omits_beta("interleaved-thinking-2025-05-14")] + fn safeguards_carry_the_dangerous_tool_use_beta(#[case] header: &str) { + let (body, _) = sent( + json!({"safeguards": {"mode": "block"}}), + MessagesModelCapabilities::default(), + Some(header), + false, + ); + assert_eq!( + body["anthropic_beta"], + json!(["dangerous-tool-use-2026-09-03"]) + ); + assert_eq!(body["safeguards"], json!({"mode": "block"})); + } + + #[rstest] + #[case::flagged(true, json!(["per-turn-control-2026-07-01"]), Some("per-turn-control-2026-07-01"))] + #[case::not_flagged(false, Value::Null, None)] + fn per_turn_control_is_kept_only_where_the_model_flags_it( + #[case] supports_per_turn_control: bool, + #[case] expected: Value, + #[case] header: Option<&str>, + ) { + let (body, headers) = sent( + json!({}), + bedrock(BedrockCapabilities { + supports_per_turn_control, + ..BedrockCapabilities::default() + }), + Some("per-turn-control-2026-07-01"), + false, + ); + assert_eq!( + body.get("anthropic_beta").cloned().unwrap_or(Value::Null), + expected + ); + assert_eq!( + headers + .iter() + .find(|(name, _)| name == "anthropic-beta") + .map(|(_, value)| value.as_str()), + header + ); + } + + #[rstest] + #[case::supported(true, json!(["tool-search-tool-2025-10-19"]))] + #[case::unsupported(false, Value::Null)] + fn tool_search_beta_follows_the_model_map( + #[case] supports_tool_search: bool, + #[case] expected: Value, + ) { + assert_eq!( + betas( + json!({"tools": [{"type": "tool_search_tool_regex_20251119", "name": "tool_search"}]}), + bedrock(BedrockCapabilities { + supports_tool_search, + ..BedrockCapabilities::default() + }), + None, + ), + expected + ); + } + + #[test] + fn tool_search_with_programmatic_calling_keeps_the_users_beta() { + assert_eq!( + betas( + json!({"tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search"}, + {"name": "t", "input_schema": {"type": "object"}, "allowed_callers": ["code_execution_20250825"]}, + ]}), + MessagesModelCapabilities::default(), + Some("advanced-tool-use-2025-11-20"), + ), + json!(["tool-search-tool-2025-10-19"]) + ); + } + + #[rstest] + #[case::computer_20250124("computer_20250124", json!(["computer-use-2025-01-24"]))] + #[case::computer_20251124("computer_20251124", json!(["computer-use-2025-11-24"]))] + #[case::computer_20241022_is_rejected("computer_20241022", Value::Null)] + fn computer_tools_add_their_beta(#[case] tool_type: &str, #[case] expected: Value) { + assert_eq!( + betas( + json!({"tools": [{"type": tool_type, "name": "computer", "display_width_px": 1, "display_height_px": 1}]}), + MessagesModelCapabilities::default(), + None, + ), + expected + ); + } + + #[test] + fn output_config_effort_adds_the_effort_beta_on_non_adaptive_models() { + assert_eq!( + betas( + json!({"output_config": {"effort": "high"}}), + MessagesModelCapabilities { + supports_output_config: true, + ..MessagesModelCapabilities::default() + }, + None, + ), + json!(["effort-2025-11-24"]) + ); + } + + #[rstest] + #[case::display_updates(json!({"type": "adaptive", "display": "updates"}), json!(["thinking-display-updates-2026-08-18"]))] + #[case::block_binding(json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}), json!(["thinking-binding-controls-2026-08-01"]))] + fn thinking_controls_add_their_betas(#[case] thinking: Value, #[case] expected: Value) { + let capabilities = MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + ..MessagesModelCapabilities::default() + }; + assert_eq!( + betas(json!({"thinking": thinking}), capabilities, None), + expected + ); + } + + #[test] + fn mid_conversation_output_config_keeps_its_auto_added_beta() { + let (body, _) = sent( + json!({"messages": [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]}, + {"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}}, + {"role": "user", "content": [{"type": "text", "text": "Go on"}]}, + ]}), + MessagesModelCapabilities { + supports_mid_conversation_system: true, + ..MessagesModelCapabilities::default() + }, + None, + true, + ); + assert_eq!( + body["anthropic_beta"], + json!(["mid-conversation-output-config-2026-07-01"]) + ); + assert_eq!( + body["messages"][2]["output_config"], + json!({"effort": "high"}) + ); + } + + #[test] + fn mid_conversation_tool_changes_add_their_beta() { + let (body, _) = sent( + json!({"messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "tool_name": "x"}}]}, + {"role": "user", "content": "Go on"}, + ]}), + MessagesModelCapabilities { + supports_mid_conversation_system: true, + ..MessagesModelCapabilities::default() + }, + None, + true, + ); + assert_eq!( + body["anthropic_beta"], + json!(["mid-conversation-tool-changes-2026-07-01"]) + ); + } + + #[rstest] + #[case::clear_at_stripped( + json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}), + true, + json!([ + {"role": "user", "content": "a"}, + {"role": "system", "content": "note"}, + {"role": "user", "content": "b"} + ]) + )] + #[case::emptied_turn_dropped( + json!({"role": "system", "content": "", "clear_at": {"type": "turns"}}), + true, + json!([{"role": "user", "content": "a"}, {"role": "user", "content": "b"}]) + )] + #[case::kept_without_drop_params( + json!({"role": "system", "content": "note", "clear_at": {"type": "turns"}}), + false, + json!([ + {"role": "user", "content": "a"}, + {"role": "system", "content": "note", "clear_at": {"type": "turns"}}, + {"role": "user", "content": "b"} + ]) + )] + fn a_system_field_the_body_betas_do_not_unlock_is_stripped_under_drop_params( + #[case] system: Value, + #[case] drop_params: bool, + #[case] expected: Value, + ) { + let (body, _) = sent( + json!({"messages": [{"role": "user", "content": "a"}, system, {"role": "user", "content": "b"}]}), + MessagesModelCapabilities { + supports_mid_conversation_system: true, + ..MessagesModelCapabilities::default() + }, + None, + drop_params, + ); + assert_eq!(body["messages"], expected); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs index 68ceaac3444..333753b6a5c 100644 --- a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs @@ -1 +1,4 @@ pub mod anthropic_claude_transformation; +mod betas; +pub mod request; +mod stream; diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/request.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/request.rs new file mode 100644 index 00000000000..d357bfd2a5d --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/request.rs @@ -0,0 +1,1088 @@ +use litellm_llms_types::{ + formats::messages::{Message, MessagesOptionalParams, MessagesRequest, ThinkingConfig}, + providers::anthropic::{AnthropicBeta, BetaProvider}, + recognized::Recognized, +}; +use serde_json::{Map, Value, json}; + +use crate::{ + Error, ErrorDetail, + anthropic::{ + beta_gated_fields::strip_beta_gated_request, common_utils::supports_effort_param, + messages::transformation::transform_messages_request, + }, + base_llm::{ + auth::Headers, + messages::{ + context::{MessagesModelCapabilities, MessagesTransformContext, ThinkingBudgets}, + normalization::normalize_system_role_messages, + }, + }, +}; + +use crate::bedrock::common_utils::{ + clamp_effort, ensure_bedrock_anthropic_messages_tool_names, is_int, + normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, + normalize_tool_input_schema_types_for_bedrock_invoke, + strip_unsupported_bedrock_invoke_output_config_keys, + structured_output::apply_bedrock_invoke_structured_output, tools_without_eager_input_streaming, +}; + +use super::betas::get_bedrock_invoke_anthropic_beta_headers; + +const BEDROCK_INVOKE_LOG_PROVIDER: &str = "bedrock_invoke"; + +pub(super) type Body = Map; + +pub const DEFAULT_BEDROCK_ANTHROPIC_API_VERSION: &str = "bedrock-2023-05-31"; + +/// Port of Python's `BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS`. +const BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS: [&str; 17] = [ + "anthropic_version", + "max_tokens", + "messages", + "anthropic_beta", + "system", + "stop_sequences", + "temperature", + "top_p", + "top_k", + "tools", + "tool_choice", + "thinking", + "metadata", + "output_config", + "safeguards", + "context_management", + "compaction", +]; + +const ADAPTIVE_THINKING_KEYS: [&str; 3] = ["type", "display", "block_binding"]; + +const EXTENDED_THINKING_PATTERNS: [&str; 12] = [ + "opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5", "opus-4.1", "opus_4.1", "opus-4-1", "opus_4_1", + "opus-4", "opus_4", "sonnet-4", "sonnet_4", +]; + +const CLEAR_THINKING_EDIT: &str = "clear_thinking_20251015"; + +fn to_body(request: &MessagesRequest) -> Result { + match serde_json::to_value(request) { + Ok(Value::Object(body)) => Ok(body), + Ok(_) => Err(Error::InvalidRequest(ErrorDetail::Expected { + subject: "Bedrock invoke request", + expected: "an object", + })), + Err(error) => Err(Error::InvalidRequest(ErrorDetail::failed( + "Bedrock invoke request serialization", + error, + ))), + } +} + +fn from_body(body: Body) -> Result { + serde_json::from_value(Value::Object(body)).map_err(|error| { + Error::InvalidRequest(ErrorDetail::invalid("Bedrock invoke request", error)) + }) +} + +/// Port of Python's `_get_bedrock_output_config_effort_ceiling`. +fn get_bedrock_output_config_effort_ceiling( + capabilities: &MessagesModelCapabilities, +) -> Option { + capabilities + .bedrock + .output_config_effort_ceiling + .map(|level| <&str>::from(level).to_string()) +} + +/// Port of Python's `_clamp_adaptive_reasoning_effort_for_bedrock`. +fn clamp_adaptive_reasoning_effort_for_bedrock( + request: MessagesRequest, + capabilities: &MessagesModelCapabilities, +) -> Result { + if !capabilities.supports_adaptive_thinking { + return Ok(request); + } + let mut body = to_body(&request)?; + let Some(clamped) = body + .get("reasoning_effort") + .and_then(Value::as_str) + .and_then(|effort| { + clamp_effort( + effort, + get_bedrock_output_config_effort_ceiling(capabilities).as_deref(), + ) + }) + else { + return Ok(request); + }; + body.insert("reasoning_effort".into(), Value::String(clamped)); + from_body(body) +} + +/// Port of Python's `AnthropicModelInfo.is_thinking_display_updates_used`. +pub(super) fn is_thinking_display_updates_used(thinking: Option<&Value>) -> bool { + let Some(Value::Object(thinking)) = thinking else { + return false; + }; + matches!( + thinking.get("type").and_then(Value::as_str), + Some("adaptive" | "enabled") + ) && thinking.get("display").and_then(Value::as_str) == Some("updates") +} + +/// Port of Python's `carried_thinking_keys`. +fn carried_thinking_keys(thinking: &Body, rebuilt: &[&str], drops_unknown: bool) -> Body { + thinking + .iter() + .filter(|(key, _)| !rebuilt.contains(&key.as_str())) + .filter(|(key, _)| ADAPTIVE_THINKING_KEYS.contains(&key.as_str()) || !drops_unknown) + .map(|(key, value)| (key.clone(), value.clone())) + .collect() +} + +fn readd_display_updates(body: &mut Body, drops_unknown: bool) { + let Some(Value::Object(thinking)) = body.get("thinking") else { + return; + }; + if thinking.get("type").and_then(Value::as_str) != Some("adaptive") { + return; + } + let mut rebuilt = carried_thinking_keys(thinking, &["display"], drops_unknown); + rebuilt.insert("display".into(), Value::String("updates".into())); + body.insert("thinking".into(), Value::Object(rebuilt)); +} + +/// Python's `strip_beta_gated_native_request` over the final body, whose `anthropic_beta` list +/// is the only one this leg sends. +fn strip_beta_gated_body(body: &mut Body, model: &str, betas: &[String]) -> Result<(), Error> { + let invalid = + |error| Error::InvalidRequest(ErrorDetail::invalid("Bedrock invoke request", error)); + let thinking: Option> = body + .get("thinking") + .cloned() + .map(serde_json::from_value) + .transpose() + .map_err(invalid)?; + let messages: Vec> = body + .get("messages") + .cloned() + .map(serde_json::from_value) + .transpose() + .map_err(invalid)? + .unwrap_or_default(); + let stripped = strip_beta_gated_request( + MessagesRequest { + model: model.to_string(), + messages, + params: MessagesOptionalParams { + thinking, + ..MessagesOptionalParams::default() + }, + }, + &betas + .iter() + .map(|beta| { + beta.parse::() + .unwrap_or_else(|never| match never {}) + }) + .collect(), + true, + BEDROCK_INVOKE_LOG_PROVIDER, + ); + let failed = |error| { + Error::InvalidRequest(ErrorDetail::failed( + "Bedrock invoke request serialization", + error, + )) + }; + if let Some(thinking) = stripped.params.thinking { + body.insert( + "thinking".into(), + serde_json::to_value(thinking).map_err(failed)?, + ); + } + if body.contains_key("messages") { + body.insert( + "messages".into(), + serde_json::to_value(stripped.messages).map_err(failed)?, + ); + } + Ok(()) +} + +/// Port of Python's `_reject_unsupported_web_search_tools`. +fn reject_unsupported_web_search_tools(body: &Body, model: &str) -> Result<(), Error> { + let Some(Value::Array(tools)) = body.get("tools") else { + return Ok(()); + }; + let web_search = tools.iter().filter_map(Value::as_object).find_map(|tool| { + tool.get("type") + .and_then(Value::as_str) + .filter(|tool_type| tool_type.starts_with("web_search_") && *tool_type != "function") + }); + match web_search { + Some(tool_type) => Err(Error::InvalidRequest( + ErrorDetail::UnsupportedBedrockWebSearchTool { + tool_type: tool_type.to_string(), + model: model.to_string(), + }, + )), + None => Ok(()), + } +} + +/// Port of Python's `_supports_extended_thinking_on_bedrock`. +pub(super) fn supports_extended_thinking_on_bedrock( + model: &str, + capabilities: &MessagesModelCapabilities, +) -> bool { + let lowered = model.to_lowercase(); + capabilities.supports_adaptive_thinking + || EXTENDED_THINKING_PATTERNS + .iter() + .any(|pattern| lowered.contains(pattern)) +} + +/// Port of Python's `_effort_from_thinking_budget`. +fn effort_from_thinking_budget(budget: i128, budgets: &ThinkingBudgets) -> &'static str { + if budget >= i128::from(budgets.xhigh) { + "xhigh" + } else if budget >= i128::from(budgets.high) { + "high" + } else if budget >= i128::from(budgets.medium) { + "medium" + } else { + "low" + } +} + +/// Port of Python's `_resolve_clear_thinking_budget_tokens`. +fn resolve_clear_thinking_budget_tokens( + value: Option<&Value>, + minimum: u64, +) -> Result { + match value { + None | Some(Value::Null) => Ok(i128::from(minimum)), + Some(Value::Bool(flag)) => Ok(i128::from(*flag)), + Some(Value::Number(number)) => number + .as_i64() + .map(i128::from) + .or_else(|| number.as_u64().map(i128::from)) + .or_else(|| { + number + .as_f64() + .filter(|value| value.is_finite()) + .map(|value| value.trunc() as i128) + }) + .ok_or_else(|| { + Error::InvalidRequest(ErrorDetail::Expected { + subject: "thinking.budget_tokens", + expected: "a finite number", + }) + }), + Some(_) => Err(Error::InvalidRequest(ErrorDetail::Expected { + subject: "thinking.budget_tokens", + expected: "an integer", + })), + } +} + +/// Port of Python's `_inject_adaptive_thinking_for_clear_thinking`. +fn inject_adaptive_thinking_for_clear_thinking( + body: &mut Body, + budget: i128, + budgets: &ThinkingBudgets, + drops_unknown: bool, +) { + let mut output_config = match body.get("output_config") { + Some(Value::Object(output_config)) => output_config.clone(), + _ => Map::new(), + }; + output_config + .entry("effort") + .or_insert_with(|| Value::String(effort_from_thinking_budget(budget, budgets).into())); + let thinking = match body.get("thinking") { + Some(Value::Object(thinking)) => thinking.clone(), + _ => Map::new(), + }; + let mut rebuilt = carried_thinking_keys(&thinking, &["type", "budget_tokens"], drops_unknown); + rebuilt.insert("type".into(), Value::String("adaptive".into())); + body.insert("output_config".into(), Value::Object(output_config)); + body.insert("thinking".into(), Value::Object(rebuilt)); +} + +/// Port of Python's `_ensure_thinking_for_clear_thinking_context_management`. +fn ensure_thinking_for_clear_thinking_context_management( + body: &mut Body, + model: &str, + capabilities: &MessagesModelCapabilities, + budgets: &ThinkingBudgets, + drops_unknown: bool, +) -> Result { + let Some(Value::Object(context_management)) = body.get("context_management") else { + return Ok(false); + }; + let Some(Value::Array(edits)) = context_management.get("edits") else { + return Ok(false); + }; + let needs_thinking = edits + .iter() + .any(|edit| edit.get("type").and_then(Value::as_str) == Some(CLEAR_THINKING_EDIT)); + if !needs_thinking || !supports_extended_thinking_on_bedrock(model, capabilities) { + return Ok(false); + } + let adaptive = capabilities.supports_adaptive_thinking; + if let Some(Value::Object(thinking)) = body.get("thinking") { + match thinking.get("type").and_then(Value::as_str) { + Some("adaptive") => return Ok(false), + Some("enabled") if !adaptive => return Ok(false), + Some("enabled") => { + let budget = resolve_clear_thinking_budget_tokens( + thinking.get("budget_tokens"), + budgets.bedrock_min, + )?; + inject_adaptive_thinking_for_clear_thinking(body, budget, budgets, drops_unknown); + return Ok(true); + } + _ => {} + } + } + let minimum = budgets.bedrock_min; + let too_small_max_tokens = body + .get("max_tokens") + .filter(|value| is_int(value)) + .and_then(|value| { + value + .as_i64() + .map(i128::from) + .or_else(|| value.as_u64().map(i128::from)) + }) + .filter(|max_tokens| *max_tokens <= i128::from(minimum)); + if let Some(max_tokens) = too_small_max_tokens { + litellm_tracing::warn!( + "Bedrock clear_thinking_20251015: max_tokens={max_tokens} is not greater than minimum thinking budget ({minimum}); cannot inject thinking safely" + ); + return Ok(false); + } + if adaptive { + inject_adaptive_thinking_for_clear_thinking( + body, + i128::from(minimum), + budgets, + drops_unknown, + ); + } else { + body.insert( + "thinking".into(), + json!({"type": "enabled", "budget_tokens": minimum}), + ); + } + Ok(true) +} + +/// Port of Python's `_remove_ttl_from_cache_control._sanitize_cache_control`. +fn sanitize_cache_control(cache_control: &mut Value, supports_ttl: bool) { + let Value::Object(cache_control) = cache_control else { + return; + }; + cache_control.shift_remove("scope"); + let keeps_ttl = supports_ttl + && matches!( + cache_control.get("ttl").and_then(Value::as_str), + Some("5m" | "1h") + ); + if !keeps_ttl { + cache_control.shift_remove("ttl"); + } +} + +/// Port of Python's `_remove_ttl_from_cache_control._process_content_list`. +fn process_content_list(blocks: &mut [Value], supports_ttl: bool) { + blocks + .iter_mut() + .filter_map(|block| block.get_mut("cache_control")) + .for_each(|cache_control| sanitize_cache_control(cache_control, supports_ttl)); +} + +/// Port of Python's `_remove_ttl_from_cache_control`. +fn remove_ttl_from_cache_control(body: &mut Body, supports_ttl: bool) { + if let Some(Value::Array(tools)) = body.get_mut("tools") { + process_content_list(tools, supports_ttl); + } + if let Some(Value::Array(system)) = body.get_mut("system") { + process_content_list(system, supports_ttl); + } + if let Some(Value::Array(messages)) = body.get_mut("messages") { + messages + .iter_mut() + .filter_map(|message| message.get_mut("content")) + .filter_map(Value::as_array_mut) + .for_each(|content| process_content_list(content, supports_ttl)); + } +} + +fn drop_compaction_without_beta(body: &mut Body, betas: &[String]) { + if body.contains_key("compaction") + && !(body.get("compaction").is_some_and(Value::is_object) + && betas.iter().any(|beta| beta == "compact-2026-09-04")) + { + litellm_tracing::debug!( + "Bedrock Invoke: dropping `compaction`; compact-2026-09-04 is not served on this route" + ); + body.shift_remove("compaction"); + } +} + +fn pop_output_config_under_global_drop( + body: &mut Body, + global_drop_params: bool, + supports_effort: bool, +) { + let has_non_format_keys = matches!( + body.get("output_config"), + Some(Value::Object(output_config)) if output_config.keys().any(|key| key != "format") + ); + if global_drop_params && has_non_format_keys && !supports_effort { + litellm_tracing::warn!( + "Bedrock Invoke: dropping `output_config` the model does not accept" + ); + body.shift_remove("output_config"); + } +} + +/// Port of Python's `_strip_unsupported_bedrock_invoke_fields`. +fn strip_unsupported_bedrock_invoke_fields(body: Body) -> Body { + let stripped: Vec<&String> = body + .keys() + .filter(|key| !BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS.contains(&key.as_str())) + .collect(); + if !stripped.is_empty() { + litellm_tracing::debug!( + "Bedrock Invoke: stripping unsupported top-level request fields: {stripped:?}" + ); + } + body.into_iter() + .filter(|(key, _)| BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS.contains(&key.as_str())) + .collect() +} + +/// Port of Python's `AmazonAnthropicClaudeMessagesConfig.transform_anthropic_messages_request`. +pub(super) fn transform_anthropic_messages_request( + request: MessagesRequest, + context: &MessagesTransformContext, + headers: Headers, +) -> Result<(MessagesRequest, Headers), Error> { + let model = request.model.clone(); + let stream = request.params.stream.clone(); + let capabilities = &context.thinking.capabilities; + let drops_unknown_thinking = context.drop_params || context.global_drop_params; + let requested_display_updates = is_thinking_display_updates_used( + serde_json::to_value(&request.params.thinking).ok().as_ref(), + ); + let request = clamp_adaptive_reasoning_effort_for_bedrock(request, capabilities)?; + let request = normalize_system_role_messages( + transform_messages_request(request, context)?, + capabilities.supports_mid_conversation_system, + context.drops_unsupported_params, + BetaProvider::Bedrock, + ); + let mut body = to_body(&request)?; + let shared_params = body.clone(); + if requested_display_updates { + readd_display_updates(&mut body, drops_unknown_thinking); + } + reject_unsupported_web_search_tools(&body, &model)?; + body.entry("anthropic_version") + .or_insert_with(|| Value::String(DEFAULT_BEDROCK_ANTHROPIC_API_VERSION.into())); + body.shift_remove("stream"); + body.shift_remove("model"); + let injected_thinking = ensure_thinking_for_clear_thinking_context_management( + &mut body, + &model, + capabilities, + &context.thinking.budgets, + drops_unknown_thinking, + )?; + remove_ttl_from_cache_control(&mut body, capabilities.bedrock.supports_prompt_cache_ttl); + apply_bedrock_invoke_structured_output(&mut body, &model, capabilities.bedrock.native_output)?; + let ceiling = get_bedrock_output_config_effort_ceiling(capabilities); + normalize_bedrock_opus_output_config_effort(body.get_mut("output_config"), ceiling.as_deref()); + let supports_effort = supports_effort_param(capabilities); + strip_unsupported_bedrock_invoke_output_config_keys( + &model, + &mut body, + capabilities.bedrock.supports_output_config || supports_effort, + ); + normalize_custom_field_on_tools(&mut body); + normalize_tool_input_schema_types_for_bedrock_invoke(&mut body); + ensure_bedrock_anthropic_messages_tool_names(&mut body); + let (betas, headers) = get_bedrock_invoke_anthropic_beta_headers( + &mut body, + &shared_params, + headers, + &model, + capabilities, + injected_thinking, + )?; + if !betas.is_empty() { + body.insert( + "anthropic_beta".into(), + Value::Array(betas.iter().cloned().map(Value::String).collect()), + ); + } + drop_compaction_without_beta(&mut body, &betas); + if context.drops_unsupported_params { + strip_beta_gated_body(&mut body, &model, &betas)?; + } + tools_without_eager_input_streaming(&mut body); + pop_output_config_under_global_drop(&mut body, context.global_drop_params, supports_effort); + let mut body = strip_unsupported_bedrock_invoke_fields(body); + body.insert("model".into(), Value::String(model)); + if let Some(stream) = stream { + body.insert( + "stream".into(), + serde_json::to_value(stream).unwrap_or(Value::Null), + ); + } + Ok((from_body(body)?, headers)) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::*; + use crate::{ + base_llm::messages::{ + context::{BedrockCapabilities, NativeOutputSupport, SupportedEffortTiers}, + transformation::BaseMessagesConfig, + }, + bedrock::messages::invoke_transformations::anthropic_claude_transformation::AmazonAnthropicClaudeMessagesConfig, + }; + use litellm_llms_types::formats::messages::EffortLevel; + + fn body(value: Value) -> Body { + value.as_object().unwrap().clone() + } + + fn adaptive() -> MessagesModelCapabilities { + MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { + low: true, + medium: true, + high: true, + max: true, + xhigh: true, + ..SupportedEffortTiers::default() + }, + ..MessagesModelCapabilities::default() + } + } + + fn with_bedrock( + capabilities: MessagesModelCapabilities, + bedrock: BedrockCapabilities, + ) -> MessagesModelCapabilities { + MessagesModelCapabilities { + bedrock, + ..capabilities + } + } + + fn context( + capabilities: MessagesModelCapabilities, + drop_params: bool, + ) -> MessagesTransformContext { + MessagesTransformContext { + drops_unsupported_params: drop_params, + ..MessagesTransformContext::new(capabilities, drop_params) + } + } + + fn transformed_with( + request: Value, + context: &MessagesTransformContext, + headers: Headers, + ) -> Result<(Value, Headers), Error> { + let request: MessagesRequest = serde_json::from_value(request).unwrap(); + let (request, headers) = + AmazonAnthropicClaudeMessagesConfig.finalize_request(request, headers, context)?; + Ok(( + AmazonAnthropicClaudeMessagesConfig.wire_body(serde_json::to_value(&request).unwrap()), + headers, + )) + } + + fn rejection(error: Error) -> String { + match error { + Error::InvalidRequest(detail) => detail.to_string(), + other => panic!("expected a request rejection, got {other:?}"), + } + } + + fn transformed(request: Value, capabilities: MessagesModelCapabilities) -> Value { + transformed_with(request, &context(capabilities, false), Vec::new()) + .unwrap() + .0 + } + + fn hello(model: &str, params: Value) -> Value { + let mut request = json!({ + "model": model, + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + "max_tokens": 4096, + }); + request + .as_object_mut() + .unwrap() + .extend(params.as_object().unwrap().clone()); + request + } + + #[test] + fn allowlist_filters_anthropic_only_fields() { + let wire = transformed( + hello( + "anthropic.claude-opus-4-7", + json!({ + "temperature": 0.5, + "mcp_servers": [{"type": "url", "url": "https://example.com"}], + "container": {"skills": []}, + "inference_geo": "us", + "output_config": {"effort": "low"}, + "context_management": {"edits": []}, + "stream": true, + }), + ), + MessagesModelCapabilities { + supports_sampling_params: true, + ..adaptive() + }, + ); + let fields = wire.as_object().unwrap(); + assert!( + fields + .keys() + .all(|key| BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS.contains(&key.as_str())), + "{wire}" + ); + assert_eq!(wire["output_config"], json!({"effort": "low"})); + assert_eq!(wire["max_tokens"], json!(4096)); + assert_eq!(wire["temperature"], json!(0.5)); + assert_eq!( + wire["anthropic_version"], + json!(DEFAULT_BEDROCK_ANTHROPIC_API_VERSION) + ); + } + + #[test] + fn a_callers_anthropic_version_wins() { + let wire = transformed( + hello("anthropic.claude-3", json!({"anthropic_version": "custom"})), + MessagesModelCapabilities::default(), + ); + assert_eq!(wire["anthropic_version"], json!("custom")); + } + + #[rstest] + #[case::ttl_dropped_without_support(false, json!({"type": "ephemeral", "ttl": "1h", "scope": "global"}), json!({"type": "ephemeral"}))] + #[case::ttl_kept_with_support(true, json!({"type": "ephemeral", "ttl": "1h", "scope": "global"}), json!({"type": "ephemeral", "ttl": "1h"}))] + #[case::unknown_ttl_dropped_with_support(true, json!({"type": "ephemeral", "ttl": "2h"}), json!({"type": "ephemeral"}))] + fn cache_control_keeps_only_supported_ttl_on_tools_system_and_messages( + #[case] supports_ttl: bool, + #[case] cache_control: Value, + #[case] expected: Value, + ) { + let mut request = body(json!({ + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}, "cache_control": cache_control.clone()}], + "system": [{"type": "text", "text": "You are helpful.", "cache_control": cache_control.clone()}], + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": cache_control}]}], + })); + remove_ttl_from_cache_control(&mut request, supports_ttl); + assert_eq!( + ( + &request["tools"][0]["cache_control"], + &request["system"][0]["cache_control"], + &request["messages"][0]["content"][0]["cache_control"], + ), + (&expected, &expected, &expected) + ); + } + + #[test] + fn eager_input_streaming_adds_its_beta_and_is_stripped_from_tools() { + let wire = transformed( + hello( + "anthropic.claude-sonnet-4-5", + json!({"tools": [{"name": "t", "input_schema": {"type": "object"}, "eager_input_streaming": true}]}), + ), + MessagesModelCapabilities::default(), + ); + assert_eq!( + wire["anthropic_beta"], + json!(["fine-grained-tool-streaming-2025-05-14"]) + ); + assert_eq!( + wire["tools"], + json!([{"name": "t", "input_schema": {"type": "object"}}]) + ); + } + + #[test] + fn a_non_bool_eager_input_streaming_is_refused() { + let error = transformed_with( + hello( + "anthropic.claude-sonnet-4-5", + json!({"tools": [{"name": "t", "input_schema": {"type": "object"}, "eager_input_streaming": "yes"}]}), + ), + &context(MessagesModelCapabilities::default(), false), + Vec::new(), + ) + .unwrap_err(); + assert_eq!(rejection(error), "eager_input_streaming must be a boolean"); + } + + #[test] + fn output_config_format_is_inlined_without_native_output() { + let schema = json!({"type": "object", "properties": {"answer": {"type": "string"}}}); + let wire = transformed( + hello( + "anthropic.claude-opus-4-7", + json!({"output_config": {"effort": "xhigh", "format": {"type": "json_schema", "schema": schema}}}), + ), + adaptive(), + ); + assert_eq!(wire["output_config"], json!({"effort": "xhigh"})); + let content = wire["messages"][0]["content"].as_array().unwrap(); + assert_eq!( + content.last().unwrap()["text"], + json!(r#"{"type": "object", "properties": {"answer": {"type": "string"}}}"#) + ); + } + + #[rstest] + #[case::opus_4_5_ceiling_high(Some(EffortLevel::High), "high")] + #[case::opus_4_6_ceiling_max(Some(EffortLevel::Max), "max")] + #[case::opus_4_7_no_ceiling(None, "xhigh")] + fn output_config_effort_is_clamped_to_the_bedrock_ceiling( + #[case] ceiling: Option, + #[case] expected: &str, + ) { + let capabilities = with_bedrock( + MessagesModelCapabilities { + supports_output_config: true, + ..MessagesModelCapabilities::default() + }, + BedrockCapabilities { + output_config_effort_ceiling: ceiling, + ..BedrockCapabilities::default() + }, + ); + let wire = transformed( + hello( + "anthropic.claude-opus-4-5", + json!({"output_config": {"effort": "xhigh"}}), + ), + capabilities, + ); + assert_eq!(wire["output_config"], json!({"effort": expected})); + } + + #[test] + fn global_drop_params_pops_output_config_the_model_does_not_take() { + let context = MessagesTransformContext { + global_drop_params: true, + ..context(MessagesModelCapabilities::default(), false) + }; + let mut request = + body(json!({"output_config": {"effort": "high", "format": {"type": "json_schema"}}})); + pop_output_config_under_global_drop(&mut request, context.global_drop_params, false); + assert_eq!(request.get("output_config"), None); + } + + #[rstest] + #[case::adaptive_without_thinking(adaptive(), json!({"max_tokens": 32000}), true, Some(json!({"type": "adaptive"})), Some("low"))] + #[case::adaptive_already_adaptive(adaptive(), json!({"max_tokens": 32000, "thinking": {"type": "adaptive"}}), false, Some(json!({"type": "adaptive"})), None)] + #[case::adaptive_legacy_budget(adaptive(), json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 4096}}), true, Some(json!({"type": "adaptive"})), Some("high"))] + #[case::adaptive_explicit_zero_budget(adaptive(), json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 0}}), true, Some(json!({"type": "adaptive"})), Some("low"))] + #[case::non_adaptive_keeps_enabled(MessagesModelCapabilities::default(), json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 2048}}), false, Some(json!({"type": "enabled", "budget_tokens": 2048})), None)] + #[case::non_adaptive_injects_enabled(MessagesModelCapabilities::default(), json!({"max_tokens": 32000}), true, Some(json!({"type": "enabled", "budget_tokens": 1024})), None)] + #[case::max_tokens_too_small(MessagesModelCapabilities::default(), json!({"max_tokens": 1024}), false, None, None)] + #[case::display_updates_preserved(adaptive(), json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 8192, "display": "updates"}}), true, Some(json!({"display": "updates", "type": "adaptive"})), Some("xhigh"))] + fn clear_thinking_injects_the_thinking_bedrock_requires( + #[case] capabilities: MessagesModelCapabilities, + #[case] request: Value, + #[case] injected: bool, + #[case] thinking: Option, + #[case] effort: Option<&str>, + ) { + let mut request = body(request); + request.insert( + "context_management".into(), + json!({"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}), + ); + let changed = ensure_thinking_for_clear_thinking_context_management( + &mut request, + "us.anthropic.claude-opus-4-5", + &capabilities, + &ThinkingBudgets::default(), + false, + ) + .unwrap(); + assert_eq!( + ( + changed, + request.get("thinking").cloned(), + request + .get("output_config") + .and_then(|config| config.get("effort")) + .and_then(Value::as_str) + ), + (injected, thinking, effort) + ); + } + + #[rstest] + #[case::xhigh(8192, "xhigh")] + #[case::high(4096, "high")] + #[case::medium(2048, "medium")] + #[case::low(2047, "low")] + #[case::zero(0, "low")] + fn effort_follows_thinking_budget_tiers(#[case] budget: i128, #[case] expected: &str) { + assert_eq!( + effort_from_thinking_budget(budget, &ThinkingBudgets::default()), + expected + ); + } + + #[rstest] + #[case::sonnet_4("anthropic.claude-sonnet-4-20250514-v1:0", false, true)] + #[case::opus_4_1("anthropic.claude-opus-4-1", false, true)] + #[case::haiku_3("anthropic.claude-3-haiku", false, false)] + #[case::adaptive_flag("anthropic.claude-fable-5", true, true)] + fn extended_thinking_follows_adaptive_flag_then_names( + #[case] model: &str, + #[case] supports_adaptive_thinking: bool, + #[case] expected: bool, + ) { + let capabilities = MessagesModelCapabilities { + supports_adaptive_thinking, + ..MessagesModelCapabilities::default() + }; + assert_eq!( + supports_extended_thinking_on_bedrock(model, &capabilities), + expected + ); + } + + #[test] + fn clear_thinking_edit_is_never_forwarded() { + let wire = transformed( + hello( + "anthropic.claude-sonnet-4-5", + json!({"max_tokens": 32000, "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}, {"type": "clear_tool_uses_20250919"}]}}), + ), + MessagesModelCapabilities::default(), + ); + assert_eq!( + wire["context_management"], + json!({"edits": [{"type": "clear_tool_uses_20250919"}]}) + ); + assert_eq!( + wire["thinking"], + json!({"type": "enabled", "budget_tokens": 1024}) + ); + assert_eq!( + wire["anthropic_beta"], + json!(["context-management-2025-06-27"]) + ); + } + + #[rstest] + #[case::non_list_edits_dropped(json!({"edits": "x"}), None, json!(null))] + #[case::unsupported_edits_dropped(json!({"edits": [{"type": "clear_thinking_20251015"}]}), None, json!(null))] + #[case::compact_kept(json!({"edits": [{"type": "compact_20260112"}]}), Some(json!({"edits": [{"type": "compact_20260112"}]})), json!(["compact-2026-01-12"]))] + fn context_management_keeps_bedrock_edits_with_their_betas( + #[case] context_management: Value, + #[case] expected: Option, + #[case] betas: Value, + ) { + let wire = transformed( + hello( + "anthropic.claude-3", + json!({"context_management": context_management}), + ), + MessagesModelCapabilities::default(), + ); + assert_eq!(wire.get("context_management").cloned(), expected); + assert_eq!( + wire.get("anthropic_beta").cloned().unwrap_or(Value::Null), + betas + ); + } + + #[rstest] + #[case::object_compaction_kept(json!({"type": "auto"}), Some(json!({"type": "auto"})))] + #[case::string_compaction_dropped(json!("auto"), None)] + #[case::list_compaction_dropped(json!([]), None)] + fn compaction_needs_an_object_and_its_beta( + #[case] compaction: Value, + #[case] expected: Option, + ) { + let wire = transformed( + hello("anthropic.claude-3", json!({"compaction": compaction})), + MessagesModelCapabilities::default(), + ); + assert_eq!(wire.get("compaction").cloned(), expected); + } + + #[test] + fn leading_system_turns_are_hoisted_and_string_system_becomes_blocks() { + let wire = transformed( + json!({ + "model": "anthropic.claude-3", + "max_tokens": 10, + "system": "base", + "messages": [ + {"role": "system", "content": "first"}, + {"role": "user", "content": "hi"}, + ], + }), + MessagesModelCapabilities::default(), + ); + assert_eq!( + wire["system"], + json!([{"type": "text", "text": "base"}, {"type": "text", "text": "first"}]) + ); + assert_eq!(wire["messages"], json!([{"role": "user", "content": "hi"}])); + } + + #[rstest] + #[case::kept_in_place(true, "system")] + #[case::converted_to_user(false, "user")] + fn mid_conversation_system_turns_follow_model_support( + #[case] supports_mid_conversation_system: bool, + #[case] role: &str, + ) { + let wire = transformed( + json!({ + "model": "anthropic.claude-3", + "max_tokens": 10, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "system", "content": "reminder"}, + ], + }), + MessagesModelCapabilities { + supports_mid_conversation_system, + ..MessagesModelCapabilities::default() + }, + ); + assert_eq!(wire["messages"][2]["role"], json!(role)); + } + + #[rstest] + #[case::web_search_20250305("web_search_20250305")] + #[case::web_search_20260209("web_search_20260209")] + fn server_web_search_tools_are_rejected(#[case] tool_type: &str) { + let error = transformed_with( + hello( + "anthropic.claude-sonnet-4-5", + json!({"tools": [{"type": tool_type, "name": "web_search"}]}), + ), + &context(MessagesModelCapabilities::default(), false), + Vec::new(), + ) + .unwrap_err(); + assert_eq!( + rejection(error), + format!( + "Bedrock does not support Anthropic's server-side web search tool (tool type '{tool_type}', model 'anthropic.claude-sonnet-4-5'). To use web search with this model, enable LiteLLM's web search interception so the proxy executes the search instead: https://docs.litellm.ai/docs/integrations/websearch_interception. Alternatively, remove the web_search tool from the request." + ) + ); + } + + #[test] + fn a_converted_websearch_function_tool_is_allowed() { + let wire = transformed( + hello( + "anthropic.claude-sonnet-4-5", + json!({"tools": [{"name": "litellm_web_search", "input_schema": {"type": "object"}}]}), + ), + MessagesModelCapabilities::default(), + ); + assert_eq!(wire["tools"][0]["name"], json!("litellm_web_search")); + } + + #[test] + fn reasoning_effort_is_clamped_before_the_effort_gate_on_adaptive_models() { + let capabilities = with_bedrock( + MessagesModelCapabilities { + effort_tiers: SupportedEffortTiers { + low: true, + medium: true, + high: true, + max: true, + ..SupportedEffortTiers::default() + }, + ..adaptive() + }, + BedrockCapabilities { + output_config_effort_ceiling: Some(EffortLevel::Max), + ..BedrockCapabilities::default() + }, + ); + let wire = transformed( + hello( + "anthropic.claude-opus-4-6", + json!({"reasoning_effort": "xhigh"}), + ), + capabilities, + ); + assert_eq!(wire["output_config"]["effort"], json!("max")); + assert_eq!(wire.get("reasoning_effort"), None); + } + + #[test] + fn native_output_forwards_a_cleaned_schema() { + let capabilities = with_bedrock( + adaptive(), + BedrockCapabilities { + native_output: NativeOutputSupport { + enabled: true, + mapped: true, + ..NativeOutputSupport::default() + }, + ..BedrockCapabilities::default() + }, + ); + let wire = transformed( + hello( + "anthropic.claude-opus-4-7", + json!({"output_config": {"format": {"type": "json_schema", "schema": {"type": "object", "properties": {"n": {"type": "integer", "minimum": 1}}, "required": ["n"]}}}}), + ), + capabilities, + ); + assert_eq!( + wire["output_config"], + json!({"format": {"type": "json_schema", "schema": {"type": "object", "properties": {"n": {"description": "Note: minimum value: 1.", "type": "integer"}}, "required": ["n"], "additionalProperties": false}}}) + ); + assert_eq!(wire["messages"][0]["content"].as_array().unwrap().len(), 1); + } + + #[test] + fn model_and_stream_leave_the_wire_body() { + let wire = transformed( + hello("anthropic.claude-3", json!({"stream": true})), + MessagesModelCapabilities::default(), + ); + assert_eq!((wire.get("model"), wire.get("stream")), (None, None)); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/stream.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/stream.rs new file mode 100644 index 00000000000..b5531bcfc08 --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/stream.rs @@ -0,0 +1,443 @@ +use std::collections::VecDeque; + +use bytes::Bytes; +use futures_util::{Stream, StreamExt, stream}; +use litellm_llms_types::providers::bedrock::INVOCATION_METRICS_KEY; +use litellm_python_compat::{Value, json, repr, truthy::truthy}; + +use crate::{ + Error, ErrorDetail, + base_llm::messages::streaming::{ByteStream, FrameStream}, +}; + +use crate::bedrock::chat::invoke_handler::{BedrockStreamRaise, aiter_bytes, raised}; + +const METRICS_USAGE_KEYS: [(&str, &str); 4] = [ + ("input_tokens", "inputTokenCount"), + ("output_tokens", "outputTokenCount"), + ("cache_read_input_tokens", "cacheReadInputTokenCount"), + ("cache_creation_input_tokens", "cacheWriteInputTokenCount"), +]; + +const CACHE_FIELDS: [&str; 2] = ["cache_creation_input_tokens", "cache_read_input_tokens"]; + +type Fields = Vec<(Value, Value)>; + +fn key(name: &str) -> Value { + Value::Str(name.to_string()) +} + +fn position(fields: &Fields, name: &str) -> Option { + fields + .iter() + .position(|(key, _)| matches!(key, Value::Str(key) if key == name)) +} + +fn get<'a>(fields: &'a Fields, name: &str) -> Option<&'a Value> { + position(fields, name).map(|at| &fields[at].1) +} + +fn set(fields: &mut Fields, name: &str, value: Value) { + match position(fields, name) { + Some(at) => fields[at].1 = value, + None => fields.push((key(name), value)), + } +} + +fn set_pair(fields: &mut Fields, name: Value, value: Value) { + match fields.iter().position(|(key, _)| *key == name) { + Some(at) => fields[at].1 = value, + None => fields.push((name, value)), + } +} + +fn pop(fields: &mut Fields, name: &str) -> Option { + position(fields, name).map(|at| fields.remove(at).1) +} + +fn is_type(fields: &Fields, event_type: &str) -> bool { + matches!(get(fields, "type"), Some(Value::Str(value)) if value == event_type) +} + +fn dict_or_empty(value: Option<&Value>) -> Fields { + match value { + Some(Value::Dict(fields)) => fields.clone(), + _ => Fields::new(), + } +} + +/// `AmazonAnthropicClaudeMessagesStreamDecoder._chunk_parser`: Bedrock's invocation metrics +/// become Anthropic usage, under the chunk's own usage. +fn chunk_parser(chunk: Value) -> Result { + let mut fields = match chunk { + Value::Dict(fields) => fields, + other => return Err(raised(BedrockStreamRaise::NotAnObject(dumped(&other)?))), + }; + let Some(metrics) = pop(&mut fields, INVOCATION_METRICS_KEY).filter(truthy) else { + return Ok(fields); + }; + let metrics = dict_or_empty(Some(&metrics)); + let preserved = dict_or_empty(get(&fields, "usage")); + let usage = preserved.into_iter().fold( + METRICS_USAGE_KEYS + .iter() + .filter_map(|(anthropic, metric)| { + Some((key(anthropic), get(&metrics, metric)?.clone())) + }) + .collect::(), + |mut usage, (name, value)| { + set_pair(&mut usage, name, value); + usage + }, + ); + set(&mut fields, "usage", Value::Dict(usage)); + Ok(fields) +} + +/// `AmazonAnthropicClaudeMessagesConfig._promote_message_stop_usage`, on the decoded chunks. +#[derive(Default)] +struct UsagePromoter { + pending_delta: Option, + start_usage: Option, +} + +impl UsagePromoter { + /// Port of Python's `_merge_message_start_cache_into_delta_usage`. + fn merge_message_start_cache_into_delta_usage(&self, delta_usage: &mut Fields) { + let Some(start) = self.start_usage.as_ref().filter(|start| !start.is_empty()) else { + return; + }; + for field in CACHE_FIELDS.iter().chain(["cache_creation"].iter()) { + if get(delta_usage, field).is_none() + && let Some(value) = get(start, field).filter(|value| **value != Value::None) + { + set(delta_usage, field, value.clone()); + } + } + } + + fn completed(&self, mut delta: Fields, stop: Option<&Fields>) -> Fields { + let mut delta_usage = dict_or_empty(get(&delta, "usage").filter(|usage| truthy(usage))); + if let Some(stop) = stop { + for field in CACHE_FIELDS { + if let Some(value) = get(stop, field) { + set(&mut delta_usage, field, value.clone()); + } + } + if let Some(raw) = get(stop, "input_tokens").filter(|raw| **raw != Value::None) { + let tokens = match raw { + Value::Int(_) | Value::Bool(_) => raw.clone(), + _ => Value::from(0), + }; + set(&mut delta_usage, "input_tokens", tokens); + } + } + self.merge_message_start_cache_into_delta_usage(&mut delta_usage); + if !delta_usage.is_empty() { + set(&mut delta, "usage", Value::Dict(delta_usage)); + } + delta + } + + fn feed(&mut self, chunk: Fields) -> Vec { + if is_type(&chunk, "message_start") { + let message = dict_or_empty(get(&chunk, "message").filter(|message| truthy(message))); + if let Some(Value::Dict(usage)) = get(&message, "usage") { + self.start_usage = Some(usage.clone()); + } + return self + .pending_delta + .take() + .into_iter() + .chain([chunk]) + .collect(); + } + if is_type(&chunk, "message_delta") { + self.pending_delta = Some(chunk); + return Vec::new(); + } + if is_type(&chunk, "message_stop") + && let Some(delta) = self.pending_delta.take() + { + let stop_usage = dict_or_empty(get(&chunk, "usage").filter(|usage| truthy(usage))); + return vec![self.completed(delta, Some(&stop_usage)), chunk]; + } + self.pending_delta + .take() + .into_iter() + .chain([chunk]) + .collect() + } + + fn finish(&mut self) -> Option { + let delta = self.pending_delta.take()?; + Some(self.completed(delta, None)) + } +} + +fn dumped(value: &Value) -> Result { + json::dumps(value) + .map_err(|error| Error::InvalidResponse(ErrorDetail::failed("SSE event", error))) +} + +/// `BaseAnthropicMessagesStreamingIterator._convert_chunk_to_sse_format` for a decoded chunk. +fn sse_frame(chunk: Fields) -> Result { + let event_type = get(&chunk, "type").map_or_else(|| "message".to_string(), repr::to_str); + let data = dumped(&Value::Dict(chunk))?; + Ok(Bytes::from(format!( + "event: {event_type}\ndata: {data}\n\n" + ))) +} + +struct Relay { + chunks: S, + promoter: UsagePromoter, + queued: VecDeque, + finished: bool, +} + +impl> + Unpin> Relay { + fn queue(&mut self, chunks: impl IntoIterator) -> Result<(), Error> { + for chunk in chunks { + self.queued.push_back(sse_frame(chunk)?); + } + Ok(()) + } + + async fn advance(&mut self) -> Result<(), Error> { + match self.chunks.next().await { + Some(chunk) => { + let chunk = chunk_parser(chunk?)?; + let promoted = self.promoter.feed(chunk); + self.queue(promoted) + } + None => { + self.finished = true; + let flushed = self.promoter.finish(); + self.queue(flushed) + } + } + } +} + +/// Bedrock InvokeModel's event stream as the SSE frames Python's Bedrock Messages stream yields: +/// each decoded chunk re-serialized with `json.dumps`, with usage promoted onto `message_delta`. +pub fn get_async_streaming_response_iterator( + bytes: ByteStream, + headers: Vec<(String, String)>, +) -> FrameStream { + let relay = Relay { + chunks: Box::pin(aiter_bytes(bytes, headers)), + promoter: UsagePromoter::default(), + queued: VecDeque::new(), + finished: false, + }; + stream::unfold(Some(relay), |relay| async move { + let mut relay = relay?; + loop { + if let Some(frame) = relay.queued.pop_front() { + return Some((Ok(frame), Some(relay))); + } + if relay.finished { + return None; + } + if let Err(error) = relay.advance().await { + return Some((Err(error), None)); + } + } + }) + .boxed() +} + +#[cfg(test)] +mod tests { + use aws_smithy_eventstream::frame::write_message_to; + use aws_smithy_types::event_stream::{Header, HeaderValue, Message}; + use base64::{Engine, engine::general_purpose::STANDARD}; + use futures_util::TryStreamExt; + use rstest::rstest; + + use super::*; + + fn aws_frame(chunk: &str) -> Vec { + let payload = format!(r#"{{"bytes": "{}"}}"#, STANDARD.encode(chunk)); + let message = Message::new(Bytes::from(payload)) + .add_header(Header::new( + ":event-type", + HeaderValue::String("chunk".into()), + )) + .add_header(Header::new( + ":message-type", + HeaderValue::String("event".into()), + )); + let mut wire = Vec::new(); + write_message_to(&message, &mut wire).unwrap(); + wire + } + + fn in_pieces(chunks: &[&str]) -> ByteStream { + let wire: Vec = chunks.iter().flat_map(|chunk| aws_frame(chunk)).collect(); + stream::iter( + wire.chunks(7) + .map(|piece| Ok(Bytes::copy_from_slice(piece))) + .collect::>(), + ) + .boxed() + } + + async fn relayed(chunks: &[&str]) -> Result { + let frames: Vec = + get_async_streaming_response_iterator(in_pieces(chunks), Vec::new()) + .try_collect() + .await?; + Ok(frames + .iter() + .map(|frame| String::from_utf8(frame.to_vec()).unwrap()) + .collect()) + } + + const MESSAGE_STOP: &str = "event: message_stop\ndata: {\"type\": \"message_stop\"}\n\n"; + + #[rstest] + #[case::stop_sequence_null_is_kept( + r#"{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":4}}"#, + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\", \"stop_sequence\": null}, \"usage\": {\"output_tokens\": 4}}\n\n" + )] + #[case::an_absent_stop_sequence_stays_absent( + r#"{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":4}}"#, + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 4}}\n\n" + )] + #[case::unknown_keys_and_non_ascii_are_dumped_as_python_does( + r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hé"},"extra":[1.0,2]}"#, + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"h\\u00e9\"}, \"extra\": [1.0, 2]}\n\n" + )] + #[tokio::test] + async fn each_chunk_is_relayed_as_json_dumps_writes_it( + #[case] chunk: &str, + #[case] expected: &str, + ) { + assert_eq!( + relayed(&[chunk, r#"{"type":"message_stop"}"#]) + .await + .unwrap(), + format!("{expected}{MESSAGE_STOP}") + ); + } + + #[tokio::test] + async fn a_chunk_without_a_type_is_named_message() { + assert_eq!( + relayed(&[r#"{"message":"x"}"#, r#"{"type":"message_stop"}"#]) + .await + .unwrap(), + format!("event: message\ndata: {{\"message\": \"x\"}}\n\n{MESSAGE_STOP}") + ); + } + + #[rstest] + #[case::cache_fields_on_message_stop( + r#"{"input_tokens": 10, "output_tokens": 0}"#, + r#"{"output_tokens": 5}"#, + r#"{"input_tokens": 3, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20}"#, + r#"{"output_tokens": 5, "cache_creation_input_tokens": 20, "cache_read_input_tokens": 100, "input_tokens": 3}"# + )] + #[case::cache_only_on_message_start( + r#"{"input_tokens": 10, "output_tokens": 0, "cache_read_input_tokens": 80, "cache_creation_input_tokens": 4, "cache_creation": {"ephemeral_5m_input_tokens": 4}}"#, + r#"{"output_tokens": 5}"#, + r#"{"input_tokens": 10}"#, + r#"{"output_tokens": 5, "input_tokens": 10, "cache_creation_input_tokens": 4, "cache_read_input_tokens": 80, "cache_creation": {"ephemeral_5m_input_tokens": 4}}"# + )] + #[case::a_float_input_count_is_zeroed( + r#"{"input_tokens": 10}"#, + r#"{"output_tokens": 5}"#, + r#"{"input_tokens": 3.5}"#, + r#"{"output_tokens": 5, "input_tokens": 0}"# + )] + #[tokio::test] + async fn message_delta_usage_is_completed_from_stop_then_start( + #[case] start: &str, + #[case] delta: &str, + #[case] stop: &str, + #[case] expected: &str, + ) { + let start = format!(r#"{{"type": "message_start", "message": {{"usage": {start}}}}}"#); + let delta = format!(r#"{{"type": "message_delta", "usage": {delta}}}"#); + let stop = format!(r#"{{"type": "message_stop", "usage": {stop}}}"#); + + let relayed = relayed(&[&start, &delta, &stop]).await.unwrap(); + + assert!( + relayed.contains(&format!( + "event: message_delta\ndata: {{\"type\": \"message_delta\", \"usage\": {expected}}}\n\n" + )), + "{relayed}" + ); + } + + #[rstest] + #[case::metrics_fill_missing_usage( + r#"{"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "outputTokenCount": 9}}"#, + r#"{"type": "message_stop", "usage": {"input_tokens": 3, "output_tokens": 9}}"# + )] + #[case::the_chunks_own_usage_wins_in_place( + r#"{"type": "message_stop", "usage": {"input_tokens": 1}, "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "cacheReadInputTokenCount": 40}, "amazon-bedrock-trace": {}}"#, + r#"{"type": "message_stop", "usage": {"input_tokens": 1, "cache_read_input_tokens": 40}, "amazon-bedrock-trace": {}}"# + )] + #[case::empty_metrics_are_dropped( + r#"{"type": "message_stop", "amazon-bedrock-invocationMetrics": {}}"#, + r#"{"type": "message_stop"}"# + )] + #[tokio::test] + async fn invocation_metrics_become_anthropic_usage( + #[case] chunk: &str, + #[case] expected: &str, + ) { + assert_eq!( + relayed(&[chunk]).await.unwrap(), + format!("event: message_stop\ndata: {expected}\n\n") + ); + } + + #[rstest] + #[case::no_message_stop( + &[r#"{"type":"message_start","message":{}}"#], + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {}}\n\n" + )] + #[case::a_held_delta_is_flushed( + &[r#"{"type":"message_delta","usage":{"output_tokens":1}}"#], + "event: message_delta\ndata: {\"type\": \"message_delta\", \"usage\": {\"output_tokens\": 1}}\n\n" + )] + #[tokio::test] + async fn a_stream_without_message_stop_ends_with_its_own_frames( + #[case] chunks: &[&str], + #[case] expected: &str, + ) { + assert_eq!(relayed(chunks).await.unwrap(), expected); + } + + #[tokio::test] + async fn a_provider_error_event_is_terminal() { + let error = r#"{"type":"error","error":{"type":"overloaded_error","message":"busy"}}"#; + + assert_eq!( + relayed(&[error]).await.unwrap(), + "event: error\ndata: {\"type\": \"error\", \"error\": {\"type\": \"overloaded_error\", \"message\": \"busy\"}}\n\n" + ); + } + + #[rstest] + #[case::a_list(r#"[1]"#, BedrockStreamRaise::NotAnObject("[1]".into()))] + #[case::not_json(r#"{"type": "#, BedrockStreamRaise::InvalidJson(r#"{"type": "#.into()))] + #[tokio::test] + async fn a_payload_python_cannot_parse_raises_what_python_raises( + #[case] chunk: &str, + #[case] expected: BedrockStreamRaise, + ) { + let error = relayed(&[chunk]).await.unwrap_err(); + + assert_eq!( + litellm_error::find::(&error), + Some(&expected) + ); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock/mod.rs b/litellm/rust/crates/llms/src/bedrock/mod.rs index b2dc8937a8a..7c825c85812 100644 --- a/litellm/rust/crates/llms/src/bedrock/mod.rs +++ b/litellm/rust/crates/llms/src/bedrock/mod.rs @@ -3,3 +3,4 @@ pub mod chat; pub mod claude_platform; pub mod common_utils; pub mod messages; +pub mod request_metadata; diff --git a/litellm/rust/crates/llms/src/bedrock/request_metadata.rs b/litellm/rust/crates/llms/src/bedrock/request_metadata.rs new file mode 100644 index 00000000000..ae066ca66f6 --- /dev/null +++ b/litellm/rust/crates/llms/src/bedrock/request_metadata.rs @@ -0,0 +1,336 @@ +use litellm_python_compat::{json as python_json, repr}; +use serde_json::{Map, Value}; + +use crate::base_llm::auth::Headers; + +pub const REQUEST_METADATA_FIELDS_PARAM: &str = "bedrock_request_metadata_fields"; +pub const METADATA_PARAMS: [&str; 2] = ["metadata", "litellm_metadata"]; + +pub const REQUEST_METADATA_HEADER: &str = "X-Amzn-Bedrock-Request-Metadata"; +const REQUEST_METADATA_MAX_PAIRS: usize = 16; +const REQUEST_METADATA_IDENTITY_PREFIX: &str = "user_api_key_"; +const REQUEST_METADATA_CLIENT_FIELD: &str = "spend_logs_metadata"; +const REQUEST_METADATA_HEADER_MAX_CHARS: usize = 8500; +const REQUEST_METADATA_MAX_CHARS: usize = 256; + +fn is_allowed_character(character: char) -> bool { + character.is_ascii_alphanumeric() + || matches!( + character, + ' ' | '\t' + | '\n' + | '\r' + | '\x0b' + | '\x0c' + | ':' + | '_' + | '@' + | '$' + | '#' + | '=' + | '/' + | '+' + | ',' + | '.' + | '-' + ) +} + +fn matches_constraint(text: &str, minimum: usize) -> bool { + let count = text.chars().count(); + (minimum..=REQUEST_METADATA_MAX_CHARS).contains(&count) + && text.chars().all(is_allowed_character) +} + +fn is_forwardable(key: &str, value: &str) -> bool { + matches_constraint(key, 1) && matches_constraint(value, 0) +} + +fn is_reserved(key: &str) -> bool { + key.trim() + .to_lowercase() + .starts_with(REQUEST_METADATA_IDENTITY_PREFIX) +} + +fn text_pairs(source: Option<&Value>) -> Vec<(&str, &str)> { + match source { + Some(Value::Object(fields)) => fields + .iter() + .filter_map(|(key, value)| Some((key.as_str(), value.as_str()?))) + .collect(), + _ => Vec::new(), + } +} + +fn allowed_fields(configured: Option<&Value>) -> Vec { + let Some(Value::Array(fields)) = configured else { + return Vec::new(); + }; + let fields: Vec = fields + .iter() + .map(|field| match field { + Value::String(text) => text.clone(), + other => repr::to_str(&python_json::from_json(other.clone())), + }) + .collect(); + fields + .iter() + .enumerate() + .filter(|(index, field)| !fields[..*index].contains(field)) + .map(|(_, field)| field.clone()) + .collect() +} + +/// Port of Python's `resolve_bedrock_request_metadata`. +fn resolve_bedrock_request_metadata( + sources: &[&Map], + allowed: &[String], +) -> Vec<(String, String)> { + let identity: Vec<(String, String)> = allowed + .iter() + .filter(|field| field.starts_with(REQUEST_METADATA_IDENTITY_PREFIX)) + .filter_map(|field| { + let value = sources + .iter() + .find_map(|source| source.get(field.as_str()).and_then(Value::as_str))?; + is_forwardable(field, value).then(|| (field.clone(), value.to_string())) + }) + .take(REQUEST_METADATA_MAX_PAIRS) + .collect(); + let spend_logs_pairs: Vec<(&str, &str)> = if allowed + .iter() + .any(|field| field == REQUEST_METADATA_CLIENT_FIELD) + { + sources + .iter() + .flat_map(|source| text_pairs(source.get(REQUEST_METADATA_CLIENT_FIELD))) + .collect() + } else { + Vec::new() + }; + let mut seen: Vec<&str> = Vec::new(); + let client: Vec<(String, String)> = spend_logs_pairs + .into_iter() + .filter(|(key, value)| !is_reserved(key) && is_forwardable(key, value)) + .filter(|(key, _)| { + let first = !seen.contains(key); + seen.push(key); + first + }) + .take(REQUEST_METADATA_MAX_PAIRS - identity.len()) + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + identity + .into_iter() + .chain(client) + .fold(Vec::new(), |mut resolved, (key, value)| { + match resolved.iter_mut().find(|(existing, _)| *existing == key) { + Some(slot) => slot.1 = value, + None => resolved.push((key, value)), + } + resolved + }) +} + +fn header_value(pairs: &[(String, String)]) -> String { + let object: Map = pairs + .iter() + .map(|(key, value)| (key.clone(), Value::String(value.clone()))) + .collect(); + python_json::dumps(&python_json::from_json(Value::Object(object))) + .map(|text| compact_python_json(&text)) + .unwrap_or_default() +} + +fn compact_python_json(text: &str) -> String { + let mut compact = String::with_capacity(text.len()); + let mut in_string = false; + let mut escaped = false; + let mut characters = text.chars().peekable(); + while let Some(character) = characters.next() { + if in_string { + compact.push(character); + match (escaped, character) { + (true, _) => escaped = false, + (false, '\\') => escaped = true, + (false, '"') => in_string = false, + _ => {} + } + continue; + } + match character { + '"' => { + in_string = true; + compact.push(character); + } + ',' | ':' if characters.peek() == Some(&' ') => { + compact.push(character); + characters.next(); + } + _ => compact.push(character), + } + } + compact +} + +/// Port of Python's `_longest_fitting_prefix`. +fn longest_fitting_prefix(pairs: &[(String, String)]) -> Option { + (1..=pairs.len()) + .rev() + .map(|count| header_value(&pairs[..count])) + .find(|value| value.chars().count() <= REQUEST_METADATA_HEADER_MAX_CHARS) +} + +/// Port of Python's `bedrock_request_metadata_headers`. +pub(crate) fn bedrock_request_metadata_headers( + extra: &std::collections::BTreeMap, +) -> Option> { + let allowed = allowed_fields(extra.get(REQUEST_METADATA_FIELDS_PARAM)); + if allowed.is_empty() { + return None; + } + let sources: Vec<&Map> = METADATA_PARAMS + .iter() + .filter_map(|name| extra.get(*name).and_then(Value::as_object)) + .collect(); + let resolved = resolve_bedrock_request_metadata(&sources, &allowed); + Some( + longest_fitting_prefix(&resolved).map(|value| (REQUEST_METADATA_HEADER.to_string(), value)), + ) +} + +/// Port of Python's `merge_bedrock_invoke_headers`. +pub(crate) fn merge_bedrock_invoke_headers( + headers: Headers, + caller_owned: Vec<(String, String)>, + proxy_owned: Option>, +) -> Headers { + let owned_names: Vec = proxy_owned + .as_ref() + .map(|_| vec![REQUEST_METADATA_HEADER.to_lowercase()]) + .unwrap_or_default(); + if caller_owned.is_empty() && owned_names.is_empty() { + return headers; + } + let existing: Vec = headers + .iter() + .map(|(name, _)| name.to_lowercase()) + .collect(); + let kept = headers + .into_iter() + .filter(|(name, _)| !owned_names.contains(&name.to_lowercase())); + let added = caller_owned + .into_iter() + .filter(|(name, _)| !existing.contains(&name.to_lowercase())); + kept.chain(added).chain(proxy_owned.flatten()).fold( + Vec::new(), + |mut merged: Headers, (name, value)| { + match merged.iter_mut().find(|(existing, _)| *existing == name) { + Some(slot) => slot.1 = value, + None => merged.push((name, value)), + } + merged + }, + ) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::bedrock::chat::invoke_transformations::base_invoke_transformation::{ + GUARDRAIL_IDENTIFIER_HEADER, GUARDRAIL_VERSION_HEADER, + }; + + fn extra(value: Value) -> std::collections::BTreeMap { + value + .as_object() + .unwrap() + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect() + } + + #[test] + fn a_callers_guardrail_header_wins() { + let merged = merge_bedrock_invoke_headers( + vec![("x-amzn-bedrock-guardrailidentifier".into(), "caller".into())], + vec![ + (GUARDRAIL_IDENTIFIER_HEADER.into(), "deployment".into()), + (GUARDRAIL_VERSION_HEADER.into(), "1".into()), + ], + None, + ); + assert_eq!( + merged, + vec![ + ( + "x-amzn-bedrock-guardrailidentifier".to_string(), + "caller".to_string() + ), + (GUARDRAIL_VERSION_HEADER.to_string(), "1".to_string()), + ] + ); + } + + #[rstest] + #[case::disabled_by_default(json!({}), None)] + #[case::identity_then_spend_logs( + json!({ + "bedrock_request_metadata_fields": ["user_api_key_team_id", "spend_logs_metadata", "user_api_key_team_id"], + "metadata": {"user_api_key_team_id": "team-1", "spend_logs_metadata": {"project": "alpha", "user_api_key_alias": "forged", "bad key!": "x"}}, + }), + Some(Some(r#"{"user_api_key_team_id":"team-1","project":"alpha"}"#)), + )] + #[case::litellm_metadata_is_read_too( + json!({"bedrock_request_metadata_fields": ["user_api_key_user_id"], "litellm_metadata": {"user_api_key_user_id": "u 1"}}), + Some(Some(r#"{"user_api_key_user_id":"u 1"}"#)), + )] + #[case::owned_but_empty( + json!({"bedrock_request_metadata_fields": ["user_api_key_team_id"], "metadata": {}}), + Some(None), + )] + fn request_metadata_header_follows_the_allow_list( + #[case] params: Value, + #[case] expected: Option>, + ) { + assert_eq!( + bedrock_request_metadata_headers(&extra(params)), + expected.map(|header| header + .map(|value| (REQUEST_METADATA_HEADER.to_string(), value.to_string()))) + ); + } + + #[test] + fn the_proxy_owns_the_request_metadata_header() { + let merged = merge_bedrock_invoke_headers( + vec![ + ("x-amzn-bedrock-request-metadata".into(), "forged".into()), + ("x-keep".into(), "1".into()), + ], + Vec::new(), + Some(None), + ); + assert_eq!(merged, vec![("x-keep".to_string(), "1".to_string())]); + } + + #[test] + fn the_header_keeps_the_longest_prefix_that_fits() { + let long = "v".repeat(256); + let spend_logs: Map = (0..40) + .map(|index| (format!("key{index:02}"), Value::String(long.clone()))) + .collect(); + let params = extra(json!({ + "bedrock_request_metadata_fields": ["spend_logs_metadata"], + "metadata": {"spend_logs_metadata": spend_logs}, + })); + let Some(Some((_, value))) = bedrock_request_metadata_headers(¶ms) else { + panic!("expected a header"); + }; + let pairs: Map = serde_json::from_str(&value).unwrap(); + assert_eq!(pairs.len(), 16); + assert!(value.len() <= REQUEST_METADATA_HEADER_MAX_CHARS); + } +} diff --git a/litellm/rust/crates/llms/src/bedrock_mantle/messages/transformation.rs b/litellm/rust/crates/llms/src/bedrock_mantle/messages/transformation.rs index 0e67ea1a139..51238c6539c 100644 --- a/litellm/rust/crates/llms/src/bedrock_mantle/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock_mantle/messages/transformation.rs @@ -5,8 +5,8 @@ use std::sync::LazyLock; use litellm_auth::{AwsParams, CredentialPlacement}; use litellm_secret_value::SecretValue; -use litellm_auth_aws::AwsCredentialSource; use litellm_auth_aws::bedrock_model_id_and_region; +use litellm_auth_aws::{AwsCredentialSource, CallerSigningHeaders}; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{with_header, without_headers}; use litellm_llms_types::{ @@ -102,6 +102,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { region: mantle_region(params.api_base.as_deref(), model, params, env)?, service: "bedrock", credentials: Box::new(AwsCredentialSource::from_params(¶ms.aws, env)), + caller_headers: CallerSigningHeaders::Botocore, }, }; Ok(ValidatedEnvironment { headers, auth }) diff --git a/litellm/rust/crates/llms/src/error.rs b/litellm/rust/crates/llms/src/error.rs index af7ceb7da17..3aa1cc6d9df 100644 --- a/litellm/rust/crates/llms/src/error.rs +++ b/litellm/rust/crates/llms/src/error.rs @@ -220,6 +220,10 @@ pub enum ErrorDetail { #[source] cause: Source, }, + #[error( + "Bedrock does not support Anthropic's server-side web search tool (tool type '{tool_type}', model '{model}'). To use web search with this model, enable LiteLLM's web search interception so the proxy executes the search instead: https://docs.litellm.ai/docs/integrations/websearch_interception. Alternatively, remove the web_search tool from the request." + )] + UnsupportedBedrockWebSearchTool { tool_type: String, model: String }, #[error("{operation} failed: {detail}")] RemoteFailure { operation: &'static str, diff --git a/litellm/rust/crates/python-bridge/Cargo.toml b/litellm/rust/crates/python-bridge/Cargo.toml index 4a52308cef1..8d7d714cd60 100644 --- a/litellm/rust/crates/python-bridge/Cargo.toml +++ b/litellm/rust/crates/python-bridge/Cargo.toml @@ -21,6 +21,7 @@ tiktoken = ["litellm-token-counter/tiktoken"] [dependencies] rustls.workspace = true litellm-secret-value.workspace = true +litellm-db.workspace = true fancy-regex.workspace = true litellm-tracing.workspace = true litellm-spend-clickhouse.workspace = true @@ -46,6 +47,7 @@ litellm-auth-aws.workspace = true litellm-auth-azure.workspace = true azure_core = "1.0.0" litellm-callbacks-legacy-python.workspace = true +litellm-callbacks-budgets.workspace = true litellm-callbacks.workspace = true litellm-callbacks-python.workspace = true litellm-callbacks-s3-log.workspace = true @@ -92,6 +94,7 @@ tempfile.workspace = true tokio-tungstenite.workspace = true wiremock.workspace = true aws-sdk-secretsmanager = "1.117.0" +aws-credential-types = "1.3.0" [[bench]] name = "serialization" diff --git a/litellm/rust/crates/python-bridge/src/budgets.rs b/litellm/rust/crates/python-bridge/src/budgets.rs new file mode 100644 index 00000000000..09f4ee61d7a --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/budgets.rs @@ -0,0 +1,95 @@ +use std::sync::Arc; + +use litellm_callbacks_budgets::{ + BudgetCounterSpend, BudgetPlan, BudgetReservation, Budgets, Check, EngineError, ErrorKind, + Settle, decide, +}; +use litellm_host_python::{Pythonized, from_py_argument, json_from_py, run_async_value, to_py}; +use pyo3::{ + exceptions::{PyRuntimeError, PyTypeError, PyValueError}, + prelude::*, +}; +use serde::de::DeserializeOwned; + +use crate::redis::store::spend_counters::NativeSpendCounters; + +fn from_model(model: &Bound<'_, PyAny>) -> PyResult { + from_py_argument(&model.call_method0("model_dump")?) +} + +fn from_record(record: &Bound<'_, PyAny>) -> PyResult { + serde_json::from_value(json_from_py(&record.call_method0("model_dump")?)?) + .map_err(|error| PyValueError::new_err(error.to_string())) +} + +fn settle_error(error: EngineError) -> PyErr { + match error { + EngineError::Reservation(error) => match error.kind { + ErrorKind::Type => PyTypeError::new_err(error.message), + ErrorKind::Value => PyValueError::new_err(error.message), + }, + other => PyRuntimeError::new_err(other.to_string()), + } +} + +#[pyclass(frozen, name = "NativeBudgets")] +pub(crate) struct NativeBudgets(Arc); + +#[pymethods] +impl NativeBudgets { + #[new] + fn new(py: Python<'_>, spend_counters: PyRef<'_, NativeSpendCounters>) -> Self { + Self(Arc::new( + Budgets::new(spend_counters.store()).with_logger(crate::logger::capture(py)), + )) + } + + fn reserve<'py>( + &self, + py: Python<'py>, + #[pyo3(from_py_with = from_record)] plan: BudgetPlan, + #[pyo3(from_py_with = from_py_argument)] observed: Vec, + ) -> PyResult> { + let budgets = Arc::clone(&self.0); + run_async_value(py, async move { + budgets + .admit(plan, &observed) + .await + .map(Pythonized) + .map_err(PyRuntimeError::new_err) + }) + } + + fn settle<'py>( + &self, + py: Python<'py>, + #[pyo3(from_py_with = from_record)] reservation: BudgetReservation, + window_resets: Vec>, + observed: Vec>, + actual: f64, + apply_consistent: bool, + ) -> PyResult> { + let budgets = Arc::clone(&self.0); + run_async_value(py, async move { + budgets + .settle(&Settle { + reservation: &reservation, + window_resets: &window_resets, + observed: &observed, + actual, + apply_consistent, + }) + .await + .map(Pythonized) + .map_err(settle_error) + }) + } +} + +#[pyfunction] +pub(crate) fn budget_check_decision( + py: Python<'_>, + #[pyo3(from_py_with = from_model)] check: Check, +) -> PyResult> { + to_py(py, &decide(&check)) +} diff --git a/litellm/rust/crates/python-bridge/src/db.rs b/litellm/rust/crates/python-bridge/src/db.rs index 03fb30b986f..c14317219c8 100644 --- a/litellm/rust/crates/python-bridge/src/db.rs +++ b/litellm/rust/crates/python-bridge/src/db.rs @@ -1,19 +1,147 @@ -use litellm_host_python::ready_future; -use pyo3::{prelude::*, types::PyNone}; +use std::{sync::Arc, time::UNIX_EPOCH}; + +use litellm_db::{Call, Endpoint, Observer, Outcome, Pools, QueryEvent}; +use litellm_secret_value::SecretValue; +use pyo3::{exceptions::PyValueError, prelude::*}; + +const HEALTH_CHECK: Call = Call { + call_type: "health_check", + table: None, + operation: None, +}; + +struct PythonEvents(Py); + +impl Observer for PythonEvents { + fn record(&self, event: QueryEvent) { + let started_at = event + .started_at + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + let failure = match event.outcome { + Outcome::Succeeded => None, + Outcome::Failed(message) => Some(message), + }; + Python::attach(|py| { + let events = self.0.bind(py); + if let Err(error) = events.call_method1( + "record", + ( + event.call.call_type, + event.call.table, + event.call.operation, + started_at, + event.duration.as_secs_f64(), + failure, + ), + ) { + error.write_unraisable(py, Some(events)); + } + }); + } +} #[pyclass(frozen)] -pub(crate) struct NativeDatabase {} +pub(crate) struct NativeDatabase { + writer_endpoint: Endpoint, + reader_endpoint: Option, + pools: Pools, + events: Arc, +} + +pyo3::create_exception!( + _native, + NativeDatabaseFailure, + pyo3::exceptions::PyException, + "A native database operation failed. The kind attribute is litellm-db's ErrorKind, and unavailable and requeue_safe are the error's is_unavailable and is_requeue_safe." +); + +fn map_error(error: litellm_db::Error) -> PyErr { + let kind: &'static str = error.kind().into(); + let failure = NativeDatabaseFailure::new_err(litellm_error::Report::new(&error).to_string()); + let decided = Python::attach(|py| { + let value = failure.value(py); + value.setattr("kind", kind)?; + value.setattr("unavailable", error.is_unavailable())?; + value.setattr("requeue_safe", error.is_requeue_safe()) + }); + match decided { + Ok(()) => failure, + Err(error) => error, + } +} -// TODO: take typed writer/reader settings and a token provider, then open a sqlx PgPool, once the first -// repository moves to native. The Prisma DATABASE_URL carries Prisma-only params and rotating IAM tokens #[pymethods] impl NativeDatabase { #[staticmethod] - fn connect(py: Python<'_>) -> PyResult> { - ready_future(py, Bound::new(py, NativeDatabase {})?.as_any()) + #[pyo3(signature = (events, writer, reader=None))] + fn connect( + py: Python<'_>, + events: Py, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] writer: Endpoint, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] reader: Option, + ) -> PyResult> { + crate::execution::run_async_value(py, async move { + let pools = Pools::new(&writer, reader.as_ref()).map_err(map_error)?; + Ok(NativeDatabase { + writer_endpoint: writer, + reader_endpoint: reader, + pools, + events: Arc::new(PythonEvents(events)), + }) + }) + } + + fn set_writer_password(&self, password: String) { + self.pools.writer().set_connect_options( + self.writer_endpoint + .with_password(SecretValue::new(password)) + .connect_options(), + ); + } + + fn set_reader_password(&self, password: String) -> PyResult<()> { + let (Some(endpoint), Some(pool)) = (&self.reader_endpoint, self.pools.reader_pool()) else { + return Err(PyValueError::new_err("no read replica is configured")); + }; + pool.set_connect_options( + endpoint + .with_password(SecretValue::new(password)) + .connect_options(), + ); + Ok(()) + } + + fn health_check<'py>(&self, py: Python<'py>) -> PyResult> { + let writer = self.pools.writer().clone(); + let events = Arc::clone(&self.events); + let socket_timeout = self.writer_endpoint.socket_timeout().map_err(map_error)?; + // Boxed so each future wrapper between here and the tokio worker holds a pointer instead of a copy of + // the SQLx connect state, which overflowed the worker's 2 MiB stack in unoptimized builds + crate::execution::run_async( + py, + Box::pin(async move { + litellm_db::observe( + events.as_ref(), + HEALTH_CHECK, + litellm_db::within(socket_timeout, litellm_db::health_check(&writer)), + ) + .await + }), + map_error, + ) } fn close<'py>(&self, py: Python<'py>) -> PyResult> { - ready_future(py, PyNone::get(py).as_any()) + let pools = self.pools.clone(); + crate::execution::run_async( + py, + async move { + pools.close().await; + Ok(()) + }, + |error: PyErr| error, + ) } } diff --git a/litellm/rust/crates/python-bridge/src/errors.rs b/litellm/rust/crates/python-bridge/src/errors.rs index e19cd8880e1..6ca10c38a54 100644 --- a/litellm/rust/crates/python-bridge/src/errors.rs +++ b/litellm/rust/crates/python-bridge/src/errors.rs @@ -1,6 +1,6 @@ use std::error::Error as StdError; -use litellm_error::Report; +use litellm_error::{Report, Source}; use litellm_inference::Error; use pyo3::{ exceptions::{PyRuntimeError, PyTypeError, PyValueError}, @@ -42,28 +42,81 @@ const MESSAGES_PYTHON_RAISED: &str = "messages_python_raised"; /// The text of a refusal Python's provider config raises from its request transform, which its /// HTTP handler maps as a provider error. const MESSAGES_PROVIDER_REJECTION: &str = "messages_provider_rejection"; +/// Python's provider config raises the refusal as litellm's `BadRequestError`. +const MESSAGES_PROVIDER_BAD_REQUEST: &str = "messages_provider_bad_request"; +/// Which failure Python's AWS signing raises for a credential Rust could not acquire, or `replay` +/// when only running Python's signing again gives its exception. +const MESSAGES_CREDENTIAL_FAILURE: &str = "messages_credential_failure"; -fn provider_rejection(rejection: &(dyn StdError + 'static)) -> Option { +struct ProviderRejection { + text: String, + bad_request: bool, +} + +fn credential_failure(error: &Error) -> Option<&'static str> { + let (Error::Auth(auth) | Error::Llm(litellm_llms::Error::Auth(auth))) = error else { + return None; + }; + match auth { + litellm_auth::Error::Provider(_) => Some(aws_credential_failure(auth)), + litellm_auth::Error::CredentialAcquisition(_) + | litellm_auth::Error::CredentialChain { .. } => Some("replay"), + litellm_auth::Error::InvalidConfiguration(_) + | litellm_auth::Error::EmptyCallerCredential { .. } + | litellm_auth::Error::MissingApiKey { .. } + | litellm_auth::Error::MissingParam { .. } + | litellm_auth::Error::MissingApiBase { .. } + | litellm_auth::Error::InvalidHeader => None, + } +} + +fn aws_credential_failure(error: &(dyn StdError + 'static)) -> &'static str { + let innermost = std::iter::successors(Some(error), |&cause| cause.source()) + .filter_map(|cause| { + cause + .downcast_ref::() + .or_else(|| cause.downcast_ref::()?.downcast_ref()) + }) + .last(); + match innermost { + Some(litellm_auth_aws::Error::WebIdentityTokenReference) => "web_identity_reference", + Some(litellm_auth_aws::Error::MissingWebIdentityToken) => "missing_web_identity_token", + Some(failure) if failure.is_empty_default_chain() => "no_credentials", + _ => "replay", + } +} + +fn provider_rejection(rejection: &(dyn StdError + 'static)) -> Option { match rejection.downcast_ref::() { Some(litellm_llms::Error::MissingField("max_tokens")) => { - return Some("max_tokens is required for Anthropic /v1/messages API".to_owned()); + return Some(ProviderRejection { + text: "max_tokens is required for Anthropic /v1/messages API".to_owned(), + bad_request: false, + }); } Some(_) => return None, None => {} } match rejection.downcast_ref::() { Some( - detail @ litellm_llms::ErrorDetail::InvalidChoice { + detail @ (litellm_llms::ErrorDetail::InvalidChoice { field: "reasoning effort", .. - }, - ) => Some(format!("litellm.BadRequestError: {detail}")), + } + | litellm_llms::ErrorDetail::UnsupportedBedrockWebSearchTool { .. }), + ) => Some(ProviderRejection { + text: detail.to_string(), + bad_request: true, + }), Some( detail @ (litellm_llms::ErrorDetail::InvalidReasoningEffort { .. } | litellm_llms::ErrorDetail::UnsupportedValue { field: "effort", .. }), - ) => Some(detail.to_string()), + ) => Some(ProviderRejection { + text: detail.to_string(), + bad_request: false, + }), _ => None, } } @@ -182,8 +235,11 @@ pub(crate) fn route_failure( ) { value.setattr(MESSAGES_LATE_REJECTION, true)?; } - if let Some(text) = provider_rejection(rejection) { - value.setattr(MESSAGES_PROVIDER_REJECTION, text)?; + if let Some(provider) = provider_rejection(rejection) { + value.setattr(MESSAGES_PROVIDER_REJECTION, provider.text)?; + if provider.bad_request { + value.setattr(MESSAGES_PROVIDER_BAD_REQUEST, true)?; + } } native } @@ -204,6 +260,9 @@ pub(crate) fn route_failure( if error.is_missing_credential() { value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?; } + if let Some(failure) = credential_failure(&error) { + value.setattr(MESSAGES_CREDENTIAL_FAILURE, failure)?; + } native } (RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error), @@ -389,7 +448,7 @@ mod tests { #[rstest::rstest] #[case::missing_max_tokens( litellm_llms::Error::MissingField("max_tokens").into(), - Some("max_tokens is required for Anthropic /v1/messages API") + Some(("max_tokens is required for Anthropic /v1/messages API", false)) )] #[case::unmapped_effort( Error::invalid_request(litellm_llms::ErrorDetail::InvalidChoice { @@ -397,7 +456,17 @@ mod tests { actual: "'ultra'".into(), choices: vec!["low", "none"], }), - Some("litellm.BadRequestError: Unmapped reasoning effort: 'ultra'. Must be one of: 'low', 'none'.") + Some(("Unmapped reasoning effort: 'ultra'. Must be one of: 'low', 'none'.", true)) + )] + #[case::bedrock_web_search( + Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedBedrockWebSearchTool { + tool_type: "web_search_20250305".into(), + model: "claude".into(), + }), + Some(( + "Bedrock does not support Anthropic's server-side web search tool (tool type 'web_search_20250305', model 'claude'). To use web search with this model, enable LiteLLM's web search interception so the proxy executes the search instead: https://docs.litellm.ai/docs/integrations/websearch_interception. Alternatively, remove the web_search tool from the request.", + true + )) )] #[case::other_missing_field(litellm_llms::Error::MissingField("model").into(), None)] #[case::unsupported_param( @@ -411,17 +480,169 @@ mod tests { )] fn provider_config_rejections_carry_python_text( #[case] error: Error, - #[case] expected: Option<&str>, + #[case] expected: Option<(&str, bool)>, ) { Python::initialize(); Python::attach(|py| { let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); - let text = native - .value(py) + let value = native.value(py); + let text = value .getattr_opt(MESSAGES_PROVIDER_REJECTION) .unwrap() .map(|value| value.extract::().unwrap()); - assert_eq!(text.as_deref(), expected); + let bad_request = value + .getattr_opt(MESSAGES_PROVIDER_BAD_REQUEST) + .unwrap() + .is_some_and(|value| value.extract::().unwrap()); + assert_eq!(text.as_deref().map(|text| (text, bad_request)), expected); + assert!(text.is_some() || !bad_request); + }); + } + + fn web_identity_failure(failure: litellm_auth_aws::Error) -> litellm_auth_aws::Error { + litellm_auth_aws::Error::Credentials { + flow: litellm_auth_aws::CredentialFlow::WebIdentity, + source: Source::new( + aws_credential_types::provider::error::CredentialsError::provider_error(failure), + ), + } + } + + #[rstest::rstest] + #[case::environment_reference( + litellm_auth_aws::Error::WebIdentityTokenReference, + "web_identity_reference" + )] + #[case::unresolved_token( + web_identity_failure(litellm_auth_aws::Error::MissingWebIdentityToken), + "missing_web_identity_token" + )] + #[case::empty_default_chain( + litellm_auth_aws::Error::Credentials { + flow: litellm_auth_aws::CredentialFlow::DefaultChain, + source: Source::new(aws_credential_types::provider::error::CredentialsError::not_loaded( + "no providers in chain provided credentials", + )), + }, + "no_credentials" + )] + #[case::unreadable_token( + web_identity_failure(litellm_auth_aws::Error::WebIdentityToken(Source::new( + std::io::Error::other("secret source unavailable"), + ))), + "replay" + )] + #[case::sts_rejection( + web_identity_failure(litellm_auth_aws::Error::MissingWebIdentityCredentials), + "replay" + )] + #[case::assume_role( + litellm_auth_aws::Error::Credentials { + flow: litellm_auth_aws::CredentialFlow::AssumeRole, + source: Source::new(aws_credential_types::provider::error::CredentialsError::provider_error( + std::io::Error::other("AccessDenied"), + )), + }, + "replay" + )] + #[case::empty_profile( + litellm_auth_aws::Error::Credentials { + flow: litellm_auth_aws::CredentialFlow::Profile, + source: Source::new(aws_credential_types::provider::error::CredentialsError::not_loaded( + "profile not found", + )), + }, + "replay" + )] + #[case::expired(litellm_auth_aws::Error::ExpiredCredentials, "replay")] + fn an_aws_credential_failure_names_what_python_signing_raises( + #[case] failure: litellm_auth_aws::Error, + #[case] expected: &str, + #[values(false, true)] through_llm: bool, + ) { + let auth = litellm_auth::Error::from(failure); + let error = match through_llm { + true => Error::Llm(litellm_llms::Error::Auth(auth)), + false => Error::Auth(auth), + }; + Python::initialize(); + Python::attach(|py| { + let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); + assert!(native.is_instance_of::(py)); + let marked = native + .value(py) + .getattr(MESSAGES_CREDENTIAL_FAILURE) + .unwrap() + .extract::() + .unwrap(); + assert_eq!(marked, expected); + }); + } + + #[rstest::rstest] + #[case::acquisition( + litellm_auth::Error::CredentialAcquisition(litellm_auth::ErrorDetail::Empty { + subject: "token" + }), + Some("replay") + )] + #[case::chain( + litellm_auth::Error::CredentialChain { + last: Box::new(litellm_auth::Error::CredentialAcquisition( + litellm_auth::ErrorDetail::Empty { subject: "token" }, + )), + earlier: Vec::new(), + }, + Some("replay") + )] + #[case::missing_key( + litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: "ANTHROPIC_API_KEY", + }, + None + )] + #[case::invalid_header(litellm_auth::Error::InvalidHeader, None)] + fn only_credential_acquisition_failures_carry_the_credential_marker( + #[case] auth: litellm_auth::Error, + #[case] expected: Option<&str>, + ) { + Python::initialize(); + Python::attach(|py| { + let unmarked = + route_failure(py, Error::Auth(auth.clone()), RequestMarker::Unmarked, Ok).unwrap(); + assert!( + unmarked + .value(py) + .getattr_opt(MESSAGES_CREDENTIAL_FAILURE) + .unwrap() + .is_none() + ); + let native = route_failure(py, Error::Auth(auth), RequestMarker::Messages, Ok).unwrap(); + let marked = native + .value(py) + .getattr_opt(MESSAGES_CREDENTIAL_FAILURE) + .unwrap() + .map(|value| value.extract::().unwrap()); + assert_eq!(marked.as_deref(), expected); + }); + } + + #[test] + fn a_wrapped_credential_failure_keeps_the_native_mapping() { + let error = Error::post_call(Error::Auth(litellm_auth::Error::from( + litellm_auth_aws::Error::WebIdentityTokenReference, + ))); + Python::initialize(); + Python::attach(|py| { + let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); + assert!( + native + .value(py) + .getattr_opt(MESSAGES_CREDENTIAL_FAILURE) + .unwrap() + .is_none() + ); }); } diff --git a/litellm/rust/crates/python-bridge/src/http.rs b/litellm/rust/crates/python-bridge/src/http.rs index e7cc8e35172..806d883a33e 100644 --- a/litellm/rust/crates/python-bridge/src/http.rs +++ b/litellm/rust/crates/python-bridge/src/http.rs @@ -80,11 +80,16 @@ fn decode_ssl_verify(field: &Field<'_>) -> Result, ProjectionE Err(field.invalid("a Boolean, Boolean string, CA path, or None")) } -static RESOURCES: LazyLock = LazyLock::new(|| { - litellm_inference::resources::CoreResources::new(Arc::new(HttpClientPool::new(Arc::new( - PublicDnsResolver, - )))) -}); +static RESOURCES: LazyLock = + LazyLock::new(|| litellm_inference::resources::CoreResources { + pool: Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))), + auth: Arc::new(litellm_auth::AuthServices { + aws: litellm_auth_aws::AwsAuthService::with_web_identity_tokens(Arc::new( + crate::secrets::HostWebIdentityTokens, + )), + ..litellm_auth::AuthServices::default() + }), + }); pub(crate) fn resources() -> &'static litellm_inference::resources::CoreResources { &RESOURCES diff --git a/litellm/rust/crates/python-bridge/src/lib.rs b/litellm/rust/crates/python-bridge/src/lib.rs index cdf3ace6cca..83b1e253c08 100644 --- a/litellm/rust/crates/python-bridge/src/lib.rs +++ b/litellm/rust/crates/python-bridge/src/lib.rs @@ -1,3 +1,4 @@ +mod budgets; mod cache; mod callable; mod coercion; @@ -27,7 +28,11 @@ mod _native { use pyo3::{prelude::*, types::PyModule}; #[pymodule_export] - use crate::db::NativeDatabase; + use crate::budgets::NativeBudgets; + #[pymodule_export] + use crate::budgets::budget_check_decision; + #[pymodule_export] + use crate::db::{NativeDatabase, NativeDatabaseFailure}; #[cfg(feature = "panic-test")] #[pymodule_export] use crate::diagnostics::_panic_for_test; @@ -184,6 +189,7 @@ mod tests { "NativeClickHouseSpendConfig", "NativeClickHouseSpendStorage", "NativeDatabase", + "NativeDatabaseFailure", "trace_encode_error", "bundled_model_cost_map", "NativeModelCostMap", @@ -206,6 +212,8 @@ mod tests { "process_state_started", "reserve_process_for_forking", "unsupported_tls_settings", + "NativeBudgets", + "budget_check_decision", "NativeS3Log", ]; #[cfg(feature = "huggingface")] diff --git a/litellm/rust/crates/python-bridge/src/logger/machine.rs b/litellm/rust/crates/python-bridge/src/logger/machine.rs index 54f8f3d4b3f..12d9d184786 100644 --- a/litellm/rust/crates/python-bridge/src/logger/machine.rs +++ b/litellm/rust/crates/python-bridge/src/logger/machine.rs @@ -26,7 +26,9 @@ impl Machine for LoggedMachine { type Complete = M::Complete; fn resume(&mut self) -> Step<'_, Self> { - let logger = self.logger.get_or_init(|| Python::attach(super::capture)); + let logger = self + .logger + .get_or_init(|| Python::attach(super::capture_call)); Box::pin(logger.instrument(logger.scope(|| self.machine.resume()))) } @@ -34,7 +36,9 @@ impl Machine for LoggedMachine { &mut self, failure: HostFailure<::Error>, ) -> Interrupted<'_, Self> { - let logger = self.logger.get_or_init(|| Python::attach(super::capture)); + let logger = self + .logger + .get_or_init(|| Python::attach(super::capture_call)); Box::pin(logger.instrument(logger.scope(|| self.machine.interrupt(failure)))) } } diff --git a/litellm/rust/crates/python-bridge/src/logger/mod.rs b/litellm/rust/crates/python-bridge/src/logger/mod.rs index 70d323bbd84..1dae55262a5 100644 --- a/litellm/rust/crates/python-bridge/src/logger/mod.rs +++ b/litellm/rust/crates/python-bridge/src/logger/mod.rs @@ -2,6 +2,8 @@ mod machine; pub(crate) use machine::LoggedMachine; +use std::sync::atomic::{AtomicU8, Ordering}; + use litellm_host_python::{Pythonized, release_gil}; use litellm_tracing::{ DiagnosticInput, Level, Logger, Metadata, Policy, Processor, Record, Sink, TokenRedactor, @@ -118,6 +120,40 @@ fn processing_error(error: fancy_regex::Error) -> PyErr { struct PythonSink { correlation: (String, String), + levels: Option, +} + +#[derive(Default)] +struct Levels([AtomicU8; 4]); + +const UNKNOWN: u8 = 0; +const DISABLED: u8 = 1; +const ENABLED: u8 = 2; + +impl Levels { + fn enabled(&self, level: u8, query: impl FnOnce() -> bool) -> bool { + let slot = &self.0[usize::from(level / 10 - 1)]; + match slot.load(Ordering::Relaxed) { + UNKNOWN => { + let enabled = query(); + slot.store(if enabled { ENABLED } else { DISABLED }, Ordering::Relaxed); + enabled + } + state => state == ENABLED, + } + } +} + +fn python_enabled(level: u8) -> bool { + Python::try_attach(|py| { + report( + py, + py.import(MODULE) + .and_then(|module| module.call_method1("enabled", (level,))) + .and_then(|enabled| enabled.extract()), + ) + }) + .unwrap_or(false) } fn level(level: &Level) -> u8 { @@ -145,15 +181,11 @@ impl Sink for PythonSink { { return false; } - Python::try_attach(|py| { - report( - py, - py.import(MODULE) - .and_then(|module| module.call_method1("enabled", (level(metadata.level()),))) - .and_then(|enabled| enabled.extract()), - ) - }) - .unwrap_or(false) + let level = level(metadata.level()); + match &self.levels { + Some(levels) => levels.enabled(level, || python_enabled(level)), + None => python_enabled(level), + } } fn emit(&self, record: &Record) { @@ -182,12 +214,25 @@ impl Sink for PythonSink { } pub(crate) fn capture(py: Python<'_>) -> Logger { + logger(py, None) +} + +pub(crate) fn capture_call(py: Python<'_>) -> Logger { + logger(py, Some(Levels::default())) +} + +fn logger(py: Python<'_>, levels: Option) -> Logger { report( py, py.import(MODULE) .and_then(|module| module.call_method0("context")) .and_then(|value| value.extract()) - .map(|correlation| Logger::new(PythonSink { correlation })), + .map(|correlation| { + Logger::new(PythonSink { + correlation, + levels, + }) + }), ) } diff --git a/litellm/rust/crates/python-bridge/src/logger/tests.rs b/litellm/rust/crates/python-bridge/src/logger/tests.rs index 49e0cbf07f2..81af9ca883b 100644 --- a/litellm/rust/crates/python-bridge/src/logger/tests.rs +++ b/litellm/rust/crates/python-bridge/src/logger/tests.rs @@ -1,4 +1,4 @@ -use std::{process::Command, task::Poll}; +use std::{cell::Cell, process::Command, task::Poll}; use litellm_host::{ machine::{HostFailure, Interrupted, Machine, MachineStep, Step}, @@ -39,6 +39,51 @@ impl Machine for DiagnosticMachine { } } +struct ChunkMachine { + raise_level: bool, +} + +impl Machine for ChunkMachine { + type Protocol = DiagnosticMachine; + type Complete = (); + + fn resume(&mut self) -> Step<'_, Self> { + let raise_level = self.raise_level; + Box::pin(async move { + for index in 0..3 { + litellm_tracing::debug!(index, "stream chunk"); + if raise_level { + Python::attach(|py| { + py.import("litellm._logging")? + .getattr("verbose_logger")? + .call_method1("setLevel", (10,)) + .map(|_| ()) + }) + .map_err(|error| error.to_string())?; + } + tokio::task::yield_now().await; + } + Ok(MachineStep::Complete(())) + }) + } + + fn interrupt(&mut self, _: HostFailure) -> Interrupted<'_, Self> { + Box::pin(async { Ok(()) }) + } +} + +#[pyfunction] +fn machine_chunks(py: Python<'_>, raise_level: bool) -> PyResult> { + let mut machine = super::LoggedMachine::new(ChunkMachine { raise_level }); + litellm_host_python::run_async_value(py, async move { + machine + .resume() + .await + .map(|_| ()) + .map_err(pyo3::exceptions::PyValueError::new_err) + }) +} + #[pyfunction] fn machine_warning(py: Python<'_>) -> PyResult> { let mut machine = super::LoggedMachine::new(DiagnosticMachine); @@ -122,6 +167,24 @@ fn http_warning(py: Python<'_>) -> PyResult<()> { crate::http::call_config(py, &PyDict::new(py), false).map(|_| ()) } +#[rstest::rstest] +#[case::debug(10)] +#[case::info(20)] +#[case::warning(30)] +#[case::error(40)] +fn call_levels_ask_python_once_per_level(#[case] level: u8, #[values(true, false)] enabled: bool) { + let levels = super::Levels::default(); + let queries = Cell::new(0); + let query = || { + queries.set(queries.get() + 1); + enabled + }; + assert_eq!(levels.enabled(level, query), enabled); + assert_eq!(levels.enabled(level, query), enabled); + assert_eq!(levels.enabled(level, query), enabled); + assert_eq!(queries.get(), 1); +} + #[rstest::rstest] fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication() { if std::env::var_os("LITELLM_LOGGER_TEST_PROCESS").is_none() { @@ -157,6 +220,12 @@ fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication wrap_pyfunction!(machine_warning, py).unwrap(), ) .unwrap(); + locals + .set_item( + "machine_chunks", + wrap_pyfunction!(machine_chunks, py).unwrap(), + ) + .unwrap(); locals .set_item( "synchronous_failure", @@ -281,6 +350,42 @@ try: ('warn', logging.WARNING), ('error', logging.ERROR), ] + import litellm.rust_bridge.host.logger as host_logger + original_enabled = host_logger.enabled + queries = [] + def counting_enabled(level): + queries.append(level) + return original_enabled(level) + host_logger.enabled = counting_enabled + async def chunks(raise_level): + await machine_chunks(raise_level) + try: + verbose_logger.setLevel(logging.WARNING) + before_chunks = len(capture.records) + asyncio.run(chunks(False)) + assert capture.records[before_chunks:] == [] + assert queries == [logging.DEBUG] + + queries.clear() + verbose_logger.setLevel(logging.DEBUG) + asyncio.run(chunks(False)) + assert [(r.getMessage(), r.levelno, r.rust_fields) for r in capture.records[before_chunks:]] == [ + ('stream chunk', logging.DEBUG, {'index': 0}), + ('stream chunk', logging.DEBUG, {'index': 1}), + ('stream chunk', logging.DEBUG, {'index': 2}), + ] + assert queries == [logging.DEBUG] * 4 + + queries.clear() + verbose_logger.setLevel(logging.WARNING) + before_raise = len(capture.records) + asyncio.run(chunks(True)) + assert capture.records[before_raise:] == [] + assert queries == [logging.DEBUG] + finally: + host_logger.enabled = original_enabled + verbose_logger.setLevel(logging.DEBUG) + before_spans = len(capture.records) async def traced_request(): session = session_id_var.set('span-session') diff --git a/litellm/rust/crates/python-bridge/src/redis/store/spend_counters.rs b/litellm/rust/crates/python-bridge/src/redis/store/spend_counters.rs index 9da6a0e00e5..d6adccf7d9d 100644 --- a/litellm/rust/crates/python-bridge/src/redis/store/spend_counters.rs +++ b/litellm/rust/crates/python-bridge/src/redis/store/spend_counters.rs @@ -61,6 +61,12 @@ fn numbers(values: Option>>) -> Option>> #[pyclass(frozen, name = "NativeSpendCounters")] pub(crate) struct NativeSpendCounters(Arc); +impl NativeSpendCounters { + pub(crate) fn store(&self) -> Arc { + Arc::clone(&self.0) + } +} + #[pymethods] impl NativeSpendCounters { #[new] diff --git a/litellm/rust/crates/python-bridge/src/routes/messages/host.rs b/litellm/rust/crates/python-bridge/src/routes/messages/host.rs index 915e0581793..a31e0ea8061 100644 --- a/litellm/rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm/rust/crates/python-bridge/src/routes/messages/host.rs @@ -21,6 +21,7 @@ use litellm_llms::{ CALLABLE_TOKEN_PROVIDER, CALLER_TOKEN_PROVIDER_PARAM, ENTRA_ID_PARAMS, }, base_llm::messages::context::MessagesModelCapabilities, + bedrock::chat::invoke_handler::BedrockStreamRaise, }; use litellm_llms_types::headers::ProviderSpecificHeaders; use litellm_router_types::LitellmParams; @@ -51,14 +52,14 @@ type MergedHeaders = (Option>, Option>); const ANTHROPIC_BETA_HEADER: &str = "anthropic-beta"; -/// Python filters `anthropic-beta` before its client encodes it, so the wire check decides it. -/// Only for `azure_ai`: by the owner's ruling `anthropic` keeps main's hold-back of the header. +/// A config whose `should_filter_anthropic_beta_headers` holds filters `anthropic-beta` before +/// Python's client encodes it, so the wire check decides it. fn is_filtered_beta_header( name: &Bound<'_, PyAny>, value: &Bound<'_, PyAny>, - provider_filters_beta: bool, + filters_beta: bool, ) -> bool { - provider_filters_beta + filters_beta && name .extract::() .is_ok_and(|name| name.eq_ignore_ascii_case(ANTHROPIC_BETA_HEADER)) @@ -68,7 +69,7 @@ fn is_filtered_beta_header( fn ascii_headers<'py>( headers: &Bound<'py, PyAny>, unencodable: &mut Option>, - provider_filters_beta: bool, + filters_beta: bool, ) -> PyResult> { let Ok(headers) = headers.cast::() else { return Ok(headers.clone()); @@ -77,7 +78,7 @@ fn ascii_headers<'py>( let kept = PyDict::new(py); for (name, value) in headers.iter() { if value.is_instance_of::() - && !is_filtered_beta_header(&name, &value, provider_filters_beta) + && !is_filtered_beta_header(&name, &value, filters_beta) && let Err(error) = value.call_method1("encode", ("ascii",)) { if !error.is_instance_of::(py) { @@ -99,7 +100,7 @@ const PROVIDER_SPECIFIC_HEADERS_MODULE: &str = fn ascii_scoped_headers<'py>( value: &Bound<'py, PyAny>, - provider_filters_beta: bool, + filters_beta: bool, ) -> PyResult> { let ascii_entry = |entry: Bound<'py, PyAny>| -> PyResult> { let Ok(entry) = entry.cast::() else { @@ -111,7 +112,7 @@ fn ascii_scoped_headers<'py>( let kept = entry.copy()?; kept.set_item( "extra_headers", - ascii_headers(&headers, &mut None, provider_filters_beta)?, + ascii_headers(&headers, &mut None, filters_beta)?, )?; Ok(kept.into_any()) }; @@ -146,7 +147,8 @@ fn json_messages(messages: &Bound<'_, PyAny>) -> PyResult { } pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages"; -const MODEL_CAPABILITIES_MODULE: &str = "litellm.rust_bridge.host.model_capabilities"; +const MESSAGES_STREAM_FAILURE: &str = "messages_stream_failure"; +pub(super) const MODEL_CAPABILITIES_MODULE: &str = "litellm.rust_bridge.host.model_capabilities"; fn merge_headers( forwarded: Option>, @@ -162,6 +164,63 @@ const FEDERATING_PROVIDER: &str = "anthropic"; const AZURE_AI_PROVIDER: &str = "azure_ai"; const ENABLE_TOKEN_REFRESH: &str = "enable_azure_ad_token_refresh"; +const BEDROCK_PROVIDER: &str = "bedrock"; +const BEDROCK_PASSTHROUGH_PARAMS: [&str; 2] = ["model_id", "guardrailConfig"]; +const BEDROCK_METADATA_PARAMS: [&str; 2] = ["metadata", "litellm_metadata"]; +const BEDROCK_SPEND_LOGS_METADATA: &str = "spend_logs_metadata"; +const BEDROCK_REQUEST_METADATA_FIELDS: &str = "bedrock_request_metadata_fields"; + +fn text_pairs(value: &Bound<'_, PyAny>) -> PyResult> { + let Ok(mapping) = value.cast::() else { + return Ok(Map::new()); + }; + mapping + .items()? + .iter() + .filter_map(|item| { + let (key, value): (Bound<'_, PyAny>, Bound<'_, PyAny>) = match item.extract() { + Ok(pair) => pair, + Err(error) => return Some(Err(error)), + }; + let key = key.cast::().ok()?.to_str().ok()?.to_string(); + if key == BEDROCK_SPEND_LOGS_METADATA { + return Some(text_pairs(&value).map(|pairs| (key, Value::Object(pairs)))); + } + let value = value.cast::().ok()?.to_str().ok()?.to_string(); + Some(Ok((key, Value::String(value)))) + }) + .collect() +} + +fn bedrock_litellm_params<'py>( + py: Python<'py>, + argument: impl Fn(&str) -> PyResult>>, +) -> PyResult> { + let mut extra = project_optional_fields(BEDROCK_PASSTHROUGH_PARAMS, &argument)?; + for name in BEDROCK_METADATA_PARAMS { + if let Some(value) = argument(name)? { + extra.insert(name.to_string(), Value::Object(text_pairs(&value)?)); + } + } + if let Some(configured) = module_global(py, BEDROCK_REQUEST_METADATA_FIELDS)? + && (configured.is_instance_of::() || configured.is_instance_of::()) + { + let fields = configured + .try_iter()? + .map(|field| Ok(Value::String(field?.str()?.to_str()?.to_string()))) + .collect::>>()?; + extra.insert( + BEDROCK_REQUEST_METADATA_FIELDS.to_string(), + Value::Array(fields), + ); + } + Ok(extra) +} + +fn global_drop_params(py: Python<'_>) -> PyResult { + Ok(module_global(py, "drop_params")?.is_some_and(|value| value.is(PyBool::new(py, true)))) +} + fn workload_identity_fields(py: Python<'_>) -> PyResult<&'static [String]> { static FIELDS: PyOnceLock> = PyOnceLock::new(); FIELDS @@ -266,6 +325,16 @@ impl litellm_host::protocol::Protocol for MessagesHosted { type HostCall = MessagesHostCall; type Chunk = Bytes; type StreamHead = MessagesStreamHead; + + fn detached_drain(head: &MessagesStreamHead) -> Option { + ::detached_drain(head) + } + + fn unobserved_tail( + head: &MessagesStreamHead, + ) -> Option { + ::unobserved_tail(head) + } } pub(super) enum MessagesHostCall { @@ -325,6 +394,7 @@ pub(super) struct MessagesPythonHost { pending_short_circuit: Option<(bool, Reply)>, answer: Option>, answered: Option, + stream_opened: bool, } impl MessagesPythonHost { @@ -354,6 +424,7 @@ impl MessagesPythonHost { pending_short_circuit: None, answer: None, answered: None, + stream_opened: false, } } @@ -550,32 +621,41 @@ impl MessagesPythonHost { .call1((&model, api_base.as_deref()))? .extract()?, }; + let filters_beta: bool = py + .import(ROUTE_HOST_MODULE)? + .getattr("filters_beta_headers")? + .call1((&model, custom_llm_provider.as_deref(), api_base.as_deref()))? + .extract()?; let (extra_headers, mut unencodable_request_header) = match settings.extra_headers { Some(headers) => (Some(headers), None), - None => self.merged_headers( - py, - arguments, - custom_llm_provider.as_deref() == Some(AZURE_AI_PROVIDER), - )?, + None => self.merged_headers(py, arguments, filters_beta)?, }; let provider_specific_header = self.provider_specific_header( py, arguments, custom_llm_provider.as_deref(), + filters_beta, &mut unencodable_request_header, )?; let azure = match custom_llm_provider.as_deref() { Some(AZURE_AI_PROVIDER) => Some(azure_module_settings(|name| module_global(py, name))?), _ => None, }; + let bedrock = custom_llm_provider.as_deref() == Some(BEDROCK_PROVIDER); + let mut capabilities = self.capabilities(py, &model, custom_llm_provider.as_deref())?; + if bedrock { + capabilities.bedrock = + super::model_catalog::bedrock_capabilities(py, &model, argument("model_info")?)?; + } let shaping = MessagesShaping { - capabilities: self.capabilities(py, &model, custom_llm_provider.as_deref())?, + capabilities, settings: MessagesSettings { whole_sse_frames: settings.stream_cost_model.is_some(), presanitized, intercepted: self.may_intercept(py)?, unencodable_request_header, module_api_key: azure.as_ref().and_then(|azure| azure.api_key.clone()), + global_drop_params: global_drop_params(py)?, ..settings.settings }, }; @@ -587,47 +667,42 @@ impl MessagesPythonHost { Some(AZURE_AI_PROVIDER) => ENTRA_ID_PARAMS.to_vec(), _ => Vec::new(), }; - let settled = match azure { - Some(azure) => azure_settled_params(azure, argument(CALLER_TOKEN_PROVIDER_PARAM)?)?, - None => Map::new(), + let settled = match (azure, bedrock) { + (Some(azure), _) => { + azure_settled_params(azure, argument(CALLER_TOKEN_PROVIDER_PARAM)?)? + } + (None, true) => bedrock_litellm_params(py, argument)?, + (None, false) => Map::new(), }; let litellm_params = project_litellm_params(provider_fields, settled, argument, |name| { module_global(py, name) })?; - Ok( - messages_body(body, custom_llm_provider.as_deref()).and_then(|body| { - Ok(MessagesCall { - body, - api_key, - api_base, - extra_headers, - provider_specific_header, - custom_llm_provider, - litellm_params: litellm_params?, - timeout: optional_timeout(timeout), - shaping, - }) - }), - ) + Ok(messages_body(body).and_then(|body| { + Ok(MessagesCall { + body, + api_key, + api_base, + extra_headers, + provider_specific_header, + custom_llm_provider, + litellm_params: litellm_params?, + timeout: optional_timeout(timeout), + shaping, + }) + })) } fn merged_headers( &self, py: Python<'_>, arguments: &Bound<'_, PyDict>, - provider_filters_beta: bool, + filters_beta: bool, ) -> PyResult { let mut unencodable = None; let mut mapping = |name: &str| -> PyResult>> { self.view .argument(py, arguments, name)? - .map(|value| { - from_py(&ascii_headers( - &value, - &mut unencodable, - provider_filters_beta, - )?) - }) + .map(|value| from_py(&ascii_headers(&value, &mut unencodable, filters_beta)?)) .transpose() }; let headers = mapping("headers")?; @@ -640,6 +715,7 @@ impl MessagesPythonHost { py: Python<'_>, arguments: &Bound<'_, PyDict>, provider: Option<&str>, + filters_beta: bool, unencodable: &mut Option>, ) -> PyResult> { let Some(value) = self @@ -655,13 +731,9 @@ impl MessagesPythonHost { .getattr("ProviderSpecificHeaderUtils")? .call_method1("get_provider_specific_headers", (&value, provider, true)) { - ascii_headers(&scoped, unencodable, provider == AZURE_AI_PROVIDER)?; + ascii_headers(&scoped, unencodable, filters_beta)?; } - from_py(&ascii_scoped_headers( - &value, - provider == Some(AZURE_AI_PROVIDER), - )?) - .map(Some) + from_py(&ascii_scoped_headers(&value, filters_beta)?).map(Some) } fn capabilities( @@ -723,6 +795,89 @@ impl MessagesPythonHost { Err(_) => error, } } + + fn bedrock_stream_raise(&self, py: Python<'_>, raise: &BedrockStreamRaise) -> PyResult { + let (kind, arguments) = match raise { + BedrockStreamRaise::ChecksumMismatch { + expected, + calculated, + } => ( + "ChecksumMismatch", + serde_json::json!([expected, calculated]), + ), + BedrockStreamRaise::InvalidHeadersLength(length) => { + ("InvalidHeadersLength", serde_json::json!([length])) + } + BedrockStreamRaise::InvalidPayloadLength(length) => { + ("InvalidPayloadLength", serde_json::json!([length])) + } + BedrockStreamRaise::DuplicateHeader(name) => { + ("DuplicateHeader", serde_json::json!([name])) + } + BedrockStreamRaise::InvalidJson(text) => ("InvalidJson", serde_json::json!([text])), + BedrockStreamRaise::NotAnObject(chunk) => ("NotAnObject", serde_json::json!([chunk])), + }; + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("bedrock_stream_raise")? + .call1((kind, to_py(py, &arguments)?))?; + Ok(PyErr::from_value(raised)) + } + + fn mapped_error(&self, py: Python<'_>, error: Error) -> PyResult { + if let Some(raise) = litellm_error::find::(&error) { + return self.bedrock_stream_raise(py, raise); + } + if let Some((raised, raw)) = python_failure(py, &error) { + return if raw { + Ok(raised) + } else { + self.map_transport_failure(py, raised) + }; + } + if let Some(raised) = super::workload_identity::python_error(py, &error) + .or_else(|| super::workload_identity::entra_token_error(py, &error)) + { + let mapped = py + .import(ROUTE_HOST_MODULE)? + .getattr("map_token_failure")? + .call1(( + raised.value(py), + self.view.request(py)?, + self.provider(py), + self.asynchronous, + ))?; + return Ok(PyErr::from_value(mapped)); + } + if let Some(unencodable) = litellm_error::find::(&error) { + let (value, received) = match unencodable { + UnencodableHeader::Sent(value) => (value, false), + UnencodableHeader::Received(value) => (value, true), + }; + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("unencodable_header")? + .call1((PyBytes::new(py, value), received))?; + return self.map_transport_failure(py, PyErr::from_value(raised)); + } + if let Some(undecodable) = litellm_error::find::(&error) { + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("decoding_failure")? + .call1(( + undecodable.to_string(), + self.view.request(py)?, + self.provider(py), + ))?; + return Ok(PyErr::from_value(raised)); + } + route_failure(py, error, RequestMarker::Messages, |native| { + if self.stream_opened { + native.value(py).setattr(MESSAGES_STREAM_FAILURE, true)?; + } + Ok(self.map_failure(py, native)) + }) + } } impl MessagesPythonHost { @@ -839,6 +994,9 @@ impl PythonBinding for MessagesPythonHost { return Ok(answer); } let public = public_json_response(py, ROUTE_HOST_MODULE, &response)?; + py.import(ROUTE_HOST_MODULE)? + .getattr("record_guardrail")? + .call1((&public, self.view.request(py)?))?; self.safeguards.published(py, public) } @@ -859,15 +1017,12 @@ impl PythonBinding for MessagesPythonHost { py: Python<'_>, head: MessagesStreamHead, ) -> PyResult> { + self.stream_opened = true; let module = py.import(ROUTE_HOST_MODULE)?; let head = module .getattr("stream_hidden_params")? - .call1((to_py(py, &head.headers)?,))?; - if let Some(arguments) = self - .arguments - .as_ref() - .filter(|_| self.asynchronous && self.provider(py) == AZURE_AI_PROVIDER) - { + .call1((to_py(py, &head.headers)?, head.unobserved_tail.is_some()))?; + if let Some(arguments) = self.arguments.as_ref().filter(|_| self.asynchronous) { module .getattr("python_stream_types")? .call1((&head, arguments))?; @@ -896,52 +1051,15 @@ impl PythonBinding for MessagesPythonHost { } fn map_error(&self, py: Python<'_>, error: Error) -> PyResult { - if let Some((raised, raw)) = python_failure(py, &error) { - return if raw { - Ok(raised) - } else { - self.map_transport_failure(py, raised) - }; + let mapped = self.mapped_error(py, error)?; + if !self.stream_opened { + return Ok(mapped); } - if let Some(raised) = super::workload_identity::python_error(py, &error) - .or_else(|| super::workload_identity::entra_token_error(py, &error)) - { - let mapped = py - .import(ROUTE_HOST_MODULE)? - .getattr("map_token_failure")? - .call1(( - raised.value(py), - self.view.request(py)?, - self.provider(py), - self.asynchronous, - ))?; - return Ok(PyErr::from_value(mapped)); - } - if let Some(unencodable) = litellm_error::find::(&error) { - let (value, received) = match unencodable { - UnencodableHeader::Sent(value) => (value, false), - UnencodableHeader::Received(value) => (value, true), - }; - let raised = py - .import(ROUTE_HOST_MODULE)? - .getattr("unencodable_header")? - .call1((PyBytes::new(py, value), received))?; - return self.map_transport_failure(py, PyErr::from_value(raised)); - } - if let Some(undecodable) = litellm_error::find::(&error) { - let raised = py - .import(ROUTE_HOST_MODULE)? - .getattr("decoding_failure")? - .call1(( - undecodable.to_string(), - self.view.request(py)?, - self.provider(py), - ))?; - return Ok(PyErr::from_value(raised)); - } - route_failure(py, error, RequestMarker::Messages, |native| { - Ok(self.map_failure(py, native)) - }) + let described = py + .import(ROUTE_HOST_MODULE)? + .getattr("stream_failure")? + .call1((mapped.value(py), self.view.request(py)?, self.provider(py)))?; + Ok(PyErr::from_value(described)) } fn host_error(error: &PyErr) -> Error { @@ -1115,7 +1233,7 @@ mod tests { )] fn a_header_python_cannot_encode_is_held_back_unless_the_provider_filters_it( #[case] headers: &std::ffi::CStr, - #[case] provider_filters_beta: bool, + #[case] filters_beta: bool, #[case] kept: Value, #[case] unencodable: Option>, ) { @@ -1124,8 +1242,7 @@ mod tests { let headers = py.eval(headers, None, None).unwrap(); let mut held_back = None; let sent: Value = - from_py(&ascii_headers(&headers, &mut held_back, provider_filters_beta).unwrap()) - .unwrap(); + from_py(&ascii_headers(&headers, &mut held_back, filters_beta).unwrap()).unwrap(); assert_eq!((sent, held_back), (kept, unencodable)); }); } @@ -1143,6 +1260,14 @@ mod tests { json!({"model": "m", "vertex_project": "p", "vertex_ai_location": "us-east5", "vertex_credentials": {"type": "service_account"}}), Ok(json!({"model": "m", "vertex_project": "p", "vertex_ai_location": "us-east5", "vertex_credentials": {"type": "service_account"}})), )] + #[case::aws_session_tags_are_projected( + json!({"model": "m", "aws_role_name": "r", "aws_session_tags": [{"Key": "team", "Value": "genai"}]}), + Ok(json!({"model": "m", "aws_role_name": "r", "aws_session_tags": [{"Key": "team", "Value": "genai"}]})), + )] + #[case::malformed_aws_session_tags_are_a_request_error( + json!({"model": "m", "aws_session_tags": [{"Key": "team"}]}), + Err(()), + )] #[case::an_explicit_none_is_absent(json!({"model": "m", "aws_region_name": null}), Ok(json!({"model": "m"})))] #[case::a_wrong_type_is_a_request_error(json!({"model": "m", "aws_region_name": 7}), Err(()))] #[case::a_missing_model_is_a_request_error(json!({"aws_region_name": "eu-central-1"}), Err(()))] diff --git a/litellm/rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm/rust/crates/python-bridge/src/routes/messages/mod.rs index 2df94cf911c..fe4be84d2d3 100644 --- a/litellm/rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm/rust/crates/python-bridge/src/routes/messages/mod.rs @@ -1,6 +1,7 @@ mod agentic; mod answer; mod host; +mod model_catalog; mod modify_params; mod pre_request; mod requests_reason; diff --git a/litellm/rust/crates/python-bridge/src/routes/messages/model_catalog.rs b/litellm/rust/crates/python-bridge/src/routes/messages/model_catalog.rs new file mode 100644 index 00000000000..c2100c068be --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/routes/messages/model_catalog.rs @@ -0,0 +1,62 @@ +use std::sync::{Arc, Mutex, PoisonError}; + +use litellm_host_python::json_from_py; +use litellm_llms::{ + anthropic::model_capabilities::bedrock_model_capabilities, + base_llm::messages::context::BedrockCapabilities, +}; +use litellm_model_catalog::Catalog; +use pyo3::{ + exceptions::PyValueError, + prelude::*, + types::{PyBool, PyMapping}, +}; + +use super::host::MODEL_CAPABILITIES_MODULE; + +type Version = (usize, usize, u64); + +static LIVE: Mutex)>> = Mutex::new(None); + +fn live_catalog(py: Python<'_>) -> PyResult> { + let known = LIVE.lock().unwrap_or_else(PoisonError::into_inner).clone(); + let (version, document): (Version, Option>) = py + .import(MODEL_CAPABILITIES_MODULE)? + .getattr("model_cost_document")? + .call1((known.as_ref().map(|(version, _)| *version),))? + .extract()?; + let Some(document) = document else { + return known + .map(|(_, catalog)| catalog) + .ok_or_else(|| PyValueError::new_err("the model map came back without a document")); + }; + let body = serde_json::to_vec(&json_from_py(&document)?) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let catalog = Arc::new( + py.detach(|| Catalog::parse(&body)) + .map_err(|error| PyValueError::new_err(error.to_string()))? + .value, + ); + *LIVE.lock().unwrap_or_else(PoisonError::into_inner) = Some((version, Arc::clone(&catalog))); + Ok(catalog) +} + +pub(super) fn bedrock_capabilities( + py: Python<'_>, + model: &str, + model_info: Option>, +) -> PyResult { + let model_info = model_info.and_then(|info| info.cast_into::().ok()); + let declared = |name: &str| { + model_info + .as_ref() + .and_then(|info| info.get_item(name).ok()) + .and_then(|value| value.cast_into::().ok()) + .map(|flag| flag.is_true()) + }; + Ok(bedrock_model_capabilities( + &*live_catalog(py)?, + model, + &declared, + )) +} diff --git a/litellm/rust/crates/python-bridge/src/secrets/local.rs b/litellm/rust/crates/python-bridge/src/secrets/local.rs new file mode 100644 index 00000000000..701dc5e6f5b --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/secrets/local.rs @@ -0,0 +1,238 @@ +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + sync::Arc, +}; + +use futures_util::future::{BoxFuture, FutureExt, try_join_all}; +use litellm_core_utils::{serde_compat::parse_str_bool, settings::Lookup}; +use litellm_secret_value::SecretValue; +use litellm_secrets::{ + Error, + source::{SecretSource, Secrets}, +}; + +const ENVIRONMENT_PREFIX: &str = "os.environ/"; +const OIDC_PREFIX: &str = "oidc/"; + +/// Python's `get_secret_str` while no secret manager is readable: names come from the process +/// environment, read in Rust when the route looks them up, and `oidc/` references go to `python`. +pub(super) struct LocalSecrets { + python: Arc, +} + +impl LocalSecrets { + pub(super) fn new(python: Arc) -> Self { + Self { python } + } +} + +fn python_name(name: &str) -> Cow<'_, str> { + if name.starts_with(ENVIRONMENT_PREFIX) { + Cow::Owned(name.replace(ENVIRONMENT_PREFIX, "")) + } else { + Cow::Borrowed(name) + } +} + +fn is_oidc(name: &str) -> bool { + python_name(name).starts_with(OIDC_PREFIX) +} + +fn environment_str(name: &str) -> Option { + let name = python_name(name); + if name.contains(['=', '\0']) { + return None; + } + std::env::var(name.as_ref()) + .ok() + .filter(|value| parse_str_bool(value).is_none()) + .map(SecretValue::new) +} + +impl SecretSource for LocalSecrets { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> BoxFuture<'a, Result, Error>> { + if is_oidc(name) { + return self.python.get_secret_str(name); + } + futures_util::future::ready(Ok(environment_str(name))).boxed() + } + + fn resolve<'a>(&'a self, names: &'a [&str]) -> BoxFuture<'a, Result> { + Box::pin(async move { + let (oidc, environment): (Vec<&str>, Vec<&str>) = + names.iter().copied().partition(|name| is_oidc(name)); + let oidc = try_join_all(oidc.into_iter().map(|name| async move { + self.python + .get_secret_str(name) + .await + .map(|value| (name.to_owned(), value)) + })) + .await? + .into_iter() + .collect(); + Ok(Arc::new(LocalLookup { + environment: environment.into_iter().map(str::to_owned).collect(), + oidc, + }) as Secrets) + }) + } +} + +struct LocalLookup { + environment: HashSet, + oidc: HashMap>, +} + +impl Lookup for LocalLookup { + fn get(&self, name: &str) -> Option { + let value = if self.environment.contains(name) { + environment_str(name) + } else { + self.oidc.get(name).cloned().flatten() + }; + value.map(|value| value.expose().to_owned()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use futures_util::future::{BoxFuture, FutureExt}; + use litellm_secret_value::SecretValue; + use litellm_secrets::{Error, source::SecretSource}; + use rstest::rstest; + + use super::LocalSecrets; + + #[derive(Default)] + struct Recording(Mutex>); + + impl Recording { + fn reads(&self) -> Vec { + self.0.lock().unwrap().clone() + } + } + + impl SecretSource for Recording { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> BoxFuture<'a, Result, Error>> { + self.0.lock().unwrap().push(name.to_owned()); + futures_util::future::ready(Ok(Some(SecretValue::new(format!("python-{name}"))))) + .boxed() + } + } + + fn local() -> (LocalSecrets, Arc) { + let python = Arc::new(Recording::default()); + ( + LocalSecrets::new(Arc::clone(&python) as Arc), + python, + ) + } + + fn set(name: &str, value: Option<&str>) { + match value { + Some(value) => unsafe { std::env::set_var(name, value) }, + None => unsafe { std::env::remove_var(name) }, + } + } + + #[rstest] + #[case::plain( + "LITELLM_LOCAL_PLAIN", + "LITELLM_LOCAL_PLAIN", + Some("value"), + Some("value") + )] + #[case::empty("LITELLM_LOCAL_EMPTY", "LITELLM_LOCAL_EMPTY", Some(""), Some(""))] + #[case::unset("LITELLM_LOCAL_UNSET", "LITELLM_LOCAL_UNSET", None, None)] + #[case::reference( + "LITELLM_LOCAL_REFERENCE", + "os.environ/LITELLM_LOCAL_REFERENCE", + Some("value"), + Some("value") + )] + #[case::every_marker_removed( + "LITELLM_LOCAL_TWICE", + "os.environ/os.environ/LITELLM_LOCAL_TWICE", + Some("value"), + Some("value") + )] + #[case::marker_inside_only_when_prefixed( + "LITELLM_LOCAL_INNER", + "LITELLM_LOCAL_os.environ/INNER", + Some("value"), + None + )] + #[case::boolean_true("LITELLM_LOCAL_TRUE", "LITELLM_LOCAL_TRUE", Some(" TRUE\n"), None)] + #[case::boolean_false("LITELLM_LOCAL_FALSE", "LITELLM_LOCAL_FALSE", Some("False"), None)] + #[case::boolean_like("LITELLM_LOCAL_ONE", "LITELLM_LOCAL_ONE", Some("1"), Some("1"))] + #[case::equals_in_name("LITELLM_LOCAL_EQ", "LITELLM_LOCAL_EQ=x", Some("x=y"), None)] + #[tokio::test] + async fn reads_what_get_secret_str_reads_from_the_environment( + #[case] variable: &str, + #[case] name: &str, + #[case] value: Option<&str>, + #[case] expected: Option<&str>, + ) { + set(variable, value); + let (secrets, python) = local(); + + let read = secrets.get_secret_str(name).await.unwrap(); + let resolved = secrets.resolve(&[name]).await.unwrap().get(name); + set(variable, None); + + assert_eq!(read.as_ref().map(SecretValue::expose), expected); + assert_eq!(resolved.as_deref(), expected); + assert!(python.reads().is_empty()); + } + + #[rstest] + #[case::direct("oidc/env/LITELLM_LOCAL_TOKEN")] + #[case::referenced("os.environ/oidc/env/LITELLM_LOCAL_TOKEN")] + #[tokio::test] + async fn oidc_references_are_left_to_python(#[case] name: &str) { + let (secrets, python) = local(); + + let read = secrets.get_secret_str(name).await.unwrap(); + let resolved = secrets + .resolve(&[name, "LITELLM_LOCAL_BESIDE"]) + .await + .unwrap(); + + let expected = format!("python-{name}"); + assert_eq!( + read.as_ref().map(SecretValue::expose), + Some(expected.as_str()) + ); + assert_eq!(resolved.get(name).as_deref(), Some(expected.as_str())); + assert_eq!(python.reads(), vec![name.to_owned(), name.to_owned()]); + } + + #[tokio::test] + async fn resolved_names_are_read_when_looked_up_and_others_are_absent() { + let name = "LITELLM_LOCAL_LAZY"; + let other = "LITELLM_LOCAL_NOT_RESOLVED"; + set(name, None); + set(other, Some("present")); + let (secrets, python) = local(); + + let resolved = secrets.resolve(&[name]).await.unwrap(); + set(name, Some("set-after-resolve")); + let looked_up = resolved.get(name); + let unrequested = resolved.get(other); + set(name, None); + set(other, None); + + assert_eq!(looked_up.as_deref(), Some("set-after-resolve")); + assert_eq!(unrequested, None); + assert!(python.reads().is_empty()); + } +} diff --git a/litellm/rust/crates/python-bridge/src/secrets/mod.rs b/litellm/rust/crates/python-bridge/src/secrets/mod.rs index c77fd16bebb..dd0b6944b30 100644 --- a/litellm/rust/crates/python-bridge/src/secrets/mod.rs +++ b/litellm/rust/crates/python-bridge/src/secrets/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod callback; pub(crate) mod config; mod error; +mod local; mod mutation; mod operations; mod provider; @@ -9,6 +10,7 @@ pub(crate) mod resolved; pub(crate) mod runtime; pub(crate) mod seeded; mod vault; +mod web_identity; use std::sync::Arc; @@ -16,33 +18,98 @@ use std::sync::Arc; pub(crate) use error::read_error; pub(crate) use error::{ManagerError, python_error}; use litellm_secrets::source::{EnvironmentSecrets, SecretSource}; +use local::LocalSecrets; use pyo3::prelude::*; use python::PythonSecrets; use resolved::ResolvedSecrets; +pub(crate) use web_identity::HostWebIdentityTokens; -use crate::{coercion::FieldSpec, python_settings::PythonSettings}; +use crate::{ + coercion::FieldSpec, + python_settings::{PythonSettings, Snapshot}, +}; const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager"; +const READABLE: FieldSpec = FieldSpec::new("readable", |field| field.schema_bool()); const NATIVE: FieldSpec = FieldSpec::new("native", |field| field.schema_bool()); -/// Where a Rust route reads provider secrets from. Python's `get_secret_str` when the settings -/// projection reports no configured secret manager, otherwise the resolved manager: the native -/// backend when the configured client captures one, the Python callback when it does not. A bare -/// extension module without the litellm package reads the process environment. +#[derive(Debug, PartialEq, Eq)] +enum Selection { + Local, + Native, + Python, +} + +fn select(snapshot: &Snapshot<'_>) -> PyResult { + Ok(if !snapshot.read(&READABLE)? { + Selection::Local + } else if snapshot.read(&NATIVE)? { + Selection::Native + } else { + Selection::Python + }) +} + +/// Where a Rust route reads provider secrets from, decided per call from the settings projection. +/// With no readable secret manager Python's `get_secret_str` only reads the environment, so the +/// route reads it in Rust. A readable manager is the native backend when the configured client +/// captures one, otherwise every read goes through Python's `get_secret_str`. A bare extension +/// module without the litellm package reads the process environment. pub(crate) fn source(py: Python<'_>) -> PyResult> { let Some(snapshot) = PythonSettings::SecretManager.read_or_unset(py)? else { return Ok(Arc::new(EnvironmentSecrets::python_compatible( crate::http::host_client(py, litellm_http::ClientVariant::Provider)?, ))); }; - if snapshot.read(&NATIVE)? { - let context = litellm_host_python::PythonContext::capture(py)?; - let client = crate::http::host_client(py, litellm_http::ClientVariant::Provider)?; - return Ok(Arc::new(ResolvedSecrets::new( - config::read(py)?, - context, - client, - ))); + match select(&snapshot)? { + Selection::Local => Ok(Arc::new(LocalSecrets::new(Arc::new(PythonSecrets::new( + py, + )?)))), + Selection::Native => { + let context = litellm_host_python::PythonContext::capture(py)?; + let client = crate::http::host_client(py, litellm_http::ClientVariant::Provider)?; + Ok(Arc::new(ResolvedSecrets::new( + config::read(py)?, + context, + client, + ))) + } + Selection::Python => Ok(Arc::new(PythonSecrets::new(py)?)), + } +} + +#[cfg(test)] +mod tests { + use pyo3::{prelude::*, types::PyDict}; + use rstest::rstest; + + use super::{Selection, select}; + use crate::python_settings::PythonSettings; + + #[rstest] + #[case::no_manager(false, false, Selection::Local)] + #[case::unreadable_native(false, true, Selection::Local)] + #[case::python_manager(true, false, Selection::Python)] + #[case::native_manager(true, true, Selection::Native)] + fn the_projection_selects_where_secrets_are_read( + #[case] readable: bool, + #[case] native: bool, + #[case] expected: Selection, + ) { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("readable", readable).unwrap(); + locals.set_item("native", native).unwrap(); + let value = py + .eval( + c"__import__('types').SimpleNamespace(readable=readable, native=native)", + None, + Some(&locals), + ) + .unwrap(); + let selected = select(&PythonSettings::SecretManager.snapshot(value)).unwrap(); + assert_eq!(selected, expected); + }); } - Ok(Arc::new(PythonSecrets::new(py)?)) } diff --git a/litellm/rust/crates/python-bridge/src/secrets/seeded.rs b/litellm/rust/crates/python-bridge/src/secrets/seeded.rs index 596a562d89b..9d934d4f1f5 100644 --- a/litellm/rust/crates/python-bridge/src/secrets/seeded.rs +++ b/litellm/rust/crates/python-bridge/src/secrets/seeded.rs @@ -4,8 +4,12 @@ use std::{ }; use futures_util::future::{BoxFuture, FutureExt}; +use litellm_core_utils::settings::Lookup; use litellm_secret_value::SecretValue; -use litellm_secrets::{Error, source::SecretSource}; +use litellm_secrets::{ + Error, + source::{SecretSource, Secrets}, +}; /// A call's secret source that answers the secrets the host already read for the call with the /// values it read, so a secret Python reads once is not read a second time. @@ -49,6 +53,39 @@ impl SecretSource for SeededSecrets { None => self.source.get_secret_str(name), } } + + fn resolve<'a>(&'a self, names: &'a [&str]) -> BoxFuture<'a, Result> { + let (seeded, unseeded) = { + let read = self.read.lock().unwrap_or_else(PoisonError::into_inner); + let (seeded, unseeded): (Vec<&str>, Vec<&str>) = names + .iter() + .copied() + .partition(|name| read.contains_key(*name)); + let seeded: HashMap> = seeded + .into_iter() + .map(|name| (name.to_owned(), read.get(name).cloned().flatten())) + .collect(); + (seeded, unseeded) + }; + Box::pin(async move { + let source = self.source.resolve(&unseeded).await?; + Ok(Arc::new(SeededLookup { seeded, source }) as Secrets) + }) + } +} + +struct SeededLookup { + seeded: HashMap>, + source: Secrets, +} + +impl Lookup for SeededLookup { + fn get(&self, name: &str) -> Option { + match self.seeded.get(name) { + Some(value) => value.as_ref().map(|value| value.expose().to_owned()), + None => self.source.get(name), + } + } } #[cfg(test)] @@ -98,4 +135,49 @@ mod tests { ); assert_eq!(source.0.load(Ordering::SeqCst), 1); } + + #[tokio::test] + async fn resolve_answers_seeded_names_and_leaves_the_rest_to_the_source() { + let source = Arc::new(Counting::default()); + let secrets = SeededSecrets::new(Arc::clone(&source) as Arc); + secrets.seed([ + ( + "AZURE_API_BASE".to_owned(), + Some("http://seeded".to_owned()), + ), + ("UNSET".to_owned(), None), + ]); + + let resolved = secrets + .resolve(&["AZURE_API_BASE", "UNSET", "OTHER"]) + .await + .unwrap(); + + assert_eq!( + resolved.get("AZURE_API_BASE").as_deref(), + Some("http://seeded") + ); + assert_eq!(resolved.get("UNSET"), None); + assert_eq!(resolved.get("OTHER").as_deref(), Some("source-OTHER")); + assert_eq!(resolved.get("NOT_REQUESTED"), None); + assert_eq!(source.0.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn resolve_keeps_a_lazy_source_lazy() { + let name = "LITELLM_SEEDED_LAZY"; + unsafe { std::env::remove_var(name) }; + let python = Arc::new(Counting::default()); + let secrets = SeededSecrets::new(Arc::new(crate::secrets::local::LocalSecrets::new( + Arc::clone(&python) as Arc, + ))); + + let resolved = secrets.resolve(&[name]).await.unwrap(); + unsafe { std::env::set_var(name, "set-after-resolve") }; + let looked_up = resolved.get(name); + unsafe { std::env::remove_var(name) }; + + assert_eq!(looked_up.as_deref(), Some("set-after-resolve")); + assert_eq!(python.0.load(Ordering::SeqCst), 0); + } } diff --git a/litellm/rust/crates/python-bridge/src/secrets/web_identity.rs b/litellm/rust/crates/python-bridge/src/secrets/web_identity.rs new file mode 100644 index 00000000000..3dbe9700538 --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/secrets/web_identity.rs @@ -0,0 +1,30 @@ +use litellm_auth_aws::{WebIdentityTokenFuture, WebIdentityTokenSource}; +use litellm_error::Source; +use litellm_host_python::{PythonContext, attach_blocking}; +use pyo3::prelude::*; + +use super::error::external_error; + +#[derive(Debug)] +pub(crate) struct HostWebIdentityTokens; + +impl WebIdentityTokenSource for HostWebIdentityTokens { + fn web_identity_token<'a>(&'a self, reference: &'a str) -> WebIdentityTokenFuture<'a> { + Box::pin(async move { + let context = Python::attach(|py| { + PythonContext::empty(py).map_err(|error| external_error(py, error)) + }) + .map_err(Source::new)?; + let source = match attach_blocking(context, |py| { + super::source(py).map_err(|error| external_error(py, error)) + }) + .await + { + Ok(source) => source, + Err(error) => Err(Python::attach(|py| external_error(py, error))), + } + .map_err(Source::new)?; + source.get_secret_str(reference).await.map_err(Source::new) + }) + } +} diff --git a/litellm/rust/crates/python-compat/AGENTS.md b/litellm/rust/crates/python-compat/AGENTS.md index 55049daf612..7f8185d04d0 100644 --- a/litellm/rust/crates/python-compat/AGENTS.md +++ b/litellm/rust/crates/python-compat/AGENTS.md @@ -6,6 +6,7 @@ - `cache-disk`: diskcache's pickled values (`pickle`), falsy-is-miss (`truthy`) - Cache-key derivation: sha256 over `str(value)` must match Python byte for byte (`repr::to_str`) - Byte-identical writes where Python compares raw values (`json::dumps`, `repr`) + - `callbacks-s3-log`: settings parsed as pydantic parses them (`pydantic`), object keys quoted as `urllib.parse.quote` quotes them (`urllib`), UTC key dates (`datetime`) - Relation to [`py_literal`](https://docs.rs/py_literal/latest/py_literal/): replaced, do not reintroduce - Its pest grammar backtracks: parse time doubles per nested `[`/`{` (105 ms at depth 16). Ours is linear (19 µs at depth 128) - Its formatter is not `repr` (`2e-1`, always single quotes, escapes non-ASCII). It also lost `-0.0`, `(1+2j)`, `set()` @@ -15,6 +16,7 @@ - Accepted limits: ints beyond i64, `tuple`/`set`/`frozenset` decode as lists, class references (`GLOBAL`/`REDUCE`) fail. Writes protocol 3 - Every behavior is pinned by CPython output, not by reasoning. Everything under `generated/` is script output, never hand-edited - Regenerate `generated/values.json` with `scripts/generate_fixtures.py`. Add a corpus row before changing behavior + - Regenerate `generated/stdlib.json` (`urllib.parse.quote`, UTC `datetime` fields) with `scripts/generate_stdlib_fixtures.py`, and `generated/pydantic.json` (lax `bool`/`int`) with `scripts/generate_pydantic_fixtures.py` at the pydantic version in `uv.lock` - Divergences go in `KNOWN` in `tests/fixtures.rs` with a reason. An entry that starts matching fails until deleted - Regenerate `generated/nonprintable.rs` with `scripts/generate_nonprintable.py` when the target Python's Unicode version changes - `scripts/verify_rust_pickles.py` checks CPython reads Rust pickles, with class resolution disabled. CI does not run Python diff --git a/litellm/rust/crates/python-compat/generated/.gitattributes b/litellm/rust/crates/python-compat/generated/.gitattributes new file mode 100644 index 00000000000..807ab52b2c8 --- /dev/null +++ b/litellm/rust/crates/python-compat/generated/.gitattributes @@ -0,0 +1 @@ +* linguist-generated=true -diff diff --git a/litellm/rust/crates/python-compat/generated/pydantic.json b/litellm/rust/crates/python-compat/generated/pydantic.json new file mode 100644 index 00000000000..23ebee71515 --- /dev/null +++ b/litellm/rust/crates/python-compat/generated/pydantic.json @@ -0,0 +1,485 @@ +{ + "pydantic": "2.13.4", + "rows": [ + { + "input": "null", + "bool": null, + "int": null + }, + { + "input": "true", + "bool": true, + "int": "1" + }, + { + "input": "false", + "bool": false, + "int": "0" + }, + { + "input": "0", + "bool": false, + "int": "0" + }, + { + "input": "1", + "bool": true, + "int": "1" + }, + { + "input": "2", + "bool": null, + "int": "2" + }, + { + "input": "-1", + "bool": null, + "int": "-1" + }, + { + "input": "9223372036854775807", + "bool": null, + "int": "9223372036854775807" + }, + { + "input": "9223372036854775808", + "bool": null, + "int": "9223372036854775808" + }, + { + "input": "18446744073709551616", + "bool": null, + "int": "18446744073709551616" + }, + { + "input": "-9223372036854775808", + "bool": null, + "int": "-9223372036854775808" + }, + { + "input": "0.0", + "bool": false, + "int": "0" + }, + { + "input": "-0.0", + "bool": false, + "int": "0" + }, + { + "input": "1.0", + "bool": true, + "int": "1" + }, + { + "input": "0.5", + "bool": null, + "int": null + }, + { + "input": "1.5", + "bool": null, + "int": null + }, + { + "input": "2.0", + "bool": null, + "int": "2" + }, + { + "input": "-3.0", + "bool": null, + "int": "-3" + }, + { + "input": "1000000000000000.0", + "bool": null, + "int": "1000000000000000" + }, + { + "input": "1e+20", + "bool": null, + "int": null + }, + { + "input": "1e+300", + "bool": null, + "int": null + }, + { + "input": "4.611686018427388e+18", + "bool": null, + "int": "4611686018427387904" + }, + { + "input": "9.223372036854776e+18", + "bool": null, + "int": null + }, + { + "input": "-9.223372036854776e+18", + "bool": null, + "int": null + }, + { + "input": "9007199254740992.0", + "bool": null, + "int": "9007199254740992" + }, + { + "input": "Infinity", + "bool": null, + "int": null + }, + { + "input": "-Infinity", + "bool": null, + "int": null + }, + { + "input": "NaN", + "bool": null, + "int": null + }, + { + "input": "[]", + "bool": null, + "int": null + }, + { + "input": "[1]", + "bool": null, + "int": null + }, + { + "input": "{}", + "bool": null, + "int": null + }, + { + "input": "{\"a\": 1}", + "bool": null, + "int": null + }, + { + "input": "\"\"", + "bool": null, + "int": null + }, + { + "input": "\" \"", + "bool": null, + "int": null + }, + { + "input": "\"0\"", + "bool": false, + "int": "0" + }, + { + "input": "\"1\"", + "bool": true, + "int": "1" + }, + { + "input": "\"2\"", + "bool": null, + "int": "2" + }, + { + "input": "\"-1\"", + "bool": null, + "int": "-1" + }, + { + "input": "\"+1\"", + "bool": null, + "int": "1" + }, + { + "input": "\"-0\"", + "bool": null, + "int": "0" + }, + { + "input": "\"00\"", + "bool": null, + "int": "0" + }, + { + "input": "\"007\"", + "bool": null, + "int": "7" + }, + { + "input": "\"1_000\"", + "bool": null, + "int": "1000" + }, + { + "input": "\"1__000\"", + "bool": null, + "int": null + }, + { + "input": "\"_1\"", + "bool": null, + "int": null + }, + { + "input": "\"1_\"", + "bool": null, + "int": null + }, + { + "input": "\"1.\"", + "bool": null, + "int": null + }, + { + "input": "\".5\"", + "bool": null, + "int": null + }, + { + "input": "\"1.0\"", + "bool": null, + "int": "1" + }, + { + "input": "\"1.00\"", + "bool": null, + "int": "1" + }, + { + "input": "\"1.5\"", + "bool": null, + "int": null + }, + { + "input": "\"-1.0\"", + "bool": null, + "int": "-1" + }, + { + "input": "\"1e3\"", + "bool": null, + "int": null + }, + { + "input": "\"1E3\"", + "bool": null, + "int": null + }, + { + "input": "\"0x10\"", + "bool": null, + "int": null + }, + { + "input": "\"0b1\"", + "bool": null, + "int": null + }, + { + "input": "\"0o7\"", + "bool": null, + "int": null + }, + { + "input": "\" 5 \"", + "bool": null, + "int": "5" + }, + { + "input": "\"\\t5\\n\"", + "bool": null, + "int": "5" + }, + { + "input": "\"5 0\"", + "bool": null, + "int": null + }, + { + "input": "\"\\u0661\\u0662\"", + "bool": null, + "int": null + }, + { + "input": "\"inf\"", + "bool": null, + "int": null + }, + { + "input": "\"nan\"", + "bool": null, + "int": null + }, + { + "input": "\"99999999999999999999\"", + "bool": null, + "int": "99999999999999999999" + }, + { + "input": "\"-99999999999999999999\"", + "bool": null, + "int": "-99999999999999999999" + }, + { + "input": "\"true\"", + "bool": true, + "int": null + }, + { + "input": "\"True\"", + "bool": true, + "int": null + }, + { + "input": "\"TRUE\"", + "bool": true, + "int": null + }, + { + "input": "\" true \"", + "bool": null, + "int": null + }, + { + "input": "\"false\"", + "bool": false, + "int": null + }, + { + "input": "\"False\"", + "bool": false, + "int": null + }, + { + "input": "\"t\"", + "bool": true, + "int": null + }, + { + "input": "\"f\"", + "bool": false, + "int": null + }, + { + "input": "\"T\"", + "bool": true, + "int": null + }, + { + "input": "\"y\"", + "bool": true, + "int": null + }, + { + "input": "\"n\"", + "bool": false, + "int": null + }, + { + "input": "\"yes\"", + "bool": true, + "int": null + }, + { + "input": "\"Yes\"", + "bool": true, + "int": null + }, + { + "input": "\"no\"", + "bool": false, + "int": null + }, + { + "input": "\"on\"", + "bool": true, + "int": null + }, + { + "input": "\"On\"", + "bool": true, + "int": null + }, + { + "input": "\"off\"", + "bool": false, + "int": null + }, + { + "input": "\"OFF\"", + "bool": false, + "int": null + }, + { + "input": "\"maybe\"", + "bool": null, + "int": null + }, + { + "input": "\"truthy\"", + "bool": null, + "int": null + }, + { + "input": "\"nul\"", + "bool": null, + "int": null + }, + { + "input": "\"+\"", + "bool": null, + "int": null + }, + { + "input": "\"-\"", + "bool": null, + "int": null + }, + { + "input": "\"+1.0\"", + "bool": null, + "int": "1" + }, + { + "input": "\"-0.0\"", + "bool": null, + "int": "0" + }, + { + "input": "\"1_0.0\"", + "bool": null, + "int": "10" + }, + { + "input": "\"1.0_0\"", + "bool": null, + "int": null + }, + { + "input": "\"1.0.0\"", + "bool": null, + "int": null + }, + { + "input": "\"1 \"", + "bool": null, + "int": "1" + }, + { + "input": "\"\\u00a05\"", + "bool": null, + "int": "5" + }, + { + "input": "\"\\u20035\"", + "bool": null, + "int": "5" + } + ] +} \ No newline at end of file diff --git a/litellm/rust/crates/python-compat/generated/stdlib.json b/litellm/rust/crates/python-compat/generated/stdlib.json new file mode 100644 index 00000000000..b74f1634f68 --- /dev/null +++ b/litellm/rust/crates/python-compat/generated/stdlib.json @@ -0,0 +1,966 @@ +{ + "python": "3.14.8", + "quote": [ + { + "text": "", + "safe": "/", + "quoted": "" + }, + { + "text": "", + "safe": "", + "quoted": "" + }, + { + "text": "", + "safe": ":/", + "quoted": "" + }, + { + "text": "plain", + "safe": "/", + "quoted": "plain" + }, + { + "text": "plain", + "safe": "", + "quoted": "plain" + }, + { + "text": "plain", + "safe": ":/", + "quoted": "plain" + }, + { + "text": "a/b/c", + "safe": "/", + "quoted": "a/b/c" + }, + { + "text": "a/b/c", + "safe": "", + "quoted": "a%2Fb%2Fc" + }, + { + "text": "a/b/c", + "safe": ":/", + "quoted": "a/b/c" + }, + { + "text": "a b", + "safe": "/", + "quoted": "a%20b" + }, + { + "text": "a b", + "safe": "", + "quoted": "a%20b" + }, + { + "text": "a b", + "safe": ":/", + "quoted": "a%20b" + }, + { + "text": "a+b=c&d", + "safe": "/", + "quoted": "a%2Bb%3Dc%26d" + }, + { + "text": "a+b=c&d", + "safe": "", + "quoted": "a%2Bb%3Dc%26d" + }, + { + "text": "a+b=c&d", + "safe": ":/", + "quoted": "a%2Bb%3Dc%26d" + }, + { + "text": "~_.-", + "safe": "/", + "quoted": "~_.-" + }, + { + "text": "~_.-", + "safe": "", + "quoted": "~_.-" + }, + { + "text": "~_.-", + "safe": ":/", + "quoted": "~_.-" + }, + { + "text": "100%", + "safe": "/", + "quoted": "100%25" + }, + { + "text": "100%", + "safe": "", + "quoted": "100%25" + }, + { + "text": "100%", + "safe": ":/", + "quoted": "100%25" + }, + { + "text": "a/b:c d%#?+=&~\u00e9\"q", + "safe": "/", + "quoted": "a/b%3Ac%20d%25%23%3F%2B%3D%26~%C3%A9%22q" + }, + { + "text": "a/b:c d%#?+=&~\u00e9\"q", + "safe": "", + "quoted": "a%2Fb%3Ac%20d%25%23%3F%2B%3D%26~%C3%A9%22q" + }, + { + "text": "a/b:c d%#?+=&~\u00e9\"q", + "safe": ":/", + "quoted": "a/b:c%20d%25%23%3F%2B%3D%26~%C3%A9%22q" + }, + { + "text": "id-\u00e9-\ud83d\ude00", + "safe": "/", + "quoted": "id-%C3%A9-%F0%9F%98%80" + }, + { + "text": "id-\u00e9-\ud83d\ude00", + "safe": "", + "quoted": "id-%C3%A9-%F0%9F%98%80" + }, + { + "text": "id-\u00e9-\ud83d\ude00", + "safe": ":/", + "quoted": "id-%C3%A9-%F0%9F%98%80" + }, + { + "text": "\u65e5\u672c", + "safe": "/", + "quoted": "%E6%97%A5%E6%9C%AC" + }, + { + "text": "\u65e5\u672c", + "safe": "", + "quoted": "%E6%97%A5%E6%9C%AC" + }, + { + "text": "\u65e5\u672c", + "safe": ":/", + "quoted": "%E6%97%A5%E6%9C%AC" + }, + { + "text": "\u0000\u001f\u007f", + "safe": "/", + "quoted": "%00%1F%7F" + }, + { + "text": "\u0000\u001f\u007f", + "safe": "", + "quoted": "%00%1F%7F" + }, + { + "text": "\u0000\u001f\u007f", + "safe": ":/", + "quoted": "%00%1F%7F" + }, + { + "text": "!*'();:@&=+$,/?#[]", + "safe": "/", + "quoted": "%21%2A%27%28%29%3B%3A%40%26%3D%2B%24%2C/%3F%23%5B%5D" + }, + { + "text": "!*'();:@&=+$,/?#[]", + "safe": "", + "quoted": "%21%2A%27%28%29%3B%3A%40%26%3D%2B%24%2C%2F%3F%23%5B%5D" + }, + { + "text": "!*'();:@&=+$,/?#[]", + "safe": ":/", + "quoted": "%21%2A%27%28%29%3B:%40%26%3D%2B%24%2C/%3F%23%5B%5D" + }, + { + "text": "<>\\^`{|}", + "safe": "/", + "quoted": "%3C%3E%5C%5E%60%7B%7C%7D" + }, + { + "text": "<>\\^`{|}", + "safe": "", + "quoted": "%3C%3E%5C%5E%60%7B%7C%7D" + }, + { + "text": "<>\\^`{|}", + "safe": ":/", + "quoted": "%3C%3E%5C%5E%60%7B%7C%7D" + } + ], + "utc": [ + { + "seconds": 0, + "micros": 0, + "fields": [ + 1970, + 1, + 1, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 1, + "micros": 999999, + "fields": [ + 1970, + 1, + 1, + 0, + 0, + 1, + 999999 + ] + }, + { + "seconds": 59, + "micros": 0, + "fields": [ + 1970, + 1, + 1, + 0, + 0, + 59, + 0 + ] + }, + { + "seconds": 86399, + "micros": 123456, + "fields": [ + 1970, + 1, + 1, + 23, + 59, + 59, + 123456 + ] + }, + { + "seconds": 86400, + "micros": 0, + "fields": [ + 1970, + 1, + 2, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 951782400, + "micros": 0, + "fields": [ + 2000, + 2, + 29, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 951868800, + "micros": 0, + "fields": [ + 2000, + 3, + 1, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 1700000000, + "micros": 42, + "fields": [ + 2023, + 11, + 14, + 22, + 13, + 20, + 42 + ] + }, + { + "seconds": 1757840707, + "micros": 42, + "fields": [ + 2025, + 9, + 14, + 9, + 5, + 7, + 42 + ] + }, + { + "seconds": 4102444800, + "micros": 0, + "fields": [ + 2100, + 1, + 1, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 253402300799, + "micros": 999999, + "fields": [ + 9999, + 12, + 31, + 23, + 59, + 59, + 999999 + ] + } + ], + "fromisoformat": [ + { + "text": "2025-11-13T12:00:00+00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00Z", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00.000001+00:00", + "aware": 1763035200000001 + }, + { + "text": "2025-11-13T12:00:00.5+00:00", + "aware": 1763035200500000 + }, + { + "text": "2025-11-13T12:00:00.123+00:00", + "aware": 1763035200123000 + }, + { + "text": "2025-11-13T12:00:00.1234567+00:00", + "aware": 1763035200123456 + }, + { + "text": "2025-11-13T12:00:00,5+00:00", + "aware": 1763035200500000 + }, + { + "text": "2025-11-13T12:00:00.+00:00" + }, + { + "text": "2025-11-13T13:30:00+01:30", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T11:00:00-01:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00-00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00+0130", + "aware": 1763029800000000 + }, + { + "text": "2025-11-13T12:00:00+01", + "aware": 1763031600000000 + }, + { + "text": "2025-11-13T12:00:00+01:30:15", + "aware": 1763029785000000 + }, + { + "text": "2025-11-13T12:00:00+01:30:15.5", + "aware": 1763029784500000 + }, + { + "text": "2025-11-13T12:00:00+23:59", + "aware": 1762948860000000 + }, + { + "text": "2025-11-13T12:00:00+24:00" + }, + { + "text": "2025-11-13 12:00+00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13X12:00:00+00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13\u00e912:00:00+00:00", + "aware": 1763035200000000 + }, + { + "text": "20251113T120000+0000", + "aware": 1763035200000000 + }, + { + "text": "20251113T120000", + "naive": 1763035200000000 + }, + { + "text": "20251113", + "naive": 1762992000000000 + }, + { + "text": "2025-1113" + }, + { + "text": "202511-13" + }, + { + "text": "2025-11-13", + "naive": 1762992000000000 + }, + { + "text": "2025-11-13T12", + "naive": 1763035200000000 + }, + { + "text": "2025-11-13T12:00", + "naive": 1763035200000000 + }, + { + "text": "2025-11-13T1200", + "naive": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00", + "naive": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00.5", + "naive": 1763035200500000 + }, + { + "text": "2025-11-13T12.5" + }, + { + "text": "2025-11-13T12:00.5" + }, + { + "text": "2024-02-29T00:00:00+00:00", + "aware": 1709164800000000 + }, + { + "text": "0001-01-01T00:00:00+00:00", + "aware": -62135596800000000 + }, + { + "text": "9999-12-31T23:59:59.999999+00:00", + "aware": 253402300799999999 + }, + { + "text": "1969-12-31T23:59:59.999999+00:00", + "aware": -1 + }, + { + "text": "soon" + }, + { + "text": "" + }, + { + "text": " 2025-11-13" + }, + { + "text": "2025-02-30T12:00:00+00:00" + }, + { + "text": "2025-13-01T12:00:00+00:00" + }, + { + "text": "2025-00-01T12:00:00+00:00" + }, + { + "text": "0000-01-01T00:00:00+00:00" + }, + { + "text": "2025-11-13T24:00:00+00:00", + "aware": 1763078400000000 + }, + { + "text": "2025-11-13T12:60:00+00:00" + }, + { + "text": "2025-11-13T12:00:60+00:00" + }, + { + "text": "2025-11-13T12:00:00+00:00 " + }, + { + "text": "2025-11-13T12:00:00 +00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00ZZ" + }, + { + "text": "2025-11-13T12:00:00+00:00Z" + }, + { + "text": "2025-11" + }, + { + "text": "2025" + }, + { + "text": "+002025-11-13" + }, + { + "text": "2025-317" + }, + { + "text": "2025-W46", + "naive": 1762732800000000 + }, + { + "text": "2025-W46-4", + "naive": 1762992000000000 + }, + { + "text": "2025W464", + "naive": 1762992000000000 + }, + { + "text": "2025-W53-1" + }, + { + "text": "2025-W46-4T12:00:00+00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00+1:00" + }, + { + "text": "2025-11-13T1:00:00+00:00" + }, + { + "text": "2025-11-13T12:00:00 " + }, + { + "text": "2025-11-13T12:00:00 +00:00" + }, + { + "text": "2025-11-13T12:00:00\t+00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00 +00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12 +00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00.5 +00:00" + }, + { + "text": "2025-11-13T12:00:00x+00:00", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00 Z", + "aware": 1763035200000000 + }, + { + "text": "2025-11-13T12:00:00+0" + }, + { + "text": "2025-11-13T12:00:00+000" + }, + { + "text": "2025-11-13T12:00:00+01:30:15." + }, + { + "text": "2025-11-13T12:00:00+013015", + "aware": 1763029785000000 + }, + { + "text": "2025-11-13T12:00:00+013015.5", + "aware": 1763029784500000 + }, + { + "text": "2025-11-13T12:00:00+23:59:59.999999", + "aware": 1762948800000001 + }, + { + "text": "2025-11-13T12:00:00-23:59:59.999999", + "aware": 1763121599999999 + }, + { + "text": "2025-11-13T12:00:00+00:99", + "aware": 1763029260000000 + }, + { + "text": "2025-11-13T12:00:00+99" + }, + { + "text": "9999-12-31T24:00:00+00:00" + }, + { + "text": "2025-12-31T24:00:00", + "naive": 1767225600000000 + }, + { + "text": "2024-02-28T24:00", + "naive": 1709164800000000 + }, + { + "text": "2025-02-30T24:00:00" + }, + { + "text": "2025-11-13T24:00:00.000001" + }, + { + "text": "2025-11-13T12:00:00.1234567890123", + "naive": 1763035200123456 + }, + { + "text": "2025-11-13T12:00:00.12a" + }, + { + "text": "2025-11-13T12:00:00-" + }, + { + "text": "2025-11-13T12:00:00Z+00:00" + }, + { + "text": "2025-11-13T12:3" + }, + { + "text": "2025-11-13T12:00:0" + }, + { + "text": "2025-11-13T12:00:00," + }, + { + "text": "2025-11-13T12:0000" + }, + { + "text": "2025-11-13T1200:00" + }, + { + "text": "20251113T120000123+0000", + "aware": 1763035200123000 + }, + { + "text": "20251113T120000.5", + "naive": 1763035200500000 + }, + { + "text": "2025-11-13T12:00:00\u0000", + "naive": 1763035200000000 + }, + { + "text": "2025-W464" + }, + { + "text": "2025W46-4" + }, + { + "text": "2025W46", + "naive": 1762732800000000 + }, + { + "text": "2025-W4" + }, + { + "text": "2025-W46-" + }, + { + "text": "2025-W46-41" + }, + { + "text": "2020-W53-1", + "naive": 1609113600000000 + }, + { + "text": "2026-W53-1", + "naive": 1798416000000000 + }, + { + "text": "2025-W00-1" + }, + { + "text": "2025-W46-8" + }, + { + "text": "2025-W46-0" + }, + { + "text": "2025-W46T12", + "naive": 1762776000000000 + }, + { + "text": "2025-W46-4T12", + "naive": 1763035200000000 + }, + { + "text": "2025W464T12", + "naive": 1763035200000000 + }, + { + "text": "2025W4612" + }, + { + "text": "9999-W52-6" + }, + { + "text": "0001-W01-1", + "naive": -62135596800000000 + }, + { + "text": "2025-11-13T" + }, + { + "text": "2025-11-13TT" + }, + { + "text": "2025-11-1" + }, + { + "text": "2025-11-13 " + }, + { + "text": "2025-11-13\u00e9" + }, + { + "text": "2025-11-13\ud83d\ude0012:00", + "naive": 1763035200000000 + }, + { + "text": "1234567" + }, + { + "text": "2025-11-13T12:00:00+05:30", + "aware": 1763015400000000 + } + ], + "fixed": [ + { + "value": 0.125, + "precision": 2, + "text": "0.12" + }, + { + "value": 0.375, + "precision": 2, + "text": "0.38" + }, + { + "value": 2.5, + "precision": 0, + "text": "2" + }, + { + "value": 3.5, + "precision": 0, + "text": "4" + }, + { + "value": 1.0, + "precision": 4, + "text": "1.0000" + }, + { + "value": 4.9999e-05, + "precision": 4, + "text": "0.0000" + }, + { + "value": -0.005, + "precision": 2, + "text": "-0.01" + }, + { + "value": -0.0, + "precision": 2, + "text": "-0.00" + }, + { + "value": 0.0, + "precision": 0, + "text": "0" + }, + { + "value": 99.999, + "precision": 2, + "text": "100.00" + }, + { + "value": 1e+22, + "precision": 2, + "text": "10000000000000000000000.00" + }, + { + "value": 1e-07, + "precision": 4, + "text": "0.0000" + }, + { + "value": 123.456789, + "precision": 4, + "text": "123.4568" + }, + { + "value": 0.1, + "precision": 20, + "text": "0.10000000000000000555" + }, + { + "value": 5e-324, + "precision": 2, + "text": "0.00" + }, + { + "value": 1.7976931348623157e+308, + "precision": 2, + "text": "179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.00" + }, + { + "value": "nan", + "precision": 2, + "text": "nan" + }, + { + "value": "inf", + "precision": 2, + "text": "inf" + }, + { + "value": "-inf", + "precision": 4, + "text": "-inf" + } + ], + "number_str": [ + { + "value": 0, + "str": "0" + }, + { + "value": 1, + "str": "1" + }, + { + "value": -5, + "str": "-5" + }, + { + "value": 9007199254740993, + "str": "9007199254740993" + }, + { + "value": 9223372036854775807, + "str": "9223372036854775807" + }, + { + "value": -9223372036854775808, + "str": "-9223372036854775808" + }, + { + "value": 0.0, + "str": "0.0" + }, + { + "value": -0.0, + "str": "-0.0" + }, + { + "value": 1.0, + "str": "1.0" + }, + { + "value": 0.1, + "str": "0.1" + }, + { + "value": 1.5, + "str": "1.5" + }, + { + "value": 1e+16, + "str": "1e+16" + }, + { + "value": 1e-05, + "str": "1e-05" + }, + { + "value": 123456789.123, + "str": "123456789.123" + }, + { + "float": "inf", + "str": "inf" + }, + { + "float": "-inf", + "str": "-inf" + } + ], + "isspace": [ + 9, + 10, + 11, + 12, + 13, + 28, + 29, + 30, + 31, + 32, + 133, + 160, + 5760, + 8192, + 8193, + 8194, + 8195, + 8196, + 8197, + 8198, + 8199, + 8200, + 8201, + 8202, + 8232, + 8233, + 8239, + 8287, + 12288 + ] +} \ No newline at end of file diff --git a/litellm/rust/crates/python-compat/scripts/generate_pydantic_fixtures.py b/litellm/rust/crates/python-compat/scripts/generate_pydantic_fixtures.py new file mode 100644 index 00000000000..139f48d4832 --- /dev/null +++ b/litellm/rust/crates/python-compat/scripts/generate_pydantic_fixtures.py @@ -0,0 +1,142 @@ +"""Regenerate generated/pydantic.json: what pydantic's lax `bool` and `int` validation return. + + uv run --no-project --with pydantic==2.13.4 python scripts/generate_pydantic_fixtures.py > generated/pydantic.json + +Each row records `TypeAdapter(bool).validate_python(value)` and `TypeAdapter(int).validate_python(value)` +for one input written as JSON, or null where pydantic raises `ValidationError`. Inputs are what a +JSON-shaped setting can carry: None, booleans, integers, floats, strings, lists and dicts. +""" + +import json +import sys + +import pydantic +from pydantic import TypeAdapter, ValidationError + +TEXTS = [ + "", + " ", + "0", + "1", + "2", + "-1", + "+1", + "-0", + "00", + "007", + "1_000", + "1__000", + "_1", + "1_", + "1.", + ".5", + "1.0", + "1.00", + "1.5", + "-1.0", + "1e3", + "1E3", + "0x10", + "0b1", + "0o7", + " 5 ", + "\t5\n", + "5 0", + "١٢", + "inf", + "nan", + "99999999999999999999", + "-99999999999999999999", + "true", + "True", + "TRUE", + " true ", + "false", + "False", + "t", + "f", + "T", + "y", + "n", + "yes", + "Yes", + "no", + "on", + "On", + "off", + "OFF", + "maybe", + "truthy", + "nul", + "+", + "-", + "+1.0", + "-0.0", + "1_0.0", + "1.0_0", + "1.0.0", + "1 ", + "\u00a05", + "\u20035", +] + +VALUES = [ + None, + True, + False, + 0, + 1, + 2, + -1, + 2**63 - 1, + 2**63, + 2**64, + -(2**63), + 0.0, + -0.0, + 1.0, + 0.5, + 1.5, + 2.0, + -3.0, + 1e15, + 1e20, + 1e300, + 2.0**62, + 2.0**63, + -(2.0**63), + 9007199254740993.0, + float("inf"), + float("-inf"), + float("nan"), + [], + [1], + {}, + {"a": 1}, + *TEXTS, +] + + +def validated(adapter, value): + try: + return adapter.validate_python(value) + except ValidationError: + return None + + +def row(value): + as_int = validated(TypeAdapter(int), value) + return { + "input": json.dumps(value), + "bool": validated(TypeAdapter(bool), value), + "int": None if as_int is None else str(as_int), + } + + +if __name__ == "__main__": + json.dump( + {"pydantic": pydantic.VERSION, "rows": [row(value) for value in VALUES]}, + sys.stdout, + indent=2, + ensure_ascii=True, + ) diff --git a/litellm/rust/crates/python-compat/scripts/generate_stdlib_fixtures.py b/litellm/rust/crates/python-compat/scripts/generate_stdlib_fixtures.py new file mode 100644 index 00000000000..9afdf33a133 --- /dev/null +++ b/litellm/rust/crates/python-compat/scripts/generate_stdlib_fixtures.py @@ -0,0 +1,245 @@ +"""Regenerate generated/stdlib.json: `urllib.parse.quote`, `datetime`, float formatting and +`str.isspace` in CPython. + + python scripts/generate_stdlib_fixtures.py > generated/stdlib.json + +`quote` rows record `urllib.parse.quote(text, safe=safe)`. `utc` rows record the fields of +`datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=micros)`. `fromisoformat` +rows record `datetime.fromisoformat(text)` as microseconds since the epoch, `aware` when it +has an offset and `naive` (read as UTC) when not, or neither when it raises `ValueError`. +`fixed` rows record `format(value, f".{precision}f")`, `number_str` rows `str(value)` for an +`int` or `float`, and `isspace` lists every code point whose `str.isspace()` is true. +""" + +import json +import math +import sys +from datetime import datetime, timedelta, timezone +from urllib.parse import quote + +QUOTE_TEXTS = [ + "", + "plain", + "a/b/c", + "a b", + "a+b=c&d", + "~_.-", + "100%", + "a/b:c d%#?+=&~é\"q", + "id-é-\U0001f600", + "日本", + "\x00\x1f\x7f", + "!*'();:@&=+$,/?#[]", + "<>\\^`{|}", +] + +SAFES = ["/", "", ":/"] + +TIMESTAMPS = [ + (0, 0), + (1, 999999), + (59, 0), + (86399, 123456), + (86400, 0), + (951782400, 0), + (951868800, 0), + (1700000000, 42), + (1757840707, 42), + (4102444800, 0), + (253402300799, 999999), +] + +ISO_TEXTS = [ + "2025-11-13T12:00:00+00:00", + "2025-11-13T12:00:00Z", + "2025-11-13T12:00:00.000001+00:00", + "2025-11-13T12:00:00.5+00:00", + "2025-11-13T12:00:00.123+00:00", + "2025-11-13T12:00:00.1234567+00:00", + "2025-11-13T12:00:00,5+00:00", + "2025-11-13T12:00:00.+00:00", + "2025-11-13T13:30:00+01:30", + "2025-11-13T11:00:00-01:00", + "2025-11-13T12:00:00-00:00", + "2025-11-13T12:00:00+0130", + "2025-11-13T12:00:00+01", + "2025-11-13T12:00:00+01:30:15", + "2025-11-13T12:00:00+01:30:15.5", + "2025-11-13T12:00:00+23:59", + "2025-11-13T12:00:00+24:00", + "2025-11-13 12:00+00:00", + "2025-11-13X12:00:00+00:00", + "2025-11-13\u00e912:00:00+00:00", + "20251113T120000+0000", + "20251113T120000", + "20251113", + "2025-1113", + "202511-13", + "2025-11-13", + "2025-11-13T12", + "2025-11-13T12:00", + "2025-11-13T1200", + "2025-11-13T12:00:00", + "2025-11-13T12:00:00.5", + "2025-11-13T12.5", + "2025-11-13T12:00.5", + "2024-02-29T00:00:00+00:00", + "0001-01-01T00:00:00+00:00", + "9999-12-31T23:59:59.999999+00:00", + "1969-12-31T23:59:59.999999+00:00", + "soon", + "", + " 2025-11-13", + "2025-02-30T12:00:00+00:00", + "2025-13-01T12:00:00+00:00", + "2025-00-01T12:00:00+00:00", + "0000-01-01T00:00:00+00:00", + "2025-11-13T24:00:00+00:00", + "2025-11-13T12:60:00+00:00", + "2025-11-13T12:00:60+00:00", + "2025-11-13T12:00:00+00:00 ", + "2025-11-13T12:00:00 +00:00", + "2025-11-13T12:00:00ZZ", + "2025-11-13T12:00:00+00:00Z", + "2025-11", + "2025", + "+002025-11-13", + "2025-317", + "2025-W46", + "2025-W46-4", + "2025W464", + "2025-W53-1", + "2025-W46-4T12:00:00+00:00", + "2025-11-13T12:00:00+1:00", + "2025-11-13T1:00:00+00:00", + "2025-11-13T12:00:00 ", + "2025-11-13T12:00:00 +00:00", + "2025-11-13T12:00:00\t+00:00", + "2025-11-13T12:00 +00:00", + "2025-11-13T12 +00:00", + "2025-11-13T12:00:00.5 +00:00", + "2025-11-13T12:00:00x+00:00", + "2025-11-13T12:00:00 Z", + "2025-11-13T12:00:00+0", + "2025-11-13T12:00:00+000", + "2025-11-13T12:00:00+01:30:15.", + "2025-11-13T12:00:00+013015", + "2025-11-13T12:00:00+013015.5", + "2025-11-13T12:00:00+23:59:59.999999", + "2025-11-13T12:00:00-23:59:59.999999", + "2025-11-13T12:00:00+00:99", + "2025-11-13T12:00:00+99", + "9999-12-31T24:00:00+00:00", + "2025-12-31T24:00:00", + "2024-02-28T24:00", + "2025-02-30T24:00:00", + "2025-11-13T24:00:00.000001", + "2025-11-13T12:00:00.1234567890123", + "2025-11-13T12:00:00.12a", + "2025-11-13T12:00:00-", + "2025-11-13T12:00:00Z+00:00", + "2025-11-13T12:3", + "2025-11-13T12:00:0", + "2025-11-13T12:00:00,", + "2025-11-13T12:0000", + "2025-11-13T1200:00", + "20251113T120000123+0000", + "20251113T120000.5", + "2025-11-13T12:00:00\x00", + "2025-W464", + "2025W46-4", + "2025W46", + "2025-W4", + "2025-W46-", + "2025-W46-41", + "2020-W53-1", + "2026-W53-1", + "2025-W00-1", + "2025-W46-8", + "2025-W46-0", + "2025-W46T12", + "2025-W46-4T12", + "2025W464T12", + "2025W4612", + "9999-W52-6", + "0001-W01-1", + "2025-11-13T", + "2025-11-13TT", + "2025-11-1", + "2025-11-13 ", + "2025-11-13\u00e9", + "2025-11-13\U0001f600" "12:00", + "1234567", + "2025-11-13T12:00:00+05:30", +] + +FIXED = [ + (0.125, 2), + (0.375, 2), + (2.5, 0), + (3.5, 0), + (1.0, 4), + (0.000049999, 4), + (-0.005, 2), + (-0.0, 2), + (0.0, 0), + (99.999, 2), + (1e22, 2), + (1e-7, 4), + (123.456789, 4), + (0.1, 20), + (5e-324, 2), + (1.7976931348623157e308, 2), + (math.nan, 2), + (math.inf, 2), + (-math.inf, 4), +] + +NUMBERS = [0, 1, -5, 2**53 + 1, 2**63 - 1, -(2**63), 0.0, -0.0, 1.0, 0.1, 1.5, 1e16, 1e-5, 123456789.123, math.inf, -math.inf] + +EPOCH = datetime(1970, 1, 1) + + +def fromisoformat(text): + try: + moment = datetime.fromisoformat(text) + except ValueError: + return {"text": text} + if moment.tzinfo is None: + return {"text": text, "naive": (moment - EPOCH) // timedelta(microseconds=1)} + return {"text": text, "aware": (moment - EPOCH.replace(tzinfo=timezone.utc)) // timedelta(microseconds=1)} + + +def number_str(value): + return {"value": value, "str": str(value)} if math.isfinite(value) else {"float": repr(value), "str": str(value)} + + +def fixed(value, precision): + shown = value if math.isfinite(value) else repr(value) + return {"value": shown, "precision": precision, "text": format(value, f".{precision}f")} + + +def utc(seconds, micros): + moment = datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=micros) + return { + "seconds": seconds, + "micros": micros, + "fields": [moment.year, moment.month, moment.day, moment.hour, moment.minute, moment.second, moment.microsecond], + } + + +if __name__ == "__main__": + json.dump( + { + "python": sys.version.split()[0], + "quote": [{"text": text, "safe": safe, "quoted": quote(text, safe=safe)} for text in QUOTE_TEXTS for safe in SAFES], + "utc": [utc(seconds, micros) for seconds, micros in TIMESTAMPS], + "fromisoformat": [fromisoformat(text) for text in ISO_TEXTS], + "fixed": [fixed(value, precision) for value, precision in FIXED], + "number_str": [number_str(value) for value in NUMBERS], + "isspace": [code for code in range(sys.maxunicode + 1) if chr(code).isspace()], + }, + sys.stdout, + indent=2, + ensure_ascii=True, + ) diff --git a/litellm/rust/crates/python-compat/src/datetime.rs b/litellm/rust/crates/python-compat/src/datetime.rs new file mode 100644 index 00000000000..0e862000e97 --- /dev/null +++ b/litellm/rust/crates/python-compat/src/datetime.rs @@ -0,0 +1,360 @@ +//! The fields of `datetime.fromtimestamp(seconds, timezone.utc)`, and `datetime.fromisoformat(text)`. + +/// A UTC `datetime`'s fields, as Python's `datetime` exposes them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct UtcDateTime { + pub year: i64, + pub month: u32, + pub day: u32, + pub hour: u32, + pub minute: u32, + pub second: u32, + pub microsecond: u32, +} + +impl UtcDateTime { + /// `datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=microsecond)`. + pub fn from_timestamp(seconds: i64, microsecond: u32) -> Self { + let (year, month, day) = civil_from_days(seconds.div_euclid(86_400)); + let of_day = seconds.rem_euclid(86_400); + Self { + year, + month, + day, + hour: (of_day / 3600) as u32, + minute: (of_day / 60 % 60) as u32, + second: (of_day % 60) as u32, + microsecond, + } + } +} + +/// Days since 1970-01-01 to a proleptic Gregorian `(year, month, day)`. +/// +/// Howard Hinnant's `civil_from_days`, : +/// shifts the epoch to 0000-03-01 so the leap day ends the year, then counts 400-year eras +/// of 146097 days. +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let shifted = days + 719_468; + let era = shifted.div_euclid(146_097); + let day_of_era = shifted.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + } as u32; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +/// Days since 1970-01-01 of a proleptic Gregorian date, the inverse of [`civil_from_days`]. +fn days_from_civil(year: i64, month: u32, day: u32) -> i64 { + let year = year - i64::from(month <= 2); + let era = year.div_euclid(400); + let year_of_era = year.rem_euclid(400); + let month_index = i64::from(if month > 2 { month - 3 } else { month + 9 }); + let day_of_year = (153 * month_index + 2) / 5 + i64::from(day) - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - 719_468 +} + +fn is_leap(year: i64) -> bool { + year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) +} + +fn days_in_month(year: i64, month: u32) -> u32 { + match month { + 2 if is_leap(year) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +/// What `datetime.fromisoformat(text)` returns, in microseconds since 1970-01-01. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IsoDateTime { + /// A naive `datetime`, its fields read as UTC. + Naive(i64), + /// An aware `datetime`, at its UTC instant. + Aware(i64), +} + +/// `datetime.fromisoformat(text)` as CPython 3.14's C parser reads it, or `None` where it +/// raises `ValueError`. Python before 3.11 accepts only `isoformat()` output. +/// +/// This follows `Modules/_datetimemodule.c` rather than the documented grammar, quirks +/// included: any one character separates date from time, one stray character may sit +/// between the time and its offset, and `T24:00` rolls over to the next midnight. +pub fn fromisoformat(text: &str) -> Option { + if text.chars().count() < 7 { + return None; + } + let bytes = Bytes(text.as_bytes()); + let separator = bytes.separator(); + let (mut year, mut month, mut day) = bytes.date(separator)?; + let mut time = Clock::default(); + let mut offset = None; + if bytes.len() > separator { + let lead = bytes.at(separator); + let width = match lead { + _ if lead & 0x80 == 0 => 1, + _ if lead & 0xf0 == 0xe0 => 3, + _ if lead & 0xf0 == 0xf0 => 4, + _ => 2, + }; + let start = separator + width; + if start > bytes.len() { + return None; + } + (time, offset) = bytes.time(start)?; + } + if time.hour == 24 && (1..=12).contains(&month) && day <= days_in_month(year, month) { + if time.minute != 0 || time.second != 0 || time.microsecond != 0 { + return None; + } + time.hour = 0; + day += 1; + if day > days_in_month(year, month) { + day = 1; + month += 1; + if month > 12 { + month = 1; + year += 1; + } + } + } + if !(1..=9999).contains(&year) + || !(1..=12).contains(&month) + || !(1..=days_in_month(year, month)).contains(&day) + || time.hour > 23 + || time.minute > 59 + || time.second > 59 + { + return None; + } + let seconds = days_from_civil(year, month, day) * 86_400 + + i64::from(time.hour) * 3600 + + i64::from(time.minute) * 60 + + i64::from(time.second); + let local = seconds * 1_000_000 + i64::from(time.microsecond); + Some(match offset { + None => IsoDateTime::Naive(local), + Some(offset) if offset.abs() >= 86_400_000_000 => return None, + Some(offset) => IsoDateTime::Aware(local - offset), + }) +} + +#[derive(Clone, Copy, Default)] +struct Clock { + hour: u32, + minute: u32, + second: u32, + microsecond: u32, +} + +/// Where `parse_hh_mm_ss_ff` stopped: at the end of the input, or with more after it. +enum Rest { + End, + More, +} + +/// The UTF-8 bytes C reads, with the NUL terminator it relies on past the end. +struct Bytes<'a>(&'a [u8]); + +impl Bytes<'_> { + fn len(&self) -> usize { + self.0.len() + } + + fn at(&self, index: usize) -> u8 { + self.0.get(index).copied().unwrap_or(0) + } + + fn digits(&self, at: usize, count: usize) -> Option { + (at..at + count).try_fold(0, |value, index| { + let digit = self.at(index).wrapping_sub(b'0'); + (digit <= 9).then_some(value * 10 + u32::from(digit)) + }) + } + + /// `_find_isoformat_datetime_separator`: where the date ends, from its shape. + fn separator(&self) -> usize { + let len = self.len(); + if len == 7 { + return 7; + } + if self.at(4) == b'-' { + if self.at(5) != b'W' { + return 10; + } + if len > 8 && self.at(8) == b'-' { + if len == 9 { + return usize::MAX; + } + return if len > 10 && self.at(10).is_ascii_digit() { + 8 + } else { + 10 + }; + } + return 8; + } + if self.at(4) != b'W' { + return 8; + } + let digits_end = (7..len) + .find(|&index| !self.at(index).is_ascii_digit()) + .unwrap_or(len); + match digits_end { + end if end < 9 => end, + end if end % 2 == 0 => 7, + _ => 8, + } + } + + /// `parse_isoformat_date`: calendar `YYYY-MM-DD` or ISO week `YYYY-Www-D`, unvalidated. + fn date(&self, len: usize) -> Option<(i64, u32, u32)> { + let year = self.digits(0, 4)?; + let dashed = self.at(4) == b'-'; + let mut at = 4 + usize::from(dashed); + if self.at(at) == b'W' { + let week = self.digits(at + 1, 2)?; + at += 3; + let weekday = if at < len { + if dashed { + if self.at(at) != b'-' { + return None; + } + at += 1; + } + self.digits(at, 1)? + } else { + 1 + }; + return iso_week_date(i64::from(year), week, weekday); + } + let month = self.digits(at, 2)?; + at += 2; + if dashed { + if self.at(at) != b'-' { + return None; + } + at += 1; + } + Some((i64::from(year), month, self.digits(at, 2)?)) + } + + /// `parse_isoformat_time`: the clock, and the offset in microseconds when there is one. + fn time(&self, start: usize) -> Option<(Clock, Option)> { + let end = self.len(); + let zone = (start..end) + .find(|&index| matches!(self.at(index), b'Z' | b'+' | b'-')) + .unwrap_or(if start == end { start + 1 } else { end }); + let (clock, rest) = self.clock(start, zone)?; + if zone >= end { + return matches!(rest, Rest::End).then_some((clock, None)); + } + if self.at(zone) == b'Z' { + return (self.at(zone + 1) == 0).then_some((clock, Some(0))); + } + let sign = if self.at(zone) == b'-' { -1 } else { 1 }; + let (offset, rest) = self.clock(zone + 1, end)?; + if matches!(rest, Rest::More) { + return None; + } + let seconds = i64::from(offset.hour) * 3600 + + i64::from(offset.minute) * 60 + + i64::from(offset.second); + Some(( + clock, + Some(sign * (seconds * 1_000_000 + i64::from(offset.microsecond))), + )) + } + + /// `parse_hh_mm_ss_ff`: `HH[:?MM[:?SS[{.,}f+]]]` from `start` up to `end`. + fn clock(&self, start: usize, end: usize) -> Option<(Clock, Rest)> { + let mut fields = [0; 3]; + let mut at = start; + let mut colons = true; + for (index, field) in fields.iter_mut().enumerate() { + *field = self.digits(at, 2)?; + let next = self.at(at + 2); + at += 3; + if index == 0 { + colons = next == b':'; + } + if matches!(next, b'.' | b',') { + if index < 2 || at >= end { + return None; + } + break; + } else if at >= end { + let [hour, minute, second] = fields; + let clock = Clock { + hour, + minute, + second, + microsecond: 0, + }; + return Some((clock, if next == 0 { Rest::End } else { Rest::More })); + } else if colons && next == b':' { + if index == 2 { + return None; + } + } else if !colons { + at -= 1; + } else { + return None; + } + } + let count = (end - at).min(6); + let mut microsecond = self.digits(at, count)?; + microsecond *= 10_u32.pow(u32::try_from(6 - count).ok()?); + at += count; + while self.at(at).is_ascii_digit() { + at += 1; + } + let [hour, minute, second] = fields; + let clock = Clock { + hour, + minute, + second, + microsecond, + }; + Some(( + clock, + if self.at(at) == 0 { + Rest::End + } else { + Rest::More + }, + )) + } +} + +/// `iso_to_ymd`: the calendar date of ISO year, week and weekday, as `date.fromisocalendar`. +fn iso_week_date(year: i64, week: u32, weekday: u32) -> Option<(i64, u32, u32)> { + if !(1..=9999).contains(&year) { + return None; + } + let first = days_from_civil(year, 1, 1); + // 0 for Monday, as `date.weekday()`; 1970-01-01 was a Thursday. + let first_weekday = (first + 3).rem_euclid(7); + let long_year = first_weekday == 3 || (first_weekday == 2 && is_leap(year)); + if !(1..=52).contains(&week) && !(week == 53 && long_year) { + return None; + } + if !(1..=7).contains(&weekday) { + return None; + } + let week_one_monday = first - first_weekday + if first_weekday > 3 { 7 } else { 0 }; + Some(civil_from_days( + week_one_monday + i64::from((week - 1) * 7 + weekday - 1), + )) +} diff --git a/litellm/rust/crates/python-compat/src/format.rs b/litellm/rust/crates/python-compat/src/format.rs new file mode 100644 index 00000000000..f6ebfc61c73 --- /dev/null +++ b/litellm/rust/crates/python-compat/src/format.rs @@ -0,0 +1,61 @@ +//! `format(value, f".{precision}f")` for a `float`. + +fn round_half_even(digits: &str, precision: usize) -> String { + let (whole, fraction) = digits.split_once('.').unwrap_or((digits, "")); + let mut kept: Vec = whole + .bytes() + .chain( + fraction + .bytes() + .chain(std::iter::repeat(b'0')) + .take(precision), + ) + .collect(); + let rest = fraction.get(precision..).unwrap_or(""); + let first = rest.bytes().next().unwrap_or(b'0'); + let tail_nonzero = rest.bytes().skip(1).any(|digit| digit != b'0'); + let last_odd = kept.last().is_some_and(|digit| (digit - b'0') % 2 == 1); + if first > b'5' || (first == b'5' && (tail_nonzero || last_odd)) { + let mut index = kept.len(); + loop { + if index == 0 { + kept.insert(0, b'1'); + break; + } + index -= 1; + if kept[index] == b'9' { + kept[index] = b'0'; + } else { + kept[index] += 1; + break; + } + } + } + let split = kept.len() - precision; + let (head, tail) = kept.split_at(split); + let head = String::from_utf8_lossy(head); + if precision == 0 { + head.into_owned() + } else { + format!("{head}.{}", String::from_utf8_lossy(tail)) + } +} + +/// `format(value, f".{precision}f")`: the exact binary value rounded half to even at +/// `precision` digits, which is what CPython's `float.__format__` prints. +pub fn fixed(value: f64, precision: usize) -> String { + if value.is_nan() { + return "nan".to_owned(); + } + if value.is_infinite() { + return if value > 0.0 { "inf" } else { "-inf" }.to_owned(); + } + let exact = format!("{:.1100}", value.abs()); + let rounded = round_half_even(&exact, precision); + let negative = value.is_sign_negative(); + if negative { + format!("-{rounded}") + } else { + rounded + } +} diff --git a/litellm/rust/crates/python-compat/src/lib.rs b/litellm/rust/crates/python-compat/src/lib.rs index e23dbdb285e..b079f8ed555 100644 --- a/litellm/rust/crates/python-compat/src/lib.rs +++ b/litellm/rust/crates/python-compat/src/lib.rs @@ -11,7 +11,12 @@ //! | [`json`] | `json.dumps(value)` and `json.loads(text)` | //! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data | //! | [`truthy`] | `bool(value)` | -//! | [`number`] | `int(text)`, `float(text)`, and `int(seconds)` as a `Duration` | +//! | [`number`] | `int(text)`, `float(text)`, `int(seconds)` as a `Duration`, and `str()` of an `int` or `float` | +//! | [`format`] | `format(value, f".{precision}f")` | +//! | [`unicode`] | `str.isspace()` for one character | +//! | [`pydantic`] | `TypeAdapter(bool)` and `TypeAdapter(int)` lax validation | +//! | [`urllib`] | `urllib.parse.quote(text, safe=safe)` | +//! | [`datetime`] | `datetime.fromtimestamp(seconds, timezone.utc)` fields and `datetime.fromisoformat(text)` | //! //! [`Value`] is the closed data model these formats share. Live Python objects //! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the @@ -24,14 +29,19 @@ //! - [`pickle::loads`] decodes `tuple`, `set`, and `frozenset` as lists. //! - [`number`] reads ASCII digits only, while Python's `int` and `float` accept any Unicode decimal digit. +pub mod datetime; mod error; +pub mod format; pub mod json; pub mod literal; pub mod number; pub mod pickle; +pub mod pydantic; pub mod repr; pub mod reprlib; pub mod truthy; +pub mod unicode; +pub mod urllib; mod value; pub use error::Error; diff --git a/litellm/rust/crates/python-compat/src/number.rs b/litellm/rust/crates/python-compat/src/number.rs index c84e8dd613f..f653ebdf739 100644 --- a/litellm/rust/crates/python-compat/src/number.rs +++ b/litellm/rust/crates/python-compat/src/number.rs @@ -1,6 +1,9 @@ use std::{str::FromStr, time::Duration}; use num_bigint::BigInt; +use serde::{Deserialize, Serialize, Serializer}; + +use crate::{format::fixed, repr::float_repr}; /// `int(text)` in base 10, or `None` where Python raises `ValueError`. pub fn int(text: &str) -> Option { @@ -62,6 +65,47 @@ pub fn seconds(value: f64) -> Result { } } +#[derive(Clone, Copy, Debug, Deserialize, PartialEq)] +#[serde(untagged)] +/// A Python `int` or `float`, kept apart so it prints as `str(value)` does. +pub enum PyNumber { + Int(i64), + Float(f64), +} + +impl PyNumber { + /// `float(value)`. + pub fn value(self) -> f64 { + match self { + #[allow(clippy::cast_precision_loss)] + Self::Int(value) => value as f64, + Self::Float(value) => value, + } + } + + /// `str(value)`. + pub fn text(self) -> String { + match self { + Self::Int(value) => value.to_string(), + Self::Float(value) => float_repr(value), + } + } + + /// `format(value, f".{precision}f")`. + pub fn fixed(self, precision: usize) -> String { + fixed(self.value(), precision) + } +} + +impl Serialize for PyNumber { + fn serialize(&self, serializer: S) -> Result { + match self { + Self::Int(value) => serializer.serialize_i64(*value), + Self::Float(value) => serializer.serialize_f64(*value), + } + } +} + fn parse(text: &str) -> Option { let text = text.trim(); let digit_at = |index: Option| { diff --git a/litellm/rust/crates/python-compat/src/pydantic.rs b/litellm/rust/crates/python-compat/src/pydantic.rs new file mode 100644 index 00000000000..35a12890dda --- /dev/null +++ b/litellm/rust/crates/python-compat/src/pydantic.rs @@ -0,0 +1,70 @@ +//! Pydantic's lax `bool` and `int` validation: `TypeAdapter(bool)` and `TypeAdapter(int)`. + +use num_bigint::BigInt; + +use crate::Value; + +const TWO_POW_63: f64 = 9_223_372_036_854_775_808.0; + +/// `TypeAdapter(bool).validate_python(value)`, or `None` where pydantic raises `ValidationError`. +pub fn lax_bool(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(*flag), + Value::Int(number) => match i8::try_from(number) { + Ok(0) => Some(false), + Ok(1) => Some(true), + _ => None, + }, + Value::Float(number) if *number == 0.0 => Some(false), + Value::Float(number) if *number == 1.0 => Some(true), + Value::Str(text) => match text.to_ascii_lowercase().as_str() { + "0" | "off" | "f" | "false" | "n" | "no" => Some(false), + "1" | "on" | "t" | "true" | "y" | "yes" => Some(true), + _ => None, + }, + _ => None, + } +} + +/// `TypeAdapter(int).validate_python(value)`, or `None` where pydantic raises `ValidationError`. +pub fn lax_int(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(BigInt::from(u8::from(*flag))), + Value::Int(number) => Some(number.clone()), + Value::Float(number) + if number.fract() == 0.0 && (-TWO_POW_63 < *number && *number < TWO_POW_63) => + { + Some(BigInt::from(*number as i64)) + } + Value::Str(text) => int_text(text.trim()), + _ => None, + } +} + +fn int_text(text: &str) -> Option { + let whole = match text.split_once('.') { + Some((whole, fraction)) + if !fraction.is_empty() && fraction.bytes().all(|byte| byte == b'0') => + { + whole + } + Some(_) => return None, + None => text, + }; + let (negative, digits) = match whole.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, whole.strip_prefix('+').unwrap_or(whole)), + }; + let well_formed = !digits.is_empty() + && !digits.starts_with('_') + && !digits.ends_with('_') + && !digits.contains("__") + && digits + .bytes() + .all(|byte| byte.is_ascii_digit() || byte == b'_'); + if !well_formed { + return None; + } + let magnitude = BigInt::parse_bytes(digits.replace('_', "").as_bytes(), 10)?; + Some(if negative { -magnitude } else { magnitude }) +} diff --git a/litellm/rust/crates/python-compat/src/unicode.rs b/litellm/rust/crates/python-compat/src/unicode.rs new file mode 100644 index 00000000000..9737092f4e2 --- /dev/null +++ b/litellm/rust/crates/python-compat/src/unicode.rs @@ -0,0 +1,20 @@ +//! Character classes from Python's `str` methods. + +/// `c.isspace()` for one character: Unicode `White_Space` plus the four separators +/// `\x1c`..`\x1f`, which Python counts and `char::is_whitespace` does not. +pub fn isspace(c: char) -> bool { + matches!( + c, + '\u{9}'..='\u{d}' + | '\u{1c}'..='\u{20}' + | '\u{85}' + | '\u{a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + ) +} diff --git a/litellm/rust/crates/python-compat/src/urllib.rs b/litellm/rust/crates/python-compat/src/urllib.rs new file mode 100644 index 00000000000..6d68249c693 --- /dev/null +++ b/litellm/rust/crates/python-compat/src/urllib.rs @@ -0,0 +1,18 @@ +//! `urllib.parse.quote(text, safe=safe)` over the UTF-8 bytes of `text`. + +/// `urllib.parse.quote(text, safe=safe)`: ASCII letters, digits and `_.-~` are never quoted, +/// the ASCII characters in `safe` are kept, and every other byte becomes `%XX`. +pub fn quote(text: &str, safe: &str) -> String { + let mut encoded = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() + || matches!(byte, b'_' | b'.' | b'-' | b'~') + || (byte.is_ascii() && safe.as_bytes().contains(&byte)) + { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} diff --git a/litellm/rust/crates/python-compat/tests/pydantic.rs b/litellm/rust/crates/python-compat/tests/pydantic.rs new file mode 100644 index 00000000000..bf4d690f6f6 --- /dev/null +++ b/litellm/rust/crates/python-compat/tests/pydantic.rs @@ -0,0 +1,48 @@ +//! Replays `generated/pydantic.json`, which pydantic wrote with `scripts/generate_pydantic_fixtures.py`. + +use litellm_python_compat::{ + json, + pydantic::{lax_bool, lax_int}, +}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Fixtures { + rows: Vec, +} + +#[derive(Deserialize)] +struct Row { + input: String, + bool: Option, + int: Option, +} + +fn fixtures() -> Fixtures { + serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/generated/pydantic.json" + ))) + .expect("pydantic.json matches the fixture schema") +} + +#[test] +fn lax_bool_and_int_match_pydantic() { + let mismatches: Vec = fixtures() + .rows + .iter() + .flat_map(|row| { + let value = json::loads(&row.input).expect("inputs are json.dumps output"); + let as_bool = lax_bool(&value); + let as_int = lax_int(&value).map(|number| number.to_string()); + [ + (as_bool != row.bool) + .then(|| format!("bool {}: {as_bool:?} != {:?}", row.input, row.bool)), + (as_int != row.int) + .then(|| format!("int {}: {as_int:?} != {:?}", row.input, row.int)), + ] + }) + .flatten() + .collect(); + assert!(mismatches.is_empty(), "{mismatches:#?}"); +} diff --git a/litellm/rust/crates/python-compat/tests/stdlib.rs b/litellm/rust/crates/python-compat/tests/stdlib.rs new file mode 100644 index 00000000000..9455bdcbd2b --- /dev/null +++ b/litellm/rust/crates/python-compat/tests/stdlib.rs @@ -0,0 +1,157 @@ +//! Replays `generated/stdlib.json`, which CPython wrote with `scripts/generate_stdlib_fixtures.py`. + +use litellm_python_compat::{ + datetime::{IsoDateTime, UtcDateTime, fromisoformat}, + format::fixed, + number::{self, PyNumber}, + unicode::isspace, + urllib::quote, +}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Fixtures { + quote: Vec, + utc: Vec, + fromisoformat: Vec, + fixed: Vec, + number_str: Vec, + isspace: Vec, +} + +#[derive(Deserialize)] +struct FromIsoFormat { + text: String, + naive: Option, + aware: Option, +} + +/// A finite `float`, or the `repr` of a non-finite one, which JSON cannot hold. +#[derive(Deserialize)] +#[serde(untagged)] +enum Float { + Finite(f64), + Repr(String), +} + +impl Float { + fn value(&self) -> f64 { + match self { + Self::Finite(value) => *value, + Self::Repr(text) => number::float(text).expect("a float repr"), + } + } +} + +#[derive(Deserialize)] +struct Fixed { + value: Float, + precision: usize, + text: String, +} + +#[derive(Deserialize)] +struct NumberStr { + value: Option, + float: Option, + str: String, +} + +#[derive(Deserialize)] +struct Quote { + text: String, + safe: String, + quoted: String, +} + +#[derive(Deserialize)] +struct Utc { + seconds: i64, + micros: u32, + fields: [i64; 7], +} + +fn fixtures() -> Fixtures { + serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/generated/stdlib.json" + ))) + .expect("stdlib.json matches the fixture schema") +} + +#[test] +fn quote_matches_urllib() { + for row in fixtures().quote { + assert_eq!( + quote(&row.text, &row.safe), + row.quoted, + "{:?} safe={:?}", + row.text, + row.safe + ); + } +} + +#[test] +fn utc_fields_match_datetime() { + for row in fixtures().utc { + let moment = UtcDateTime::from_timestamp(row.seconds, row.micros); + let fields = [ + moment.year, + i64::from(moment.month), + i64::from(moment.day), + i64::from(moment.hour), + i64::from(moment.minute), + i64::from(moment.second), + i64::from(moment.microsecond), + ]; + assert_eq!(fields, row.fields, "{} {}", row.seconds, row.micros); + } +} + +#[test] +fn fromisoformat_matches_datetime() { + for row in fixtures().fromisoformat { + let expected = match (row.naive, row.aware) { + (Some(micros), None) => Some(IsoDateTime::Naive(micros)), + (None, Some(micros)) => Some(IsoDateTime::Aware(micros)), + _ => None, + }; + assert_eq!(fromisoformat(&row.text), expected, "{:?}", row.text); + } +} + +#[test] +fn fixed_matches_format() { + for row in fixtures().fixed { + let value = row.value.value(); + assert_eq!( + fixed(value, row.precision), + row.text, + "{value:?} .{}f", + row.precision + ); + } +} + +#[test] +fn number_text_matches_str() { + for row in fixtures().number_str { + let number = match (row.value, row.float) { + (Some(number), None) => number, + (None, Some(text)) => PyNumber::Float(number::float(&text).expect("a float repr")), + _ => panic!("a number_str row holds a value or a float repr"), + }; + assert_eq!(number.text(), row.str); + } +} + +#[test] +fn isspace_matches_str_isspace() { + let ours: Vec = (0..=0x0010_ffff_u32) + .filter_map(char::from_u32) + .filter(|c| isspace(*c)) + .map(u32::from) + .collect(); + assert_eq!(ours, fixtures().isspace); +} diff --git a/litellm/rust/crates/router-types/tests/litellm_params.rs b/litellm/rust/crates/router-types/tests/litellm_params.rs index c5c0d16ee8f..fb564e2d884 100644 --- a/litellm/rust/crates/router-types/tests/litellm_params.rs +++ b/litellm/rust/crates/router-types/tests/litellm_params.rs @@ -126,7 +126,10 @@ fn a_param_of_the_wrong_type_is_rejected(#[case] value: Value) { #[rstest] fn fields_fill_the_model_the_credentials_and_every_group() { let filled: Value = LitellmParams::fields() - .map(|name| (name.to_string(), Value::from(name))) + .map(|name| match name { + AwsParams::SESSION_TAGS => (name.to_string(), json!([{"Key": "k", "Value": "v"}])), + _ => (name.to_string(), Value::from(name)), + }) .collect::>() .into(); let typed = params(filled); diff --git a/litellm/rust/crates/secrets-aws/src/auth.rs b/litellm/rust/crates/secrets-aws/src/auth.rs index 6bec8ad96a1..0e8923dc599 100644 --- a/litellm/rust/crates/secrets-aws/src/auth.rs +++ b/litellm/rust/crates/secrets-aws/src/auth.rs @@ -49,6 +49,8 @@ impl Credentials { profile_name: settings.aws_profile_name.clone(), web_identity_token: settings.aws_web_identity_token.as_ref().cloned(), sts_endpoint: settings.aws_sts_endpoint.clone(), + session_tags: None, + session_policy: None, }, environment, } diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 331fad62c12..a4067705c7e 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -9,8 +9,11 @@ from typing_extensions import disjoint_base from litellm.integrations.custom_logger_v2.rust import RustHandle from litellm.redis.store_settings import RedisBreakerSettings, RedisStoreSettings from litellm.redis.stores.router_session_pins.keys import PinKey +from litellm.rust_bridge.database import NativeDatabaseEndpoint, NativeDatabaseEvents from litellm.rust_bridge.execution import Execution from litellm.rust_bridge.public_call import NativeCall +from litellm.types.integrations.custom_logger_v2 import BudgetPlan, BudgetReservation +from litellm.types.proxy.native_budgets import BudgetCheck _T_co = TypeVar("_T_co", covariant=True) @@ -20,10 +23,16 @@ class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... class NativeRedisStoreError(Exception): ... +class NativeDatabaseFailure(Exception): + kind: str + unavailable: bool + requeue_safe: bool + def trace_encode_error(message: str) -> bytes: ... def bundled_model_cost_map() -> bytes: ... def native_redis_breaker_states() -> list[str]: ... def unsupported_tls_settings(ssl_security_level: str | None = None, ssl_ecdh_curve: str | None = None) -> list[str]: ... + @final class NativeCostMapSource: def __new__(cls, url: str, *, use_http_settings: bool) -> NativeCostMapSource: ... @@ -563,6 +572,21 @@ class PostCallSettlement: @property def settled(self) -> bool: ... +@final +class NativeBudgets: + def __new__(cls, spend_counters: NativeSpendCounters) -> NativeBudgets: ... + def reserve(self, plan: BudgetPlan, observed: tuple[Mapping[str, object], ...]) -> Future[JsonValue]: ... + def settle( + self, + reservation: BudgetReservation, + window_resets: tuple[str | None, ...], + observed: tuple[float | None, ...], + actual: float, + apply_consistent: bool, + ) -> Future[JsonValue]: ... + +def budget_check_decision(check: BudgetCheck) -> JsonValue: ... + @final class NativeSpendCounters: def __new__(cls, client: NativeRedisClient | None) -> NativeSpendCounters: ... @@ -596,7 +620,12 @@ class NativeSpendCounters: @final class NativeDatabase: @staticmethod - def connect() -> Future[NativeDatabase]: ... + def connect( + events: NativeDatabaseEvents, writer: NativeDatabaseEndpoint, reader: NativeDatabaseEndpoint | None = None + ) -> Future[NativeDatabase]: ... + def set_writer_password(self, password: str) -> None: ... + def set_reader_password(self, password: str) -> None: ... + def health_check(self) -> Future[None]: ... def close(self) -> Future[None]: ... @final @@ -793,10 +822,12 @@ __all__ = [ "LocalSizes", "NativeAdmission", "NativeAuthCache", + "NativeBudgets", "NativeClickHouseSpendConfig", "NativeClickHouseSpendStorage", "NativeCostMapSource", "NativeDatabase", + "NativeDatabaseFailure", "NativeDeploymentHealth", "NativeDiagnosticProcessor", "NativeFetchFailed", @@ -841,6 +872,7 @@ __all__ = [ "aocr", "aresponses", "atranscription", + "budget_check_decision", "bundled_model_cost_map", "completion", "embedding", diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 3e5b9977df6..089d8476d61 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -30,9 +30,14 @@ class Target(str, Enum): MESSAGES = "messages" MESSAGES_PASSTHROUGH = "messages_passthrough" """A deployment's own /v1/messages, which Python's OpenAI-compatible config forwards untranslated.""" + BEDROCK_INVOKE = "bedrock_invoke" + """Claude on Bedrock InvokeModel, an Anthropic Messages body behind the InvokeModel API.""" RESPONSES = "responses" +NATIVE_MESSAGES_TARGETS: Final = frozenset({Target.MESSAGES, Target.MESSAGES_PASSTHROUGH, Target.BEDROCK_INVOKE}) + + class Feature(str, Enum): MOCK_RESPONSE = "mock_response" """mock_response short-circuits before any provider call.""" @@ -202,6 +207,12 @@ RULES: Final[Rules] = ( ), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=MESSAGES_PYTHON_FEATURES), RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED, providers=frozenset({"anthropic", "azure_ai"})), + RouteRule( + Route.MESSAGES, + Rollout.RUST_REQUIRED, + providers=frozenset({"bedrock"}), + targets=frozenset({Target.BEDROCK_INVOKE}), + ), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY, features=RESPONSES_PYTHON_FEATURES), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), diff --git a/litellm/rust_bridge/database.py b/litellm/rust_bridge/database.py new file mode 100644 index 00000000000..c4971003821 --- /dev/null +++ b/litellm/rust_bridge/database.py @@ -0,0 +1,45 @@ +from typing import Literal, Protocol, TypeAlias + +from typing_extensions import ReadOnly, TypedDict + +SslMode: TypeAlias = Literal["disable", "prefer", "require", "verify-ca", "verify-full"] + + +class NativePoolLimits(TypedDict): + max_connections: ReadOnly[int] + acquire_timeout_seconds: ReadOnly[float | None] + idle_timeout_seconds: ReadOnly[float | None] + socket_timeout_seconds: ReadOnly[float | None] + + +class NativeClientIdentity(TypedDict): + cert_pem: ReadOnly[str] + key_pem: ReadOnly[str] + + +class NativeDatabaseEndpoint(TypedDict): + host: ReadOnly[str] + port: ReadOnly[int] + socket: ReadOnly[str | None] + username: ReadOnly[str] + password: ReadOnly[str | None] + database: ReadOnly[str] + ssl_mode: ReadOnly[SslMode] + ssl_root_cert: ReadOnly[str | None] + ssl_client: ReadOnly[NativeClientIdentity | None] + application_name: ReadOnly[str | None] + options: ReadOnly[tuple[tuple[str, str], ...]] + statement_cache_capacity: ReadOnly[int] + pool: ReadOnly[NativePoolLimits] + + +class NativeDatabaseEvents(Protocol): + def record( + self, + call_type: str, + table: str | None, + operation: str | None, + started_at: float, + duration_seconds: float, + failure: str | None, + ) -> None: ... diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index 7147e99ac27..10cfa286bd3 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -707,18 +707,26 @@ def _log_superseded_stream_failure( verbose_proxy_logger.error("Error scheduling stream failure logging: %s", e) +def _relayed(head: object) -> bool: + from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead + + return isinstance(head, MessagesStreamHead) and head.relayed + + def stream_failure( logger: Logging, endpoint_type: str, request_body: dict[str, object], chunks: list[bytes], error: Exception, + head: object, ) -> Coroutine[object, object, None]: from litellm._logging import verbose_proxy_logger from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType - verbose_proxy_logger.error("Error in chunk_processor: %s", error) + if not _relayed(head): + verbose_proxy_logger.error("Error in chunk_processor: %s", error) # Python's stream fails before its success handler records `cache_hit`; only a cache hit # stamps it earlier. if logger.model_call_details.get("cache_hit") is not True: diff --git a/litellm/rust_bridge/host/model_capabilities.py b/litellm/rust_bridge/host/model_capabilities.py index 00a00b9e79b..a26dae0ede0 100644 --- a/litellm/rust_bridge/host/model_capabilities.py +++ b/litellm/rust_bridge/host/model_capabilities.py @@ -1,11 +1,14 @@ from __future__ import annotations +import json from collections.abc import Mapping from functools import lru_cache from typing import Final, TypeAlias import litellm -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils import fallback_generalizations +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping +from litellm.litellm_core_utils.get_model_cost_map import FALLBACK_GENERALIZATIONS_KEY from litellm.litellm_core_utils.model_cost_table import ModelCostVersion, model_cost_version @@ -22,6 +25,7 @@ class _Held: return isinstance(other, _Held) and other.value is self.value +_CatalogVersion: TypeAlias = tuple[int, int, int] _Fingerprint: TypeAlias = tuple[str, str | None, _Held, ModelCostVersion, int, tuple[_Held, ...]] @@ -95,3 +99,18 @@ def _capabilities(fingerprint: _Fingerprint, resolved_model: str, provider: str) "supports_mid_conversation_system": supports("supports_mid_conversation_system"), "effort_tiers": {level: tier(level) for level in ("minimal", "low", "medium", "high", "xhigh", "max")}, } + + +def model_cost_document(known: _CatalogVersion | None) -> tuple[_CatalogVersion, Mapping[str, object] | None]: + """The live model map and its active fallback rules as one document, which the native catalog reads, or + ``None`` when the map is still at version ``known``.""" + version: Final = model_cost_version() + current: Final = (version.table, version.size, version.writes) + if current == known: + return current, None + rows: Final = as_str_mapping(litellm.model_cost) or {} # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the model map is an untyped dict of rows + rules: Final = as_object_list(fallback_generalizations.get_fallback_generalization_rules()) or [] # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the rule registry keeps the raw untyped list + document: Final = json.loads( + json.dumps({**rows, FALLBACK_GENERALIZATIONS_KEY: {"rules": list(rules)}}, default=str) + ) + return current, as_str_mapping(document) diff --git a/litellm/rust_bridge/host/model_cost.py b/litellm/rust_bridge/host/model_cost.py index cfedbb64592..ad743c63498 100644 --- a/litellm/rust_bridge/host/model_cost.py +++ b/litellm/rust_bridge/host/model_cost.py @@ -46,6 +46,11 @@ def merged( def kept(previous: dict[str, object] | None, registration: Mapping[str, object]) -> dict[str, object]: """The registration ``register_model(persist_across_reloads=True)`` keeps for a key: ``registration`` merged into the one kept before it.""" - from litellm.utils import update_dictionary + from litellm.utils import base_fields_replaced_by, update_dictionary - return update_dictionary(dict(previous or {}), registration) + replaced: Final = base_fields_replaced_by(registration) + return { + name: value + for name, value in update_dictionary(dict(previous or {}), registration).items() + if name not in replaced + } diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 4a890c8ac63..cda1ab5b7ee 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -9,6 +9,7 @@ import http import http.client import inspect import ipaddress +import json import os import socket import ssl @@ -18,12 +19,17 @@ from typing import Final, TypeAlias, TypeVar, cast # noqa: TID251 # narrows na from urllib.parse import SplitResult, urlsplit import httpx +from aiohttp import ClientPayloadError from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger from litellm.caching.caching_handler import LLMCachingHandler +from litellm.constants import ( + ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS, + ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils import exception_mapping_utils from litellm.litellm_core_utils.core_helpers import ( @@ -42,7 +48,7 @@ from litellm.litellm_core_utils.model_latest_aliases import is_model_latest_alia from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout -from litellm.litellm_core_utils.url_utils import ensure_api_route +from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.anthropic.common_utils import ( AnthropicError, AnthropicModelInfo, @@ -70,7 +76,13 @@ from litellm.llms.anthropic.messages.native.handler import ( _with_deployment_output_effort as _deployment_output_effort, # pyright: ignore[reportPrivateUsage] # the handler's own effort rounding ) from litellm.llms.anthropic.messages.native.interceptors import MessagesInterceptor, get_messages_interceptors -from litellm.llms.anthropic.messages.native.utils import normalize_native_messages, prepare_native_messages +from litellm.llms.anthropic.messages.native.utils import ( + continues_message_thread, + message_thread_not_found_body, + message_thread_unsupported_body, + normalize_native_messages, + prepare_native_messages, +) from litellm.llms.anthropic.messages.safeguards import ( SafeguardsEvaluator, StreamedSafeguardResults, @@ -81,6 +93,11 @@ from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enab from litellm.llms.anthropic.wif import get_anthropic_wif_token from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.bedrock.common_utils import ( + ensure_bedrock_anthropic_messages_tool_names, # pyright: ignore[reportUnknownVariableType] # takes an untyped request dict + normalize_custom_field_on_tools, # pyright: ignore[reportUnknownVariableType] # takes an untyped request dict + normalize_tool_input_schema_types_for_bedrock_invoke, # pyright: ignore[reportUnknownVariableType] # takes an untyped request dict +) from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # its params dict is untyped @@ -152,6 +169,8 @@ class MessagesSettings: ascii_response_headers: bool = False joined_reads: bool = False ascii_request_headers: bool = True + detached_drain_timeout: float | None = None + detached_drain_limit: int | None = None def response(value: Mapping[str, object]) -> AnthropicMessagesResponse: @@ -191,15 +210,100 @@ def _anthropic_metadata(params: Mapping[str, object]) -> Mapping[str, object]: def _body_fields(request: Mapping[str, object]) -> Mapping[str, object]: params: Final = _anthropic_metadata( - known_params(_BODY_FIELDS.difference(handler.additional_drop_paths(request)), request) + known_params(_BODY_FIELDS.difference(handler.additional_drop_paths(request), _dropped_thread(request)), request) ) if deployment_reasoning_effort_levels(request.get("model_info")) is None: return params return _with_deployment_effort(params, request) +def _thread_provider(request: Mapping[str, object]) -> str | None: + """The provider of a call carrying a thread its Messages config cannot serve, as Python's handler + resolves it, or None when the call carries no thread or its config serves threads.""" + if request.get("thread") is None: + return None + model, provider, _, _ = _requested_provider( + str(request["model"]), optional_str(request.get("custom_llm_provider")), _handler_params(request) + ) + config: Final = native_messages_config( + model, provider, request.get("model_info"), optional_str(request.get("api_base")) + ) + if config is not None and config.supports_message_threads(): + return None + return str(provider) + + +def _dropped_thread(request: Mapping[str, object]) -> tuple[str, ...]: + return () if _thread_provider(request) is None else ("thread",) + + +def _thread_refusal(thread: object, provider: str, drop_params: object) -> AnthropicError | None: + if continues_message_thread(thread): + return AnthropicError( + status_code=404, + message=message_thread_not_found_body( + f"message threads are not supported for provider {provider}, " + "so this thread cannot be continued. Send the full conversation to start over" + ), + ) + if not drops_unsupported_params(drop_params): + return AnthropicError( + status_code=400, + message=message_thread_unsupported_body( + f"message threads are not supported for provider {provider}. " + "Send the request without thread, or set drop_params to serve it without a server-side thread" + ), + ) + verbose_logger.warning( + "Dropping thread for provider %s: message threads are not supported there, " + "so the request is served without a server-side thread", + provider, + ) + return None + + +def _refuse_unsupported_thread(request: Mapping[str, object], asynchronous: bool) -> None: + """Python's handler refuses a thread its provider cannot serve, or drops it under drop_params.""" + provider: Final = _thread_provider(request) + if provider is None: + return + refusal: Final = _thread_refusal(request.get("thread"), provider, request.get("drop_params")) + if refusal is None: + return + _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) + try: + raise refusal + except AnthropicError as provider_error: + _exception_type( + model=str(request["model"]), + custom_llm_provider=provider, + original_exception=provider_error, + extra_kwargs={name: value for name, value in request.items() if name not in _ENTRY_PARAMS}, + ) + raise + + +def _shape_bedrock_tools(tools: object, model: str) -> None: + """Python's Bedrock invoke transform shapes the caller's own tool dicts in place, so its logs show them shaped.""" + if not isinstance(tools, list): + return + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( + AmazonAnthropicClaudeMessagesConfig, + ) + + body: Final[dict[str, object]] = {"tools": tools} # mutable-ok: the Bedrock transform helpers shape it in place + AmazonAnthropicClaudeMessagesConfig()._remove_ttl_from_cache_control( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # the transform's own in-place step + anthropic_messages_request=body, model=model + ) + normalize_custom_field_on_tools(body) + normalize_tool_input_schema_types_for_bedrock_invoke(body) + ensure_bedrock_anthropic_messages_tool_names(body) + + def body_params(request: Mapping[str, object], asynchronous: bool, hooked: bool = False) -> Mapping[str, object]: fields: Final = _body_fields(request) + if _resolved_provider(request) == "bedrock": + _shape_bedrock_tools(fields.get("tools"), str(request["model"]).removeprefix("bedrock/")) if not detectors.cache_control_injection(NativeCall(args=(), kwargs=request, base={})): return {**fields, "messages": request["messages"]} if hooked else fields messages, system, tools = _prompt_cached(request, asynchronous, hooked) @@ -493,12 +597,18 @@ def answer(owner: MessagesInterceptor, request: Mapping[str, object], stream: bo ) -def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: +def stream_hidden_params(headers: Sequence[tuple[str, str]], relayed: bool = False) -> Mapping[str, object]: + """The stream's head. `relayed` says the stream runs through Python's native Messages relay, + `BaseAnthropicMessagesStreamingIterator.async_sse_wrapper`, rather than its pass-through.""" from litellm.llms.anthropic.messages.native.streaming_iterator import ( anthropic_messages_stream_hidden_params, ) - return MessagesStreamHead(anthropic_messages_stream_hidden_params(callbacks_legacy_python.wire_headers(headers))) + head: Final = MessagesStreamHead( + anthropic_messages_stream_hidden_params(callbacks_legacy_python.wire_headers(headers)) + ) + head.relayed = relayed + return head async def _unrequested_stream_delivered( @@ -557,6 +667,7 @@ def unrequested_stream( _UPSTREAM_ARGS: Final = TypeAdapter(tuple[int, str]) _UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]]) +_INVOCATION_METRICS: Final = "amazon-bedrock-invocationMetrics" _BODY_CUT: Final = TypeAdapter(tuple[int, int | None, bool]) _ERROR_CLASS: TypeAlias = Callable[..., Exception] _validate_metadata: Final = cast( # cast-ok: the handler's untyped metadata validator @@ -601,12 +712,18 @@ def _rejection(error: Exception, request: Mapping[str, object], request_provider provider_rejection: Final = getattr(error, "messages_provider_rejection", None) if isinstance(provider_rejection, str): _log_resolved_call(request, request_provider) - _flag_azure_ai_call(request, request_provider, asynchronous) - return _public_provider_error( - _config_error(400, provider_rejection, (), request, request_provider), error, request, request_provider + _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) + rejection: Final = _provider_rejection( + provider_rejection, + getattr(error, "messages_provider_bad_request", False) is True, + request, + request_provider, ) + if isinstance(rejection, litellm.BadRequestError): + return rejection + return _public_provider_error(rejection, error, request, request_provider) if getattr(error, "messages_unsupported_param", False): - _flag_azure_ai_call(request, request_provider, asynchronous) + _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) return litellm.UnsupportedParamsError(message=str(error)) return litellm.BadRequestError( message=str(error), @@ -615,6 +732,23 @@ def _rejection(error: Exception, request: Mapping[str, object], request_provider ) +def _provider_rejection( + text: str, bad_request: bool, request: Mapping[str, object], request_provider: str +) -> Exception: + """The Bedrock invoke config raises its own `BadRequestError`, which other configs' HTTP handler + maps into their error class by its text.""" + if not bad_request: + return _config_error(400, text, (), request, request_provider) + raised: Final = litellm.BadRequestError( + message=text, + model=str(request["model"]).removeprefix(f"{request_provider}/"), + llm_provider=request_provider, + ) + if request_provider == "bedrock": + return raised + return _config_error(400, str(raised), (), request, request_provider) + + def _timeout_text(phase: str, request: Mapping[str, object]) -> str: """`litellm.Timeout`'s text as Python's HTTP client raises it for the same timeout.""" seconds: Final = request.get("timeout") @@ -652,21 +786,115 @@ def _body_cut(received: int, expected: int | None, reset: bool) -> httpx.Transpo f"Not enough data to satisfy content length header (received {received} of {expected} bytes)." ) ) - text: Final = f"Response payload is not completed: {payload!r}" - if not reset: - return httpx.ReadError(text) - return httpx.ReadError(f"{text}. {ConnectionResetError(errno.ECONNRESET, os.strerror(errno.ECONNRESET))!r}") + completed: Final = f"Response payload is not completed: {payload!r}" + text: Final = ( + f"{completed}. {ConnectionResetError(errno.ECONNRESET, os.strerror(errno.ECONNRESET))!r}" + if reset + else completed + ) + payload_error: Final = ClientPayloadError(text) + payload_error.__cause__ = payload + read_error: Final = httpx.ReadError(text) + read_error.__cause__ = payload_error + read_error.__context__ = payload_error + return read_error + + +@dataclass(frozen=True, slots=True) +class _ProviderCall: + config: BaseAnthropicMessagesConfig + model: str + params: dict[str, object] # mutable-ok: the config hooks take a params dict + + +def _provider_call(request: Mapping[str, object]) -> _ProviderCall | None: + """The config, model and params Python's handler hands its provider config for the call.""" + model, provider, _, _ = _requested_provider( + str(request["model"]), optional_str(request.get("custom_llm_provider")), _handler_params(request) + ) + config: Final = native_messages_config( + model, provider, request.get("model_info"), optional_str(request.get("api_base")) + ) + if config is None: + return None + return _ProviderCall(config=config, model=model, params=dict(_python_handler_params(request, provider))) + + +def _complete_url(call: _ProviderCall, request: Mapping[str, object]) -> str: + return call.config.get_complete_url( # pyright: ignore[reportUnknownMemberType] # the config's untyped params dicts + api_base=optional_str(request.get("api_base")), + api_key=optional_str(request.get("api_key")), + model=call.model, + optional_params=call.params, + litellm_params=call.params, + stream=request.get("stream") is True, + ) + + +def _bedrock_url(request: Mapping[str, object]) -> str | None: + """The URL Python's bedrock Messages config builds for the call.""" + call: Final = _provider_call(request) + return None if call is None else _complete_url(call, request) + + +def _signing_failure(kind: str) -> Exception | None: + """The exception Python's AWS signing raises for a credential failure whose outcome needs no STS, + IMDS or secret manager call. botocore's environment provider raises `PartialCredentialsError` + for an access key without its secret, where Rust's default chain finds nothing.""" + from botocore.exceptions import NoCredentialsError + + from litellm.llms.bedrock.base_aws_llm import AwsAuthError + + if kind == "web_identity_reference": + return AwsAuthError(status_code=400, message="Invalid web identity token reference.") + if kind == "missing_web_identity_token": + return AwsAuthError(status_code=401, message="OIDC token could not be retrieved from secret manager.") + if kind == "no_credentials" and not os.environ.get("AWS_ACCESS_KEY_ID"): + return NoCredentialsError() + return None + + +def _bedrock_credential_failure(error: Exception, request: Mapping[str, object]) -> Exception | None: + """The exception Python's bedrock config raises while signing a call whose credentials Rust + could not acquire, from botocore's `NoCredentialsError` to an STS or `AwsAuthError` failure.""" + kind: Final = getattr(error, "messages_credential_failure", None) + if not isinstance(kind, str): + return None + known: Final = _signing_failure(kind) + if known is not None: + return _raised(known) + call: Final = _provider_call(request) + if call is None: + return None + try: + call.config.sign_request( # pyright: ignore[reportUnknownMemberType] # the config's untyped params dicts + headers={}, + optional_params=call.params, + request_data={}, + api_base=_complete_url(call, request), + api_key=optional_str(request.get("api_key")), + model=call.model, + stream=request.get("stream") is True, + fake_stream=False, + ) + except Exception as python_failure: # noqa: BLE001 # the exception Python's signing raises is the answer + return python_failure + return None def _sent_base(request: Mapping[str, object], request_provider: str, secrets_read: _SecretsRead) -> str | None: """The base the call was sent to: azure_ai's falls back to the `AZURE_API_BASE` the host read - to send it, and anthropic's to its default.""" + to send it, anthropic's to its default and bedrock's to the URL its config builds.""" api_base: Final = request.get("api_base") + if isinstance(api_base, str) and (api_base or request_provider != "azure_ai"): + return api_base if request_provider == "anthropic": - return api_base if isinstance(api_base, str) else anthropic_pass_through_api_base() - if request_provider == "azure_ai" and not api_base: + return anthropic_pass_through_api_base() + if request_provider == "azure_ai": return dict(secrets_read).get("AZURE_API_BASE") - return api_base if isinstance(api_base, str) else None + if request_provider == "bedrock": + return _bedrock_url(request) + return None def _connect_failure( @@ -869,6 +1097,58 @@ def _raised_like_python(error: _RaisedT, request_provider: str) -> _RaisedT: return error if request_provider in _MAIN_NATIVE_ERROR_PROVIDERS else _raised(error) +def _chunk_parser_failure(text: str) -> Exception: + try: + json.loads(text).pop(_INVOCATION_METRICS, {}) + except Exception as error: # noqa: BLE001 # the chunk parser's own failure is what Python raised + return error + return ValueError(text) + + +def _json_failure(text: str) -> Exception: + try: + json.loads(text) + except json.JSONDecodeError as error: + return error + return ValueError(text) + + +def bedrock_stream_raise(kind: str, arguments: Sequence[object]) -> Exception: + """The exception Python's Bedrock stream decoder raised unmapped for a body it could not parse.""" + from botocore import eventstream + + raises: Final[Mapping[str, Callable[..., Exception]]] = { + "ChecksumMismatch": eventstream.ChecksumMismatch, + "InvalidHeadersLength": eventstream.InvalidHeadersLength, + "InvalidPayloadLength": eventstream.InvalidPayloadLength, + "DuplicateHeader": eventstream.DuplicateHeader, + "InvalidJson": _json_failure, + "NotAnObject": _chunk_parser_failure, + } + return _raise_with(raises[kind], arguments) + + +def _raise_with(factory: Callable[..., Exception], arguments: Sequence[object]) -> Exception: + return factory(*arguments) + + +def stream_failure(error: BaseException, request: Mapping[str, object], request_provider: str) -> BaseException: + """Python's Bedrock Messages stream raises a failure after its head with the deployment's debug info.""" + if request_provider in _MAIN_NATIVE_ERROR_PROVIDERS or not isinstance(error, Exception): + return error + return _with_deployment_debug_info(error, request, request_provider) + + +def _bedrock_stream_error(error: Exception, request: Mapping[str, object]) -> Exception | None: + """Python's Bedrock stream decoder raises its `BedrockError` as built, unmapped.""" + try: + status, text = _UPSTREAM_ARGS.validate_python(error.args) + headers: Final = _UPSTREAM_HEADERS.validate_python(getattr(error, "headers", None)) + except ValidationError: + return None + return _config_error(status, text, headers, request, "bedrock") + + def map_failure( error: Exception, request: Mapping[str, object], @@ -878,6 +1158,10 @@ def map_failure( ) -> Exception: """Raises what Python's Messages handler raises for the same failure. `secrets_read` are the secrets the host read to send the call, such as azure_ai's `AZURE_API_BASE`.""" + if request_provider == "bedrock" and getattr(error, "messages_stream_failure", False): + stream_error: Final = _bedrock_stream_error(error, request) + if stream_error is not None: + return stream_error if getattr(error, "messages_timeout", None) == "Body": # Python reads a stream's body outside its HTTP client's error mapping, so the transport's # own timeout reaches the caller unmapped. @@ -915,6 +1199,10 @@ def map_failure( ), request_provider, ) + credential_failure: Final = _bedrock_credential_failure(error, request) if request_provider == "bedrock" else None + if credential_failure is not None: + _log_resolved_call(request, request_provider) + return map_token_failure(credential_failure, request, request_provider, asynchronous) provider_error: Final = _provider_error(error, request, request_provider, secrets_read or []) if provider_error is None: return failures.map_native_failure( @@ -950,6 +1238,20 @@ def failure_provider(request: Mapping[str, object]) -> str: return _resolved_provider(request) or "anthropic" +def filters_beta_headers(model: str, custom_llm_provider: str | None, api_base: str | None) -> bool: + """Whether the call's provider config filters the caller's `anthropic-beta` before Python's client encodes it.""" + config: Final = native_messages_config( + model.removeprefix(f"{custom_llm_provider}/"), custom_llm_provider, None, api_base + ) + return config is not None and config.should_filter_anthropic_beta_headers() + + +def record_guardrail(public: Mapping[str, object], request: Mapping[str, object]) -> None: + logger: Final = _logger(request) + if logger is not None: + logger.record_native_bedrock_guardrail(public) + + def _scope_provider_specific_header(request: Mapping[str, object]) -> None: scoped: Final = request.get("provider_specific_header") if scoped is None: @@ -1009,10 +1311,15 @@ def settings( drops_unsupported_params=drops_unsupported_params(kwargs.get("drop_params")), ascii_response_headers=aiohttp, joined_reads=aiohttp, + detached_drain_timeout=ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS if asynchronous else None, + detached_drain_limit=ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS if asynchronous else None, ) - return asdict( - configured if request is None else _with_call_settings(configured, request, kwargs.get("litellm_logging_obj")) - ) + if request is None: + return asdict(configured) + call_settings: Final = _with_call_settings(configured, request, kwargs.get("litellm_logging_obj")) + if call_settings.mock_response is None and not call_settings.mcp_gateway: + _refuse_unsupported_thread(request, asynchronous) + return asdict(call_settings) def _flag_messages_call(logger: object, asynchronous: bool) -> None: @@ -1021,13 +1328,6 @@ def _flag_messages_call(logger: object, asynchronous: bool) -> None: logger.model_call_details.setdefault("litellm_params", {})[CallTypes.aanthropic_messages.value] = asynchronous -def _flag_azure_ai_call(request: Mapping[str, object], request_provider: str, asynchronous: bool) -> None: - """Python's handler flags the call before it gets the credential or refuses a param. By the owner's ruling - anthropic keeps main's unflagged failure.""" - if request_provider == "azure_ai": - _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) - - @dataclass(frozen=True, slots=True) class _Answered: value: object @@ -1296,7 +1596,7 @@ def map_token_failure( ) -> Exception: """Python's handler maps a litellm exception raised while it gets the call's credential, and lets any other exception through as raised.""" - _flag_azure_ai_call(request, request_provider, asynchronous) + _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) if not isinstance(error, BaseLLMException): return error return _public_provider_error(error, error, request, request_provider) @@ -1518,13 +1818,14 @@ def unencodable_header(value: bytes, received: bool) -> Exception: def _sent_url(request: Mapping[str, object], url: str) -> str: """The native URL builder trims the caller's api_base; Python sends it as given.""" api_base: Final = request.get("api_base") - if ( - not isinstance(api_base, str) - or api_base == api_base.strip() - or not _unsendable_api_base(api_base) - or _resolved_provider(request) != "anthropic" - ): + if not isinstance(api_base, str) or api_base == api_base.strip() or not _unsendable_api_base(api_base): return url + provider: Final = _resolved_provider(request) + if provider not in ("anthropic", "bedrock"): + return url + if provider == "bedrock": + trimmed: Final = f"{api_base.strip().rstrip('/')}/" + return join_api_base(api_base, url.removeprefix(trimmed)) if url.startswith(trimmed) else url try: return _python_messages_url(api_base) if url == _python_messages_url(api_base.strip()) else url except ValueError: # a base urlsplit cannot split keeps the URL the native builder made diff --git a/litellm/rust_bridge/host/settings.py b/litellm/rust_bridge/host/settings.py index 895e7994f71..77a4a55bf4b 100644 --- a/litellm/rust_bridge/host/settings.py +++ b/litellm/rust_bridge/host/settings.py @@ -31,6 +31,8 @@ class ProviderDefaults: enable_azure_ad_token_refresh: object api_key: object azure_key: object + drop_params: object + bedrock_request_metadata_fields: object @dataclass(frozen=True, slots=True) @@ -145,6 +147,8 @@ def provider_defaults() -> ProviderDefaults: enable_azure_ad_token_refresh=litellm.enable_azure_ad_token_refresh, api_key=litellm.api_key, azure_key=litellm.azure_key, + drop_params=litellm.drop_params, + bedrock_request_metadata_fields=litellm.bedrock_request_metadata_fields, ) diff --git a/litellm/rust_bridge/messages/entrypoints.py b/litellm/rust_bridge/messages/entrypoints.py index cc708914df5..56772b6b3a7 100644 --- a/litellm/rust_bridge/messages/entrypoints.py +++ b/litellm/rust_bridge/messages/entrypoints.py @@ -14,6 +14,7 @@ class MessagesStreamHead(dict[str, object]): agentic: Callable[[execution.AsyncStream], object] | None = None deliver: Callable[[object], Awaitable[object]] | None = None python_types: bool = False + relayed: bool = False cached: object | None = None cache_writer: Callable[[object], object] | None = None diff --git a/litellm/schema.prisma b/litellm/schema.prisma index db88b24dcb0..9e1f47a5d1f 100644 --- a/litellm/schema.prisma +++ b/litellm/schema.prisma @@ -85,6 +85,8 @@ model LiteLLM_CustomerTeamTable { updated_at DateTime @default(now()) @updatedAt updated_by String @@index([parent_org_id]) + @@index([budget_id]) + @@index([budget_override_id]) @@index([spend]) } @@ -107,6 +109,9 @@ model LiteLLM_CustomerMemberTable { updated_at DateTime @default(now()) @updatedAt updated_by String @@index([parent_org_id]) + @@index([parent_team_ids], type: Gin) + @@index([budget_id]) + @@index([budget_override_id]) @@index([spend]) } @@ -2308,7 +2313,7 @@ enum AccountingDeliveryState { } model LiteLLM_SpendLogDelivery { - delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + delivery_id String @id @default(uuid()) request_id String state AccountingDeliveryState @default(pending) accounting_payload Json diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index d1415f583d2..d3024960b86 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -229,6 +229,8 @@ class UserAPIKeyLabelNames(Enum): STREAM = "stream" ORG_ID = "org_id" ORG_ALIAS = "org_alias" + CUSTOMER_ORG_ID = "customer_org_id" + CUSTOMER_ORG_ALIAS = "customer_org_alias" MCP_TOOL_NAME = "mcp_tool_name" MCP_SERVER_NAME = "mcp_server_name" SERVICE_TIER = "service_tier" @@ -725,6 +727,10 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.MODEL_GROUP.value, UserAPIKeyLabelNames.TEAM.value, UserAPIKeyLabelNames.TEAM_ALIAS.value, + UserAPIKeyLabelNames.API_KEY_HASH.value, + UserAPIKeyLabelNames.API_KEY_ALIAS.value, + UserAPIKeyLabelNames.CUSTOMER_ORG_ID.value, + UserAPIKeyLabelNames.CUSTOMER_ORG_ALIAS.value, UserAPIKeyLabelNames.PRIORITY.value, ) @@ -1125,6 +1131,8 @@ class UserAPIKeyLabelValues: stream: str | None = None org_id: str | None = None org_alias: str | None = None + customer_org_id: str | None = None + customer_org_alias: str | None = None mcp_tool_name: str | None = None mcp_server_name: str | None = None service_tier: str | None = None diff --git a/litellm/types/proxy/management_endpoints/customer_hierarchy.py b/litellm/types/proxy/management_endpoints/customer_hierarchy.py index 5550e2c1f81..0835a31ae10 100644 --- a/litellm/types/proxy/management_endpoints/customer_hierarchy.py +++ b/litellm/types/proxy/management_endpoints/customer_hierarchy.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from datetime import datetime from hashlib import sha256 from json import dumps -from typing import Final, Literal, TypeAlias +from typing import Annotated, Final, Literal, TypeAlias from pydantic import ( AwareDatetime, @@ -14,6 +14,7 @@ from pydantic import ( field_validator, model_validator, ) +from pydantic.json_schema import SkipJsonSchema from litellm.models.budget import BudgetLimits from litellm.proxy.common_utils.timezone_utils import budget_duration_error @@ -116,7 +117,7 @@ class TeamCreate(TeamPatch): parent_org_id: str -class MemberPatch(BaseModel): +class MemberFields(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") alias: str | None = None @@ -125,13 +126,28 @@ class MemberPatch(BaseModel): budget_override_expires_at: AwareDatetime | None = None -class MemberCreate(MemberPatch): +class MemberPatch(MemberFields): + parent_team_ids: Annotated[tuple[str, ...], Field(min_length=1)] | SkipJsonSchema[None] = Field( + default=None, + description="Replace the member's teams within its existing organization. First team is the default. " + "Omit to keep current teams. Member identity, budgets and historical usage are retained.", + ) + + @field_validator("parent_team_ids") + @classmethod + def valid_parent_teams(cls, value: tuple[str, ...] | None) -> tuple[str, ...]: + if value is None or any(not team.strip() for team in value) or len(set(value)) != len(value): + raise ValueError("parent_team_ids must contain unique, nonempty team IDs") + return value + + +class MemberCreate(MemberFields): customer_member_id: str | None = None parent_org_id: str | None = Field(default=None, description="Must match the parent teams' organization when set") parent_team_ids: tuple[str, ...] = Field(min_length=1, description="Teams the member belongs to, first is default") -class MemberUpsert(MemberPatch): +class MemberUpsert(MemberFields): customer_member_id: str = Field(min_length=1) customer_org_id: str | None = None customer_org_alias: str | None = None diff --git a/litellm/types/proxy/native_budgets.py b/litellm/types/proxy/native_budgets.py new file mode 100644 index 00000000000..fd2f33ab7b3 --- /dev/null +++ b/litellm/types/proxy/native_budgets.py @@ -0,0 +1,202 @@ +from typing import Annotated, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr + + +class NativeReseed(BaseModel): + model_config = ConfigDict(frozen=True) + + counter_key: str + actual_cost: float + + +class NativePendingIncrement(BaseModel): + model_config = ConfigDict(frozen=True) + + counter_key: str + increment: float + + +class NativeSettlement(BaseModel): + model_config = ConfigDict(frozen=True) + + adjustments: tuple[float | None, ...] + reseed: tuple[NativeReseed, ...] + pending: tuple[NativePendingIncrement, ...] + + +PyNumber: TypeAlias = StrictInt | StrictFloat + + +class _Check(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + +class KeyMaxCheck(_Check): + check: Literal["key_max"] + spend: PyNumber + max_budget: PyNumber + token: StrictStr | None = None + key_alias: StrictStr | None = None + key_name: StrictStr | None = None + throttle_pct: PyNumber | None = None + + +class KeyWindowCheck(_Check): + check: Literal["key_window"] + spend: PyNumber + max_budget: PyNumber + budget_duration: StrictStr + token: StrictStr | None = None + + +class KeySoftCheck(_Check): + check: Literal["key_soft"] + spend: PyNumber + soft_budget: PyNumber | None = None + + +class TeamMaxCheck(_Check): + check: Literal["team_max"] + spend: PyNumber + max_budget: PyNumber + team_id: StrictStr | None = None + + +class TeamWindowCheck(_Check): + check: Literal["team_window"] + spend: PyNumber + max_budget: PyNumber + budget_duration: StrictStr + team_id: StrictStr | None = None + + +class UserWindowCheck(_Check): + check: Literal["user_window"] + spend: PyNumber + max_budget: PyNumber + budget_duration: StrictStr + user_id: StrictStr + + +class TeamSoftCheck(_Check): + check: Literal["team_soft"] + spend: PyNumber | None = None + soft_budget: PyNumber | None = None + + +class ProjectMaxCheck(_Check): + check: Literal["project_max"] + spend: PyNumber + max_budget: PyNumber + project_id: StrictStr | None = None + + +class ProjectSoftCheck(_Check): + check: Literal["project_soft"] + spend: PyNumber | None = None + soft_budget: PyNumber | None = None + + +class OrganizationCheck(_Check): + check: Literal["organization"] + spend: PyNumber + max_budget: PyNumber + org_id: StrictStr + + +class TagCheck(_Check): + check: Literal["tag"] + spend: PyNumber + max_budget: PyNumber + tag: StrictStr + + +class ModelAccessGroupCheck(_Check): + check: Literal["model_access_group"] + spend: PyNumber + max_budget: PyNumber + group: StrictStr + + +class UserCheck(_Check): + check: Literal["user"] + spend: PyNumber + max_budget: PyNumber + user_id: StrictStr | None = None + + +class TeamMemberCheck(_Check): + check: Literal["team_member"] + spend: PyNumber + max_budget: PyNumber + user_id: StrictStr | None = None + team_id: StrictStr | None = None + + +class TeamMemberPositiveCheck(_Check): + check: Literal["team_member_positive"] + spend: PyNumber + max_budget: PyNumber + user_id: StrictStr | None = None + team_id: StrictStr | None = None + + +class EndUserCheck(_Check): + check: Literal["end_user"] + spend: PyNumber + max_budget: PyNumber + end_user_id: StrictStr | None = None + + +class ProxyCheck(_Check): + check: Literal["proxy"] + spend: PyNumber + max_budget: PyNumber + + +BudgetCheck: TypeAlias = Annotated[ + KeyMaxCheck + | KeyWindowCheck + | KeySoftCheck + | TeamMaxCheck + | TeamWindowCheck + | UserWindowCheck + | TeamSoftCheck + | ProjectMaxCheck + | ProjectSoftCheck + | OrganizationCheck + | TagCheck + | ModelAccessGroupCheck + | UserCheck + | TeamMemberCheck + | TeamMemberPositiveCheck + | EndUserCheck + | ProxyCheck, + Field(discriminator="check"), +] + +__all__ = [ + "BudgetCheck", + "EndUserCheck", + "KeyMaxCheck", + "KeySoftCheck", + "KeyWindowCheck", + "ModelAccessGroupCheck", + "NativePendingIncrement", + "NativeReseed", + "NativeSettlement", + "OrganizationCheck", + "ProjectMaxCheck", + "ProjectSoftCheck", + "ProxyCheck", + "PyNumber", + "TagCheck", + "TeamMaxCheck", + "TeamMemberCheck", + "TeamMemberPositiveCheck", + "TeamSoftCheck", + "TeamWindowCheck", + "UserCheck", + "UserWindowCheck", +] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 99ab0f1fb4b..02153a833ad 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -5402,6 +5402,37 @@ class PriorityReservationDict(TypedDict, total=False): value: float +ClassShareEntity: TypeAlias = Literal["team", "key", "customer_organization", "customer_member"] + + +ModelLimitBorrowingEntity: TypeAlias = Literal["team", "key", "customer_organization"] + + +class ModelLimitBorrowingConfig(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + borrow_idle_capacity: tuple[ModelLimitBorrowingEntity, ...] = Field( + default=(), + description="Entity types whose model_tpm_limit and model_rpm_limit on a model this limiter covers stop being ceilings. Each may run past them on capacity other callers are not using while the model is below saturation_threshold, up to its borrow cap when one is set, and is held to them once the model saturates. A customer organization's usage counts across every key its requests use.", + ) + max_borrow_multiple: float | None = Field( + default=None, + ge=1, + le=1_000_000, + allow_inf_nan=False, + description="Most a borrower may use on a model while it borrows idle capacity, as a multiple of its model_tpm_limit or model_rpm_limit. It applies to every listed entity type. A team or key that sets model_limit_max_borrow_multiple in its metadata uses that value instead. None lets a borrower without its own value use everything up to the model limit.", + ) + + +class PriorityClassShareConfig(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + split_equally_among_active: tuple[ClassShareEntity, ...] = Field( + default=(), + description="Entity types that each take an equal slice of their priority class's share of a model, renormalized over the classes active on the model in the current window. Once the model saturates each is held to its own slice instead of the class pool. A request that matches several listed types counts under one of them: its team first, so customer headers cannot move a team's key off the team's slice, then customer_member, then customer_organization, then key.", + ) + + class PriorityReservationSettings(LiteLLMBaseModel): """ Settings for priority-based rate limiting reservation. @@ -5419,15 +5450,23 @@ class PriorityReservationSettings(LiteLLMBaseModel): default=0.50, description="Saturation threshold (0.0-1.0) at which strict priority enforcement begins. Below this threshold, generous mode allows priority borrowing. Above this threshold, strict mode enforces normalized priority limits.", ) - team_model_limit_borrows_idle_capacity_from_other_teams: bool = Field( - default=False, - description="When true, a team's model_tpm_limit and model_rpm_limit on a model this limiter covers stop being ceilings. The team may run past them on capacity other teams are not using while the model is below saturation_threshold, and is held to them once the model saturates.", + model_limit_borrowing_config: ModelLimitBorrowingConfig = Field( + default_factory=ModelLimitBorrowingConfig, + description="Which callers have their per-model limits treated as guaranteed shares that borrow idle capacity.", ) - priority_class_share_splits_equally_among_active_teams: bool = Field( - default=False, - description="When true, a priority class's share of a model is renormalized over the classes whose teams used the model in the current window and split equally among that class's active teams, and once the model saturates each team is held to its own share instead of the class pool. Keys without a team keep the class pool.", + priority_class_share_config: PriorityClassShareConfig = Field( + default_factory=PriorityClassShareConfig, + description="Which callers take their own equal slice of a priority class's share of a model, instead of sharing the class pool.", ) + @property + def model_limit_borrowing_entities(self) -> frozenset[ModelLimitBorrowingEntity]: + return frozenset(self.model_limit_borrowing_config.borrow_idle_capacity) + + @property + def class_share_split_entities(self) -> frozenset[ClassShareEntity]: + return frozenset(self.priority_class_share_config.split_equally_among_active) + saturation_check_cache_ttl: int = Field( default=60, description="TTL in seconds for local cache when reading saturation check values from Redis.", diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.integration.test.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.integration.test.tsx index ebc3ede926f..b8426673b83 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.integration.test.tsx @@ -40,7 +40,7 @@ beforeEach(() => { creations.push(await request.json()); if (freeTierReached) return Response.json( - { detail: "Free customer portal limit reached. Upgrade to Enterprise." }, + { detail: "Customer Management free tier limit reached. Upgrade to Enterprise." }, { status: 403 }, ); return rejectCreation @@ -65,7 +65,7 @@ describe("CreateCustomerDialog", () => { fireEvent.click(screen.getByRole("button", { name: "Create customer" })); await waitFor(() => expect(toast.error).toHaveBeenCalledWith( - expect.stringContaining("Free customer portal limit reached. Upgrade to Enterprise."), + expect.stringContaining("Customer Management free tier limit reached. Upgrade to Enterprise."), ), ); }); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerHierarchyDialog.test.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerHierarchyDialog.test.tsx index 5db40e0ce5a..c08b6bc7359 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerHierarchyDialog.test.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerHierarchyDialog.test.tsx @@ -164,7 +164,9 @@ describe("CreateCustomerHierarchyDialog", () => { }); it.each(["team", "member"] as const)("surfaces the free-tier 403 detail when creating a %s", async (kind) => { - fetchMock.mockResolvedValue(json({ detail: "Free customer portal limit reached. Upgrade to Enterprise." }, 403)); + fetchMock.mockResolvedValue( + json({ detail: "Customer Management free tier limit reached. Upgrade to Enterprise." }, 403), + ); render( { fireEvent.change(screen.getByRole("textbox", { name: "Display name" }), { target: { value: "Example" } }); fireEvent.click(screen.getByRole("button", { name: "Create" })); expect(await screen.findByRole("alert")).toHaveTextContent( - "Free customer portal limit reached. Upgrade to Enterprise.", + "Customer Management free tier limit reached. Upgrade to Enterprise.", ); expect(onOpenChange).not.toHaveBeenCalled(); expect(onCreated).not.toHaveBeenCalled(); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetColumn.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetColumn.tsx new file mode 100644 index 00000000000..b388b14f139 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetColumn.tsx @@ -0,0 +1,35 @@ +import { ArrowDown, ArrowUp, ArrowUpDown } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { formatUsd } from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { Button } from "@/components/ui/button"; +import { TableCell, TableHead } from "@/components/ui/table"; + +export function CustomerBudgetSortHeader({ + sort, + onSort, +}: { + sort: "budget_asc" | "budget_desc" | null; + onSort: () => void; +}) { + const { t } = useTranslation(); + const SortIcon = sort === "budget_desc" ? ArrowDown : ArrowUp; + const Icon = sort ? SortIcon : ArrowUpDown; + const direction = sort === "budget_desc" ? "descending" : "ascending"; + return ( + + + + ); +} + +export function CustomerBudgetLimitCell({ limit }: { limit: number | null }) { + const { t } = useTranslation(); + return ( + + {limit === null ? t("customerPortal.analytics.noLimit") : formatUsd(limit)} + + ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetSummary.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetSummary.tsx index 69c98d614ec..6384f14d5ab 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetSummary.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetSummary.tsx @@ -1,6 +1,6 @@ import { useTranslation } from "react-i18next"; import type { CustomerUsageEntity } from "./customerUsageApi"; -import CustomerBudgetSpend, { CustomerBudgetWindow } from "./CustomerBudgetSpend"; +import { CustomerBudgetWindow } from "./CustomerBudgetSpend"; import { CUSTOMER_BUDGET_LABELS, customerBudgetPeriod, customerBudgetUsage } from "./customerBudgetUsage"; export default function CustomerBudgetSummary({ @@ -35,10 +35,10 @@ export default function CustomerBudgetSummary({ ); })} - {budgets.length === 0 && ( - + {budgets.length === 0 &&

{t("customerPortal.analytics.noBudget")}

} + {budgets.length > 0 && ( +

{t("customerPortal.analytics.budgetUsageHint")}

)} -

{t("customerPortal.analytics.budgetUsageHint")}

); } diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchyManager.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchyManager.tsx index ad75a9b62f5..8a642f39de5 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchyManager.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchyManager.tsx @@ -1,24 +1,34 @@ "use client"; import { useState } from "react"; -import { ChevronRight, Pencil, Plus } from "lucide-react"; +import { ChevronRight, MoreHorizontal, Pencil, Plus } from "lucide-react"; import { useRouter } from "next/navigation"; import { useTranslation } from "react-i18next"; -import { formatUsd } from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; import { ErrorState } from "@/components/shared/ErrorState"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import MoveCustomerMemberDialog from "./MoveCustomerMemberDialog"; +import type { Member } from "./customerHierarchyForms"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { extractProxyErrorMessage } from "@/lib/http/client"; -import { uiHref } from "@/utils/uiHref"; -import CustomerHierarchyNavigation, { customerHierarchyHref } from "./CustomerHierarchyNavigation"; +import CustomerHierarchyNavigation, { useCustomerHierarchyLinks } from "./CustomerHierarchyNavigation"; import CustomerHierarchySettings from "./CustomerHierarchySettings"; import CustomerBudgetGroups from "./CustomerBudgetGroups"; import CustomerBudgetSummary from "./CustomerBudgetSummary"; -import type { CustomerUsageEntity } from "./customerUsageApi"; +import CustomerMemberSearch from "./CustomerMemberSearch"; +import { customerUsageEntity, isCustomerUsageIdentifier, type CustomerUsageEntity } from "./customerUsageApi"; +import { CustomerSpendCells, CustomerSpendHeaders, CustomerSpendIdentity } from "./CustomerSpendCells"; import CreateCustomerHierarchyDialog from "./CreateCustomerHierarchyDialog"; import { customerEntityId, customerEntityLabel, type CreatedCustomerHierarchyEntity } from "./customerHierarchyForms"; import useCustomerHierarchy, { type CustomerHierarchyContext } from "./useCustomerHierarchy"; import type { CustomerHierarchyScope } from "./customerHierarchyApi"; +import { customerBudgetLimit } from "./customerBudgetUsage"; +import { CustomerBudgetLimitCell, CustomerBudgetSortHeader } from "./CustomerBudgetColumn"; type Hierarchy = ReturnType; const CHILD_HINTS = { @@ -39,6 +49,7 @@ const DESCRIPTION_KEYS = { function HierarchyHeader({ hierarchy, selected }: Pick) { const { t } = useTranslation(); + const { customerHierarchyHref, customersHref } = useCustomerHierarchyLinks(); const entity = selected.entity; const team = selected.team; const label = customerEntityLabel(entity); @@ -70,7 +81,7 @@ function HierarchyHeader({ hierarchy, selected }: Pick - + {t("customerPortal.entities.organizations")}