diff --git a/litellm/batches/batch_line_items.py b/litellm/batches/batch_line_items.py
index 120c1d958e1..8f1e3efaaf3 100644
--- a/litellm/batches/batch_line_items.py
+++ b/litellm/batches/batch_line_items.py
@@ -7,7 +7,7 @@ from collections import Counter
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass
from itertools import accumulate, chain, count
-from typing import Final, Literal, TypeAlias
+from typing import TYPE_CHECKING, Final, Literal, TypeAlias
import litellm
from litellm._logging import verbose_logger
@@ -23,6 +23,9 @@ from litellm.types.utils import (
StandardBatchLineItemUsage,
)
+if TYPE_CHECKING:
+ from litellm.integrations.s3_v3 import S3Logger
+
BatchFileProvider: TypeAlias = Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"]
_VERTEX_AI_UNKNOWN_CUSTOM_ID: Final = "unknown" # what a Vertex AI batch row that carries no custom_id is read as
@@ -238,9 +241,11 @@ async def _deliver(
)
-def _resolve_logger(callback: litellm.CALLBACK_TYPES) -> CustomLogger | None:
+def _resolve_logger(callback: "litellm.CALLBACK_TYPES | S3Logger") -> "CustomLogger | S3Logger | None":
+ from litellm.integrations.s3_v3 import S3Logger
+
match callback:
- case CustomLogger():
+ case CustomLogger() | S3Logger():
return callback
case "s3_v2" | "gcs_bucket":
return init_custom_logger_compatible_class(callback, internal_usage_cache=None, llm_router=None)
@@ -248,7 +253,7 @@ def _resolve_logger(callback: litellm.CALLBACK_TYPES) -> CustomLogger | None:
return None
-def _logger_turned_off_message_logging(logger: CustomLogger) -> bool:
+def _logger_turned_off_message_logging(logger: object) -> bool:
return (
getattr(logger, "turn_off_message_logging", False) is True
or getattr(logger, "message_logging", True) is not True
diff --git a/litellm/constants.py b/litellm/constants.py
index 166300b8145..68331e3199e 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -130,7 +130,6 @@ S3_PREFIX_DIGEST_CHARS: Final = 16
# s3 allows 2048 bytes of combined metadata headers, which Content-Disposition counts against
MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY"
-S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
MAX_FILE_LIST_LIMIT: Final = 10000
ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
diff --git a/litellm/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py b/litellm/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py
index 6a97a9de414..c28fa74cd98 100644
--- a/litellm/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py
+++ b/litellm/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py
@@ -4,6 +4,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.constants import X_LITELLM_DISABLE_CALLBACKS
from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
from litellm.litellm_core_utils.llm_request_utils import (
get_proxy_server_request_headers,
)
@@ -45,7 +46,7 @@ class EnterpriseCallbackControls:
if callback.lower() in disabled_callbacks:
verbose_logger.debug(f"Not logging to {callback} because it is disabled via {X_LITELLM_DISABLE_CALLBACKS}")
return True
- elif isinstance(callback, CustomLogger):
+ elif isinstance(callback, CustomLogger | CustomLoggerV2):
# get the string name of the callback
callback_str = CustomLoggerRegistry.get_callback_str_from_class_type(callback.__class__)
if callback_str is not None and callback_str.lower() in disabled_callbacks:
diff --git a/litellm/integrations/adaptive_concurrency.py b/litellm/integrations/adaptive_concurrency.py
deleted file mode 100644
index e0c6b730dda..00000000000
--- a/litellm/integrations/adaptive_concurrency.py
+++ /dev/null
@@ -1,78 +0,0 @@
-"""
-Adaptive in-flight concurrency limiter (AIMD, Vector ARC style).
-
-Grows the limit additively after `limit` consecutive clean completions and
-halves it only on an explicit throttle signal (429, 503, SlowDown, or a
-transport error out of the PUT). With floor == ceiling it degenerates to a
-fixed-width semaphore.
-"""
-
-import asyncio
-from collections import deque
-from contextlib import suppress
-from dataclasses import dataclass
-from typing import Final
-
-
-@dataclass(frozen=True, slots=True)
-class PutSample:
- throttled: bool
-
-
-class AdaptiveConcurrencyLimiter:
- """AIMD in-flight limiter used as `async with limiter:`."""
-
- def __init__(self, initial: int, floor: int, ceiling: int) -> None:
- if not 1 <= floor <= ceiling:
- raise ValueError(f"adaptive limiter bounds must satisfy 1 <= floor <= ceiling, got {floor}..{ceiling}")
- self._limit: int = min(max(initial, floor), ceiling)
- self._floor: Final[int] = floor
- self._ceiling: Final[int] = ceiling
- self._clean_streak: int = 0
- self._in_flight: int = 0
- self._waiters: deque[asyncio.Future[None]] = deque() # mutable-ok: waiters queue up behind a full limit
-
- @property
- def limit(self) -> int:
- return self._limit
-
- async def __aenter__(self) -> "AdaptiveConcurrencyLimiter":
- if self._in_flight < self._limit:
- self._in_flight += 1
- return self
- waiter: Final = asyncio.get_running_loop().create_future()
- self._waiters.append(waiter)
- try:
- await waiter
- except asyncio.CancelledError:
- if waiter.done() and not waiter.cancelled():
- self._in_flight -= 1
- self._grant()
- else:
- with suppress(ValueError):
- self._waiters.remove(waiter)
- raise
- return self
-
- def _grant(self) -> None:
- while self._in_flight < self._limit and self._waiters:
- waiter = self._waiters.popleft()
- if waiter.done():
- continue
- self._in_flight += 1
- waiter.set_result(None)
-
- async def __aexit__(self, *_: object) -> None:
- self._in_flight -= 1
- self._grant()
-
- def record(self, sample: PutSample) -> None:
- if sample.throttled:
- self._limit = max(self._floor, self._limit // 2)
- self._clean_streak = 0
- return
- self._clean_streak += 1
- if self._clean_streak >= self._limit and self._limit < self._ceiling:
- self._limit += 1
- self._clean_streak = 0
- self._grant()
diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py
index 962064127fa..b55bd8b30c2 100644
--- a/litellm/integrations/s3.py
+++ b/litellm/integrations/s3.py
@@ -17,7 +17,6 @@ from litellm.constants import (
MAX_S3_OBJECT_KEY_FILENAME_BYTES,
S3_BOUNDED_OBJECT_KEY_HEAD_BYTES,
S3_LOG_PROMPTS_ONLY_ENV_VAR,
- S3_PARTITION_GRANULARITY_ENV_VAR,
S3_PREFIX_DIGEST_CHARS,
)
from litellm.types.integrations.s3_v2 import S3PartitionGranularity
@@ -39,18 +38,6 @@ def resolve_s3_log_prompts_only(configured: object, environ: Mapping[str, str] |
return True
-def resolve_s3_partition_granularity(
- configured: object, environ: Mapping[str, str] | None = None
-) -> S3PartitionGranularity:
- env: Final = os.environ if environ is None else environ
- raw: Final = env.get(S3_PARTITION_GRANULARITY_ENV_VAR) if configured is None else configured
- if raw == "hour":
- return "hour"
- if raw is not None and raw not in ("", "day"):
- verbose_logger.warning("s3 logging: s3_partition_granularity=%r is not one of day, hour, using day", raw)
- return "day"
-
-
def _resolve_positive_int(setting: str, configured: object, fallback: int, *, reject_bool: bool) -> int:
if configured is None or configured == "":
return fallback
@@ -70,10 +57,6 @@ def _resolve_positive_int(setting: str, configured: object, fallback: int, *, re
return bound
-def resolve_s3_max_concurrent_uploads(configured: object, fallback: int) -> int:
- return _resolve_positive_int("s3_max_concurrent_uploads", configured, fallback, reject_bool=False)
-
-
def resolve_s3_max_queue_size(configured: object, fallback: int) -> int:
return _resolve_positive_int("s3_max_queue_size", configured, fallback, reject_bool=True)
@@ -103,48 +86,6 @@ def resolve_s3_max_retry_age_seconds(configured: object, fallback: int | None) -
return bound or None
-def resolve_s3_max_adaptive_concurrency(configured: object, fallback: int) -> int:
- return _resolve_positive_int("s3_max_adaptive_concurrency", configured, fallback, reject_bool=True)
-
-
-def resolve_s3_drop_on_terminal_error(configured: object) -> bool:
- if configured is None or configured == "":
- return True
- try:
- return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
- except ValidationError:
- verbose_logger.warning(
- "s3 logging: s3_drop_on_terminal_error=%r is not a boolean, dropping terminal-failed uploads",
- configured,
- )
- return True
-
-
-def resolve_s3_adaptive_concurrency(configured: object) -> bool:
- if configured is None or configured == "":
- return False
- try:
- return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
- except ValidationError:
- verbose_logger.warning(
- "s3 logging: s3_adaptive_concurrency=%r is not a boolean, keeping the fixed upload width",
- configured,
- )
- return False
-
-
-def resolve_s3_batch_file_upload(configured: object) -> bool:
- if configured is None or configured == "":
- return False
- try:
- return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
- except ValidationError:
- verbose_logger.warning(
- "s3 logging: s3_batch_file_upload=%r is not a boolean, keeping per-request objects", configured
- )
- return False
-
-
def prompts_only_payload(payload: StandardLoggingPayload) -> StandardLoggingPayload:
return {**payload, "response": None}
diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py
index 6c35b53c92d..98892a7fdf4 100644
--- a/litellm/integrations/s3_v2.py
+++ b/litellm/integrations/s3_v2.py
@@ -1,1073 +1,3 @@
-"""
-s3 Bucket Logging Integration
+from litellm.integrations.s3_v3 import S3BatchUploadError, S3Logger
-async_log_success_event: Processes the event, stores it in memory for DEFAULT_S3_FLUSH_INTERVAL_SECONDS seconds or until DEFAULT_S3_BATCH_SIZE and then flushes to s3
-async_log_failure_event: Processes the event, stores it in memory for DEFAULT_S3_FLUSH_INTERVAL_SECONDS seconds or until DEFAULT_S3_BATCH_SIZE and then flushes to s3
-NOTE 1: S3 does not provide a BATCH PUT API endpoint; by default each element is uploaded concurrently with the fixed s3_max_concurrent_uploads bound (or an adaptive bound when s3_adaptive_concurrency is on, backing off only on throttling), or with s3_batch_file_upload the whole flush is written as one .jsonl file
-"""
-
-import asyncio
-import contextvars
-import logging
-import os
-import random
-import re
-import time
-from collections.abc import Awaitable, Callable, Mapping
-from dataclasses import dataclass
-from datetime import datetime, timezone
-from functools import partial
-from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, cast
-from urllib.parse import quote
-from uuid import uuid4
-
-import httpx
-
-import litellm
-from litellm._logging import print_verbose, verbose_logger
-from litellm.constants import (
- DEFAULT_S3_BATCH_SIZE,
- DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
- DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
- DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
- S3_PARTITION_GRANULARITY_ENV_VAR,
- S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
-)
-from litellm.exceptions import Timeout as LiteLLMTimeout
-from litellm.integrations.adaptive_concurrency import AdaptiveConcurrencyLimiter, PutSample
-from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name
-from litellm.integrations.s3 import (
- get_s3_object_download_filename,
- get_s3_object_key,
- prompts_only_payload,
- resolve_s3_adaptive_concurrency,
- resolve_s3_batch_file_upload,
- resolve_s3_drop_on_terminal_error,
- resolve_s3_log_prompts_only,
- resolve_s3_max_adaptive_concurrency,
- resolve_s3_max_concurrent_uploads,
- resolve_s3_max_queue_size,
- resolve_s3_max_retry_age_seconds,
- resolve_s3_partition_granularity,
- resolve_sse_params,
-)
-from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
-from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
-from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
-from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
-from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, WebIdentitySessionPolicy
-from litellm.llms.custom_httpx.http_handler import (
- get_async_httpx_client,
- get_httpx_client,
- httpxSpecialProvider,
-)
-from litellm.types.integrations.s3_v2 import S3PartitionGranularity, s3BatchLoggingElement
-from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload
-
-from .custom_batch_logger import CustomBatchLogger
-
-if TYPE_CHECKING:
- from botocore.credentials import Credentials
-
-
-UploadOutcome = Literal["delivered", "retry", "dropped"]
-
-_TERMINAL_ERROR_CODES: Final = frozenset(
- {
- "EntityTooLarge",
- "InvalidArgument",
- "MalformedXML",
- "InvalidDigest",
- "KeyTooLongError",
- "BadDigest",
- "InvalidRequest",
- }
-)
-_BODY_CODED_STATUSES: Final = frozenset({400, 403})
-_RETRYABLE_STATUSES: Final = frozenset({403, 429, 500, 503})
-_S3_ERROR_CODE: Final = re.compile(r"([^<]+)")
-_TRANSIENT_ERRORS: Final = (httpx.TransportError, LiteLLMTimeout)
-_UPLOAD_ATTEMPTS: Final = 3
-_UPLOAD_ATTEMPT_TIMEOUT: Final = httpx.Timeout(S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS)
-
-PutOutcome: TypeAlias = httpx.Response | Exception
-
-
-@dataclass(frozen=True, slots=True)
-class _PreparedPut:
- json_string: str
- headers: Mapping[str, str]
-
-
-def _s3_error_code(response: httpx.Response) -> str | None:
- text: Final = response.text
- match: Final = _S3_ERROR_CODE.search(text) if isinstance(text, str) else None
- return match.group(1) if match else None
-
-
-def _is_terminal(response: httpx.Response) -> bool:
- """True only for object-specific, unrecoverable rejections (400/403 with a terminal XML code).
- Unknown codes, empty or non-XML bodies, and every other status fail safe toward retry."""
- return response.status_code in _BODY_CODED_STATUSES and _s3_error_code(response) in _TERMINAL_ERROR_CODES
-
-
-def _s3_key_parent(s3_object_key: str) -> str:
- return s3_object_key.rsplit("/", 1)[0] if "/" in s3_object_key else ""
-
-
-def _backoff_seconds(attempt: int) -> float:
- return (1 << attempt) * (0.5 + random.random())
-
-
-def _log_retry(level: int, attempt: int, reason: str, wait: float, s3_object_key: str) -> None:
- verbose_logger.log(
- level,
- "S3 upload attempt %s/%s failed with %s, retrying in %.2fs key=%s",
- attempt + 1,
- _UPLOAD_ATTEMPTS,
- reason,
- wait,
- s3_object_key,
- )
-
-
-def _response_or_raise(outcome: PutOutcome) -> httpx.Response:
- if isinstance(outcome, Exception):
- raise outcome
- outcome.raise_for_status()
- return outcome
-
-
-def _throttled(outcome: PutOutcome) -> bool:
- if isinstance(outcome, Exception):
- return True
- return outcome.status_code in (429, 503) or _s3_error_code(outcome) == "SlowDown"
-
-
-class S3BatchUploadError(Exception):
- def __init__(self, failed: int, total: int) -> None:
- self.failed = failed
- self.total = total
- super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush")
-
-
-_in_flush: Final[contextvars.ContextVar[bool]] = contextvars.ContextVar("s3_v2_in_flush", default=False)
-
-
-class S3Logger(CustomBatchLogger, BaseAWSLLM):
- _web_identity_session_policy: ClassVar[WebIdentitySessionPolicy] = S3_LOGGING_SESSION_POLICY
- preserve_events_added_during_flush = True
- _flush_retries: int = 0
- _requeued_count: int = 0
- _upload_limiter: asyncio.Semaphore | AdaptiveConcurrencyLimiter | None = None
- s3_drop_on_terminal_error: bool = True
- s3_max_retry_age_seconds: int | None = 3600
- s3_partition_granularity: object = None
- _partition_granularity_cache: tuple[object, S3PartitionGranularity] | None = None
-
- def __init__(
- self,
- s3_bucket_name: str | None = None,
- s3_path: str | None = None,
- s3_region_name: str | None = None,
- s3_api_version: str | None = None,
- s3_use_ssl: bool = True,
- s3_verify: bool | None = None,
- s3_endpoint_url: str | None = None,
- s3_aws_access_key_id: str | None = None,
- s3_aws_secret_access_key: str | None = None,
- s3_aws_session_token: str | None = None,
- s3_aws_session_name: str | None = None,
- s3_aws_profile_name: str | None = None,
- s3_aws_role_name: str | None = None,
- s3_aws_web_identity_token: str | None = None,
- s3_aws_sts_endpoint: str | None = None,
- s3_flush_interval: int | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
- s3_batch_size: int | None = DEFAULT_S3_BATCH_SIZE,
- s3_config=None,
- s3_use_team_prefix: bool = False,
- s3_strip_base64_files: bool = False,
- s3_use_key_prefix: bool = False,
- s3_use_virtual_hosted_style: bool = False,
- s3_server_side_encryption: str | None = None,
- s3_sse_kms_key_id: str | None = None,
- s3_log_prompts_only: bool | None = None,
- s3_partition_granularity: str | None = None,
- s3_max_concurrent_uploads: int = DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
- s3_max_queue_size: int | None = None,
- s3_max_retry_age_seconds: int | None = 3600,
- s3_drop_on_terminal_error: bool = True,
- s3_adaptive_concurrency: bool = False,
- s3_max_adaptive_concurrency: int | None = None,
- s3_batch_file_upload: bool = False,
- s3_callback_params_override: dict | None = None,
- **kwargs,
- ):
- try:
- _masker: Final = SensitiveDataMasker()
- if s3_callback_params_override is not None:
- verbose_logger.debug(
- "in init s3 logger (audit override) - %s", _masker.mask_dict(dict(s3_callback_params_override))
- )
- else:
- verbose_logger.debug(
- "in init s3 logger - s3_callback_params %s",
- _masker.mask_dict(dict(litellm.s3_callback_params or {})),
- )
-
- # Initialize S3 params first to get the correct s3_verify value
- self._init_s3_params(
- params_source=s3_callback_params_override,
- s3_bucket_name=s3_bucket_name,
- s3_region_name=s3_region_name,
- s3_api_version=s3_api_version,
- s3_use_ssl=s3_use_ssl,
- s3_verify=s3_verify,
- s3_endpoint_url=s3_endpoint_url,
- s3_aws_access_key_id=s3_aws_access_key_id,
- s3_aws_secret_access_key=s3_aws_secret_access_key,
- s3_aws_session_token=s3_aws_session_token,
- s3_aws_session_name=s3_aws_session_name,
- s3_aws_profile_name=s3_aws_profile_name,
- s3_aws_role_name=s3_aws_role_name,
- s3_aws_web_identity_token=s3_aws_web_identity_token,
- s3_aws_sts_endpoint=s3_aws_sts_endpoint,
- s3_config=s3_config,
- s3_path=s3_path,
- s3_use_team_prefix=s3_use_team_prefix,
- s3_strip_base64_files=s3_strip_base64_files,
- s3_use_key_prefix=s3_use_key_prefix,
- s3_use_virtual_hosted_style=s3_use_virtual_hosted_style,
- s3_server_side_encryption=s3_server_side_encryption,
- s3_sse_kms_key_id=s3_sse_kms_key_id,
- s3_log_prompts_only=s3_log_prompts_only,
- s3_partition_granularity=s3_partition_granularity,
- s3_max_concurrent_uploads=s3_max_concurrent_uploads,
- s3_max_queue_size=s3_max_queue_size,
- s3_max_retry_age_seconds=s3_max_retry_age_seconds,
- s3_drop_on_terminal_error=s3_drop_on_terminal_error,
- s3_adaptive_concurrency=s3_adaptive_concurrency,
- s3_max_adaptive_concurrency=s3_max_adaptive_concurrency,
- s3_batch_file_upload=s3_batch_file_upload,
- )
- self._upload_limiter = (
- AdaptiveConcurrencyLimiter(
- initial=self.s3_max_concurrent_uploads,
- floor=self.s3_max_concurrent_uploads,
- ceiling=max(self.s3_max_concurrent_uploads, self.s3_max_adaptive_concurrency),
- )
- if self.s3_adaptive_concurrency
- else asyncio.Semaphore(self.s3_max_concurrent_uploads)
- )
- verbose_logger.debug("s3 logger using endpoint url %s", s3_endpoint_url)
-
- # IMPORTANT
- # Create httpx client AFTER _init_s3_params so we have the correct s3_verify value
- verbose_logger.debug("s3_v2 logger creating async httpx client with s3_verify=%s", self.s3_verify)
- self.async_httpx_client = get_async_httpx_client(
- llm_provider=httpxSpecialProvider.LoggingCallback,
- params={"ssl_verify": self.s3_verify},
- )
-
- asyncio.create_task(self.periodic_flush())
- self.flush_lock = asyncio.Lock()
-
- verbose_logger.debug("s3 flush interval: %s, s3 batch size: %s", s3_flush_interval, s3_batch_size)
- # Call CustomLogger's __init__
- CustomBatchLogger.__init__(
- self,
- flush_lock=self.flush_lock,
- flush_interval=s3_flush_interval,
- batch_size=s3_batch_size,
- max_queue_size=self.s3_max_queue_size,
- )
- self.log_queue: list[s3BatchLoggingElement] = []
- self._requeued_count = 0
- self._flush_retries = 0
- self._flush_dropped: dict[int, s3BatchLoggingElement] = {}
-
- # Call BaseAWSLLM's __init__
- BaseAWSLLM.__init__(self)
-
- except Exception as e:
- print_verbose(f"Got exception on init s3 client {e}")
- raise e
-
- def _init_s3_params(
- self,
- s3_bucket_name: str | None = None,
- s3_region_name: str | None = None,
- s3_api_version: str | None = None,
- s3_use_ssl: bool = True,
- s3_verify: bool | None = None,
- s3_endpoint_url: str | None = None,
- s3_aws_access_key_id: str | None = None,
- s3_aws_secret_access_key: str | None = None,
- s3_aws_session_token: str | None = None,
- s3_aws_session_name: str | None = None,
- s3_aws_profile_name: str | None = None,
- s3_aws_role_name: str | None = None,
- s3_aws_web_identity_token: str | None = None,
- s3_aws_sts_endpoint: str | None = None,
- s3_config=None,
- s3_path: str | None = None,
- s3_use_team_prefix: bool = False,
- s3_strip_base64_files: bool = False,
- s3_use_key_prefix: bool = False,
- s3_use_virtual_hosted_style: bool = False,
- s3_server_side_encryption: str | None = None,
- s3_sse_kms_key_id: str | None = None,
- s3_log_prompts_only: bool | None = None,
- s3_partition_granularity: str | None = None,
- s3_max_concurrent_uploads: int = DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
- s3_max_queue_size: int | None = None,
- s3_max_retry_age_seconds: int | None = 3600,
- s3_drop_on_terminal_error: bool = True,
- s3_adaptive_concurrency: bool = False,
- s3_max_adaptive_concurrency: int | None = None,
- s3_batch_file_upload: bool = False,
- params_source: dict | None = None,
- ):
- """
- Initialize the s3 params for this logging callback. Reads from
- `params_source` if given (e.g. `s3_audit_callback_params` for the
- audit-log instance), otherwise falls back to `litellm.s3_callback_params`.
- Resolves `os.environ/X` markers into a local dict; never mutates the source.
- """
- if params_source is None:
- params_source = litellm.s3_callback_params or {}
- params: Final[dict] = {
- key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value)
- for key, value in params_source.items()
- }
-
- self.s3_bucket_name = params.get("s3_bucket_name") or s3_bucket_name
- self.s3_region_name = params.get("s3_region_name") or s3_region_name
- self.s3_api_version = params.get("s3_api_version") or s3_api_version
- self.s3_use_ssl = params.get("s3_use_ssl", True) if params.get("s3_use_ssl") is not None else s3_use_ssl
- self.s3_verify = params.get("s3_verify") if params.get("s3_verify") is not None else s3_verify
- self.s3_endpoint_url = params.get("s3_endpoint_url") or s3_endpoint_url
- self.s3_aws_access_key_id = params.get("s3_aws_access_key_id") or s3_aws_access_key_id
-
- self.s3_aws_secret_access_key = params.get("s3_aws_secret_access_key") or s3_aws_secret_access_key
-
- self.s3_aws_session_token = params.get("s3_aws_session_token") or s3_aws_session_token
-
- self.s3_aws_session_name = params.get("s3_aws_session_name") or s3_aws_session_name
-
- self.s3_aws_profile_name = params.get("s3_aws_profile_name") or s3_aws_profile_name
-
- self.s3_aws_role_name = params.get("s3_aws_role_name") or s3_aws_role_name
-
- self.s3_aws_web_identity_token = params.get("s3_aws_web_identity_token") or s3_aws_web_identity_token
-
- self.s3_aws_sts_endpoint = params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
-
- self.s3_config = params.get("s3_config") or s3_config
- self.s3_path = params.get("s3_path") or s3_path
- self.s3_use_team_prefix = bool(params.get("s3_use_team_prefix", False)) or s3_use_team_prefix
-
- self.s3_use_key_prefix = bool(params.get("s3_use_key_prefix", False)) or s3_use_key_prefix
-
- self.s3_strip_base64_files = bool(params.get("s3_strip_base64_files", False)) or s3_strip_base64_files
-
- self.s3_use_virtual_hosted_style = (
- bool(params.get("s3_use_virtual_hosted_style", False)) or s3_use_virtual_hosted_style
- )
-
- self.s3_log_prompts_only: object = (
- params.get("s3_log_prompts_only") if s3_log_prompts_only is None else s3_log_prompts_only
- )
-
- self.s3_partition_granularity = (
- params.get("s3_partition_granularity") if s3_partition_granularity is None else s3_partition_granularity
- )
- self._partition_granularity_cache = None
-
- self.s3_server_side_encryption, self.s3_sse_kms_key_id = resolve_sse_params(
- params.get("s3_server_side_encryption") or s3_server_side_encryption,
- params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id,
- )
-
- configured_bound: Final = params.get("s3_max_concurrent_uploads")
- self.s3_max_concurrent_uploads = resolve_s3_max_concurrent_uploads(
- s3_max_concurrent_uploads if configured_bound is None or configured_bound == "" else configured_bound,
- DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
- )
-
- configured_queue_size: Final = params.get("s3_max_queue_size")
- constructor_queue_size: Final = resolve_s3_max_queue_size(
- s3_max_queue_size, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE
- )
- self.s3_max_queue_size = resolve_s3_max_queue_size(configured_queue_size, constructor_queue_size)
-
- configured_retry_age: Final = params.get("s3_max_retry_age_seconds")
- constructor_retry_age: Final = resolve_s3_max_retry_age_seconds(s3_max_retry_age_seconds, 3600)
- self.s3_max_retry_age_seconds = (
- constructor_retry_age
- if configured_retry_age is None or configured_retry_age == ""
- else resolve_s3_max_retry_age_seconds(configured_retry_age, constructor_retry_age)
- )
-
- configured_drop: Final = params.get("s3_drop_on_terminal_error")
- self.s3_drop_on_terminal_error = resolve_s3_drop_on_terminal_error(
- configured_drop if configured_drop is not None else s3_drop_on_terminal_error
- )
-
- self.s3_adaptive_concurrency = s3_adaptive_concurrency or resolve_s3_adaptive_concurrency(
- params.get("s3_adaptive_concurrency")
- )
-
- configured_adaptive_ceiling: Final = params.get("s3_max_adaptive_concurrency")
- self.s3_max_adaptive_concurrency = resolve_s3_max_adaptive_concurrency(
- s3_max_adaptive_concurrency
- if configured_adaptive_ceiling is None or configured_adaptive_ceiling == ""
- else configured_adaptive_ceiling,
- DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
- )
-
- self.s3_batch_file_upload = s3_batch_file_upload or resolve_s3_batch_file_upload(
- params.get("s3_batch_file_upload")
- )
-
- def _build_object_url(self, s3_object_key: str) -> str:
- """
- Build the exact URL that is both signed and sent, with the key percent-encoded once.
-
- S3SigV4Auth signs the path verbatim while S3 canonicalizes the received path with reserved
- characters encoded, so an unencoded `=`, `+`, `&`, `#`, `?`, `%` or space in the key makes
- the two signatures disagree (403 SignatureDoesNotMatch).
- """
- encoded_key: Final = quote(s3_object_key, safe="/")
- if self.s3_endpoint_url and self.s3_bucket_name:
- if self.s3_use_virtual_hosted_style:
- endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
- protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
- return f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{encoded_key}"
- return f"{self.s3_endpoint_url}/{self.s3_bucket_name}/{encoded_key}"
- return (
- f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}."
- f"{get_aws_dns_suffix(self.s3_region_name)}/{encoded_key}"
- )
-
- def _sign_put(
- self, credentials: "Credentials", url: str, json_string: str, headers: Mapping[str, str]
- ) -> dict[str, str]: # mutable-ok: [LIT001] AsyncHTTPHandler.put/HTTPHandler.put only accept dict headers
- """
- ``RefreshableCredentials`` (IMDS roles) may refresh between the access key, secret and token
- reads SigV4 performs, producing a mixed-generation signature that S3 rejects with 403.
- Freezing first makes the three values one atomic snapshot.
- """
- from botocore.auth import S3SigV4Auth
- from botocore.awsrequest import AWSRequest
- from botocore.credentials import RefreshableCredentials
-
- frozen: Final = (
- credentials.get_frozen_credentials() if isinstance(credentials, RefreshableCredentials) else credentials
- )
- aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=dict(headers))
- aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
- S3SigV4Auth(frozen, "s3", aws_region_name).add_auth(aws_request)
- return dict(aws_request.headers.items())
-
- def _sse_headers(self) -> Mapping[str, str]:
- candidates: Final = {
- "x-amz-server-side-encryption": self.s3_server_side_encryption,
- "x-amz-server-side-encryption-aws-kms-key-id": self.s3_sse_kms_key_id,
- }
- return {key: value for key, value in candidates.items() if value}
-
- def _prepare_put(self, batch_logging_element: s3BatchLoggingElement) -> _PreparedPut:
- try:
- import base64
- import hashlib
- except ImportError:
- raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
-
- json_string: Final = (
- batch_logging_element.body
- if batch_logging_element.body is not None
- else safe_dumps(batch_logging_element.payload)
- )
- content_hash: Final = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
- content_md5: Final = base64.b64encode(
- hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
- ).decode()
- return _PreparedPut(
- json_string=json_string,
- headers={
- "Content-Type": batch_logging_element.content_type,
- "Content-MD5": content_md5,
- "x-amz-content-sha256": content_hash,
- "Content-Language": "en",
- "Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
- "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
- **self._sse_headers(),
- },
- )
-
- async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
- await self._async_log_event_base(
- kwargs=kwargs,
- response_obj=response_obj,
- start_time=start_time,
- end_time=end_time,
- )
-
- async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
- await self._async_log_event_base(
- kwargs=kwargs,
- response_obj=response_obj,
- start_time=start_time,
- end_time=end_time,
- )
-
- async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None:
- """Batch audit logs and upload to S3 under audit_logs/ prefix."""
- try:
- from datetime import timezone
-
- now: Final = datetime.now(timezone.utc)
- audit_log_id: Final = audit_log.get("id", "unknown")
-
- s3_object_key: Final = get_s3_object_key(
- cast(str | None, self.s3_path) or "",
- "audit_logs/",
- now,
- f"{now.strftime('%H-%M-%S')}_{audit_log_id}",
- partition_granularity=self.resolve_partition_granularity(),
- )
-
- element: Final = s3BatchLoggingElement(
- payload=dict(audit_log),
- s3_object_key=s3_object_key,
- s3_object_download_filename=f"audit-{audit_log_id}.json",
- )
-
- self.log_queue.append(element)
-
- if len(self.log_queue) >= self.batch_size:
- await self.flush_queue()
- except Exception as e:
- verbose_logger.exception("S3 audit log error: %s", e)
-
- async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
- """Upload one part of a batch's lines as an NDJSON object under /batch_line_items/."""
- object_name: Final = batch_line_items_object_name(payload)
- s3_path: Final = cast(str | None, self.s3_path) # cast-ok: s3_path is set from untyped callback params
- s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name
- element: Final = s3BatchLoggingElement(
- payload={},
- body=await asyncio.to_thread(batch_line_items_ndjson, payload),
- content_type="application/x-ndjson",
- s3_object_key=s3_object_key,
- s3_object_download_filename="-".join(object_name.rsplit("/", 2)[-2:]),
- )
- # hashing a large object is CPU-bound too, so it is also prepared off the event loop
- prepared_put: Final = await asyncio.to_thread(self._prepare_put, element)
- delivered: Final = await self.async_upload_data_to_s3(element, prepared_put)
- if not delivered:
- # only a log-queue flush reads and clears the drop marks, and this upload is not part of one
- self._flush_dropped.pop(id(element), None)
- verbose_logger.error(
- "s3 logging: the %d line items of part %d of batch %s were not stored at %s",
- len(payload["lines"]),
- payload["part"],
- payload["batch_id"],
- s3_object_key,
- )
-
- async def _async_log_event_base(self, kwargs, response_obj, start_time, end_time):
- try:
- verbose_logger.debug("s3 Logging - Enters logging function for model %s", kwargs)
-
- s3_batch_logging_element: Final = self.create_s3_batch_logging_element(
- start_time=start_time,
- standard_logging_payload=kwargs.get("standard_logging_object", None),
- )
-
- # afile_delete and other non-model call types never produce a standard_logging_object,
- # so s3_batch_logging_element is None. Skip gracefully instead of raising ValueError.
- if s3_batch_logging_element is None:
- verbose_logger.debug(
- "s3 Logging - skipping event, no standard_logging_object for call_type=%s",
- kwargs.get("call_type", "unknown"),
- )
- return
-
- verbose_logger.debug("\ns3 Logger - Logging payload = %s", s3_batch_logging_element)
-
- self.log_queue.append(s3_batch_logging_element)
- verbose_logger.debug(
- "s3 logging: queue length %s, batch size %s",
- len(self.log_queue),
- self.batch_size,
- )
- except Exception as e:
- verbose_logger.exception("s3 Layer Error - %s", e)
- self.handle_callback_failure(callback_name="S3Logger")
-
- @property
- def _upload_semaphore(self) -> asyncio.Semaphore | AdaptiveConcurrencyLimiter:
- limiter: Final = self._upload_limiter
- if limiter is None:
- raise AttributeError("_upload_semaphore")
- return limiter
-
- @_upload_semaphore.setter
- def _upload_semaphore(self, value: asyncio.Semaphore | AdaptiveConcurrencyLimiter) -> None:
- self._upload_limiter = value
-
- async def async_upload_data_to_s3(
- self,
- batch_logging_element: s3BatchLoggingElement,
- prepared_put: _PreparedPut | None = None,
- ) -> bool:
- try:
- from litellm.litellm_core_utils.asyncify import asyncify
-
- asyncified_get_credentials: Final = asyncify(self.get_credentials)
-
- verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
- verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify)
-
- url: Final = self._build_object_url(batch_logging_element.s3_object_key)
-
- async def signed_put(prepared: _PreparedPut) -> PutOutcome:
- credentials: Final = await asyncified_get_credentials(
- aws_access_key_id=self.s3_aws_access_key_id,
- aws_secret_access_key=self.s3_aws_secret_access_key,
- aws_session_token=self.s3_aws_session_token,
- aws_region_name=self.s3_region_name,
- aws_session_name=self.s3_aws_session_name,
- aws_profile_name=self.s3_aws_profile_name,
- aws_role_name=self.s3_aws_role_name,
- aws_web_identity_token=self.s3_aws_web_identity_token,
- aws_sts_endpoint=self.s3_aws_sts_endpoint,
- )
- signed_headers: Final = await run_aws_signing(
- self._sign_put, credentials, url, prepared.json_string, prepared.headers
- )
- try:
- return await self.async_httpx_client.put(
- url, data=prepared.json_string, headers=signed_headers, timeout=_UPLOAD_ATTEMPT_TIMEOUT
- )
- except httpx.HTTPStatusError as error:
- return error.response
- except _TRANSIENT_ERRORS as error:
- return error
-
- prepared: Final = prepared_put if prepared_put is not None else self._prepare_put(batch_logging_element)
- await self._put_with_retries(partial(signed_put, prepared), batch_logging_element.s3_object_key)
- except Exception as e:
- verbose_logger.exception("Error uploading to s3: %s", e)
- self.handle_callback_failure(callback_name="S3Logger")
- if isinstance(e, httpx.HTTPStatusError) and self.s3_drop_on_terminal_error and _is_terminal(e.response):
- verbose_logger.warning(
- "s3 logging: dropping object %s after terminal status %s",
- batch_logging_element.s3_object_key,
- e.response.status_code,
- )
- self._flush_dropped[id(batch_logging_element)] = batch_logging_element
- return False
- return True
-
- async def async_send_batch(self) -> None:
- """
- Sends runs from self.log_queue.
-
- Raises S3BatchUploadError when any upload failed; CustomBatchLogger.flush_queue
- keeps the surviving queue entries for the next flush.
- """
- batch: Final = tuple(self.log_queue)
- if not batch:
- return
- verbose_logger.debug("s3_v2 logger - sending batch of %s", len(batch))
-
- #########################################################
- # Flush the log queue to s3
- # the log queue can be bounded by DEFAULT_S3_BATCH_SIZE
- # see custom_batch_logger.py which triggers the flush
- #########################################################
- uploads: Final = self._batch_file_elements(batch) if self._batch_file_mode_active() else batch
- self._flush_retries = 0
- self._flush_dropped = {}
- stale: Final = min(self._requeued_count, len(uploads)) if len(uploads) == len(batch) else 0
- order: Final = (*range(stale, len(uploads)), *range(stale))
- ordered: Final = await asyncio.gather(*(self._upload_outcome(uploads[i]) for i in order))
- outcomes: Final = dict(zip(order, ordered, strict=True))
- results: Final = tuple(outcomes[i] for i in range(len(uploads)))
- if self._flush_retries:
- verbose_logger.warning(
- "s3 logging: %s in-call retries across %s uploads this flush",
- self._flush_retries,
- len(uploads),
- )
- delivered: Final = sum(1 for outcome in results if outcome == "delivered")
- bucket_wide: Final = delivered == 0
- failed: Final = tuple(
- (element, outcome) for element, outcome in zip(uploads, results, strict=True) if outcome != "delivered"
- )
- now: Final = time.monotonic()
- requeued: Final = (
- tuple(element for element, _ in failed)
- if bucket_wide
- else tuple(
- element
- if element.retrying_since is not None or self.s3_max_retry_age_seconds is None
- else element.model_copy(update={"retrying_since": now})
- for element, outcome in failed
- if outcome != "dropped"
- and not (
- self.s3_max_retry_age_seconds is not None
- and element.retrying_since is not None
- and now - element.retrying_since > self.s3_max_retry_age_seconds
- )
- )
- )
- dropped: Final = len(failed) - len(requeued)
- if dropped:
- terminal: Final = sum(1 for _, outcome in failed if outcome == "dropped")
- verbose_logger.warning(
- "s3 logging: %s uploads dropped this flush: %s rejected as terminal by S3, "
- "%s retrying longer than s3_max_retry_age_seconds=%s",
- dropped,
- terminal,
- dropped - terminal,
- self.s3_max_retry_age_seconds,
- )
- if not requeued:
- self._requeued_count = 0
- return
- arrivals: Final = self.log_queue[len(batch) :]
- overflow: Final = max(0, len(requeued) + len(arrivals) - self.max_queue_size)
- if overflow:
- verbose_logger.warning(
- "s3 logging: queue exceeded max_queue_size=%s after a failed flush, dropped %s oldest events",
- self.max_queue_size,
- overflow,
- )
- self.log_queue = [
- *requeued,
- *arrivals,
- ][overflow:]
- self._requeued_count = max(0, len(requeued) - overflow)
- raise S3BatchUploadError(failed=len(failed), total=len(uploads))
-
- def _batch_file_mode_active(self) -> bool:
- if not self.s3_batch_file_upload:
- return False
- if litellm.cold_storage_custom_logger == "s3_v2":
- verbose_logger.warning(
- "s3 logging: s3_batch_file_upload is ignored because s3_v2 is the cold storage logger; "
- "per-request objects are required for spend log lookups"
- )
- return False
- return True
-
- async def _upload_bounded(self, element: s3BatchLoggingElement) -> bool:
- token: Final = _in_flush.set(True)
- try:
- async with self._upload_semaphore:
- return await self.async_upload_data_to_s3(element)
- finally:
- _in_flush.reset(token)
-
- async def _upload_outcome(self, element: s3BatchLoggingElement) -> UploadOutcome:
- delivered: Final = await self._upload_bounded(element)
- if delivered:
- return "delivered"
- if id(element) in self._flush_dropped:
- return "dropped"
- return "retry"
-
- async def _recorded_put(self, signed_put: Callable[[], Awaitable[PutOutcome]]) -> PutOutcome:
- outcome: Final = await signed_put()
- limiter: Final = self._upload_limiter
- if isinstance(limiter, AdaptiveConcurrencyLimiter):
- limiter.record(PutSample(throttled=_throttled(outcome)))
- return outcome
-
- def _retry_reason(self, outcome: PutOutcome) -> str | None:
- if isinstance(outcome, Exception):
- return type(outcome).__name__
- if outcome.status_code not in _RETRYABLE_STATUSES:
- return None
- if self.s3_drop_on_terminal_error and _is_terminal(outcome):
- return None
- return str(outcome.status_code)
-
- async def _put_with_retries(
- self, signed_put: Callable[[], Awaitable[PutOutcome]], s3_object_key: str
- ) -> httpx.Response:
- for attempt in range(_UPLOAD_ATTEMPTS - 1):
- outcome = await self._recorded_put(signed_put)
- reason = self._retry_reason(outcome)
- if reason is None:
- return _response_or_raise(outcome)
- wait = _backoff_seconds(attempt)
- _log_retry(logging.DEBUG if _in_flush.get() else logging.WARNING, attempt, reason, wait, s3_object_key)
- self._flush_retries += 1
- await asyncio.sleep(wait)
- return _response_or_raise(await self._recorded_put(signed_put))
-
- def _sync_put_with_retries(self, signed_put: Callable[[], PutOutcome], s3_object_key: str) -> httpx.Response:
- for attempt in range(_UPLOAD_ATTEMPTS - 1):
- outcome = signed_put()
- reason = self._retry_reason(outcome)
- if reason is None:
- return _response_or_raise(outcome)
- wait = _backoff_seconds(attempt)
- _log_retry(logging.WARNING, attempt, reason, wait, s3_object_key)
- time.sleep(wait)
- return _response_or_raise(signed_put())
-
- def _batch_file_elements(self, batch: tuple[s3BatchLoggingElement, ...]) -> tuple[s3BatchLoggingElement, ...]:
- now: Final = datetime.now(timezone.utc)
- groups: Final = {
- parent: tuple(
- element for element in batch if element.body is None and _s3_key_parent(element.s3_object_key) == parent
- )
- for parent in sorted({_s3_key_parent(element.s3_object_key) for element in batch if element.body is None})
- }
- return tuple(element for element in batch if element.body is not None) + tuple(
- self._build_batch_file_element(elements, parent, now) for parent, elements in groups.items()
- )
-
- def _build_batch_file_element(
- self, elements: tuple[s3BatchLoggingElement, ...], parent: str, now: datetime
- ) -> s3BatchLoggingElement:
- batch_name: Final = f"batch_{now.strftime('%H-%M-%S')}_{uuid4().hex}"
- return s3BatchLoggingElement(
- payload={},
- body="\n".join(safe_dumps(element.payload) for element in elements),
- content_type="application/x-ndjson",
- s3_object_key=f"{parent}/{batch_name}.jsonl" if parent else f"{batch_name}.jsonl",
- s3_object_download_filename=f"{batch_name}.jsonl",
- retrying_since=min(
- (element.retrying_since for element in elements if element.retrying_since is not None), default=None
- ),
- )
-
- def resolve_partition_granularity(self) -> S3PartitionGranularity:
- raw: Final = (
- os.environ.get(S3_PARTITION_GRANULARITY_ENV_VAR)
- if self.s3_partition_granularity is None
- else self.s3_partition_granularity
- )
- cached: Final = self._partition_granularity_cache
- if cached is not None and cached[0] == raw:
- return cached[1]
- resolved: Final = resolve_s3_partition_granularity(raw)
- self._partition_granularity_cache = (raw, resolved)
- return resolved
-
- def create_s3_batch_logging_element(
- self,
- start_time: datetime,
- standard_logging_payload: StandardLoggingPayload | None,
- ) -> s3BatchLoggingElement | None:
- """
- Helper function to create an s3BatchLoggingElement.
-
- Args:
- start_time (datetime): The start time of the logging event.
- standard_logging_payload (Optional[StandardLoggingPayload]): The payload to be logged.
- s3_path (Optional[str]): The S3 path prefix.
-
- Returns:
- Optional[s3BatchLoggingElement]: The created s3BatchLoggingElement, or None if payload is None.
- """
- if standard_logging_payload is None:
- return None
-
- if self.s3_strip_base64_files:
- standard_logging_payload = self._strip_base64_from_messages_sync(standard_logging_payload)
-
- # Base prefix (default empty)
- prefix_components: Final = []
- if self.s3_use_team_prefix:
- team_alias: Final = standard_logging_payload.get("metadata", {}).get("user_api_key_team_alias", None)
- if team_alias:
- prefix_components.append(team_alias)
- if self.s3_use_key_prefix:
- user_api_key_alias: Final = standard_logging_payload.get("metadata", {}).get("user_api_key_alias", None)
- if user_api_key_alias:
- prefix_components.append(user_api_key_alias)
-
- # Construct full prefix path
- prefix_path = "/".join(prefix_components)
- if prefix_path:
- prefix_path += "/"
-
- s3_file_name: Final = litellm.utils.get_logging_id(start_time, standard_logging_payload) or ""
- verbose_logger.debug(
- "Creating s3 file with prefix_components=%s,prefix_path=%s and %s",
- prefix_components,
- prefix_path,
- s3_file_name,
- )
-
- def object_key(partition_granularity: S3PartitionGranularity) -> str:
- return get_s3_object_key(
- s3_path=cast(str | None, self.s3_path) or "",
- prefix=prefix_path,
- start_time=start_time,
- s3_file_name=s3_file_name,
- partition_granularity=partition_granularity,
- )
-
- metadata: Final = standard_logging_payload.get("metadata")
- cold_storage_object_key: Final = (
- metadata.get("cold_storage_object_key")
- if metadata is not None and litellm.cold_storage_custom_logger == "s3_v2"
- else None
- )
- s3_object_key: Final = (
- cold_storage_object_key
- if cold_storage_object_key is not None
- and cold_storage_object_key in (object_key("day"), object_key("hour"))
- else object_key(self.resolve_partition_granularity())
- )
- verbose_logger.debug("s3_object_key=%s", s3_object_key)
-
- s3_object_download_filename: Final = get_s3_object_download_filename(start_time, standard_logging_payload["id"])
-
- payload: Final = (
- prompts_only_payload(standard_logging_payload)
- if resolve_s3_log_prompts_only(self.s3_log_prompts_only)
- else standard_logging_payload
- )
- return s3BatchLoggingElement(
- payload=dict(payload),
- s3_object_key=s3_object_key,
- s3_object_download_filename=s3_object_download_filename,
- )
-
- def upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement):
- try:
- verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
-
- url: Final = self._build_object_url(batch_logging_element.s3_object_key)
-
- prepared: Final = self._prepare_put(batch_logging_element)
-
- httpx_client: Final = get_httpx_client(
- params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
- )
-
- def signed_put(prepared_put: _PreparedPut) -> PutOutcome:
- credentials: Final = self.get_credentials(
- aws_access_key_id=self.s3_aws_access_key_id,
- aws_secret_access_key=self.s3_aws_secret_access_key,
- aws_session_token=self.s3_aws_session_token,
- aws_region_name=self.s3_region_name,
- )
- signed_headers: Final = self._sign_put(credentials, url, prepared_put.json_string, prepared_put.headers)
- try:
- return httpx_client.put(
- url, data=prepared_put.json_string, headers=signed_headers, timeout=_UPLOAD_ATTEMPT_TIMEOUT
- )
- except _TRANSIENT_ERRORS as error:
- return error
-
- self._sync_put_with_retries(partial(signed_put, prepared), batch_logging_element.s3_object_key)
- except Exception as e:
- verbose_logger.exception("Error uploading to s3: %s", e)
- self.handle_callback_failure(callback_name="S3Logger")
- if isinstance(e, httpx.HTTPStatusError) and self.s3_drop_on_terminal_error and _is_terminal(e.response):
- verbose_logger.warning(
- "s3 logging: dropping object %s after terminal status %s",
- batch_logging_element.s3_object_key,
- e.response.status_code,
- )
-
- async def _download_object_from_s3(self, s3_object_key: str) -> dict | None:
- """
- Download and parse JSON object from S3.
-
- Args:
- s3_object_key: The S3 object key to download
-
- Returns:
- Optional[dict]: The parsed JSON object or None if not found/error
- """
- try:
- import hashlib
-
- from botocore.auth import S3SigV4Auth
- from botocore.awsrequest import AWSRequest
- except ImportError:
- raise ImportError("Missing boto3 to call S3. Run 'pip install boto3'.")
-
- try:
- from litellm.litellm_core_utils.asyncify import asyncify
-
- # Get AWS credentials
- asyncified_get_credentials: Final = asyncify(self.get_credentials)
- credentials: Final = await asyncified_get_credentials(
- aws_access_key_id=self.s3_aws_access_key_id,
- aws_secret_access_key=self.s3_aws_secret_access_key,
- aws_session_token=self.s3_aws_session_token,
- aws_region_name=self.s3_region_name,
- aws_session_name=self.s3_aws_session_name,
- aws_profile_name=self.s3_aws_profile_name,
- aws_role_name=self.s3_aws_role_name,
- aws_web_identity_token=self.s3_aws_web_identity_token,
- aws_sts_endpoint=self.s3_aws_sts_endpoint,
- )
-
- verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key)
-
- url: Final = self._build_object_url(s3_object_key)
-
- # Prepare the request for GET operation
- # For GET requests, we need x-amz-content-sha256 with hash of empty string
- empty_string_hash: Final = hashlib.sha256(b"").hexdigest()
- headers: Final = {
- "x-amz-content-sha256": empty_string_hash,
- }
-
- # Sign the request
- aws_request: Final = AWSRequest(method="GET", url=url, headers=headers)
- await run_aws_signing(S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth, aws_request)
-
- # Prepare the signed headers
- signed_headers: Final = dict(aws_request.headers.items())
-
- response: Final = await self.async_httpx_client.get(url, headers=signed_headers)
-
- if response.status_code != 200:
- verbose_logger.exception("S3 object not found, saw response=", response.text)
- return None
-
- # Parse JSON response
- return response.json()
-
- except Exception as e:
- verbose_logger.exception("Error downloading from S3: %s", e)
- return None
-
- async def get_proxy_server_request_from_cold_storage_with_object_key(
- self,
- object_key: str,
- ) -> dict | None:
- """
- Get the proxy server request from cold storage
-
- Allows fetching a dict of the proxy server request from s3 or GCS bucket.
-
- Args:
- request_id: The unique request ID to search for
- start_time: The start time of the request (datetime or ISO string)
-
- Returns:
- Optional[dict]: The request data dictionary or None if not found
- """
- try:
- # Download and return the object from S3
- downloaded_object: Final = await self._download_object_from_s3(object_key)
- return downloaded_object
- except Exception as e:
- verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e)
- return None
+__all__ = ["S3BatchUploadError", "S3Logger"]
diff --git a/litellm/integrations/s3_v3.py b/litellm/integrations/s3_v3.py
new file mode 100644
index 00000000000..45a6f887650
--- /dev/null
+++ b/litellm/integrations/s3_v3.py
@@ -0,0 +1,481 @@
+import asyncio
+import copy
+import json
+import os
+from collections.abc import Callable, Collection, Mapping
+from typing import TYPE_CHECKING, Final
+
+from typing_extensions import Unpack
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.constants import (
+ COMPLETION_HTTP_FALLBACK_SECONDS,
+ DEFAULT_S3_BATCH_SIZE,
+ DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
+ DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
+ DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
+ HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
+ S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
+)
+from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name
+from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.custom_logger_v2.call import CallContext
+from litellm.integrations.custom_logger_v2.derivations import (
+ cold_storage_object_key,
+ log_id,
+ logged_key_identity,
+ mapping,
+ payload_copy,
+)
+from litellm.integrations.custom_logger_v2.enums import Field, Hook
+from litellm.integrations.custom_logger_v2.field_sources import HookArgs
+from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on
+from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch
+from litellm.integrations.custom_logger_v2.rust import RustLogger
+from litellm.integrations.custom_logger_v2.selection import Selection
+from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds
+from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document
+from litellm.secret_managers.main import get_secret, get_secret_str
+from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity
+from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload
+
+if TYPE_CHECKING:
+ from litellm.rust_bridge._native import NativeS3Log
+
+CALLBACK_NAME: Final = "s3_v2"
+STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files"
+_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4
+_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600
+
+
+class S3BatchUploadError(Exception):
+ def __init__(self, failed: int, total: int) -> None:
+ self.failed: Final = failed
+ self.total: Final = total
+ super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush")
+
+
+def _raise_batch_upload_error(failed: int, total: int) -> None:
+ raise S3BatchUploadError(failed=failed, total=total)
+
+
+def batch_upload_failed(failure: tuple[int, int, int] | None) -> None:
+ if failure is None:
+ return
+ queued, failed, total = failure
+ try:
+ _raise_batch_upload_error(failed, total)
+ except S3BatchUploadError:
+ verbose_logger.exception(
+ "CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued
+ )
+
+
+def _cold_storage_is_s3_v2() -> bool:
+ return litellm.cold_storage_custom_logger == "s3_v2"
+
+
+def _region_from_environment() -> str | None:
+ region_name: Final = get_secret_str("AWS_REGION_NAME", None)
+ region: Final = get_secret_str("AWS_REGION", None)
+ return region if region is not None else region_name
+
+
+class _Host:
+ def __init__(self) -> None:
+ self.failures: Final = CustomLogger()
+
+ def callback_failed(self) -> None:
+ self.failures.handle_callback_failure(callback_name="S3Logger")
+
+ def web_identity_token(self, reference: str) -> str:
+ if reference.startswith("os.environ/") or reference in os.environ:
+ raise ValueError("Invalid web identity token reference.")
+ match _read_secret(get_secret, reference):
+ case None:
+ raise ValueError("OIDC token could not be retrieved from secret manager.")
+ case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH:
+ raise ValueError(
+ "Parameter validation failed:\nInvalid length for parameter WebIdentityToken, "
+ f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}"
+ )
+ case str() as token:
+ return token
+ case other:
+ raise ValueError(
+ "Parameter validation failed:\nInvalid type for parameter WebIdentityToken, "
+ f"value: {other}, type: {type(other)}, valid types: {str}"
+ )
+
+
+def _read_secret(read: Callable[[str], object], reference: str) -> object:
+ return read(reference)
+
+
+def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust
+ configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None)
+ source: Final[Mapping[str, object] | Mapping[object, object]] = (
+ params_source if params_source is not None else (configured or {})
+ )
+ return {
+ key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value)
+ for key, value in source.items()
+ if isinstance(key, str)
+ }
+
+
+def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]:
+ return {
+ "batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE,
+ "max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
+ "max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
+ "upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
+ "download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS,
+ "download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
+ "web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY),
+ }
+
+
+def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float:
+ return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS
+
+
+_CONFIGURED_FIRST: Final = frozenset(
+ {
+ "s3_bucket_name",
+ "s3_path",
+ "s3_region_name",
+ "s3_api_version",
+ "s3_endpoint_url",
+ "s3_aws_access_key_id",
+ "s3_aws_secret_access_key",
+ "s3_aws_session_token",
+ "s3_aws_session_name",
+ "s3_aws_profile_name",
+ "s3_aws_role_name",
+ "s3_aws_web_identity_token",
+ "s3_aws_sts_endpoint",
+ "s3_config",
+ "s3_server_side_encryption",
+ "s3_sse_kms_key_id",
+ }
+)
+_CONSTRUCTOR_ENABLES: Final = frozenset(
+ {
+ "s3_use_team_prefix",
+ "s3_strip_base64_files",
+ "s3_use_key_prefix",
+ "s3_use_virtual_hosted_style",
+ "s3_adaptive_concurrency",
+ "s3_batch_file_upload",
+ }
+)
+_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"})
+_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"})
+_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"})
+_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = {
+ "s3_use_ssl": True,
+ "s3_drop_on_terminal_error": True,
+ "s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
+ "s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS,
+ "s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
+ "s3_batch_size": DEFAULT_S3_BATCH_SIZE,
+}
+
+
+def _blank(value: object) -> bool:
+ return value is None or value == ""
+
+
+def _constructor_wins(key: str, configured: object, value: object) -> bool:
+ return (
+ (key in _CONFIGURED_FIRST and not configured)
+ or (key in _CONFIGURED_UNLESS_NONE and configured is None)
+ or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured))
+ or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None)
+ )
+
+
+_KEEP_CONFIGURED: Final = object()
+
+
+def _constructor_value(key: str, configured: object, value: object) -> object:
+ if _constructor_wins(key, configured, value):
+ return value
+ if key in _CONSTRUCTOR_ENABLES and value:
+ return True
+ if key == "s3_max_queue_size" and value is not None:
+ return resolve_s3_max_queue_size(
+ configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE)
+ )
+ if key == "s3_max_retry_age_seconds":
+ fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS)
+ resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback)
+ return 0 if resolved is None else resolved
+ return _KEEP_CONFIGURED
+
+
+def _constructed_params(
+ params_source: Mapping[str, object] | None, settings: S3LoggerSettings
+) -> dict[str, object]: # mutable-ok: handed to Rust
+ params: Final = _resolved_params(params_source)
+ overrides: Final = {
+ key: replaced
+ for key, value in settings.items()
+ if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None)
+ and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED
+ }
+ return {**params, **overrides}
+
+
+class _Base64Stripper(CustomLogger):
+ def strip(self, payload: StandardLoggingPayload) -> None:
+ _ = self._strip_base64_from_messages_sync(payload)
+
+ def stripped(self, messages: list[object]) -> list[dict[str, object]]:
+ return self._process_messages(messages=[_shallow_copy(message) for message in messages])
+
+
+def _shallow_copy(message: object) -> object:
+ entries: Final = mapping(message)
+ return copy.copy(entries) if entries is not None and isinstance(message, dict) else message
+
+
+_STRIPPER: Final = _Base64Stripper()
+
+
+def legacy_should_log(kwargs: Mapping[str, object]) -> bool:
+ from litellm.litellm_core_utils import litellm_logging
+
+ if litellm.global_disable_no_log_param:
+ return True
+ litellm_params: Final = kwargs.get("litellm_params", {})
+ lookup: Final = getattr(litellm_params, "get", None)
+ if callable(lookup) and lookup("no-log", False) is True:
+ return False
+ return not litellm_logging.callback_disabled_dynamically(
+ callback=CALLBACK_NAME,
+ litellm_params=litellm_params,
+ standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check
+ )
+
+
+def _payload_messages(payload: Mapping[str, object]) -> list[object]:
+ raw: Final = payload.get("messages")
+ return [*raw] if isinstance(raw, list) else []
+
+
+class S3StripBase64Files(CustomLoggerV2):
+ selection = Selection(name=STRIP_CALLBACK_NAME)
+
+ def __init__(self, s3_logger_key: str) -> None:
+ self.s3_logger_key: Final = s3_logger_key
+
+ @on(
+ Hook.log_prepare,
+ reads={Field.deprecated_kwargs, Field.standard_logging_payload},
+ writes={Field.standard_logging_payload},
+ )
+ async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None:
+ from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
+
+ kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None))
+ payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None))
+ if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs):
+ return None
+ messages: Final = _payload_messages(payload)
+ if not messages:
+ return None
+ return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)})
+
+
+async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None:
+ while True:
+ await asyncio.sleep(interval)
+ try:
+ batch_upload_failed(await sink.flush())
+ except (asyncio.CancelledError, KeyboardInterrupt, SystemExit):
+ raise
+ except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does
+ verbose_logger.exception("CustomLogger: periodic flush failed: %s", e)
+
+
+def _excluding_fields(kwargs: object) -> Mapping[str, object] | None:
+ excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
+ details: Final = mapping(kwargs)
+ logged: Final = None if details is None else mapping(details.get("standard_logging_object"))
+ if details is None or logged is None or not isinstance(excluded, Collection) or not excluded:
+ return None
+ if logged.get("status") == "failure":
+ return None
+ return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}}
+
+
+def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object:
+ source: Final = mapping(kwargs) if details is None else details
+ return None if source is None else source.get("standard_logging_object")
+
+
+def _legacy_kwargs(view: object) -> object:
+ return getattr(view, Field.deprecated_kwargs.value, None)
+
+
+_read_attribute: Final[Callable[[object, str], object]] = getattr
+
+
+class _ExcludedView:
+ def __init__(self, view: object, details: Mapping[str, object]) -> None:
+ args: Final = HookArgs(request=details)
+ self.view: Final = view
+ self.fields: Final[Mapping[str, object]] = {
+ Field.deprecated_kwargs.value: details,
+ Field.standard_logging_payload_ref.value: payload_copy(args),
+ Field.log_id.value: log_id(args),
+ Field.cold_storage_object_key.value: cold_storage_object_key(args),
+ Field.logged_key_identity.value: logged_key_identity(args),
+ }
+
+ def __getattr__(self, name: str) -> object:
+ if name not in self.fields:
+ return _read_attribute(self.view, name)
+ value: Final = self.fields[name]
+ if value is None:
+ raise AttributeError(name)
+ return value
+
+
+_ATTEMPT_ENDED_READS: Final = frozenset(
+ {
+ Field.deprecated_kwargs,
+ Field.attempt_timing,
+ Field.log_id,
+ Field.cold_storage_object_key,
+ Field.standard_logging_payload_ref,
+ Field.logged_key_identity,
+ }
+)
+
+
+class S3Logger(RustLogger):
+ selection = Selection(name=CALLBACK_NAME, honors_no_log=True)
+
+ def __init__(
+ self,
+ s3_callback_params_override: Mapping[str, object] | None = None,
+ **settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords
+ ) -> None:
+ from litellm.rust_bridge._native import NativeS3Log
+
+ host: Final = _Host()
+ params: Final = _constructed_params(s3_callback_params_override, settings)
+ sink: Final = NativeS3Log(
+ params,
+ _constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)),
+ host,
+ _cold_storage_is_s3_v2(),
+ None if params.get("s3_region_name") else _region_from_environment(),
+ )
+ self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name")
+ self.sink: Final = sink
+ self.s3_host: Final = host
+ self.periodic_flush_task: Final = asyncio.create_task(
+ _periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS)))
+ )
+ super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None})
+ self.v2_companion: Final = (
+ S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self))
+ if sink.strip_base64_files
+ else None
+ )
+
+ @on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True)
+ async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None:
+ return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx)
+
+ async def run_rust(
+ self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object]
+ ) -> Patch | Abort | None:
+ from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
+
+ kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None)
+ if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict
+ return None
+ payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict
+ if payload is not None and not isinstance(payload, Mapping):
+ return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload
+ try:
+ raw: Final = _legacy_kwargs(view)
+ details: Final = _excluding_fields(raw)
+ logged: Final = _logged_payload(raw, details)
+ if self.sink.strip_base64_files and isinstance(logged, Mapping):
+ _STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does
+ return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx)
+ except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on
+ return self._failed(e)
+
+ def _failed(self, error: Exception) -> None:
+ verbose_logger.error("s3 Layer Error - %s", error, exc_info=error)
+ self.s3_host.callback_failed()
+
+ @property
+ def s3_path(self) -> str | None:
+ return self.sink.s3_path
+
+ @property
+ def s3_bucket_name(self) -> str | None:
+ return self.sink.s3_bucket_name
+
+ def resolve_partition_granularity(self) -> S3PartitionGranularity:
+ return "hour" if self.sink.partition_granularity() == "hour" else "day"
+
+ async def flush_queue(self) -> None:
+ batch_upload_failed(await self.sink.flush())
+
+ async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None:
+ try:
+ audit_log_id: Final = audit_log.get("id", "unknown")
+ batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}"))
+ except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger
+ verbose_logger.exception("S3 audit log error: %s", e)
+
+ async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
+ object_name: Final = batch_line_items_object_name(payload)
+ s3_path: Final = self.s3_path
+ s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name
+ body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload)
+ delivered: Final = await self.sink.upload_line_items(
+ s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:])
+ )
+ if not delivered:
+ verbose_logger.error(
+ "s3 logging: the %d line items of part %d of batch %s were not stored at %s",
+ len(payload["lines"]),
+ payload["part"],
+ payload["batch_id"],
+ s3_object_key,
+ )
+
+ async def _download_object_from_s3(self, s3_object_key: str) -> object:
+ try:
+ downloaded: Final = await self.sink.download(s3_object_key)
+ return None if downloaded is None else json.loads(downloaded)
+ except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
+ verbose_logger.exception("Error downloading from S3: %s", e)
+ return None
+
+ async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object:
+ try:
+ return await self._download_object_from_s3(object_key)
+ except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
+ verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e)
+ return None
+
+
+__all__ = [
+ "S3BatchUploadError",
+ "S3Logger",
+ "S3StripBase64Files",
+ "legacy_should_log",
+]
diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py
index 74badc2c8c1..535fef8fcec 100644
--- a/litellm/litellm_core_utils/custom_logger_registry.py
+++ b/litellm/litellm_core_utils/custom_logger_registry.py
@@ -46,7 +46,7 @@ from litellm.integrations.opik.opik import OpikLogger
from litellm.integrations.pointfive import PointFiveLogger
from litellm.integrations.posthog import PostHogLogger
from litellm.integrations.prometheus import PrometheusLogger
-from litellm.integrations.s3_v2 import S3Logger
+from litellm.integrations.s3_v3 import S3Logger
from litellm.integrations.sqs import SQSLogger
from litellm.integrations.ternary.ternary_logger import TernaryLogger
from litellm.integrations.vantage.vantage_logger import VantageLogger
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 1b682d078f0..15a95e91c52 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -15,6 +15,7 @@ import time
import traceback
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
from contextlib import AbstractContextManager
+from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime as dt_object
from enum import Enum
@@ -305,7 +306,6 @@ from ..integrations.pointfive import PointFiveLogger
from ..integrations.posthog import PostHogLogger
from ..integrations.prompt_layer import PromptLayerLogger
from ..integrations.s3 import S3Logger
-from ..integrations.s3_v2 import S3Logger as S3V2Logger
from ..integrations.supabase import Supabase
from ..integrations.traceloop import TraceloopLogger
from ..integrations.zerobus import ZerobusLogger
@@ -768,6 +768,16 @@ def _callback_control_params(litellm_params: object) -> dict[str, object]:
return litellm_params.model_dump() if isinstance(litellm_params, BaseModel) else {}
+def callback_disabled_dynamically(
+ callback: object, litellm_params: object, standard_callback_dynamic_params: StandardCallbackDynamicParams
+) -> bool:
+ return EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically(
+ callback=callback,
+ litellm_params=_callback_control_params(litellm_params),
+ standard_callback_dynamic_params=standard_callback_dynamic_params,
+ )
+
+
LOGGING_KEYS: Final[Mapping[Field, str]] = MappingProxyType(
{
Field.model: "model",
@@ -783,6 +793,9 @@ def _holds_v2(callbacks: Iterable[object]) -> bool:
return any(isinstance(callback, CustomLoggerV2) for callback in callbacks)
+V2_ASYNC_LOGGING_LINE: Final[ContextVar[bool]] = ContextVar("v2_async_logging_line", default=False)
+
+
_HOOK_DERIVED_FIELDS: Final[Mapping[Hook, Mapping[Field, FieldSource]]] = MappingProxyType(
{Hook.log_prepare: v2_derive.DERIVED_LOGGING_FIELDS, Hook.attempt_ended: v2_derive.DERIVED_ATTEMPT_FIELDS}
)
@@ -839,15 +852,19 @@ async def _run_callbacks(
args: HookArgs | None = None,
) -> None:
if include_v2 and _holds_v2(callbacks):
- _ = kwargs_after(
- await run_subscribers(
- hook_subscribers(hook, callbacks, admit=Admit.every_entry),
- _logging_fields(hook, kwargs, args, owner),
- binding,
- call=_v2_call(owner, args),
- scope=None if owner is None else owner.v2_logging_scope(),
+ line: Final = V2_ASYNC_LOGGING_LINE.set(True)
+ try:
+ _ = kwargs_after(
+ await run_subscribers(
+ hook_subscribers(hook, callbacks, admit=Admit.every_entry),
+ _logging_fields(hook, kwargs, args, owner),
+ binding,
+ call=_v2_call(owner, args),
+ scope=None if owner is None else owner.v2_logging_scope(),
+ )
)
- )
+ finally:
+ V2_ASYNC_LOGGING_LINE.reset(line)
return
for callback in callbacks:
if isinstance(callback, CustomLoggerV2):
@@ -3283,9 +3300,9 @@ class Logging(LiteLLMLoggingBaseClass):
return False
# Check for dynamically disabled callbacks via headers
- if EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically(
+ if callback_disabled_dynamically(
callback=callback,
- litellm_params=_callback_control_params(litellm_params),
+ litellm_params=litellm_params,
standard_callback_dynamic_params=self.standard_callback_dynamic_params,
):
verbose_logger.debug(
@@ -6519,6 +6536,8 @@ def build_custom_logger_compatible_class(
_in_memory_loggers.append(_gcs_bucket_logger)
return _gcs_bucket_logger
elif logging_integration == "s3_v2":
+ from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
+
for callback in _in_memory_loggers:
if isinstance(callback, S3V2Logger):
return callback
@@ -7341,6 +7360,8 @@ def get_custom_logger_compatible_class(
if isinstance(callback, GCSBucketLogger):
return callback
elif logging_integration == "s3_v2":
+ from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
+
for callback in _in_memory_loggers:
if isinstance(callback, S3V2Logger):
return callback
@@ -8046,6 +8067,8 @@ class StandardLoggingPayloadSetup:
)
if custom_logger and hasattr(custom_logger, "s3_path") and getattr(custom_logger, "s3_path"):
s3_path = getattr(custom_logger, "s3_path")
+ from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
+
if isinstance(custom_logger, S3V2Logger):
partition_granularity = custom_logger.resolve_partition_granularity()
except Exception:
diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py
index fe5340cb6fe..4f22647fee5 100644
--- a/litellm/litellm_core_utils/logging_callback_manager.py
+++ b/litellm/litellm_core_utils/logging_callback_manager.py
@@ -21,6 +21,16 @@ _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
_callback_lists_lock: Final = threading.RLock()
+def _v2_companion(callback: object) -> CustomLoggerV2 | None:
+ companion: Final[object] = getattr(callback, "v2_companion", None)
+ return companion if isinstance(companion, CustomLoggerV2) else None
+
+
+def _with_v2_companions(callbacks: Iterable[object]) -> tuple[object, ...]:
+ listed: Final = tuple(callbacks)
+ return (*listed, *(companion for callback in listed if (companion := _v2_companion(callback)) is not None))
+
+
def _instance_attributes(target: object) -> Mapping[str, object]:
return getattr(target, "__dict__", {})
@@ -292,16 +302,48 @@ class LoggingCallbackManager:
parent_list=parent_list,
)
elif isinstance(callback, CustomLoggerV2):
- v2_key: Final = self._get_custom_logger_key(callback)
- if not any(
- isinstance(existing, CustomLoggerV2) and self._get_custom_logger_key(existing) == v2_key
- for existing in parent_list
- ):
- parent_list.append(callback) # pyright: ignore[reportArgumentType] # the event lists also hold V2 callbacks
+ listed: Final = self._listed_v2(callback, parent_list)
+ if listed is None:
+ self._append_v2(callback, parent_list)
+ self._place_v2_companion(callback if listed is None else listed, parent_list)
elif callable(callback):
self._add_callback_function_to_list(callback=callback, parent_list=parent_list)
+ def _append_v2(
+ self, callback: CustomLoggerV2, parent_list: list[CustomLogger | Callable[..., object] | str]
+ ) -> None:
+ parent_list.append(callback) # pyright: ignore[reportArgumentType] # the event lists also hold V2 callbacks
+
+ def _listed_v2(self, callback: CustomLoggerV2, parent_list: Iterable[object]) -> CustomLoggerV2 | None:
+ v2_key: Final = self._get_custom_logger_key(callback)
+ return next(
+ (
+ existing
+ for existing in parent_list
+ if isinstance(existing, CustomLoggerV2) and self._get_custom_logger_key(existing) == v2_key
+ ),
+ None,
+ )
+
+ def _place_v2_companion(
+ self, callback: CustomLoggerV2, parent_list: list[CustomLogger | Callable[..., object] | str]
+ ) -> None:
+ companion: Final = _v2_companion(callback)
+ if companion is None or self._listed_v2(companion, parent_list) is not None:
+ return
+ if not self._check_callback_list_size(parent_list):
+ return
+ self._append_v2(companion, parent_list)
+ position: Final = next(index for index, existing in enumerate(parent_list) if existing is callback)
+ parent_list.insert(position, parent_list.pop())
+
+ def place_v2_companions(self, parent_list: list[CustomLogger | Callable[..., object] | str]) -> None:
+ with _callback_lists_lock:
+ for callback in tuple(parent_list):
+ if isinstance(callback, CustomLoggerV2):
+ self._place_v2_companion(callback, parent_list)
+
def _add_callback_function_to_list(
self,
callback: Callable[..., object],
@@ -364,7 +406,7 @@ class LoggingCallbackManager:
callback_lists_lock: Final = _callback_lists_lock
def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]:
- dropped_ids: Final = frozenset(map(id, callbacks))
+ dropped_ids: Final = frozenset(map(id, _with_v2_companions(callbacks)))
owner_ids: Final = frozenset(map(id, method_owners))
def keep(callback: object) -> bool:
@@ -444,6 +486,7 @@ class LoggingCallbackManager:
Remove a callback object from every callback list it may have been
promoted into, so a re-initialized callback leaves no stale instance behind.
"""
+ companion: Final = None if require_self else _v2_companion(obj)
with _callback_lists_lock:
for callback_list in (
litellm.callbacks,
@@ -454,6 +497,8 @@ class LoggingCallbackManager:
litellm._async_failure_callback,
):
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
+ if companion is not None:
+ callback_list[:] = [callback for callback in callback_list if callback is not companion]
def get_active_additional_logging_utils_from_custom_logger(
self,
@@ -536,7 +581,7 @@ class LoggingCallbackManager:
if not isinstance(callback, str)
)
- def _get_callback_string(self, callback: CustomLogger | Callable[..., object] | str) -> str:
+ def _get_callback_string(self, callback: CustomLogger | CustomLoggerV2 | Callable[..., object] | str) -> str:
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.litellm_core_utils.custom_logger_registry import (
CustomLoggerRegistry,
@@ -547,7 +592,7 @@ class LoggingCallbackManager:
return callback
elif isinstance(callback, OpenTelemetry) and callback.callback_name is not None:
return callback.callback_name
- elif isinstance(callback, CustomLogger):
+ elif isinstance(callback, CustomLogger | CustomLoggerV2):
# Try to get the string representation from the registry
callback_str: Final = CustomLoggerRegistry.get_callback_str_from_class_type(type(callback))
return callback_str if callback_str is not None else type(callback).__name__
diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py
index aa31f44dd13..65b64a297c9 100644
--- a/litellm/proxy/management_helpers/audit_logs.py
+++ b/litellm/proxy/management_helpers/audit_logs.py
@@ -4,9 +4,9 @@ Functions to create audit logs for LiteLLM Proxy
import asyncio
import json
-from collections.abc import Awaitable, Sequence
+from collections.abc import Awaitable, Mapping, Sequence
from datetime import datetime, timezone
-from typing import Final, TypeVar
+from typing import TYPE_CHECKING, Final, TypeVar
import litellm
from litellm._logging import verbose_proxy_logger
@@ -21,7 +21,10 @@ from litellm.proxy._types import (
from litellm.repositories.table_repositories import AuditLogRepository
from litellm.types.utils import StandardAuditLogPayload
-_audit_log_callback_cache: Final[dict[str, CustomLogger]] = {}
+if TYPE_CHECKING:
+ from litellm.integrations.s3_v3 import S3Logger
+
+_audit_log_callback_cache: Final[dict[str, "CustomLogger | S3Logger"]] = {}
ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY: Final = "allow_litellm_changed_by_header"
@@ -62,7 +65,7 @@ def get_audit_log_changed_by(
return user_api_key_dict.user_id or litellm_proxy_admin_name
-def _resolve_audit_log_callback(name: str) -> CustomLogger | None:
+def _resolve_audit_log_callback(name: str) -> "CustomLogger | S3Logger | None":
"""Resolve a string callback name to a CustomLogger instance, with caching.
For "s3_v2" with `litellm.s3_audit_callback_params` set, constructs a
@@ -72,11 +75,12 @@ def _resolve_audit_log_callback(name: str) -> CustomLogger | None:
if name in _audit_log_callback_cache:
return _audit_log_callback_cache[name]
- instance: CustomLogger | None
- if name == "s3_v2" and getattr(litellm, "s3_audit_callback_params", None) is not None:
- from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
+ instance: CustomLogger | S3Logger | None
+ audit_params: Final[Mapping[str, object] | None] = getattr(litellm, "s3_audit_callback_params", None)
+ if name == "s3_v2" and audit_params is not None:
+ from litellm.integrations import s3_v3
- instance = S3V2Logger(s3_callback_params_override=litellm.s3_audit_callback_params)
+ instance = s3_v3.S3Logger(s3_callback_params_override=audit_params)
else:
from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,
@@ -196,14 +200,14 @@ async def _dispatch_audit_log_to_callbacks(
for callback in litellm.audit_log_callbacks:
try:
- resolved: CustomLogger | None = callback if isinstance(callback, CustomLogger) else None
+ resolved: CustomLogger | S3Logger | None = callback if isinstance(callback, CustomLogger) else None
if isinstance(callback, str):
resolved = _resolve_audit_log_callback(callback)
if resolved is None:
verbose_proxy_logger.warning("Could not resolve audit log callback: %s", callback)
continue
- if isinstance(resolved, CustomLogger):
+ if resolved is not None:
track_audit_task(
asyncio.create_task(
resolved.async_log_audit_log_event(payload),
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index b41f240f89b..b161fef588a 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -7446,7 +7446,7 @@ class ProxyConfig:
if key == "request_timeout":
litellm.request_timeout_explicitly_set = True
if key in {"s3_audit_callback_params", "s3_callback_params"}:
- from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
+ from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
from litellm.litellm_core_utils.litellm_logging import (
_in_memory_loggers,
)
@@ -16441,12 +16441,13 @@ async def _routed_count_deployment(
if llm_router is None:
return None
try:
- return await llm_router.select(model, dict(_responses_routing_kwargs(responses_request)))
+ routed: Final = await llm_router.select(model, dict(_responses_routing_kwargs(responses_request)))
except Exception:
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment"
)
return None
+ return routed if isinstance(routed, Mapping) else None
def _with_chat_timeout(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index ce7a041e147..53863c881ae 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -2514,6 +2514,7 @@ class ProxyLogging:
# Replace string entries in litellm.callbacks with initialized instances
for idx, initialized_callback in string_callbacks_to_replace.items():
litellm.callbacks[idx] = initialized_callback
+ litellm.logging_callback_manager.place_v2_companions(litellm.callbacks)
# Fan ``litellm.callbacks`` (the "all events" registry) out into the
# success/failure event lists eagerly, at startup. ``completion()`` does
diff --git a/litellm/rust/Cargo.lock b/litellm/rust/Cargo.lock
index 3a66beb7c42..811bda4f798 100644
--- a/litellm/rust/Cargo.lock
+++ b/litellm/rust/Cargo.lock
@@ -3739,6 +3739,34 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "litellm-callbacks-s3-log"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "futures-util",
+ "litellm-auth-aws",
+ "litellm-callbacks",
+ "litellm-http",
+ "litellm-python-compat",
+ "litellm-s3",
+ "litellm-secret-value",
+ "litellm-tracing",
+ "md-5",
+ "rand 0.8.7",
+ "reqwest",
+ "rstest",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "thiserror 2.0.19",
+ "tokio",
+ "tracing",
+ "uuid",
+ "wiremock",
+]
+
[[package]]
name = "litellm-circuit-breaker"
version = "0.1.0"
@@ -4451,7 +4479,10 @@ dependencies = [
"litellm-auth-azure",
"litellm-cache",
"litellm-cache-response",
+ "litellm-callbacks",
"litellm-callbacks-legacy-python",
+ "litellm-callbacks-python",
+ "litellm-callbacks-s3-log",
"litellm-circuit-breaker",
"litellm-core-utils",
"litellm-error",
@@ -4691,11 +4722,15 @@ dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"aws-types",
+ "bytes",
"http 1.4.2",
+ "hyper 1.12.0",
+ "litellm-auth-aws",
"litellm-http",
"reqwest",
"rstest",
"thiserror 2.0.19",
+ "tokio",
]
[[package]]
diff --git a/litellm/rust/Cargo.toml b/litellm/rust/Cargo.toml
index b0dc4b2c279..3fc881102c1 100644
--- a/litellm/rust/Cargo.toml
+++ b/litellm/rust/Cargo.toml
@@ -38,6 +38,7 @@ litellm-host-native = { path = "crates/host-native" }
litellm-callbacks = { path = "crates/callbacks" }
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
litellm-callbacks-python = { path = "crates/callbacks-python" }
+litellm-callbacks-s3-log = { path = "crates/callbacks-s3-log" }
litellm-framer = { path = "crates/framer" }
litellm-auth = { path = "crates/auth" }
litellm-auth-types = { path = "crates/auth-types" }
diff --git a/litellm/rust/crates/auth-aws/Cargo.toml b/litellm/rust/crates/auth-aws/Cargo.toml
index c8ed7bb0f14..1a7392afc52 100644
--- a/litellm/rust/crates/auth-aws/Cargo.toml
+++ b/litellm/rust/crates/auth-aws/Cargo.toml
@@ -17,6 +17,7 @@ serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
sha2.workspace = true
thiserror.workspace = true
+tokio.workspace = true
url.workspace = true
veil.workspace = true
diff --git a/litellm/rust/crates/auth-aws/src/aws.rs b/litellm/rust/crates/auth-aws/src/aws.rs
index 8508ac0c47c..a94a4e8d66b 100644
--- a/litellm/rust/crates/auth-aws/src/aws.rs
+++ b/litellm/rust/crates/auth-aws/src/aws.rs
@@ -9,8 +9,8 @@ use std::{future::Future, pin::Pin, sync::Arc};
use aws_credential_types::Credentials;
use aws_credential_types::provider::ProvideCredentials;
use aws_sigv4::http_request::{
- SignableBody, SignableRequest, SignatureLocation, SigningInstructions, SigningParams,
- SigningSettings, sign,
+ PercentEncodingMode, SignableBody, SignableRequest, SignatureLocation, SigningInstructions,
+ SigningParams, SigningSettings, UriPathNormalizationMode, sign,
};
use aws_sigv4::sign::v4;
use aws_smithy_runtime_api::client::identity::Identity;
@@ -38,10 +38,15 @@ pub struct AwsAuthService {
impl Default for AwsAuthService {
fn default() -> Self {
- Self::new(Arc::new(NativeProviderLoader))
+ Self::new(Arc::new(NativeProviderLoader {
+ token: None,
+ policy: None,
+ }))
}
}
+pub type WebIdentityTokenResolver = Arc Result + Send + Sync>;
+
impl AwsAuthService {
fn new(loader: Arc) -> Self {
Self {
@@ -53,6 +58,16 @@ impl AwsAuthService {
loader,
}
}
+
+ pub fn for_web_identity(
+ resolver: WebIdentityTokenResolver,
+ session_policy: Option,
+ ) -> Self {
+ Self::new(Arc::new(NativeProviderLoader {
+ token: Some(resolver),
+ policy: session_policy,
+ }))
+ }
}
struct CredentialExpiry;
@@ -81,11 +96,19 @@ trait ProviderLoader: Send + Sync {
fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_>;
}
-struct NativeProviderLoader;
+struct NativeProviderLoader {
+ token: Option,
+ policy: Option,
+}
impl ProviderLoader for NativeProviderLoader {
fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> {
- Box::pin(build_provider(config, flow))
+ Box::pin(build_provider(
+ config,
+ flow,
+ self.token.clone(),
+ self.policy.clone(),
+ ))
}
}
@@ -271,6 +294,16 @@ impl AwsAuthService {
self.fresh_credentials(&resolved, &flow).await
}
+ pub async fn invalidate_credentials(
+ &self,
+ config: AwsAuthConfig,
+ env_lookup: &(dyn Fn(&str) -> Option + Sync),
+ ) {
+ self.credentials
+ .invalidate(&config.with_environment(env_lookup))
+ .await;
+ }
+
/// The credentials botocore's default chain finds in `region`, without LiteLLM's Bedrock settings
/// (`AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, `AWS_SESSION_NAME`) that `resolve_credentials` also reads.
pub async fn default_chain_credentials(&self, region: &str) -> Result {
@@ -347,6 +380,8 @@ impl AwsAuthFlow {
async fn build_provider(
resolved: AwsAuthConfig,
flow: AwsAuthFlow,
+ token_resolver: Option,
+ policy: Option,
) -> Result {
match flow {
AwsAuthFlow::SessionToken {
@@ -417,8 +452,10 @@ async fn build_provider(
role,
session_name,
} => Ok(SharedCredentialsProvider::new(WebIdentityProvider {
+ policy,
client: aws_sdk_sts::Client::new(&sdk_config(resolved).await),
token,
+ token_resolver,
role,
session_name,
})),
@@ -448,14 +485,27 @@ async fn sdk_config(config: AwsAuthConfig) -> aws_types::SdkConfig {
loader.load().await
}
-#[derive(Debug)]
struct WebIdentityProvider {
client: aws_sdk_sts::Client,
token: SecretValue,
+ token_resolver: Option,
+ policy: Option,
role: String,
session_name: String,
}
+impl std::fmt::Debug for WebIdentityProvider {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter
+ .debug_struct("WebIdentityProvider")
+ .field("token", &self.token)
+ .field("policy", &self.policy)
+ .field("role", &self.role)
+ .field("session_name", &self.session_name)
+ .finish_non_exhaustive()
+ }
+}
+
impl ProvideCredentials for WebIdentityProvider {
fn provide_credentials<'a>(
&'a self,
@@ -464,12 +514,31 @@ impl ProvideCredentials for WebIdentityProvider {
Self: 'a,
{
aws_credential_types::provider::future::ProvideCredentials::new(async {
- let response = self
+ let token = match &self.token_resolver {
+ Some(resolve) => {
+ let resolve = Arc::clone(resolve);
+ let reference = self.token.expose().to_owned();
+ tokio::task::spawn_blocking(move || resolve(&reference))
+ .await
+ .map_err(|error| error.to_string())
+ .and_then(|token| token)
+ .map_err(|message| {
+ CredentialsError::provider_error(std::io::Error::other(message))
+ })?
+ }
+ None => self.token.expose().to_owned(),
+ };
+ let request = self
.client
.assume_role_with_web_identity()
.role_arn(&self.role)
.role_session_name(&self.session_name)
- .web_identity_token(self.token.expose())
+ .web_identity_token(token);
+ let request = match &self.policy {
+ Some(policy) => request.policy(policy).duration_seconds(3600),
+ None => request,
+ };
+ let response = request
.send()
.await
.map_err(CredentialsError::provider_error)?;
@@ -605,6 +674,32 @@ pub fn sign_headers(
.collect()
}
+pub fn sign_s3_headers(
+ method: &str,
+ url: &str,
+ body: &[u8],
+ headers: &[(String, String)],
+ scope: SigningScope<'_>,
+) -> Result, Error> {
+ let mut settings = SigningSettings::default();
+ settings.percent_encoding_mode = PercentEncodingMode::Single;
+ settings.uri_path_normalization_mode = UriPathNormalizationMode::Disabled;
+ let instructions = sign_request(
+ method,
+ url,
+ headers
+ .iter()
+ .map(|(name, value)| (name.as_str(), value.as_str())),
+ body,
+ settings,
+ scope,
+ )?;
+ Ok(instructions
+ .headers()
+ .map(|(name, value)| (name.to_owned(), value.to_owned()))
+ .collect())
+}
+
pub fn presign_url(
url: &str,
expires_in: Duration,
diff --git a/litellm/rust/crates/callbacks-s3-log/Cargo.toml b/litellm/rust/crates/callbacks-s3-log/Cargo.toml
new file mode 100644
index 00000000000..ae4513a2a48
--- /dev/null
+++ b/litellm/rust/crates/callbacks-s3-log/Cargo.toml
@@ -0,0 +1,35 @@
+[package]
+name = "litellm-callbacks-s3-log"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-auth-aws.workspace = true
+litellm-callbacks.workspace = true
+litellm-http.workspace = true
+litellm-python-compat.workspace = true
+litellm-s3.workspace = true
+litellm-secret-value.workspace = true
+litellm-tracing.workspace = true
+
+base64.workspace = true
+bytes.workspace = true
+futures-util.workspace = true
+md-5 = "0.11.0"
+rand.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+serde_json = { workspace = true, features = ["preserve_order"] }
+sha2.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["sync", "time", "rt"] }
+tracing.workspace = true
+uuid.workspace = true
+
+[dev-dependencies]
+litellm-http = { workspace = true, features = ["test-support"] }
+rstest.workspace = true
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] }
+wiremock.workspace = true
diff --git a/litellm/rust/crates/callbacks-s3-log/src/key.rs b/litellm/rust/crates/callbacks-s3-log/src/key.rs
new file mode 100644
index 00000000000..a0f3fc12b10
--- /dev/null
+++ b/litellm/rust/crates/callbacks-s3-log/src/key.rs
@@ -0,0 +1,367 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use litellm_callbacks::Moment;
+use sha2::{Digest, Sha256};
+
+use crate::settings::{Granularity, Settings};
+
+const MAX_KEY_BYTES: usize = 1024;
+const MAX_FILE_NAME_BYTES: usize = 255;
+const MAX_DOWNLOAD_NAME_BYTES: usize = 1024;
+const BOUNDED_HEAD_BYTES: i64 = 64;
+const MIN_BOUNDED_FILE_NAME_BYTES: usize = 64;
+const PREFIX_DIGEST_CHARS: usize = 16;
+const EXTENSION: &str = ".json";
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Stamp {
+ pub year: i64,
+ pub month: u32,
+ pub day: u32,
+ pub hour: u32,
+ pub minute: u32,
+ pub second: u32,
+ pub micros: u32,
+}
+
+impl From<[i64; 7]> for Stamp {
+ fn from(parts: [i64; 7]) -> Self {
+ let [year, month, day, hour, minute, second, micros] = parts;
+ let part = |value: i64| u32::try_from(value).unwrap_or_default();
+ Self {
+ year,
+ month: part(month),
+ day: part(day),
+ hour: part(hour),
+ minute: part(minute),
+ second: part(second),
+ micros: part(micros),
+ }
+ }
+}
+
+impl From<&Moment> for Stamp {
+ fn from(moment: &Moment) -> Self {
+ Self {
+ year: i64::from(moment.year),
+ month: u32::from(moment.month),
+ day: u32::from(moment.day),
+ hour: u32::from(moment.hour),
+ minute: u32::from(moment.minute),
+ second: u32::from(moment.second),
+ micros: moment.microsecond,
+ }
+ }
+}
+
+impl Stamp {
+ pub fn utc(time: SystemTime) -> Self {
+ let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default();
+ let seconds = elapsed.as_secs();
+ let (year, month, day) = civil_from_days((seconds / 86_400) as i64);
+ let of_day = seconds % 86_400;
+ Self {
+ year,
+ month,
+ day,
+ hour: (of_day / 3600) as u32,
+ minute: (of_day / 60 % 60) as u32,
+ second: (of_day % 60) as u32,
+ micros: elapsed.subsec_micros(),
+ }
+ }
+
+ pub fn date(&self) -> String {
+ format!("{:04}-{:02}-{:02}", self.year, self.month, self.day)
+ }
+
+ pub fn clock(&self) -> String {
+ format!("{:02}-{:02}-{:02}", self.hour, self.minute, self.second)
+ }
+
+ fn clock_micros(&self) -> String {
+ format!("{}-{:06}", self.clock(), self.micros)
+ }
+}
+
+fn civil_from_days(days: i64) -> (i64, u32, u32) {
+ let shifted = days + 719_468;
+ let era = shifted.div_euclid(146_097);
+ let day_of_era = shifted.rem_euclid(146_097);
+ let year_of_era =
+ (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
+ let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
+ let month_index = (5 * day_of_year + 2) / 153;
+ let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32;
+ let month = if month_index < 10 {
+ month_index + 3
+ } else {
+ month_index - 9
+ } as u32;
+ let year = year_of_era + era * 400 + i64::from(month <= 2);
+ (year, month, day)
+}
+
+pub fn logging_id(start: &Stamp, id: &str) -> String {
+ format!("time-{}_{id}", start.clock_micros())
+}
+
+pub fn object_key(
+ path: &str,
+ prefix: &str,
+ start: &Stamp,
+ file_name: &str,
+ granularity: Granularity,
+) -> String {
+ let sanitized = file_name.replace(['/', ':'], "_");
+ let configured = if path.is_empty() {
+ prefix.to_owned()
+ } else {
+ format!("{}/{prefix}", path.trim_end_matches('/'))
+ };
+ let date = match granularity {
+ Granularity::Day => format!("{}/", start.date()),
+ Granularity::Hour => format!("{}/{:02}/", start.date(), start.hour),
+ };
+ let file_name_budget = MAX_FILE_NAME_BYTES - EXTENSION.len();
+ let key = format!("{configured}{date}{sanitized}{EXTENSION}");
+ if key.len() <= MAX_KEY_BYTES && sanitized.len() <= file_name_budget {
+ return key;
+ }
+ let budget = MAX_KEY_BYTES - date.len() - EXTENSION.len();
+ if configured.len() + MIN_BOUNDED_FILE_NAME_BYTES <= budget {
+ let bounded = bounded_name(
+ file_name,
+ &sanitized,
+ file_name_budget.min(budget - configured.len()) as i64,
+ );
+ return format!("{configured}{date}{bounded}{EXTENSION}");
+ }
+ let shortest = bounded_name(file_name, &sanitized, MIN_BOUNDED_FILE_NAME_BYTES as i64);
+ let prefix = bounded_prefix(&configured, budget as i64 - shortest.len() as i64);
+ format!("{prefix}{date}{shortest}{EXTENSION}")
+}
+
+pub fn download_name(start: &Stamp, id: &str) -> String {
+ let stamp = format!("time-{}T{}", start.date(), start.clock_micros());
+ let name = format!("{stamp}_{id}");
+ let sanitized = format!("{stamp}_{}", id.replace(['/', '"'], "_"));
+ let budget = MAX_DOWNLOAD_NAME_BYTES - EXTENSION.len();
+ if sanitized.len() <= budget {
+ return format!("{sanitized}{EXTENSION}");
+ }
+ format!(
+ "{}{EXTENSION}",
+ bounded_name(&name, &sanitized, budget as i64)
+ )
+}
+
+fn bounded_name(name: &str, sanitized: &str, max_bytes: i64) -> String {
+ let digest = hex_sha256(name);
+ let head = truncate(
+ sanitized,
+ BOUNDED_HEAD_BYTES.min(max_bytes - digest.len() as i64 - 1),
+ );
+ if head.is_empty() {
+ digest
+ } else {
+ format!("{head}_{digest}")
+ }
+}
+
+fn bounded_prefix(prefix: &str, max_bytes: i64) -> String {
+ let digest = format!("{}/", &hex_sha256(prefix)[..PREFIX_DIGEST_CHARS]);
+ if max_bytes < digest.len() as i64 {
+ return String::new();
+ }
+ let head = truncate(prefix, max_bytes - digest.len() as i64 - 1).trim_end_matches('/');
+ if head.is_empty() {
+ digest
+ } else {
+ format!("{head}/{digest}")
+ }
+}
+
+fn truncate(value: &str, max_bytes: i64) -> &str {
+ let Ok(max_bytes) = usize::try_from(max_bytes) else {
+ return "";
+ };
+ if max_bytes == 0 {
+ return "";
+ }
+ if value.len() <= max_bytes {
+ return value;
+ }
+ let end = (0..=max_bytes)
+ .rev()
+ .find(|end| value.is_char_boundary(*end))
+ .unwrap_or(0);
+ &value[..end]
+}
+
+pub fn hex_sha256(value: impl AsRef<[u8]>) -> String {
+ hex(&Sha256::digest(value.as_ref()))
+}
+
+pub fn hex(bytes: &[u8]) -> String {
+ bytes.iter().map(|byte| format!("{byte:02x}")).collect()
+}
+
+pub fn parent(key: &str) -> &str {
+ key.rsplit_once('/').map_or("", |(parent, _)| parent)
+}
+
+pub fn quote(key: &str) -> String {
+ let mut encoded = String::with_capacity(key.len());
+ for byte in key.bytes() {
+ if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') {
+ encoded.push(byte as char);
+ } else {
+ encoded.push_str(&format!("%{byte:02X}"));
+ }
+ }
+ encoded
+}
+
+pub fn dns_suffix(region: Option<&str>) -> &'static str {
+ let Some(region) = region.filter(|region| !region.is_empty()) else {
+ return "amazonaws.com";
+ };
+ [
+ ("cn-", "amazonaws.com.cn"),
+ ("us-gov-", "amazonaws.com"),
+ ("us-isob-", "sc2s.sgov.gov"),
+ ("us-isof-", "csp.hci.ic.gov"),
+ ("us-iso-", "c2s.ic.gov"),
+ ("eu-isoe-", "cloud.adc-e.uk"),
+ ]
+ .into_iter()
+ .find(|(prefix, _)| region.starts_with(prefix))
+ .map_or("amazonaws.com", |(_, suffix)| suffix)
+}
+
+pub fn object_url(settings: &Settings, key: &str) -> String {
+ let encoded = quote(key);
+ match (&settings.endpoint_url, &settings.bucket_name) {
+ (Some(endpoint), Some(bucket)) if settings.use_virtual_hosted_style => {
+ let host = endpoint.replace("https://", "").replace("http://", "");
+ let scheme = if endpoint.starts_with("https://") {
+ "https://"
+ } else {
+ "http://"
+ };
+ format!("{scheme}{bucket}.{host}/{encoded}")
+ }
+ (Some(endpoint), Some(bucket)) => format!("{endpoint}/{bucket}/{encoded}"),
+ _ => {
+ let region = settings.url_region();
+ let suffix = dns_suffix(
+ settings
+ .region_name
+ .as_ref()
+ .and_then(serde_json::Value::as_str),
+ );
+ format!(
+ "https://{}.s3.{region}.{suffix}/{encoded}",
+ settings.bucket_name.as_deref().unwrap_or("None")
+ )
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use rstest::rstest;
+
+ use super::*;
+
+ fn stamp() -> Stamp {
+ Stamp::from([2026, 11, 13, 9, 5, 7, 42])
+ }
+
+ #[rstest]
+ #[case("", "", Granularity::Day, "2026-11-13/time-09-05-07-000042_abc.json")]
+ #[case(
+ "logs/",
+ "team/",
+ Granularity::Hour,
+ "logs/team/2026-11-13/09/time-09-05-07-000042_abc.json"
+ )]
+ #[case(
+ "a/b",
+ "",
+ Granularity::Day,
+ "a/b/2026-11-13/time-09-05-07-000042_abc.json"
+ )]
+ fn keys_are_laid_out_as_python_lays_them_out(
+ #[case] path: &str,
+ #[case] prefix: &str,
+ #[case] granularity: Granularity,
+ #[case] expected: &str,
+ ) {
+ let name = logging_id(&stamp(), "abc");
+ assert_eq!(
+ object_key(path, prefix, &stamp(), &name, granularity),
+ expected
+ );
+ }
+
+ #[rstest]
+ fn a_long_id_is_bounded_with_its_digest() {
+ let name = logging_id(&stamp(), &"x".repeat(400));
+ let key = object_key("p", "", &stamp(), &name, Granularity::Day);
+ let file = key.rsplit('/').next().unwrap();
+ assert!(file.len() <= MAX_FILE_NAME_BYTES);
+ assert!(file.ends_with(&format!("_{}.json", hex_sha256(&name))));
+ }
+
+ #[rstest]
+ fn utc_stamps_split_the_clock() {
+ let time = UNIX_EPOCH + std::time::Duration::from_micros(1_763_024_707_000_042);
+ assert_eq!(Stamp::utc(time), Stamp::from([2025, 11, 13, 9, 5, 7, 42]));
+ }
+
+ #[rstest]
+ fn keys_are_quoted_like_urllib() {
+ assert_eq!(quote("a b/c=d+e~f.json"), "a%20b/c%3Dd%2Be~f.json");
+ assert_eq!(quote("é"), "%C3%A9");
+ }
+
+ #[rstest]
+ #[case(
+ "cn-north-1",
+ "https://audit-bucket.s3.cn-north-1.amazonaws.com.cn/2025-01-01/key.json"
+ )]
+ #[case(
+ "cn-northwest-1",
+ "https://audit-bucket.s3.cn-northwest-1.amazonaws.com.cn/2025-01-01/key.json"
+ )]
+ #[case(
+ "us-east-1",
+ "https://audit-bucket.s3.us-east-1.amazonaws.com/2025-01-01/key.json"
+ )]
+ #[case(
+ "us-gov-west-1",
+ "https://audit-bucket.s3.us-gov-west-1.amazonaws.com/2025-01-01/key.json"
+ )]
+ fn object_urls_follow_the_partition_of_the_region(
+ #[case] region: &str,
+ #[case] expected: &str,
+ ) {
+ let params =
+ serde_json::json!({"s3_bucket_name": "audit-bucket", "s3_region_name": region});
+ let settings = Settings::from_params(
+ params.as_object().unwrap(),
+ &crate::settings::Constants::default(),
+ );
+ assert_eq!(object_url(&settings, "2025-01-01/key.json"), expected);
+ }
+
+ #[rstest]
+ fn download_names_replace_slashes_and_quotes() {
+ assert_eq!(
+ download_name(&stamp(), "a/b\"c"),
+ "time-2026-11-13T09-05-07-000042_a_b_c.json"
+ );
+ }
+}
diff --git a/litellm/rust/crates/callbacks-s3-log/src/lib.rs b/litellm/rust/crates/callbacks-s3-log/src/lib.rs
new file mode 100644
index 00000000000..c8328c6ace6
--- /dev/null
+++ b/litellm/rust/crates/callbacks-s3-log/src/lib.rs
@@ -0,0 +1,11 @@
+mod key;
+mod limiter;
+mod pyjson;
+mod settings;
+mod sink;
+mod upload;
+
+pub use key::{Stamp, download_name, logging_id, object_key, object_url};
+pub use settings::{Constants, Environment, Granularity, Settings};
+pub use sink::{BatchUploadFailure, Host, NAME, S3Log, S3LogParts, system_clock};
+pub use upload::{Backoff, Clock, Outcome, panic_message, python_backoff};
diff --git a/litellm/rust/crates/callbacks-s3-log/src/limiter.rs b/litellm/rust/crates/callbacks-s3-log/src/limiter.rs
new file mode 100644
index 00000000000..c3b0db22647
--- /dev/null
+++ b/litellm/rust/crates/callbacks-s3-log/src/limiter.rs
@@ -0,0 +1,304 @@
+use std::{
+ collections::VecDeque,
+ sync::{Arc, Mutex, PoisonError},
+};
+
+use tokio::sync::{OwnedSemaphorePermit, Semaphore, oneshot};
+
+pub enum Limiter {
+ Fixed(Arc),
+ Adaptive(Arc),
+}
+
+pub enum Permit {
+ Fixed(#[allow(dead_code)] OwnedSemaphorePermit),
+ Adaptive(Arc),
+}
+
+impl Drop for Permit {
+ fn drop(&mut self) {
+ if let Self::Adaptive(limiter) = self {
+ limiter.release();
+ }
+ }
+}
+
+impl Limiter {
+ pub fn new(width: usize, adaptive: bool, ceiling: usize) -> Self {
+ if adaptive {
+ Self::Adaptive(Arc::new(Adaptive::new(width, width, width.max(ceiling))))
+ } else {
+ Self::Fixed(Arc::new(Semaphore::new(width.min(Semaphore::MAX_PERMITS))))
+ }
+ }
+
+ pub async fn acquire(&self) -> Permit {
+ match self {
+ Self::Fixed(semaphore) => Permit::Fixed(
+ Arc::clone(semaphore)
+ .acquire_owned()
+ .await
+ .unwrap_or_else(|_| unreachable!("the upload semaphore is never closed")),
+ ),
+ Self::Adaptive(limiter) => {
+ limiter.acquire().await;
+ Permit::Adaptive(Arc::clone(limiter))
+ }
+ }
+ }
+
+ pub fn record(&self, throttled: bool) {
+ if let Self::Adaptive(limiter) = self {
+ limiter.record(throttled);
+ }
+ }
+
+ #[cfg(test)]
+ pub fn limit(&self) -> Option {
+ match self {
+ Self::Fixed(_) => None,
+ Self::Adaptive(limiter) => Some(limiter.state().limit),
+ }
+ }
+}
+
+struct State {
+ limit: usize,
+ clean_streak: usize,
+ in_flight: usize,
+ waiters: VecDeque>,
+}
+
+pub struct Adaptive {
+ floor: usize,
+ ceiling: usize,
+ state: Mutex,
+}
+
+impl Adaptive {
+ fn new(initial: usize, floor: usize, ceiling: usize) -> Self {
+ Self {
+ floor,
+ ceiling,
+ state: Mutex::new(State {
+ limit: initial.max(floor).min(ceiling),
+ clean_streak: 0,
+ in_flight: 0,
+ waiters: VecDeque::new(),
+ }),
+ }
+ }
+
+ fn state(&self) -> std::sync::MutexGuard<'_, State> {
+ self.state.lock().unwrap_or_else(PoisonError::into_inner)
+ }
+
+ async fn acquire(&self) {
+ let waiter = {
+ let mut state = self.state();
+ if state.in_flight < state.limit {
+ state.in_flight += 1;
+ return;
+ }
+ let (sender, receiver) = oneshot::channel();
+ state.waiters.push_back(sender);
+ receiver
+ };
+ let _ = waiter.await;
+ }
+
+ fn grant(state: &mut State) {
+ while state.in_flight < state.limit {
+ let Some(waiter) = state.waiters.pop_front() else {
+ return;
+ };
+ if waiter.send(()).is_ok() {
+ state.in_flight += 1;
+ }
+ }
+ }
+
+ fn release(&self) {
+ let mut state = self.state();
+ state.in_flight = state.in_flight.saturating_sub(1);
+ Self::grant(&mut state);
+ }
+
+ fn record(&self, throttled: bool) {
+ let mut state = self.state();
+ if throttled {
+ state.limit = self.floor.max(state.limit / 2);
+ state.clean_streak = 0;
+ return;
+ }
+ state.clean_streak += 1;
+ if state.clean_streak >= state.limit && state.limit < self.ceiling {
+ state.limit += 1;
+ state.clean_streak = 0;
+ Self::grant(&mut state);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use rstest::rstest;
+
+ use super::*;
+
+ #[rstest]
+ fn the_limit_grows_after_a_clean_streak_and_halves_on_a_throttle() {
+ let limiter = Limiter::new(2, true, 8);
+ for _ in 0..2 {
+ limiter.record(false);
+ }
+ assert_eq!(limiter.limit(), Some(3));
+ for _ in 0..3 {
+ limiter.record(false);
+ }
+ assert_eq!(limiter.limit(), Some(4));
+ limiter.record(true);
+ assert_eq!(limiter.limit(), Some(2));
+ limiter.record(true);
+ assert_eq!(limiter.limit(), Some(2));
+ }
+
+ #[rstest]
+ #[tokio::test]
+ async fn a_bound_past_the_semaphore_maximum_acts_as_no_bound() {
+ let limiter = Limiter::new(usize::MAX, false, 0);
+ let _first = limiter.acquire().await;
+ let _second = limiter.acquire().await;
+ }
+
+ #[rstest]
+ #[tokio::test]
+ async fn a_full_limit_admits_the_next_waiter_when_a_permit_drops() {
+ let limiter = Arc::new(Limiter::new(1, true, 1));
+ let first = limiter.acquire().await;
+ let waiting = {
+ let limiter = Arc::clone(&limiter);
+ tokio::spawn(async move {
+ let _permit = limiter.acquire().await;
+ })
+ };
+ tokio::task::yield_now().await;
+ assert!(!waiting.is_finished());
+ drop(first);
+ waiting.await.unwrap();
+ }
+
+ fn adaptive(initial: usize, floor: usize, ceiling: usize) -> Arc {
+ Arc::new(Limiter::Adaptive(Arc::new(Adaptive::new(
+ initial, floor, ceiling,
+ ))))
+ }
+
+ fn in_flight(limiter: &Limiter) -> usize {
+ match limiter {
+ Limiter::Fixed(_) => 0,
+ Limiter::Adaptive(adaptive) => adaptive.state().in_flight,
+ }
+ }
+
+ fn record(limiter: &Limiter, throttled: bool, times: usize) {
+ for _ in 0..times {
+ limiter.record(throttled);
+ }
+ }
+
+ #[rstest]
+ #[case(4, 1, 16, &[(false, 4)], 5)]
+ #[case(4, 1, 16, &[(false, 3)], 4)]
+ #[case(16, 1, 16, &[(true, 1)], 8)]
+ #[case(4, 4, 16, &[(true, 1)], 4)]
+ #[case(15, 1, 16, &[(false, 1000)], 16)]
+ #[case(4, 1, 32, &[(false, 3), (true, 1), (false, 1)], 2)]
+ fn the_limit_follows_the_samples(
+ #[case] initial: usize,
+ #[case] floor: usize,
+ #[case] ceiling: usize,
+ #[case] samples: &[(bool, usize)],
+ #[case] expected: usize,
+ ) {
+ let limiter = adaptive(initial, floor, ceiling);
+ for &(throttled, times) in samples {
+ record(&limiter, throttled, times);
+ }
+ assert_eq!(limiter.limit(), Some(expected));
+ }
+
+ #[rstest]
+ #[tokio::test]
+ async fn growing_the_limit_wakes_a_waiting_acquirer() {
+ let limiter = adaptive(1, 1, 4);
+ let holder = limiter.acquire().await;
+ let waiting = {
+ let limiter = Arc::clone(&limiter);
+ tokio::spawn(async move {
+ let _permit = limiter.acquire().await;
+ })
+ };
+ tokio::task::yield_now().await;
+ assert!(!waiting.is_finished());
+ limiter.record(false);
+ waiting.await.unwrap();
+ drop(holder);
+ assert_eq!(in_flight(&limiter), 0);
+ }
+
+ #[rstest]
+ #[tokio::test]
+ async fn releasing_a_slot_wakes_exactly_one_waiter() {
+ let limiter = adaptive(1, 1, 4);
+ let holder = limiter.acquire().await;
+ let acquired = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let (release, released) = tokio::sync::watch::channel(false);
+ let waiters: Vec<_> = (0..3)
+ .map(|_| {
+ let limiter = Arc::clone(&limiter);
+ let acquired = Arc::clone(&acquired);
+ let mut released = released.clone();
+ tokio::spawn(async move {
+ let _permit = limiter.acquire().await;
+ acquired.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ let _ = released.wait_for(|released| *released).await;
+ })
+ })
+ .collect();
+ tokio::task::yield_now().await;
+ drop(holder);
+ tokio::task::yield_now().await;
+ assert_eq!(acquired.load(std::sync::atomic::Ordering::SeqCst), 1);
+ for _ in 0..3 {
+ limiter.record(false);
+ tokio::task::yield_now().await;
+ }
+ assert_eq!(acquired.load(std::sync::atomic::Ordering::SeqCst), 3);
+ release.send(true).unwrap();
+ for waiter in waiters {
+ waiter.await.unwrap();
+ }
+ assert_eq!(in_flight(&limiter), 0);
+ }
+
+ #[rstest]
+ #[tokio::test]
+ async fn a_cancelled_waiter_is_skipped_and_leaves_in_flight_at_zero() {
+ let limiter = adaptive(1, 1, 1);
+ let holder = limiter.acquire().await;
+ let spawn_waiter = |limiter: Arc| {
+ tokio::spawn(async move {
+ let _permit = limiter.acquire().await;
+ })
+ };
+ let doomed = spawn_waiter(Arc::clone(&limiter));
+ let next = spawn_waiter(Arc::clone(&limiter));
+ tokio::task::yield_now().await;
+ doomed.abort();
+ assert!(doomed.await.unwrap_err().is_cancelled());
+ drop(holder);
+ next.await.unwrap();
+ assert_eq!(in_flight(&limiter), 0);
+ }
+}
diff --git a/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs b/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs
new file mode 100644
index 00000000000..c2f442c9af9
--- /dev/null
+++ b/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs
@@ -0,0 +1,214 @@
+use serde_json::Value;
+
+const HEX: &[u8; 16] = b"0123456789abcdef";
+
+pub fn write_str(out: &mut Vec, text: &str) {
+ out.push(b'"');
+ let mut units = [0u16; 2];
+ for character in text.chars() {
+ match character {
+ '"' => out.extend_from_slice(b"\\\""),
+ '\\' => out.extend_from_slice(b"\\\\"),
+ '\n' => out.extend_from_slice(b"\\n"),
+ '\r' => out.extend_from_slice(b"\\r"),
+ '\t' => out.extend_from_slice(b"\\t"),
+ '\u{8}' => out.extend_from_slice(b"\\b"),
+ '\u{c}' => out.extend_from_slice(b"\\f"),
+ ' '..='~' => out.push(character as u8),
+ _ => {
+ for unit in character.encode_utf16(&mut units) {
+ write_unit(out, *unit);
+ }
+ }
+ }
+ }
+ out.push(b'"');
+}
+
+fn write_unit(out: &mut Vec, unit: u16) {
+ out.extend_from_slice(b"\\u");
+ for shift in [12, 8, 4, 0] {
+ out.push(HEX[usize::from((unit >> shift) & 0xf)]);
+ }
+}
+
+pub fn float_repr(value: f64) -> String {
+ if value.is_nan() {
+ return "NaN".to_owned();
+ }
+ if value.is_infinite() {
+ return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned();
+ }
+ let shortest = format!("{value:e}");
+ let length = shortest
+ .split('e')
+ .next()
+ .unwrap_or_default()
+ .chars()
+ .filter(char::is_ascii_digit)
+ .count();
+ let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1));
+ let (negative, scientific) = match scientific.strip_prefix('-') {
+ Some(rest) => (true, rest),
+ None => (false, scientific.as_str()),
+ };
+ let (mantissa, exponent) = scientific.split_once('e').unwrap_or((scientific, "0"));
+ let exponent: i32 = exponent.parse().unwrap_or(0);
+ let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect();
+ let mut text = String::with_capacity(digits.len() + 8);
+ if negative {
+ text.push('-');
+ }
+ let point = exponent + 1;
+ if (-3..=16).contains(&point) {
+ let count = digits.len() as i32;
+ if point <= 0 {
+ text.push_str("0.");
+ text.extend(std::iter::repeat_n('0', (-point) as usize));
+ text.push_str(&digits);
+ } else if point >= count {
+ text.push_str(&digits);
+ text.extend(std::iter::repeat_n('0', (point - count) as usize));
+ text.push_str(".0");
+ } else {
+ let (whole, fraction) = digits.split_at(point as usize);
+ text.push_str(whole);
+ text.push('.');
+ text.push_str(fraction);
+ }
+ return text;
+ }
+ let (first, rest) = digits.split_at(1);
+ text.push_str(first);
+ if !rest.is_empty() {
+ text.push('.');
+ text.push_str(rest);
+ }
+ text.push('e');
+ text.push(if exponent < 0 { '-' } else { '+' });
+ text.push_str(&format!("{:02}", exponent.unsigned_abs()));
+ text
+}
+
+pub fn write_value(out: &mut Vec, value: &Value) {
+ match value {
+ Value::Null => out.extend_from_slice(b"null"),
+ Value::Bool(true) => out.extend_from_slice(b"true"),
+ Value::Bool(false) => out.extend_from_slice(b"false"),
+ Value::Number(number) => match (number.as_i64(), number.as_u64(), number.as_f64()) {
+ (Some(integer), _, _) => out.extend_from_slice(integer.to_string().as_bytes()),
+ (None, Some(integer), _) => out.extend_from_slice(integer.to_string().as_bytes()),
+ (None, None, Some(float)) => out.extend_from_slice(float_repr(float).as_bytes()),
+ (None, None, None) => out.extend_from_slice(number.to_string().as_bytes()),
+ },
+ Value::String(text) => write_str(out, text),
+ Value::Array(items) => {
+ out.push(b'[');
+ for (index, item) in items.iter().enumerate() {
+ if index > 0 {
+ out.extend_from_slice(b", ");
+ }
+ write_value(out, item);
+ }
+ out.push(b']');
+ }
+ Value::Object(entries) => {
+ out.push(b'{');
+ for (index, (key, item)) in entries.iter().enumerate() {
+ if index > 0 {
+ out.extend_from_slice(b", ");
+ }
+ write_str(out, key);
+ out.extend_from_slice(b": ");
+ write_value(out, item);
+ }
+ out.push(b'}');
+ }
+ }
+}
+
+pub fn dumps(value: &Value) -> String {
+ let mut out = Vec::new();
+ write_value(&mut out, value);
+ String::from_utf8(out).unwrap_or_default()
+}
+
+pub fn py_str(value: &Value) -> String {
+ match value {
+ Value::Null => "None".to_owned(),
+ Value::Bool(true) => "True".to_owned(),
+ Value::Bool(false) => "False".to_owned(),
+ Value::String(text) => text.clone(),
+ Value::Number(number) => match number.as_f64() {
+ Some(float) if number.is_f64() => float_repr(float),
+ _ => number.to_string(),
+ },
+ Value::Array(_) | Value::Object(_) => dumps(value),
+ }
+}
+
+pub fn truthy(value: &Value) -> bool {
+ match value {
+ Value::Null => false,
+ Value::Bool(flag) => *flag,
+ Value::Number(number) => number.as_f64().is_some_and(|float| float != 0.0),
+ Value::String(text) => !text.is_empty(),
+ Value::Array(items) => !items.is_empty(),
+ Value::Object(entries) => !entries.is_empty(),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use rstest::rstest;
+ use serde_json::json;
+
+ use super::*;
+
+ #[rstest]
+ #[case(0.0, "0.0")]
+ #[case(-0.0, "-0.0")]
+ #[case(1.0, "1.0")]
+ #[case(0.1, "0.1")]
+ #[case(1.5e-5, "1.5e-05")]
+ #[case(1e-4, "0.0001")]
+ #[case(1e-5, "1e-05")]
+ #[case(1e15, "1000000000000000.0")]
+ #[case(1e16, "1e+16")]
+ #[case(1.2345678901234568e17, "1.2345678901234568e+17")]
+ #[case(123.456, "123.456")]
+ #[case(1e100, "1e+100")]
+ #[case(-2.5e-300, "-2.5e-300")]
+ #[case(f64::NAN, "NaN")]
+ #[case(f64::INFINITY, "Infinity")]
+ #[case(f64::NEG_INFINITY, "-Infinity")]
+ #[case(0.30000000000000004, "0.30000000000000004")]
+ #[case(0.07872390747070312, "0.07872390747070312")]
+ #[case(1794700894.4414062, "1794700894.4414062")]
+ fn floats_print_as_python_repr_prints_them(#[case] value: f64, #[case] expected: &str) {
+ assert_eq!(float_repr(value), expected);
+ }
+
+ #[rstest]
+ #[case("plain", "\"plain\"")]
+ #[case("quote\" back\\", "\"quote\\\" back\\\\\"")]
+ #[case("\n\r\t\u{8}\u{c}\u{1}\u{7f}", "\"\\n\\r\\t\\b\\f\\u0001\\u007f\"")]
+ #[case("café", "\"caf\\u00e9\"")]
+ #[case("😀", "\"\\ud83d\\ude00\"")]
+ fn strings_are_escaped_as_ensure_ascii_escapes_them(
+ #[case] text: &str,
+ #[case] expected: &str,
+ ) {
+ let mut out = Vec::new();
+ write_str(&mut out, text);
+ assert_eq!(String::from_utf8(out).unwrap(), expected);
+ }
+
+ #[rstest]
+ fn containers_use_the_default_separators_and_keep_key_order() {
+ assert_eq!(
+ dumps(&json!({"b": [1, 2.5, null, true], "a": {"x": "y"}})),
+ "{\"b\": [1, 2.5, null, true], \"a\": {\"x\": \"y\"}}"
+ );
+ }
+}
diff --git a/litellm/rust/crates/callbacks-s3-log/src/settings.rs b/litellm/rust/crates/callbacks-s3-log/src/settings.rs
new file mode 100644
index 00000000000..c266f612c94
--- /dev/null
+++ b/litellm/rust/crates/callbacks-s3-log/src/settings.rs
@@ -0,0 +1,656 @@
+use std::time::Duration;
+
+use litellm_auth_aws::AwsAuthConfig;
+use litellm_secret_value::SecretValue;
+use serde::Deserialize;
+use serde_json::{Map, Value};
+
+use crate::pyjson::{py_str, truthy};
+
+fn py_repr(value: &Value) -> String {
+ litellm_python_compat::repr::repr(&litellm_python_compat::json::from_json(value.clone()))
+}
+
+fn py_optional(value: Option) -> String {
+ value.map_or_else(|| "None".to_owned(), |value| value.to_string())
+}
+
+fn py_type_name(value: &Value) -> &'static str {
+ match value {
+ Value::Null => "NoneType",
+ Value::Bool(_) => "bool",
+ Value::Number(number) if number.is_f64() => "float",
+ Value::Number(_) => "int",
+ Value::String(_) => "str",
+ Value::Array(_) => "list",
+ Value::Object(_) => "dict",
+ }
+}
+
+pub const PARTITION_GRANULARITY_ENV_VAR: &str = "S3_PARTITION_GRANULARITY";
+pub const LOG_PROMPTS_ONLY_ENV_VAR: &str = "S3_LOG_PROMPTS_ONLY";
+const DEFAULT_MAX_QUEUE_SIZE: usize = 50_000;
+const DEFAULT_MAX_RETRY_AGE_SECONDS: u64 = 3600;
+const DEFAULT_REGION: &str = "us-west-2";
+
+#[derive(Clone, Debug, PartialEq, Deserialize)]
+pub struct Constants {
+ pub batch_size: usize,
+ pub max_concurrent_uploads: usize,
+ pub max_adaptive_concurrency: usize,
+ pub upload_attempt_timeout_seconds: f64,
+ pub download_read_timeout_seconds: f64,
+ pub download_connect_timeout_seconds: f64,
+ pub web_identity_session_policy: Option,
+}
+
+impl Default for Constants {
+ fn default() -> Self {
+ Self {
+ batch_size: 512,
+ max_concurrent_uploads: 16,
+ max_adaptive_concurrency: 200,
+ upload_attempt_timeout_seconds: 30.0,
+ download_read_timeout_seconds: 600.0,
+ download_connect_timeout_seconds: 5.0,
+ web_identity_session_policy: None,
+ }
+ }
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum Granularity {
+ #[default]
+ Day,
+ Hour,
+}
+
+impl Granularity {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::Day => "day",
+ Self::Hour => "hour",
+ }
+ }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct Settings {
+ pub bucket_name: Option,
+ pub region_name: Option,
+ pub endpoint_url: Option,
+ pub path: Option,
+ pub verify: Option,
+ pub auth: AwsAuthConfig,
+ pub use_team_prefix: bool,
+ pub use_key_prefix: bool,
+ pub strip_base64_files: bool,
+ pub use_virtual_hosted_style: bool,
+ pub log_prompts_only: Option,
+ pub partition_granularity: Option,
+ pub server_side_encryption: Option,
+ pub sse_kms_key_id: Option,
+ pub max_concurrent_uploads: usize,
+ pub max_queue_size: usize,
+ pub max_retry_age: Option,
+ pub drop_on_terminal_error: bool,
+ pub adaptive_concurrency: bool,
+ pub max_adaptive_concurrency: usize,
+ pub batch_file_upload: bool,
+ pub batch_size: usize,
+ pub attempt_timeout: Duration,
+ pub download_read_timeout: Duration,
+ pub web_identity_session_policy: Option,
+}
+
+pub type Environment = dyn Fn(&str) -> Option + Send + Sync;
+
+fn text(params: &Map, name: &str) -> Option {
+ params.get(name).filter(|value| truthy(value)).map(py_str)
+}
+
+fn secret(params: &Map, name: &str) -> Option {
+ text(params, name).map(SecretValue::new)
+}
+
+fn present(params: &Map, name: &str) -> Option {
+ params.get(name).filter(|value| !value.is_null()).cloned()
+}
+
+fn is_blank(value: &Value) -> bool {
+ matches!(value, Value::String(text) if text.is_empty())
+}
+
+pub fn lax_bool(value: &Value) -> Option {
+ match value {
+ Value::Bool(flag) => Some(*flag),
+ Value::Number(number) => number
+ .as_f64()
+ .and_then(|float| {
+ [(0.0, false), (1.0, true)]
+ .into_iter()
+ .find(|(bound, _)| *bound == float)
+ })
+ .map(|(_, flag)| flag),
+ Value::String(text) => match text.trim().to_ascii_lowercase().as_str() {
+ "0" | "off" | "f" | "false" | "n" | "no" => Some(false),
+ "1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
+ _ => None,
+ },
+ _ => None,
+ }
+}
+
+pub fn lax_int(value: &Value) -> Option {
+ match value {
+ Value::Bool(flag) => Some(i128::from(*flag)),
+ Value::Number(number) => {
+ if let Some(integer) = number.as_i64() {
+ return Some(i128::from(integer));
+ }
+ if let Some(integer) = number.as_u64() {
+ return Some(i128::from(integer));
+ }
+ let float = number.as_f64()?;
+ (float.is_finite() && float.fract() == 0.0 && float.abs() <= i64::MAX as f64)
+ .then_some(float as i128)
+ }
+ Value::String(text) => int_text(text.trim()),
+ _ => None,
+ }
+}
+
+fn int_text(text: &str) -> Option {
+ let (whole, fraction) = text.split_once('.').unwrap_or((text, ""));
+ if !fraction.chars().all(|character| character == '0') {
+ return None;
+ }
+ if text.contains('.') && fraction.is_empty() {
+ return None;
+ }
+ let (sign, digits) = match whole.strip_prefix('-') {
+ Some(rest) => (-1, rest),
+ None => (1, whole.strip_prefix('+').unwrap_or(whole)),
+ };
+ let well_formed = !digits.is_empty()
+ && !digits.starts_with('_')
+ && !digits.ends_with('_')
+ && !digits.contains("__")
+ && digits
+ .chars()
+ .all(|character| character.is_ascii_digit() || character == '_');
+ if !well_formed {
+ return None;
+ }
+ digits
+ .replace('_', "")
+ .parse::()
+ .ok()
+ .map(|number| sign * number)
+}
+
+fn strip_text(value: &Value) -> Value {
+ match value {
+ Value::String(text) => Value::String(text.trim().to_owned()),
+ other => other.clone(),
+ }
+}
+
+fn positive_int(
+ setting: &str,
+ configured: Option<&Value>,
+ fallback: usize,
+ reject_bool: bool,
+) -> usize {
+ let Some(configured) = configured.filter(|value| !value.is_null() && !is_blank(value)) else {
+ return fallback;
+ };
+ if reject_bool && configured.is_boolean() {
+ tracing::warn!(
+ "s3 logging: {setting}={} is a boolean, not an integer, using {fallback}",
+ py_repr(configured)
+ );
+ return fallback;
+ }
+ match lax_int(&strip_text(configured)) {
+ None => {
+ tracing::warn!(
+ "s3 logging: {setting}={} is not an integer, using {fallback}",
+ py_repr(configured)
+ );
+ fallback
+ }
+ Some(bound) if bound < 1 => {
+ tracing::warn!(
+ "s3 logging: {setting}={} must be at least 1, using {fallback}",
+ py_repr(configured)
+ );
+ fallback
+ }
+ Some(bound) => usize::try_from(bound).unwrap_or(usize::MAX),
+ }
+}
+
+fn retry_age(configured: &Value, fallback: Option) -> Option {
+ if configured.is_null() || is_blank(configured) {
+ return None;
+ }
+ if configured.is_boolean() {
+ tracing::warn!(
+ "s3 logging: s3_max_retry_age_seconds={} is a boolean, not an integer, falling back to {}",
+ py_repr(configured),
+ py_optional(fallback)
+ );
+ return fallback;
+ }
+ match lax_int(&strip_text(configured)) {
+ None => {
+ tracing::warn!(
+ "s3 logging: s3_max_retry_age_seconds={} is not an integer, falling back to {}",
+ py_repr(configured),
+ py_optional(fallback)
+ );
+ fallback
+ }
+ Some(bound) if bound < 0 => {
+ tracing::warn!(
+ "s3 logging: s3_max_retry_age_seconds={} must be at least 0, falling back to {}",
+ py_repr(configured),
+ py_optional(fallback)
+ );
+ fallback
+ }
+ Some(0) => None,
+ Some(bound) => Some(u64::try_from(bound).unwrap_or(u64::MAX)),
+ }
+}
+
+fn flag(
+ setting: &str,
+ configured: Option<&Value>,
+ unset: bool,
+ invalid: bool,
+ consequence: &str,
+) -> bool {
+ let Some(configured) = configured.filter(|value| !value.is_null() && !is_blank(value)) else {
+ return unset;
+ };
+ lax_bool(&strip_text(configured)).unwrap_or_else(|| {
+ tracing::warn!(
+ "s3 logging: {setting}={} is not a boolean, {consequence}",
+ py_repr(configured)
+ );
+ invalid
+ })
+}
+
+fn sse_value(name: &str, value: Option<&Value>) -> Option {
+ match value {
+ None => None,
+ Some(Value::String(text)) => Some(text.clone()),
+ Some(other) => {
+ tracing::warn!(
+ "s3 logging: ignoring {name} because it has invalid type {}; expected a string",
+ py_type_name(other)
+ );
+ None
+ }
+ }
+}
+
+pub fn resolve_sse(
+ algorithm: Option<&Value>,
+ key_id: Option<&Value>,
+) -> (Option, Option) {
+ let valid_sse = sse_value("s3_server_side_encryption", algorithm);
+ let valid_key_id = sse_value("s3_sse_kms_key_id", key_id).filter(|key| !key.is_empty());
+ let Some(algorithm) = valid_sse
+ .filter(|algorithm| !algorithm.is_empty())
+ .or_else(|| valid_key_id.as_ref().map(|_| "aws:kms".to_owned()))
+ else {
+ return (None, None);
+ };
+ if valid_key_id.is_some() && !algorithm.starts_with("aws:kms") {
+ tracing::warn!(
+ "s3 logging: ignoring s3_sse_kms_key_id because s3_server_side_encryption is {algorithm}; set it to aws:kms to encrypt with the KMS key"
+ );
+ return (Some(algorithm), None);
+ }
+ (Some(algorithm), valid_key_id)
+}
+
+pub fn resolve_partition_granularity(raw: Option<&Value>) -> Granularity {
+ match raw {
+ Some(Value::String(text)) if text == "hour" => Granularity::Hour,
+ None => Granularity::Day,
+ Some(Value::String(text)) if text.is_empty() || text == "day" => Granularity::Day,
+ Some(other) => {
+ tracing::warn!(
+ "s3 logging: s3_partition_granularity={} is not one of day, hour, using day",
+ py_repr(other)
+ );
+ Granularity::Day
+ }
+ }
+}
+
+impl Settings {
+ pub fn from_params(params: &Map, constants: &Constants) -> Self {
+ let truthy_param = |name: &str| params.get(name).is_some_and(truthy);
+ let (server_side_encryption, sse_kms_key_id) = resolve_sse(
+ params
+ .get("s3_server_side_encryption")
+ .filter(|value| truthy(value)),
+ params
+ .get("s3_sse_kms_key_id")
+ .filter(|value| truthy(value)),
+ );
+ let max_concurrent_uploads = positive_int(
+ "s3_max_concurrent_uploads",
+ Some(
+ params
+ .get("s3_max_concurrent_uploads")
+ .filter(|value| !value.is_null() && !is_blank(value))
+ .cloned()
+ .unwrap_or_else(|| Value::from(constants.max_concurrent_uploads)),
+ )
+ .as_ref(),
+ constants.max_concurrent_uploads,
+ false,
+ );
+ let max_queue_size = positive_int(
+ "s3_max_queue_size",
+ params.get("s3_max_queue_size"),
+ DEFAULT_MAX_QUEUE_SIZE,
+ true,
+ );
+ let max_retry_age = match params.get("s3_max_retry_age_seconds") {
+ Some(value) if !value.is_null() && !is_blank(value) => {
+ retry_age(value, Some(DEFAULT_MAX_RETRY_AGE_SECONDS))
+ }
+ _ => Some(DEFAULT_MAX_RETRY_AGE_SECONDS),
+ };
+ Self {
+ bucket_name: text(params, "s3_bucket_name"),
+ region_name: params
+ .get("s3_region_name")
+ .filter(|value| truthy(value))
+ .cloned(),
+ endpoint_url: text(params, "s3_endpoint_url"),
+ path: text(params, "s3_path"),
+ verify: present(params, "s3_verify"),
+ auth: AwsAuthConfig {
+ access_key_id: secret(params, "s3_aws_access_key_id"),
+ secret_access_key: secret(params, "s3_aws_secret_access_key"),
+ session_token: secret(params, "s3_aws_session_token"),
+ region_name: text(params, "s3_region_name"),
+ session_name: text(params, "s3_aws_session_name"),
+ profile_name: text(params, "s3_aws_profile_name"),
+ role_name: text(params, "s3_aws_role_name"),
+ web_identity_token: secret(params, "s3_aws_web_identity_token"),
+ sts_endpoint: text(params, "s3_aws_sts_endpoint"),
+ external_id: None,
+ },
+ use_team_prefix: truthy_param("s3_use_team_prefix"),
+ use_key_prefix: truthy_param("s3_use_key_prefix"),
+ strip_base64_files: truthy_param("s3_strip_base64_files"),
+ use_virtual_hosted_style: truthy_param("s3_use_virtual_hosted_style"),
+ log_prompts_only: present(params, "s3_log_prompts_only"),
+ partition_granularity: present(params, "s3_partition_granularity"),
+ server_side_encryption,
+ sse_kms_key_id,
+ max_concurrent_uploads,
+ max_queue_size,
+ max_retry_age: max_retry_age.map(Duration::from_secs),
+ drop_on_terminal_error: flag(
+ "s3_drop_on_terminal_error",
+ params.get("s3_drop_on_terminal_error"),
+ true,
+ true,
+ "dropping terminal-failed uploads",
+ ),
+ adaptive_concurrency: flag(
+ "s3_adaptive_concurrency",
+ params.get("s3_adaptive_concurrency"),
+ false,
+ false,
+ "keeping the fixed upload width",
+ ),
+ max_adaptive_concurrency: positive_int(
+ "s3_max_adaptive_concurrency",
+ params.get("s3_max_adaptive_concurrency"),
+ constants.max_adaptive_concurrency,
+ true,
+ ),
+ batch_file_upload: flag(
+ "s3_batch_file_upload",
+ params.get("s3_batch_file_upload"),
+ false,
+ false,
+ "keeping per-request objects",
+ ),
+ batch_size: constants.batch_size,
+ attempt_timeout: Duration::from_secs_f64(
+ constants.upload_attempt_timeout_seconds.max(0.0),
+ ),
+ download_read_timeout: Duration::from_secs_f64(
+ constants.download_read_timeout_seconds.max(0.0),
+ ),
+ web_identity_session_policy: constants.web_identity_session_policy.clone(),
+ }
+ }
+
+ pub fn partition_granularity(&self, environment: &Environment) -> Granularity {
+ resolve_partition_granularity(self.partition_granularity_raw(environment).as_ref())
+ }
+
+ pub fn partition_granularity_raw(&self, environment: &Environment) -> Option {
+ match &self.partition_granularity {
+ Some(value) => Some(value.clone()),
+ None => environment(PARTITION_GRANULARITY_ENV_VAR).map(Value::String),
+ }
+ }
+
+ pub fn log_prompts_only(&self, environment: &Environment) -> bool {
+ let raw = match &self.log_prompts_only {
+ Some(value) => Some(value.clone()),
+ None => environment(LOG_PROMPTS_ONLY_ENV_VAR).map(Value::String),
+ };
+ let Some(raw) = raw.filter(|value| !is_blank(value)) else {
+ return false;
+ };
+ lax_bool(&strip_text(&raw)).unwrap_or_else(|| {
+ tracing::warn!(
+ "s3 logging: s3_log_prompts_only={} is not a boolean, logging prompts only",
+ py_repr(&raw)
+ );
+ true
+ })
+ }
+
+ pub fn signing_region(
+ &self,
+ from_environment: impl FnOnce() -> Result