feat(s3-log): replace the python s3_v2 logger with a rust callback (#51539)

The s3_v2 S3Logger is ported to a Rust callback, callbacks-s3-log, behind
the CustomLoggerV2 contract, with the bridge constructor NativeS3Log and the
Python glue in litellm/integrations/s3_v3.py. It is now the only
implementation: the Python logger is deleted, s3_v2.py re-exports the
Rust-backed S3Logger, and LITELLM_NATIVE_S3_V2 is removed. The callback keeps
its name, list position and settings, so no config changes.

The original s3_v2 tests are replicated against the Rust callback as
<original>_rust.py, and Python's outputs for the differential suite are
recorded in tests/unit/integrations/s3_v2_golden/python_s3_v2.json.
This commit is contained in:
Nate Armstrong 2026-11-14 15:03:58 -08:00 • committed by GitHub
parent 74fd46a59f
commit 8e92894ff2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
55 changed files with 15926 additions and 7622 deletions

View file

@ -7,7 +7,7 @@ from collections import Counter
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass
from itertools import accumulate, chain, count
from typing import Final, Literal, TypeAlias
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
import litellm
from litellm._logging import verbose_logger
@ -23,6 +23,9 @@ from litellm.types.utils import (
StandardBatchLineItemUsage,
)
if TYPE_CHECKING:
from litellm.integrations.s3_v3 import S3Logger
BatchFileProvider: TypeAlias = Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"]
_VERTEX_AI_UNKNOWN_CUSTOM_ID: Final = "unknown" # what a Vertex AI batch row that carries no custom_id is read as
@ -238,9 +241,11 @@ async def _deliver(
)
def _resolve_logger(callback: litellm.CALLBACK_TYPES) -> CustomLogger | None:
def _resolve_logger(callback: "litellm.CALLBACK_TYPES | S3Logger") -> "CustomLogger | S3Logger | None":
from litellm.integrations.s3_v3 import S3Logger
match callback:
case CustomLogger():
case CustomLogger() | S3Logger():
return callback
case "s3_v2" | "gcs_bucket":
return init_custom_logger_compatible_class(callback, internal_usage_cache=None, llm_router=None)
@ -248,7 +253,7 @@ def _resolve_logger(callback: litellm.CALLBACK_TYPES) -> CustomLogger | None:
return None
def _logger_turned_off_message_logging(logger: CustomLogger) -> bool:
def _logger_turned_off_message_logging(logger: object) -> bool:
return (
getattr(logger, "turn_off_message_logging", False) is True
or getattr(logger, "message_logging", True) is not True

View file

@ -130,7 +130,6 @@ S3_PREFIX_DIGEST_CHARS: Final = 16
# s3 allows 2048 bytes of combined metadata headers, which Content-Disposition counts against
MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY"
S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
MAX_FILE_LIST_LIMIT: Final = 10000
ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)

View file

@ -4,6 +4,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.constants import X_LITELLM_DISABLE_CALLBACKS
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
from litellm.litellm_core_utils.llm_request_utils import (
get_proxy_server_request_headers,
)
@ -45,7 +46,7 @@ class EnterpriseCallbackControls:
if callback.lower() in disabled_callbacks:
verbose_logger.debug(f"Not logging to {callback} because it is disabled via {X_LITELLM_DISABLE_CALLBACKS}")
return True
elif isinstance(callback, CustomLogger):
elif isinstance(callback, CustomLogger | CustomLoggerV2):
# get the string name of the callback
callback_str = CustomLoggerRegistry.get_callback_str_from_class_type(callback.__class__)
if callback_str is not None and callback_str.lower() in disabled_callbacks:

View file

@ -1,78 +0,0 @@
"""
Adaptive in-flight concurrency limiter (AIMD, Vector ARC style).
Grows the limit additively after `limit` consecutive clean completions and
halves it only on an explicit throttle signal (429, 503, SlowDown, or a
transport error out of the PUT). With floor == ceiling it degenerates to a
fixed-width semaphore.
"""
import asyncio
from collections import deque
from contextlib import suppress
from dataclasses import dataclass
from typing import Final
@dataclass(frozen=True, slots=True)
class PutSample:
throttled: bool
class AdaptiveConcurrencyLimiter:
"""AIMD in-flight limiter used as `async with limiter:`."""
def __init__(self, initial: int, floor: int, ceiling: int) -> None:
if not 1 <= floor <= ceiling:
raise ValueError(f"adaptive limiter bounds must satisfy 1 <= floor <= ceiling, got {floor}..{ceiling}")
self._limit: int = min(max(initial, floor), ceiling)
self._floor: Final[int] = floor
self._ceiling: Final[int] = ceiling
self._clean_streak: int = 0
self._in_flight: int = 0
self._waiters: deque[asyncio.Future[None]] = deque() # mutable-ok: waiters queue up behind a full limit
@property
def limit(self) -> int:
return self._limit
async def __aenter__(self) -> "AdaptiveConcurrencyLimiter":
if self._in_flight < self._limit:
self._in_flight += 1
return self
waiter: Final = asyncio.get_running_loop().create_future()
self._waiters.append(waiter)
try:
await waiter
except asyncio.CancelledError:
if waiter.done() and not waiter.cancelled():
self._in_flight -= 1
self._grant()
else:
with suppress(ValueError):
self._waiters.remove(waiter)
raise
return self
def _grant(self) -> None:
while self._in_flight < self._limit and self._waiters:
waiter = self._waiters.popleft()
if waiter.done():
continue
self._in_flight += 1
waiter.set_result(None)
async def __aexit__(self, *_: object) -> None:
self._in_flight -= 1
self._grant()
def record(self, sample: PutSample) -> None:
if sample.throttled:
self._limit = max(self._floor, self._limit // 2)
self._clean_streak = 0
return
self._clean_streak += 1
if self._clean_streak >= self._limit and self._limit < self._ceiling:
self._limit += 1
self._clean_streak = 0
self._grant()

View file

@ -17,7 +17,6 @@ from litellm.constants import (
MAX_S3_OBJECT_KEY_FILENAME_BYTES,
S3_BOUNDED_OBJECT_KEY_HEAD_BYTES,
S3_LOG_PROMPTS_ONLY_ENV_VAR,
S3_PARTITION_GRANULARITY_ENV_VAR,
S3_PREFIX_DIGEST_CHARS,
)
from litellm.types.integrations.s3_v2 import S3PartitionGranularity
@ -39,18 +38,6 @@ def resolve_s3_log_prompts_only(configured: object, environ: Mapping[str, str] |
return True
def resolve_s3_partition_granularity(
configured: object, environ: Mapping[str, str] | None = None
) -> S3PartitionGranularity:
env: Final = os.environ if environ is None else environ
raw: Final = env.get(S3_PARTITION_GRANULARITY_ENV_VAR) if configured is None else configured
if raw == "hour":
return "hour"
if raw is not None and raw not in ("", "day"):
verbose_logger.warning("s3 logging: s3_partition_granularity=%r is not one of day, hour, using day", raw)
return "day"
def _resolve_positive_int(setting: str, configured: object, fallback: int, *, reject_bool: bool) -> int:
if configured is None or configured == "":
return fallback
@ -70,10 +57,6 @@ def _resolve_positive_int(setting: str, configured: object, fallback: int, *, re
return bound
def resolve_s3_max_concurrent_uploads(configured: object, fallback: int) -> int:
return _resolve_positive_int("s3_max_concurrent_uploads", configured, fallback, reject_bool=False)
def resolve_s3_max_queue_size(configured: object, fallback: int) -> int:
return _resolve_positive_int("s3_max_queue_size", configured, fallback, reject_bool=True)
@ -103,48 +86,6 @@ def resolve_s3_max_retry_age_seconds(configured: object, fallback: int | None) -
return bound or None
def resolve_s3_max_adaptive_concurrency(configured: object, fallback: int) -> int:
return _resolve_positive_int("s3_max_adaptive_concurrency", configured, fallback, reject_bool=True)
def resolve_s3_drop_on_terminal_error(configured: object) -> bool:
if configured is None or configured == "":
return True
try:
return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
except ValidationError:
verbose_logger.warning(
"s3 logging: s3_drop_on_terminal_error=%r is not a boolean, dropping terminal-failed uploads",
configured,
)
return True
def resolve_s3_adaptive_concurrency(configured: object) -> bool:
if configured is None or configured == "":
return False
try:
return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
except ValidationError:
verbose_logger.warning(
"s3 logging: s3_adaptive_concurrency=%r is not a boolean, keeping the fixed upload width",
configured,
)
return False
def resolve_s3_batch_file_upload(configured: object) -> bool:
if configured is None or configured == "":
return False
try:
return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
except ValidationError:
verbose_logger.warning(
"s3 logging: s3_batch_file_upload=%r is not a boolean, keeping per-request objects", configured
)
return False
def prompts_only_payload(payload: StandardLoggingPayload) -> StandardLoggingPayload:
return {**payload, "response": None}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,481 @@
import asyncio
import copy
import json
import os
from collections.abc import Callable, Collection, Mapping
from typing import TYPE_CHECKING, Final
from typing_extensions import Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.constants import (
COMPLETION_HTTP_FALLBACK_SECONDS,
DEFAULT_S3_BATCH_SIZE,
DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
)
from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_logger_v2.call import CallContext
from litellm.integrations.custom_logger_v2.derivations import (
cold_storage_object_key,
log_id,
logged_key_identity,
mapping,
payload_copy,
)
from litellm.integrations.custom_logger_v2.enums import Field, Hook
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on
from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch
from litellm.integrations.custom_logger_v2.rust import RustLogger
from litellm.integrations.custom_logger_v2.selection import Selection
from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds
from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity
from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload
if TYPE_CHECKING:
from litellm.rust_bridge._native import NativeS3Log
CALLBACK_NAME: Final = "s3_v2"
STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files"
_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4
_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600
class S3BatchUploadError(Exception):
def __init__(self, failed: int, total: int) -> None:
self.failed: Final = failed
self.total: Final = total
super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush")
def _raise_batch_upload_error(failed: int, total: int) -> None:
raise S3BatchUploadError(failed=failed, total=total)
def batch_upload_failed(failure: tuple[int, int, int] | None) -> None:
if failure is None:
return
queued, failed, total = failure
try:
_raise_batch_upload_error(failed, total)
except S3BatchUploadError:
verbose_logger.exception(
"CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued
)
def _cold_storage_is_s3_v2() -> bool:
return litellm.cold_storage_custom_logger == "s3_v2"
def _region_from_environment() -> str | None:
region_name: Final = get_secret_str("AWS_REGION_NAME", None)
region: Final = get_secret_str("AWS_REGION", None)
return region if region is not None else region_name
class _Host:
def __init__(self) -> None:
self.failures: Final = CustomLogger()
def callback_failed(self) -> None:
self.failures.handle_callback_failure(callback_name="S3Logger")
def web_identity_token(self, reference: str) -> str:
if reference.startswith("os.environ/") or reference in os.environ:
raise ValueError("Invalid web identity token reference.")
match _read_secret(get_secret, reference):
case None:
raise ValueError("OIDC token could not be retrieved from secret manager.")
case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH:
raise ValueError(
"Parameter validation failed:\nInvalid length for parameter WebIdentityToken, "
f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}"
)
case str() as token:
return token
case other:
raise ValueError(
"Parameter validation failed:\nInvalid type for parameter WebIdentityToken, "
f"value: {other}, type: {type(other)}, valid types: {str}"
)
def _read_secret(read: Callable[[str], object], reference: str) -> object:
return read(reference)
def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust
configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None)
source: Final[Mapping[str, object] | Mapping[object, object]] = (
params_source if params_source is not None else (configured or {})
)
return {
key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value)
for key, value in source.items()
if isinstance(key, str)
}
def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]:
return {
"batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE,
"max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
"max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
"upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
"download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS,
"download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
"web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY),
}
def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float:
return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS
_CONFIGURED_FIRST: Final = frozenset(
{
"s3_bucket_name",
"s3_path",
"s3_region_name",
"s3_api_version",
"s3_endpoint_url",
"s3_aws_access_key_id",
"s3_aws_secret_access_key",
"s3_aws_session_token",
"s3_aws_session_name",
"s3_aws_profile_name",
"s3_aws_role_name",
"s3_aws_web_identity_token",
"s3_aws_sts_endpoint",
"s3_config",
"s3_server_side_encryption",
"s3_sse_kms_key_id",
}
)
_CONSTRUCTOR_ENABLES: Final = frozenset(
{
"s3_use_team_prefix",
"s3_strip_base64_files",
"s3_use_key_prefix",
"s3_use_virtual_hosted_style",
"s3_adaptive_concurrency",
"s3_batch_file_upload",
}
)
_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"})
_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"})
_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"})
_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = {
"s3_use_ssl": True,
"s3_drop_on_terminal_error": True,
"s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
"s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS,
"s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
"s3_batch_size": DEFAULT_S3_BATCH_SIZE,
}
def _blank(value: object) -> bool:
return value is None or value == ""
def _constructor_wins(key: str, configured: object, value: object) -> bool:
return (
(key in _CONFIGURED_FIRST and not configured)
or (key in _CONFIGURED_UNLESS_NONE and configured is None)
or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured))
or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None)
)
_KEEP_CONFIGURED: Final = object()
def _constructor_value(key: str, configured: object, value: object) -> object:
if _constructor_wins(key, configured, value):
return value
if key in _CONSTRUCTOR_ENABLES and value:
return True
if key == "s3_max_queue_size" and value is not None:
return resolve_s3_max_queue_size(
configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE)
)
if key == "s3_max_retry_age_seconds":
fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS)
resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback)
return 0 if resolved is None else resolved
return _KEEP_CONFIGURED
def _constructed_params(
params_source: Mapping[str, object] | None, settings: S3LoggerSettings
) -> dict[str, object]: # mutable-ok: handed to Rust
params: Final = _resolved_params(params_source)
overrides: Final = {
key: replaced
for key, value in settings.items()
if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None)
and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED
}
return {**params, **overrides}
class _Base64Stripper(CustomLogger):
def strip(self, payload: StandardLoggingPayload) -> None:
_ = self._strip_base64_from_messages_sync(payload)
def stripped(self, messages: list[object]) -> list[dict[str, object]]:
return self._process_messages(messages=[_shallow_copy(message) for message in messages])
def _shallow_copy(message: object) -> object:
entries: Final = mapping(message)
return copy.copy(entries) if entries is not None and isinstance(message, dict) else message
_STRIPPER: Final = _Base64Stripper()
def legacy_should_log(kwargs: Mapping[str, object]) -> bool:
from litellm.litellm_core_utils import litellm_logging
if litellm.global_disable_no_log_param:
return True
litellm_params: Final = kwargs.get("litellm_params", {})
lookup: Final = getattr(litellm_params, "get", None)
if callable(lookup) and lookup("no-log", False) is True:
return False
return not litellm_logging.callback_disabled_dynamically(
callback=CALLBACK_NAME,
litellm_params=litellm_params,
standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check
)
def _payload_messages(payload: Mapping[str, object]) -> list[object]:
raw: Final = payload.get("messages")
return [*raw] if isinstance(raw, list) else []
class S3StripBase64Files(CustomLoggerV2):
selection = Selection(name=STRIP_CALLBACK_NAME)
def __init__(self, s3_logger_key: str) -> None:
self.s3_logger_key: Final = s3_logger_key
@on(
Hook.log_prepare,
reads={Field.deprecated_kwargs, Field.standard_logging_payload},
writes={Field.standard_logging_payload},
)
async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None:
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None))
payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None))
if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs):
return None
messages: Final = _payload_messages(payload)
if not messages:
return None
return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)})
async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None:
while True:
await asyncio.sleep(interval)
try:
batch_upload_failed(await sink.flush())
except (asyncio.CancelledError, KeyboardInterrupt, SystemExit):
raise
except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does
verbose_logger.exception("CustomLogger: periodic flush failed: %s", e)
def _excluding_fields(kwargs: object) -> Mapping[str, object] | None:
excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
details: Final = mapping(kwargs)
logged: Final = None if details is None else mapping(details.get("standard_logging_object"))
if details is None or logged is None or not isinstance(excluded, Collection) or not excluded:
return None
if logged.get("status") == "failure":
return None
return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}}
def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object:
source: Final = mapping(kwargs) if details is None else details
return None if source is None else source.get("standard_logging_object")
def _legacy_kwargs(view: object) -> object:
return getattr(view, Field.deprecated_kwargs.value, None)
_read_attribute: Final[Callable[[object, str], object]] = getattr
class _ExcludedView:
def __init__(self, view: object, details: Mapping[str, object]) -> None:
args: Final = HookArgs(request=details)
self.view: Final = view
self.fields: Final[Mapping[str, object]] = {
Field.deprecated_kwargs.value: details,
Field.standard_logging_payload_ref.value: payload_copy(args),
Field.log_id.value: log_id(args),
Field.cold_storage_object_key.value: cold_storage_object_key(args),
Field.logged_key_identity.value: logged_key_identity(args),
}
def __getattr__(self, name: str) -> object:
if name not in self.fields:
return _read_attribute(self.view, name)
value: Final = self.fields[name]
if value is None:
raise AttributeError(name)
return value
_ATTEMPT_ENDED_READS: Final = frozenset(
{
Field.deprecated_kwargs,
Field.attempt_timing,
Field.log_id,
Field.cold_storage_object_key,
Field.standard_logging_payload_ref,
Field.logged_key_identity,
}
)
class S3Logger(RustLogger):
selection = Selection(name=CALLBACK_NAME, honors_no_log=True)
def __init__(
self,
s3_callback_params_override: Mapping[str, object] | None = None,
**settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords
) -> None:
from litellm.rust_bridge._native import NativeS3Log
host: Final = _Host()
params: Final = _constructed_params(s3_callback_params_override, settings)
sink: Final = NativeS3Log(
params,
_constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)),
host,
_cold_storage_is_s3_v2(),
None if params.get("s3_region_name") else _region_from_environment(),
)
self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name")
self.sink: Final = sink
self.s3_host: Final = host
self.periodic_flush_task: Final = asyncio.create_task(
_periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS)))
)
super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None})
self.v2_companion: Final = (
S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self))
if sink.strip_base64_files
else None
)
@on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True)
async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None:
return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx)
async def run_rust(
self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object]
) -> Patch | Abort | None:
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None)
if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict
return None
payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict
if payload is not None and not isinstance(payload, Mapping):
return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload
try:
raw: Final = _legacy_kwargs(view)
details: Final = _excluding_fields(raw)
logged: Final = _logged_payload(raw, details)
if self.sink.strip_base64_files and isinstance(logged, Mapping):
_STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does
return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx)
except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on
return self._failed(e)
def _failed(self, error: Exception) -> None:
verbose_logger.error("s3 Layer Error - %s", error, exc_info=error)
self.s3_host.callback_failed()
@property
def s3_path(self) -> str | None:
return self.sink.s3_path
@property
def s3_bucket_name(self) -> str | None:
return self.sink.s3_bucket_name
def resolve_partition_granularity(self) -> S3PartitionGranularity:
return "hour" if self.sink.partition_granularity() == "hour" else "day"
async def flush_queue(self) -> None:
batch_upload_failed(await self.sink.flush())
async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None:
try:
audit_log_id: Final = audit_log.get("id", "unknown")
batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}"))
except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger
verbose_logger.exception("S3 audit log error: %s", e)
async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
object_name: Final = batch_line_items_object_name(payload)
s3_path: Final = self.s3_path
s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name
body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload)
delivered: Final = await self.sink.upload_line_items(
s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:])
)
if not delivered:
verbose_logger.error(
"s3 logging: the %d line items of part %d of batch %s were not stored at %s",
len(payload["lines"]),
payload["part"],
payload["batch_id"],
s3_object_key,
)
async def _download_object_from_s3(self, s3_object_key: str) -> object:
try:
downloaded: Final = await self.sink.download(s3_object_key)
return None if downloaded is None else json.loads(downloaded)
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
verbose_logger.exception("Error downloading from S3: %s", e)
return None
async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object:
try:
return await self._download_object_from_s3(object_key)
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e)
return None
__all__ = [
"S3BatchUploadError",
"S3Logger",
"S3StripBase64Files",
"legacy_should_log",
]

View file

@ -46,7 +46,7 @@ from litellm.integrations.opik.opik import OpikLogger
from litellm.integrations.pointfive import PointFiveLogger
from litellm.integrations.posthog import PostHogLogger
from litellm.integrations.prometheus import PrometheusLogger
from litellm.integrations.s3_v2 import S3Logger
from litellm.integrations.s3_v3 import S3Logger
from litellm.integrations.sqs import SQSLogger
from litellm.integrations.ternary.ternary_logger import TernaryLogger
from litellm.integrations.vantage.vantage_logger import VantageLogger

View file

@ -15,6 +15,7 @@ import time
import traceback
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
from contextlib import AbstractContextManager
from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime as dt_object
from enum import Enum
@ -305,7 +306,6 @@ from ..integrations.pointfive import PointFiveLogger
from ..integrations.posthog import PostHogLogger
from ..integrations.prompt_layer import PromptLayerLogger
from ..integrations.s3 import S3Logger
from ..integrations.s3_v2 import S3Logger as S3V2Logger
from ..integrations.supabase import Supabase
from ..integrations.traceloop import TraceloopLogger
from ..integrations.zerobus import ZerobusLogger
@ -768,6 +768,16 @@ def _callback_control_params(litellm_params: object) -> dict[str, object]:
return litellm_params.model_dump() if isinstance(litellm_params, BaseModel) else {}
def callback_disabled_dynamically(
callback: object, litellm_params: object, standard_callback_dynamic_params: StandardCallbackDynamicParams
) -> bool:
return EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically(
callback=callback,
litellm_params=_callback_control_params(litellm_params),
standard_callback_dynamic_params=standard_callback_dynamic_params,
)
LOGGING_KEYS: Final[Mapping[Field, str]] = MappingProxyType(
{
Field.model: "model",
@ -783,6 +793,9 @@ def _holds_v2(callbacks: Iterable[object]) -> bool:
return any(isinstance(callback, CustomLoggerV2) for callback in callbacks)
V2_ASYNC_LOGGING_LINE: Final[ContextVar[bool]] = ContextVar("v2_async_logging_line", default=False)
_HOOK_DERIVED_FIELDS: Final[Mapping[Hook, Mapping[Field, FieldSource]]] = MappingProxyType(
{Hook.log_prepare: v2_derive.DERIVED_LOGGING_FIELDS, Hook.attempt_ended: v2_derive.DERIVED_ATTEMPT_FIELDS}
)
@ -839,15 +852,19 @@ async def _run_callbacks(
args: HookArgs | None = None,
) -> None:
if include_v2 and _holds_v2(callbacks):
_ = kwargs_after(
await run_subscribers(
hook_subscribers(hook, callbacks, admit=Admit.every_entry),
_logging_fields(hook, kwargs, args, owner),
binding,
call=_v2_call(owner, args),
scope=None if owner is None else owner.v2_logging_scope(),
line: Final = V2_ASYNC_LOGGING_LINE.set(True)
try:
_ = kwargs_after(
await run_subscribers(
hook_subscribers(hook, callbacks, admit=Admit.every_entry),
_logging_fields(hook, kwargs, args, owner),
binding,
call=_v2_call(owner, args),
scope=None if owner is None else owner.v2_logging_scope(),
)
)
)
finally:
V2_ASYNC_LOGGING_LINE.reset(line)
return
for callback in callbacks:
if isinstance(callback, CustomLoggerV2):
@ -3283,9 +3300,9 @@ class Logging(LiteLLMLoggingBaseClass):
return False
# Check for dynamically disabled callbacks via headers
if EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically(
if callback_disabled_dynamically(
callback=callback,
litellm_params=_callback_control_params(litellm_params),
litellm_params=litellm_params,
standard_callback_dynamic_params=self.standard_callback_dynamic_params,
):
verbose_logger.debug(
@ -6519,6 +6536,8 @@ def build_custom_logger_compatible_class(
_in_memory_loggers.append(_gcs_bucket_logger)
return _gcs_bucket_logger
elif logging_integration == "s3_v2":
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
for callback in _in_memory_loggers:
if isinstance(callback, S3V2Logger):
return callback
@ -7341,6 +7360,8 @@ def get_custom_logger_compatible_class(
if isinstance(callback, GCSBucketLogger):
return callback
elif logging_integration == "s3_v2":
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
for callback in _in_memory_loggers:
if isinstance(callback, S3V2Logger):
return callback
@ -8046,6 +8067,8 @@ class StandardLoggingPayloadSetup:
)
if custom_logger and hasattr(custom_logger, "s3_path") and getattr(custom_logger, "s3_path"):
s3_path = getattr(custom_logger, "s3_path")
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
if isinstance(custom_logger, S3V2Logger):
partition_granularity = custom_logger.resolve_partition_granularity()
except Exception:

View file

@ -21,6 +21,16 @@ _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
_callback_lists_lock: Final = threading.RLock()
def _v2_companion(callback: object) -> CustomLoggerV2 | None:
companion: Final[object] = getattr(callback, "v2_companion", None)
return companion if isinstance(companion, CustomLoggerV2) else None
def _with_v2_companions(callbacks: Iterable[object]) -> tuple[object, ...]:
listed: Final = tuple(callbacks)
return (*listed, *(companion for callback in listed if (companion := _v2_companion(callback)) is not None))
def _instance_attributes(target: object) -> Mapping[str, object]:
return getattr(target, "__dict__", {})
@ -292,16 +302,48 @@ class LoggingCallbackManager:
parent_list=parent_list,
)
elif isinstance(callback, CustomLoggerV2):
v2_key: Final = self._get_custom_logger_key(callback)
if not any(
isinstance(existing, CustomLoggerV2) and self._get_custom_logger_key(existing) == v2_key
for existing in parent_list
):
parent_list.append(callback) # pyright: ignore[reportArgumentType] # the event lists also hold V2 callbacks
listed: Final = self._listed_v2(callback, parent_list)
if listed is None:
self._append_v2(callback, parent_list)
self._place_v2_companion(callback if listed is None else listed, parent_list)
elif callable(callback):
self._add_callback_function_to_list(callback=callback, parent_list=parent_list)
def _append_v2(
self, callback: CustomLoggerV2, parent_list: list[CustomLogger | Callable[..., object] | str]
) -> None:
parent_list.append(callback) # pyright: ignore[reportArgumentType] # the event lists also hold V2 callbacks
def _listed_v2(self, callback: CustomLoggerV2, parent_list: Iterable[object]) -> CustomLoggerV2 | None:
v2_key: Final = self._get_custom_logger_key(callback)
return next(
(
existing
for existing in parent_list
if isinstance(existing, CustomLoggerV2) and self._get_custom_logger_key(existing) == v2_key
),
None,
)
def _place_v2_companion(
self, callback: CustomLoggerV2, parent_list: list[CustomLogger | Callable[..., object] | str]
) -> None:
companion: Final = _v2_companion(callback)
if companion is None or self._listed_v2(companion, parent_list) is not None:
return
if not self._check_callback_list_size(parent_list):
return
self._append_v2(companion, parent_list)
position: Final = next(index for index, existing in enumerate(parent_list) if existing is callback)
parent_list.insert(position, parent_list.pop())
def place_v2_companions(self, parent_list: list[CustomLogger | Callable[..., object] | str]) -> None:
with _callback_lists_lock:
for callback in tuple(parent_list):
if isinstance(callback, CustomLoggerV2):
self._place_v2_companion(callback, parent_list)
def _add_callback_function_to_list(
self,
callback: Callable[..., object],
@ -364,7 +406,7 @@ class LoggingCallbackManager:
callback_lists_lock: Final = _callback_lists_lock
def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]:
dropped_ids: Final = frozenset(map(id, callbacks))
dropped_ids: Final = frozenset(map(id, _with_v2_companions(callbacks)))
owner_ids: Final = frozenset(map(id, method_owners))
def keep(callback: object) -> bool:
@ -444,6 +486,7 @@ class LoggingCallbackManager:
Remove a callback object from every callback list it may have been
promoted into, so a re-initialized callback leaves no stale instance behind.
"""
companion: Final = None if require_self else _v2_companion(obj)
with _callback_lists_lock:
for callback_list in (
litellm.callbacks,
@ -454,6 +497,8 @@ class LoggingCallbackManager:
litellm._async_failure_callback,
):
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
if companion is not None:
callback_list[:] = [callback for callback in callback_list if callback is not companion]
def get_active_additional_logging_utils_from_custom_logger(
self,
@ -536,7 +581,7 @@ class LoggingCallbackManager:
if not isinstance(callback, str)
)
def _get_callback_string(self, callback: CustomLogger | Callable[..., object] | str) -> str:
def _get_callback_string(self, callback: CustomLogger | CustomLoggerV2 | Callable[..., object] | str) -> str:
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.litellm_core_utils.custom_logger_registry import (
CustomLoggerRegistry,
@ -547,7 +592,7 @@ class LoggingCallbackManager:
return callback
elif isinstance(callback, OpenTelemetry) and callback.callback_name is not None:
return callback.callback_name
elif isinstance(callback, CustomLogger):
elif isinstance(callback, CustomLogger | CustomLoggerV2):
# Try to get the string representation from the registry
callback_str: Final = CustomLoggerRegistry.get_callback_str_from_class_type(type(callback))
return callback_str if callback_str is not None else type(callback).__name__

View file

@ -4,9 +4,9 @@ Functions to create audit logs for LiteLLM Proxy
import asyncio
import json
from collections.abc import Awaitable, Sequence
from collections.abc import Awaitable, Mapping, Sequence
from datetime import datetime, timezone
from typing import Final, TypeVar
from typing import TYPE_CHECKING, Final, TypeVar
import litellm
from litellm._logging import verbose_proxy_logger
@ -21,7 +21,10 @@ from litellm.proxy._types import (
from litellm.repositories.table_repositories import AuditLogRepository
from litellm.types.utils import StandardAuditLogPayload
_audit_log_callback_cache: Final[dict[str, CustomLogger]] = {}
if TYPE_CHECKING:
from litellm.integrations.s3_v3 import S3Logger
_audit_log_callback_cache: Final[dict[str, "CustomLogger | S3Logger"]] = {}
ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY: Final = "allow_litellm_changed_by_header"
@ -62,7 +65,7 @@ def get_audit_log_changed_by(
return user_api_key_dict.user_id or litellm_proxy_admin_name
def _resolve_audit_log_callback(name: str) -> CustomLogger | None:
def _resolve_audit_log_callback(name: str) -> "CustomLogger | S3Logger | None":
"""Resolve a string callback name to a CustomLogger instance, with caching.
For "s3_v2" with `litellm.s3_audit_callback_params` set, constructs a
@ -72,11 +75,12 @@ def _resolve_audit_log_callback(name: str) -> CustomLogger | None:
if name in _audit_log_callback_cache:
return _audit_log_callback_cache[name]
instance: CustomLogger | None
if name == "s3_v2" and getattr(litellm, "s3_audit_callback_params", None) is not None:
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
instance: CustomLogger | S3Logger | None
audit_params: Final[Mapping[str, object] | None] = getattr(litellm, "s3_audit_callback_params", None)
if name == "s3_v2" and audit_params is not None:
from litellm.integrations import s3_v3
instance = S3V2Logger(s3_callback_params_override=litellm.s3_audit_callback_params)
instance = s3_v3.S3Logger(s3_callback_params_override=audit_params)
else:
from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,
@ -196,14 +200,14 @@ async def _dispatch_audit_log_to_callbacks(
for callback in litellm.audit_log_callbacks:
try:
resolved: CustomLogger | None = callback if isinstance(callback, CustomLogger) else None
resolved: CustomLogger | S3Logger | None = callback if isinstance(callback, CustomLogger) else None
if isinstance(callback, str):
resolved = _resolve_audit_log_callback(callback)
if resolved is None:
verbose_proxy_logger.warning("Could not resolve audit log callback: %s", callback)
continue
if isinstance(resolved, CustomLogger):
if resolved is not None:
track_audit_task(
asyncio.create_task(
resolved.async_log_audit_log_event(payload),

View file

@ -7446,7 +7446,7 @@ class ProxyConfig:
if key == "request_timeout":
litellm.request_timeout_explicitly_set = True
if key in {"s3_audit_callback_params", "s3_callback_params"}:
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
from litellm.litellm_core_utils.litellm_logging import (
_in_memory_loggers,
)
@ -16441,12 +16441,13 @@ async def _routed_count_deployment(
if llm_router is None:
return None
try:
return await llm_router.select(model, dict(_responses_routing_kwargs(responses_request)))
routed: Final = await llm_router.select(model, dict(_responses_routing_kwargs(responses_request)))
except Exception:
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment"
)
return None
return routed if isinstance(routed, Mapping) else None
def _with_chat_timeout(litellm_params: Mapping[str, object]) -> Mapping[str, object]:

View file

@ -2514,6 +2514,7 @@ class ProxyLogging:
# Replace string entries in litellm.callbacks with initialized instances
for idx, initialized_callback in string_callbacks_to_replace.items():
litellm.callbacks[idx] = initialized_callback
litellm.logging_callback_manager.place_v2_companions(litellm.callbacks)
# Fan ``litellm.callbacks`` (the "all events" registry) out into the
# success/failure event lists eagerly, at startup. ``completion()`` does

View file

@ -3739,6 +3739,34 @@ dependencies = [
"tokio",
]
[[package]]
name = "litellm-callbacks-s3-log"
version = "0.1.0"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-util",
"litellm-auth-aws",
"litellm-callbacks",
"litellm-http",
"litellm-python-compat",
"litellm-s3",
"litellm-secret-value",
"litellm-tracing",
"md-5",
"rand 0.8.7",
"reqwest",
"rstest",
"serde",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.19",
"tokio",
"tracing",
"uuid",
"wiremock",
]
[[package]]
name = "litellm-circuit-breaker"
version = "0.1.0"
@ -4451,7 +4479,10 @@ dependencies = [
"litellm-auth-azure",
"litellm-cache",
"litellm-cache-response",
"litellm-callbacks",
"litellm-callbacks-legacy-python",
"litellm-callbacks-python",
"litellm-callbacks-s3-log",
"litellm-circuit-breaker",
"litellm-core-utils",
"litellm-error",
@ -4691,11 +4722,15 @@ dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"aws-types",
"bytes",
"http 1.4.2",
"hyper 1.12.0",
"litellm-auth-aws",
"litellm-http",
"reqwest",
"rstest",
"thiserror 2.0.19",
"tokio",
]
[[package]]

View file

@ -38,6 +38,7 @@ litellm-host-native = { path = "crates/host-native" }
litellm-callbacks = { path = "crates/callbacks" }
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
litellm-callbacks-python = { path = "crates/callbacks-python" }
litellm-callbacks-s3-log = { path = "crates/callbacks-s3-log" }
litellm-framer = { path = "crates/framer" }
litellm-auth = { path = "crates/auth" }
litellm-auth-types = { path = "crates/auth-types" }

View file

@ -17,6 +17,7 @@ serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
sha2.workspace = true
thiserror.workspace = true
tokio.workspace = true
url.workspace = true
veil.workspace = true

View file

@ -9,8 +9,8 @@ use std::{future::Future, pin::Pin, sync::Arc};
use aws_credential_types::Credentials;
use aws_credential_types::provider::ProvideCredentials;
use aws_sigv4::http_request::{
SignableBody, SignableRequest, SignatureLocation, SigningInstructions, SigningParams,
SigningSettings, sign,
PercentEncodingMode, SignableBody, SignableRequest, SignatureLocation, SigningInstructions,
SigningParams, SigningSettings, UriPathNormalizationMode, sign,
};
use aws_sigv4::sign::v4;
use aws_smithy_runtime_api::client::identity::Identity;
@ -38,10 +38,15 @@ pub struct AwsAuthService {
impl Default for AwsAuthService {
fn default() -> Self {
Self::new(Arc::new(NativeProviderLoader))
Self::new(Arc::new(NativeProviderLoader {
token: None,
policy: None,
}))
}
}
pub type WebIdentityTokenResolver = Arc<dyn Fn(&str) -> Result<String, String> + Send + Sync>;
impl AwsAuthService {
fn new(loader: Arc<dyn ProviderLoader>) -> Self {
Self {
@ -53,6 +58,16 @@ impl AwsAuthService {
loader,
}
}
pub fn for_web_identity(
resolver: WebIdentityTokenResolver,
session_policy: Option<String>,
) -> Self {
Self::new(Arc::new(NativeProviderLoader {
token: Some(resolver),
policy: session_policy,
}))
}
}
struct CredentialExpiry;
@ -81,11 +96,19 @@ trait ProviderLoader: Send + Sync {
fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_>;
}
struct NativeProviderLoader;
struct NativeProviderLoader {
token: Option<WebIdentityTokenResolver>,
policy: Option<String>,
}
impl ProviderLoader for NativeProviderLoader {
fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> {
Box::pin(build_provider(config, flow))
Box::pin(build_provider(
config,
flow,
self.token.clone(),
self.policy.clone(),
))
}
}
@ -271,6 +294,16 @@ impl AwsAuthService {
self.fresh_credentials(&resolved, &flow).await
}
pub async fn invalidate_credentials(
&self,
config: AwsAuthConfig,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
) {
self.credentials
.invalidate(&config.with_environment(env_lookup))
.await;
}
/// The credentials botocore's default chain finds in `region`, without LiteLLM's Bedrock settings
/// (`AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, `AWS_SESSION_NAME`) that `resolve_credentials` also reads.
pub async fn default_chain_credentials(&self, region: &str) -> Result<Credentials, Error> {
@ -347,6 +380,8 @@ impl AwsAuthFlow {
async fn build_provider(
resolved: AwsAuthConfig,
flow: AwsAuthFlow,
token_resolver: Option<WebIdentityTokenResolver>,
policy: Option<String>,
) -> Result<SharedCredentialsProvider, Error> {
match flow {
AwsAuthFlow::SessionToken {
@ -417,8 +452,10 @@ async fn build_provider(
role,
session_name,
} => Ok(SharedCredentialsProvider::new(WebIdentityProvider {
policy,
client: aws_sdk_sts::Client::new(&sdk_config(resolved).await),
token,
token_resolver,
role,
session_name,
})),
@ -448,14 +485,27 @@ async fn sdk_config(config: AwsAuthConfig) -> aws_types::SdkConfig {
loader.load().await
}
#[derive(Debug)]
struct WebIdentityProvider {
client: aws_sdk_sts::Client,
token: SecretValue,
token_resolver: Option<WebIdentityTokenResolver>,
policy: Option<String>,
role: String,
session_name: String,
}
impl std::fmt::Debug for WebIdentityProvider {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("WebIdentityProvider")
.field("token", &self.token)
.field("policy", &self.policy)
.field("role", &self.role)
.field("session_name", &self.session_name)
.finish_non_exhaustive()
}
}
impl ProvideCredentials for WebIdentityProvider {
fn provide_credentials<'a>(
&'a self,
@ -464,12 +514,31 @@ impl ProvideCredentials for WebIdentityProvider {
Self: 'a,
{
aws_credential_types::provider::future::ProvideCredentials::new(async {
let response = self
let token = match &self.token_resolver {
Some(resolve) => {
let resolve = Arc::clone(resolve);
let reference = self.token.expose().to_owned();
tokio::task::spawn_blocking(move || resolve(&reference))
.await
.map_err(|error| error.to_string())
.and_then(|token| token)
.map_err(|message| {
CredentialsError::provider_error(std::io::Error::other(message))
})?
}
None => self.token.expose().to_owned(),
};
let request = self
.client
.assume_role_with_web_identity()
.role_arn(&self.role)
.role_session_name(&self.session_name)
.web_identity_token(self.token.expose())
.web_identity_token(token);
let request = match &self.policy {
Some(policy) => request.policy(policy).duration_seconds(3600),
None => request,
};
let response = request
.send()
.await
.map_err(CredentialsError::provider_error)?;
@ -605,6 +674,32 @@ pub fn sign_headers(
.collect()
}
pub fn sign_s3_headers(
method: &str,
url: &str,
body: &[u8],
headers: &[(String, String)],
scope: SigningScope<'_>,
) -> Result<Vec<(String, String)>, Error> {
let mut settings = SigningSettings::default();
settings.percent_encoding_mode = PercentEncodingMode::Single;
settings.uri_path_normalization_mode = UriPathNormalizationMode::Disabled;
let instructions = sign_request(
method,
url,
headers
.iter()
.map(|(name, value)| (name.as_str(), value.as_str())),
body,
settings,
scope,
)?;
Ok(instructions
.headers()
.map(|(name, value)| (name.to_owned(), value.to_owned()))
.collect())
}
pub fn presign_url(
url: &str,
expires_in: Duration,

View file

@ -0,0 +1,35 @@
[package]
name = "litellm-callbacks-s3-log"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-auth-aws.workspace = true
litellm-callbacks.workspace = true
litellm-http.workspace = true
litellm-python-compat.workspace = true
litellm-s3.workspace = true
litellm-secret-value.workspace = true
litellm-tracing.workspace = true
base64.workspace = true
bytes.workspace = true
futures-util.workspace = true
md-5 = "0.11.0"
rand.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
sha2.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["sync", "time", "rt"] }
tracing.workspace = true
uuid.workspace = true
[dev-dependencies]
litellm-http = { workspace = true, features = ["test-support"] }
rstest.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] }
wiremock.workspace = true

View file

@ -0,0 +1,367 @@
use std::time::{SystemTime, UNIX_EPOCH};
use litellm_callbacks::Moment;
use sha2::{Digest, Sha256};
use crate::settings::{Granularity, Settings};
const MAX_KEY_BYTES: usize = 1024;
const MAX_FILE_NAME_BYTES: usize = 255;
const MAX_DOWNLOAD_NAME_BYTES: usize = 1024;
const BOUNDED_HEAD_BYTES: i64 = 64;
const MIN_BOUNDED_FILE_NAME_BYTES: usize = 64;
const PREFIX_DIGEST_CHARS: usize = 16;
const EXTENSION: &str = ".json";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Stamp {
pub year: i64,
pub month: u32,
pub day: u32,
pub hour: u32,
pub minute: u32,
pub second: u32,
pub micros: u32,
}
impl From<[i64; 7]> for Stamp {
fn from(parts: [i64; 7]) -> Self {
let [year, month, day, hour, minute, second, micros] = parts;
let part = |value: i64| u32::try_from(value).unwrap_or_default();
Self {
year,
month: part(month),
day: part(day),
hour: part(hour),
minute: part(minute),
second: part(second),
micros: part(micros),
}
}
}
impl From<&Moment> for Stamp {
fn from(moment: &Moment) -> Self {
Self {
year: i64::from(moment.year),
month: u32::from(moment.month),
day: u32::from(moment.day),
hour: u32::from(moment.hour),
minute: u32::from(moment.minute),
second: u32::from(moment.second),
micros: moment.microsecond,
}
}
}
impl Stamp {
pub fn utc(time: SystemTime) -> Self {
let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default();
let seconds = elapsed.as_secs();
let (year, month, day) = civil_from_days((seconds / 86_400) as i64);
let of_day = seconds % 86_400;
Self {
year,
month,
day,
hour: (of_day / 3600) as u32,
minute: (of_day / 60 % 60) as u32,
second: (of_day % 60) as u32,
micros: elapsed.subsec_micros(),
}
}
pub fn date(&self) -> String {
format!("{:04}-{:02}-{:02}", self.year, self.month, self.day)
}
pub fn clock(&self) -> String {
format!("{:02}-{:02}-{:02}", self.hour, self.minute, self.second)
}
fn clock_micros(&self) -> String {
format!("{}-{:06}", self.clock(), self.micros)
}
}
fn civil_from_days(days: i64) -> (i64, u32, u32) {
let shifted = days + 719_468;
let era = shifted.div_euclid(146_097);
let day_of_era = shifted.rem_euclid(146_097);
let year_of_era =
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
let month_index = (5 * day_of_year + 2) / 153;
let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32;
let month = if month_index < 10 {
month_index + 3
} else {
month_index - 9
} as u32;
let year = year_of_era + era * 400 + i64::from(month <= 2);
(year, month, day)
}
pub fn logging_id(start: &Stamp, id: &str) -> String {
format!("time-{}_{id}", start.clock_micros())
}
pub fn object_key(
path: &str,
prefix: &str,
start: &Stamp,
file_name: &str,
granularity: Granularity,
) -> String {
let sanitized = file_name.replace(['/', ':'], "_");
let configured = if path.is_empty() {
prefix.to_owned()
} else {
format!("{}/{prefix}", path.trim_end_matches('/'))
};
let date = match granularity {
Granularity::Day => format!("{}/", start.date()),
Granularity::Hour => format!("{}/{:02}/", start.date(), start.hour),
};
let file_name_budget = MAX_FILE_NAME_BYTES - EXTENSION.len();
let key = format!("{configured}{date}{sanitized}{EXTENSION}");
if key.len() <= MAX_KEY_BYTES && sanitized.len() <= file_name_budget {
return key;
}
let budget = MAX_KEY_BYTES - date.len() - EXTENSION.len();
if configured.len() + MIN_BOUNDED_FILE_NAME_BYTES <= budget {
let bounded = bounded_name(
file_name,
&sanitized,
file_name_budget.min(budget - configured.len()) as i64,
);
return format!("{configured}{date}{bounded}{EXTENSION}");
}
let shortest = bounded_name(file_name, &sanitized, MIN_BOUNDED_FILE_NAME_BYTES as i64);
let prefix = bounded_prefix(&configured, budget as i64 - shortest.len() as i64);
format!("{prefix}{date}{shortest}{EXTENSION}")
}
pub fn download_name(start: &Stamp, id: &str) -> String {
let stamp = format!("time-{}T{}", start.date(), start.clock_micros());
let name = format!("{stamp}_{id}");
let sanitized = format!("{stamp}_{}", id.replace(['/', '"'], "_"));
let budget = MAX_DOWNLOAD_NAME_BYTES - EXTENSION.len();
if sanitized.len() <= budget {
return format!("{sanitized}{EXTENSION}");
}
format!(
"{}{EXTENSION}",
bounded_name(&name, &sanitized, budget as i64)
)
}
fn bounded_name(name: &str, sanitized: &str, max_bytes: i64) -> String {
let digest = hex_sha256(name);
let head = truncate(
sanitized,
BOUNDED_HEAD_BYTES.min(max_bytes - digest.len() as i64 - 1),
);
if head.is_empty() {
digest
} else {
format!("{head}_{digest}")
}
}
fn bounded_prefix(prefix: &str, max_bytes: i64) -> String {
let digest = format!("{}/", &hex_sha256(prefix)[..PREFIX_DIGEST_CHARS]);
if max_bytes < digest.len() as i64 {
return String::new();
}
let head = truncate(prefix, max_bytes - digest.len() as i64 - 1).trim_end_matches('/');
if head.is_empty() {
digest
} else {
format!("{head}/{digest}")
}
}
fn truncate(value: &str, max_bytes: i64) -> &str {
let Ok(max_bytes) = usize::try_from(max_bytes) else {
return "";
};
if max_bytes == 0 {
return "";
}
if value.len() <= max_bytes {
return value;
}
let end = (0..=max_bytes)
.rev()
.find(|end| value.is_char_boundary(*end))
.unwrap_or(0);
&value[..end]
}
pub fn hex_sha256(value: impl AsRef<[u8]>) -> String {
hex(&Sha256::digest(value.as_ref()))
}
pub fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
}
pub fn parent(key: &str) -> &str {
key.rsplit_once('/').map_or("", |(parent, _)| parent)
}
pub fn quote(key: &str) -> String {
let mut encoded = String::with_capacity(key.len());
for byte in key.bytes() {
if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') {
encoded.push(byte as char);
} else {
encoded.push_str(&format!("%{byte:02X}"));
}
}
encoded
}
pub fn dns_suffix(region: Option<&str>) -> &'static str {
let Some(region) = region.filter(|region| !region.is_empty()) else {
return "amazonaws.com";
};
[
("cn-", "amazonaws.com.cn"),
("us-gov-", "amazonaws.com"),
("us-isob-", "sc2s.sgov.gov"),
("us-isof-", "csp.hci.ic.gov"),
("us-iso-", "c2s.ic.gov"),
("eu-isoe-", "cloud.adc-e.uk"),
]
.into_iter()
.find(|(prefix, _)| region.starts_with(prefix))
.map_or("amazonaws.com", |(_, suffix)| suffix)
}
pub fn object_url(settings: &Settings, key: &str) -> String {
let encoded = quote(key);
match (&settings.endpoint_url, &settings.bucket_name) {
(Some(endpoint), Some(bucket)) if settings.use_virtual_hosted_style => {
let host = endpoint.replace("https://", "").replace("http://", "");
let scheme = if endpoint.starts_with("https://") {
"https://"
} else {
"http://"
};
format!("{scheme}{bucket}.{host}/{encoded}")
}
(Some(endpoint), Some(bucket)) => format!("{endpoint}/{bucket}/{encoded}"),
_ => {
let region = settings.url_region();
let suffix = dns_suffix(
settings
.region_name
.as_ref()
.and_then(serde_json::Value::as_str),
);
format!(
"https://{}.s3.{region}.{suffix}/{encoded}",
settings.bucket_name.as_deref().unwrap_or("None")
)
}
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
fn stamp() -> Stamp {
Stamp::from([2026, 11, 13, 9, 5, 7, 42])
}
#[rstest]
#[case("", "", Granularity::Day, "2026-11-13/time-09-05-07-000042_abc.json")]
#[case(
"logs/",
"team/",
Granularity::Hour,
"logs/team/2026-11-13/09/time-09-05-07-000042_abc.json"
)]
#[case(
"a/b",
"",
Granularity::Day,
"a/b/2026-11-13/time-09-05-07-000042_abc.json"
)]
fn keys_are_laid_out_as_python_lays_them_out(
#[case] path: &str,
#[case] prefix: &str,
#[case] granularity: Granularity,
#[case] expected: &str,
) {
let name = logging_id(&stamp(), "abc");
assert_eq!(
object_key(path, prefix, &stamp(), &name, granularity),
expected
);
}
#[rstest]
fn a_long_id_is_bounded_with_its_digest() {
let name = logging_id(&stamp(), &"x".repeat(400));
let key = object_key("p", "", &stamp(), &name, Granularity::Day);
let file = key.rsplit('/').next().unwrap();
assert!(file.len() <= MAX_FILE_NAME_BYTES);
assert!(file.ends_with(&format!("_{}.json", hex_sha256(&name))));
}
#[rstest]
fn utc_stamps_split_the_clock() {
let time = UNIX_EPOCH + std::time::Duration::from_micros(1_763_024_707_000_042);
assert_eq!(Stamp::utc(time), Stamp::from([2025, 11, 13, 9, 5, 7, 42]));
}
#[rstest]
fn keys_are_quoted_like_urllib() {
assert_eq!(quote("a b/c=d+e~f.json"), "a%20b/c%3Dd%2Be~f.json");
assert_eq!(quote("é"), "%C3%A9");
}
#[rstest]
#[case(
"cn-north-1",
"https://audit-bucket.s3.cn-north-1.amazonaws.com.cn/2025-01-01/key.json"
)]
#[case(
"cn-northwest-1",
"https://audit-bucket.s3.cn-northwest-1.amazonaws.com.cn/2025-01-01/key.json"
)]
#[case(
"us-east-1",
"https://audit-bucket.s3.us-east-1.amazonaws.com/2025-01-01/key.json"
)]
#[case(
"us-gov-west-1",
"https://audit-bucket.s3.us-gov-west-1.amazonaws.com/2025-01-01/key.json"
)]
fn object_urls_follow_the_partition_of_the_region(
#[case] region: &str,
#[case] expected: &str,
) {
let params =
serde_json::json!({"s3_bucket_name": "audit-bucket", "s3_region_name": region});
let settings = Settings::from_params(
params.as_object().unwrap(),
&crate::settings::Constants::default(),
);
assert_eq!(object_url(&settings, "2025-01-01/key.json"), expected);
}
#[rstest]
fn download_names_replace_slashes_and_quotes() {
assert_eq!(
download_name(&stamp(), "a/b\"c"),
"time-2026-11-13T09-05-07-000042_a_b_c.json"
);
}
}

View file

@ -0,0 +1,11 @@
mod key;
mod limiter;
mod pyjson;
mod settings;
mod sink;
mod upload;
pub use key::{Stamp, download_name, logging_id, object_key, object_url};
pub use settings::{Constants, Environment, Granularity, Settings};
pub use sink::{BatchUploadFailure, Host, NAME, S3Log, S3LogParts, system_clock};
pub use upload::{Backoff, Clock, Outcome, panic_message, python_backoff};

View file

@ -0,0 +1,304 @@
use std::{
collections::VecDeque,
sync::{Arc, Mutex, PoisonError},
};
use tokio::sync::{OwnedSemaphorePermit, Semaphore, oneshot};
pub enum Limiter {
Fixed(Arc<Semaphore>),
Adaptive(Arc<Adaptive>),
}
pub enum Permit {
Fixed(#[allow(dead_code)] OwnedSemaphorePermit),
Adaptive(Arc<Adaptive>),
}
impl Drop for Permit {
fn drop(&mut self) {
if let Self::Adaptive(limiter) = self {
limiter.release();
}
}
}
impl Limiter {
pub fn new(width: usize, adaptive: bool, ceiling: usize) -> Self {
if adaptive {
Self::Adaptive(Arc::new(Adaptive::new(width, width, width.max(ceiling))))
} else {
Self::Fixed(Arc::new(Semaphore::new(width.min(Semaphore::MAX_PERMITS))))
}
}
pub async fn acquire(&self) -> Permit {
match self {
Self::Fixed(semaphore) => Permit::Fixed(
Arc::clone(semaphore)
.acquire_owned()
.await
.unwrap_or_else(|_| unreachable!("the upload semaphore is never closed")),
),
Self::Adaptive(limiter) => {
limiter.acquire().await;
Permit::Adaptive(Arc::clone(limiter))
}
}
}
pub fn record(&self, throttled: bool) {
if let Self::Adaptive(limiter) = self {
limiter.record(throttled);
}
}
#[cfg(test)]
pub fn limit(&self) -> Option<usize> {
match self {
Self::Fixed(_) => None,
Self::Adaptive(limiter) => Some(limiter.state().limit),
}
}
}
struct State {
limit: usize,
clean_streak: usize,
in_flight: usize,
waiters: VecDeque<oneshot::Sender<()>>,
}
pub struct Adaptive {
floor: usize,
ceiling: usize,
state: Mutex<State>,
}
impl Adaptive {
fn new(initial: usize, floor: usize, ceiling: usize) -> Self {
Self {
floor,
ceiling,
state: Mutex::new(State {
limit: initial.max(floor).min(ceiling),
clean_streak: 0,
in_flight: 0,
waiters: VecDeque::new(),
}),
}
}
fn state(&self) -> std::sync::MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
async fn acquire(&self) {
let waiter = {
let mut state = self.state();
if state.in_flight < state.limit {
state.in_flight += 1;
return;
}
let (sender, receiver) = oneshot::channel();
state.waiters.push_back(sender);
receiver
};
let _ = waiter.await;
}
fn grant(state: &mut State) {
while state.in_flight < state.limit {
let Some(waiter) = state.waiters.pop_front() else {
return;
};
if waiter.send(()).is_ok() {
state.in_flight += 1;
}
}
}
fn release(&self) {
let mut state = self.state();
state.in_flight = state.in_flight.saturating_sub(1);
Self::grant(&mut state);
}
fn record(&self, throttled: bool) {
let mut state = self.state();
if throttled {
state.limit = self.floor.max(state.limit / 2);
state.clean_streak = 0;
return;
}
state.clean_streak += 1;
if state.clean_streak >= state.limit && state.limit < self.ceiling {
state.limit += 1;
state.clean_streak = 0;
Self::grant(&mut state);
}
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
fn the_limit_grows_after_a_clean_streak_and_halves_on_a_throttle() {
let limiter = Limiter::new(2, true, 8);
for _ in 0..2 {
limiter.record(false);
}
assert_eq!(limiter.limit(), Some(3));
for _ in 0..3 {
limiter.record(false);
}
assert_eq!(limiter.limit(), Some(4));
limiter.record(true);
assert_eq!(limiter.limit(), Some(2));
limiter.record(true);
assert_eq!(limiter.limit(), Some(2));
}
#[rstest]
#[tokio::test]
async fn a_bound_past_the_semaphore_maximum_acts_as_no_bound() {
let limiter = Limiter::new(usize::MAX, false, 0);
let _first = limiter.acquire().await;
let _second = limiter.acquire().await;
}
#[rstest]
#[tokio::test]
async fn a_full_limit_admits_the_next_waiter_when_a_permit_drops() {
let limiter = Arc::new(Limiter::new(1, true, 1));
let first = limiter.acquire().await;
let waiting = {
let limiter = Arc::clone(&limiter);
tokio::spawn(async move {
let _permit = limiter.acquire().await;
})
};
tokio::task::yield_now().await;
assert!(!waiting.is_finished());
drop(first);
waiting.await.unwrap();
}
fn adaptive(initial: usize, floor: usize, ceiling: usize) -> Arc<Limiter> {
Arc::new(Limiter::Adaptive(Arc::new(Adaptive::new(
initial, floor, ceiling,
))))
}
fn in_flight(limiter: &Limiter) -> usize {
match limiter {
Limiter::Fixed(_) => 0,
Limiter::Adaptive(adaptive) => adaptive.state().in_flight,
}
}
fn record(limiter: &Limiter, throttled: bool, times: usize) {
for _ in 0..times {
limiter.record(throttled);
}
}
#[rstest]
#[case(4, 1, 16, &[(false, 4)], 5)]
#[case(4, 1, 16, &[(false, 3)], 4)]
#[case(16, 1, 16, &[(true, 1)], 8)]
#[case(4, 4, 16, &[(true, 1)], 4)]
#[case(15, 1, 16, &[(false, 1000)], 16)]
#[case(4, 1, 32, &[(false, 3), (true, 1), (false, 1)], 2)]
fn the_limit_follows_the_samples(
#[case] initial: usize,
#[case] floor: usize,
#[case] ceiling: usize,
#[case] samples: &[(bool, usize)],
#[case] expected: usize,
) {
let limiter = adaptive(initial, floor, ceiling);
for &(throttled, times) in samples {
record(&limiter, throttled, times);
}
assert_eq!(limiter.limit(), Some(expected));
}
#[rstest]
#[tokio::test]
async fn growing_the_limit_wakes_a_waiting_acquirer() {
let limiter = adaptive(1, 1, 4);
let holder = limiter.acquire().await;
let waiting = {
let limiter = Arc::clone(&limiter);
tokio::spawn(async move {
let _permit = limiter.acquire().await;
})
};
tokio::task::yield_now().await;
assert!(!waiting.is_finished());
limiter.record(false);
waiting.await.unwrap();
drop(holder);
assert_eq!(in_flight(&limiter), 0);
}
#[rstest]
#[tokio::test]
async fn releasing_a_slot_wakes_exactly_one_waiter() {
let limiter = adaptive(1, 1, 4);
let holder = limiter.acquire().await;
let acquired = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let (release, released) = tokio::sync::watch::channel(false);
let waiters: Vec<_> = (0..3)
.map(|_| {
let limiter = Arc::clone(&limiter);
let acquired = Arc::clone(&acquired);
let mut released = released.clone();
tokio::spawn(async move {
let _permit = limiter.acquire().await;
acquired.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let _ = released.wait_for(|released| *released).await;
})
})
.collect();
tokio::task::yield_now().await;
drop(holder);
tokio::task::yield_now().await;
assert_eq!(acquired.load(std::sync::atomic::Ordering::SeqCst), 1);
for _ in 0..3 {
limiter.record(false);
tokio::task::yield_now().await;
}
assert_eq!(acquired.load(std::sync::atomic::Ordering::SeqCst), 3);
release.send(true).unwrap();
for waiter in waiters {
waiter.await.unwrap();
}
assert_eq!(in_flight(&limiter), 0);
}
#[rstest]
#[tokio::test]
async fn a_cancelled_waiter_is_skipped_and_leaves_in_flight_at_zero() {
let limiter = adaptive(1, 1, 1);
let holder = limiter.acquire().await;
let spawn_waiter = |limiter: Arc<Limiter>| {
tokio::spawn(async move {
let _permit = limiter.acquire().await;
})
};
let doomed = spawn_waiter(Arc::clone(&limiter));
let next = spawn_waiter(Arc::clone(&limiter));
tokio::task::yield_now().await;
doomed.abort();
assert!(doomed.await.unwrap_err().is_cancelled());
drop(holder);
next.await.unwrap();
assert_eq!(in_flight(&limiter), 0);
}
}

View file

@ -0,0 +1,214 @@
use serde_json::Value;
const HEX: &[u8; 16] = b"0123456789abcdef";
pub fn write_str(out: &mut Vec<u8>, text: &str) {
out.push(b'"');
let mut units = [0u16; 2];
for character in text.chars() {
match character {
'"' => out.extend_from_slice(b"\\\""),
'\\' => out.extend_from_slice(b"\\\\"),
'\n' => out.extend_from_slice(b"\\n"),
'\r' => out.extend_from_slice(b"\\r"),
'\t' => out.extend_from_slice(b"\\t"),
'\u{8}' => out.extend_from_slice(b"\\b"),
'\u{c}' => out.extend_from_slice(b"\\f"),
' '..='~' => out.push(character as u8),
_ => {
for unit in character.encode_utf16(&mut units) {
write_unit(out, *unit);
}
}
}
}
out.push(b'"');
}
fn write_unit(out: &mut Vec<u8>, unit: u16) {
out.extend_from_slice(b"\\u");
for shift in [12, 8, 4, 0] {
out.push(HEX[usize::from((unit >> shift) & 0xf)]);
}
}
pub fn float_repr(value: f64) -> String {
if value.is_nan() {
return "NaN".to_owned();
}
if value.is_infinite() {
return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned();
}
let shortest = format!("{value:e}");
let length = shortest
.split('e')
.next()
.unwrap_or_default()
.chars()
.filter(char::is_ascii_digit)
.count();
let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1));
let (negative, scientific) = match scientific.strip_prefix('-') {
Some(rest) => (true, rest),
None => (false, scientific.as_str()),
};
let (mantissa, exponent) = scientific.split_once('e').unwrap_or((scientific, "0"));
let exponent: i32 = exponent.parse().unwrap_or(0);
let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect();
let mut text = String::with_capacity(digits.len() + 8);
if negative {
text.push('-');
}
let point = exponent + 1;
if (-3..=16).contains(&point) {
let count = digits.len() as i32;
if point <= 0 {
text.push_str("0.");
text.extend(std::iter::repeat_n('0', (-point) as usize));
text.push_str(&digits);
} else if point >= count {
text.push_str(&digits);
text.extend(std::iter::repeat_n('0', (point - count) as usize));
text.push_str(".0");
} else {
let (whole, fraction) = digits.split_at(point as usize);
text.push_str(whole);
text.push('.');
text.push_str(fraction);
}
return text;
}
let (first, rest) = digits.split_at(1);
text.push_str(first);
if !rest.is_empty() {
text.push('.');
text.push_str(rest);
}
text.push('e');
text.push(if exponent < 0 { '-' } else { '+' });
text.push_str(&format!("{:02}", exponent.unsigned_abs()));
text
}
pub fn write_value(out: &mut Vec<u8>, value: &Value) {
match value {
Value::Null => out.extend_from_slice(b"null"),
Value::Bool(true) => out.extend_from_slice(b"true"),
Value::Bool(false) => out.extend_from_slice(b"false"),
Value::Number(number) => match (number.as_i64(), number.as_u64(), number.as_f64()) {
(Some(integer), _, _) => out.extend_from_slice(integer.to_string().as_bytes()),
(None, Some(integer), _) => out.extend_from_slice(integer.to_string().as_bytes()),
(None, None, Some(float)) => out.extend_from_slice(float_repr(float).as_bytes()),
(None, None, None) => out.extend_from_slice(number.to_string().as_bytes()),
},
Value::String(text) => write_str(out, text),
Value::Array(items) => {
out.push(b'[');
for (index, item) in items.iter().enumerate() {
if index > 0 {
out.extend_from_slice(b", ");
}
write_value(out, item);
}
out.push(b']');
}
Value::Object(entries) => {
out.push(b'{');
for (index, (key, item)) in entries.iter().enumerate() {
if index > 0 {
out.extend_from_slice(b", ");
}
write_str(out, key);
out.extend_from_slice(b": ");
write_value(out, item);
}
out.push(b'}');
}
}
}
pub fn dumps(value: &Value) -> String {
let mut out = Vec::new();
write_value(&mut out, value);
String::from_utf8(out).unwrap_or_default()
}
pub fn py_str(value: &Value) -> String {
match value {
Value::Null => "None".to_owned(),
Value::Bool(true) => "True".to_owned(),
Value::Bool(false) => "False".to_owned(),
Value::String(text) => text.clone(),
Value::Number(number) => match number.as_f64() {
Some(float) if number.is_f64() => float_repr(float),
_ => number.to_string(),
},
Value::Array(_) | Value::Object(_) => dumps(value),
}
}
pub fn truthy(value: &Value) -> bool {
match value {
Value::Null => false,
Value::Bool(flag) => *flag,
Value::Number(number) => number.as_f64().is_some_and(|float| float != 0.0),
Value::String(text) => !text.is_empty(),
Value::Array(items) => !items.is_empty(),
Value::Object(entries) => !entries.is_empty(),
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
#[rstest]
#[case(0.0, "0.0")]
#[case(-0.0, "-0.0")]
#[case(1.0, "1.0")]
#[case(0.1, "0.1")]
#[case(1.5e-5, "1.5e-05")]
#[case(1e-4, "0.0001")]
#[case(1e-5, "1e-05")]
#[case(1e15, "1000000000000000.0")]
#[case(1e16, "1e+16")]
#[case(1.2345678901234568e17, "1.2345678901234568e+17")]
#[case(123.456, "123.456")]
#[case(1e100, "1e+100")]
#[case(-2.5e-300, "-2.5e-300")]
#[case(f64::NAN, "NaN")]
#[case(f64::INFINITY, "Infinity")]
#[case(f64::NEG_INFINITY, "-Infinity")]
#[case(0.30000000000000004, "0.30000000000000004")]
#[case(0.07872390747070312, "0.07872390747070312")]
#[case(1794700894.4414062, "1794700894.4414062")]
fn floats_print_as_python_repr_prints_them(#[case] value: f64, #[case] expected: &str) {
assert_eq!(float_repr(value), expected);
}
#[rstest]
#[case("plain", "\"plain\"")]
#[case("quote\" back\\", "\"quote\\\" back\\\\\"")]
#[case("\n\r\t\u{8}\u{c}\u{1}\u{7f}", "\"\\n\\r\\t\\b\\f\\u0001\\u007f\"")]
#[case("café", "\"caf\\u00e9\"")]
#[case("😀", "\"\\ud83d\\ude00\"")]
fn strings_are_escaped_as_ensure_ascii_escapes_them(
#[case] text: &str,
#[case] expected: &str,
) {
let mut out = Vec::new();
write_str(&mut out, text);
assert_eq!(String::from_utf8(out).unwrap(), expected);
}
#[rstest]
fn containers_use_the_default_separators_and_keep_key_order() {
assert_eq!(
dumps(&json!({"b": [1, 2.5, null, true], "a": {"x": "y"}})),
"{\"b\": [1, 2.5, null, true], \"a\": {\"x\": \"y\"}}"
);
}
}

View file

@ -0,0 +1,656 @@
use std::time::Duration;
use litellm_auth_aws::AwsAuthConfig;
use litellm_secret_value::SecretValue;
use serde::Deserialize;
use serde_json::{Map, Value};
use crate::pyjson::{py_str, truthy};
fn py_repr(value: &Value) -> String {
litellm_python_compat::repr::repr(&litellm_python_compat::json::from_json(value.clone()))
}
fn py_optional(value: Option<u64>) -> String {
value.map_or_else(|| "None".to_owned(), |value| value.to_string())
}
fn py_type_name(value: &Value) -> &'static str {
match value {
Value::Null => "NoneType",
Value::Bool(_) => "bool",
Value::Number(number) if number.is_f64() => "float",
Value::Number(_) => "int",
Value::String(_) => "str",
Value::Array(_) => "list",
Value::Object(_) => "dict",
}
}
pub const PARTITION_GRANULARITY_ENV_VAR: &str = "S3_PARTITION_GRANULARITY";
pub const LOG_PROMPTS_ONLY_ENV_VAR: &str = "S3_LOG_PROMPTS_ONLY";
const DEFAULT_MAX_QUEUE_SIZE: usize = 50_000;
const DEFAULT_MAX_RETRY_AGE_SECONDS: u64 = 3600;
const DEFAULT_REGION: &str = "us-west-2";
#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct Constants {
pub batch_size: usize,
pub max_concurrent_uploads: usize,
pub max_adaptive_concurrency: usize,
pub upload_attempt_timeout_seconds: f64,
pub download_read_timeout_seconds: f64,
pub download_connect_timeout_seconds: f64,
pub web_identity_session_policy: Option<String>,
}
impl Default for Constants {
fn default() -> Self {
Self {
batch_size: 512,
max_concurrent_uploads: 16,
max_adaptive_concurrency: 200,
upload_attempt_timeout_seconds: 30.0,
download_read_timeout_seconds: 600.0,
download_connect_timeout_seconds: 5.0,
web_identity_session_policy: None,
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Granularity {
#[default]
Day,
Hour,
}
impl Granularity {
pub fn as_str(self) -> &'static str {
match self {
Self::Day => "day",
Self::Hour => "hour",
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Settings {
pub bucket_name: Option<String>,
pub region_name: Option<Value>,
pub endpoint_url: Option<String>,
pub path: Option<String>,
pub verify: Option<Value>,
pub auth: AwsAuthConfig,
pub use_team_prefix: bool,
pub use_key_prefix: bool,
pub strip_base64_files: bool,
pub use_virtual_hosted_style: bool,
pub log_prompts_only: Option<Value>,
pub partition_granularity: Option<Value>,
pub server_side_encryption: Option<String>,
pub sse_kms_key_id: Option<String>,
pub max_concurrent_uploads: usize,
pub max_queue_size: usize,
pub max_retry_age: Option<Duration>,
pub drop_on_terminal_error: bool,
pub adaptive_concurrency: bool,
pub max_adaptive_concurrency: usize,
pub batch_file_upload: bool,
pub batch_size: usize,
pub attempt_timeout: Duration,
pub download_read_timeout: Duration,
pub web_identity_session_policy: Option<String>,
}
pub type Environment = dyn Fn(&str) -> Option<String> + Send + Sync;
fn text(params: &Map<String, Value>, name: &str) -> Option<String> {
params.get(name).filter(|value| truthy(value)).map(py_str)
}
fn secret(params: &Map<String, Value>, name: &str) -> Option<SecretValue> {
text(params, name).map(SecretValue::new)
}
fn present(params: &Map<String, Value>, name: &str) -> Option<Value> {
params.get(name).filter(|value| !value.is_null()).cloned()
}
fn is_blank(value: &Value) -> bool {
matches!(value, Value::String(text) if text.is_empty())
}
pub fn lax_bool(value: &Value) -> Option<bool> {
match value {
Value::Bool(flag) => Some(*flag),
Value::Number(number) => number
.as_f64()
.and_then(|float| {
[(0.0, false), (1.0, true)]
.into_iter()
.find(|(bound, _)| *bound == float)
})
.map(|(_, flag)| flag),
Value::String(text) => match text.trim().to_ascii_lowercase().as_str() {
"0" | "off" | "f" | "false" | "n" | "no" => Some(false),
"1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
_ => None,
},
_ => None,
}
}
pub fn lax_int(value: &Value) -> Option<i128> {
match value {
Value::Bool(flag) => Some(i128::from(*flag)),
Value::Number(number) => {
if let Some(integer) = number.as_i64() {
return Some(i128::from(integer));
}
if let Some(integer) = number.as_u64() {
return Some(i128::from(integer));
}
let float = number.as_f64()?;
(float.is_finite() && float.fract() == 0.0 && float.abs() <= i64::MAX as f64)
.then_some(float as i128)
}
Value::String(text) => int_text(text.trim()),
_ => None,
}
}
fn int_text(text: &str) -> Option<i128> {
let (whole, fraction) = text.split_once('.').unwrap_or((text, ""));
if !fraction.chars().all(|character| character == '0') {
return None;
}
if text.contains('.') && fraction.is_empty() {
return None;
}
let (sign, digits) = match whole.strip_prefix('-') {
Some(rest) => (-1, rest),
None => (1, whole.strip_prefix('+').unwrap_or(whole)),
};
let well_formed = !digits.is_empty()
&& !digits.starts_with('_')
&& !digits.ends_with('_')
&& !digits.contains("__")
&& digits
.chars()
.all(|character| character.is_ascii_digit() || character == '_');
if !well_formed {
return None;
}
digits
.replace('_', "")
.parse::<i128>()
.ok()
.map(|number| sign * number)
}
fn strip_text(value: &Value) -> Value {
match value {
Value::String(text) => Value::String(text.trim().to_owned()),
other => other.clone(),
}
}
fn positive_int(
setting: &str,
configured: Option<&Value>,
fallback: usize,
reject_bool: bool,
) -> usize {
let Some(configured) = configured.filter(|value| !value.is_null() && !is_blank(value)) else {
return fallback;
};
if reject_bool && configured.is_boolean() {
tracing::warn!(
"s3 logging: {setting}={} is a boolean, not an integer, using {fallback}",
py_repr(configured)
);
return fallback;
}
match lax_int(&strip_text(configured)) {
None => {
tracing::warn!(
"s3 logging: {setting}={} is not an integer, using {fallback}",
py_repr(configured)
);
fallback
}
Some(bound) if bound < 1 => {
tracing::warn!(
"s3 logging: {setting}={} must be at least 1, using {fallback}",
py_repr(configured)
);
fallback
}
Some(bound) => usize::try_from(bound).unwrap_or(usize::MAX),
}
}
fn retry_age(configured: &Value, fallback: Option<u64>) -> Option<u64> {
if configured.is_null() || is_blank(configured) {
return None;
}
if configured.is_boolean() {
tracing::warn!(
"s3 logging: s3_max_retry_age_seconds={} is a boolean, not an integer, falling back to {}",
py_repr(configured),
py_optional(fallback)
);
return fallback;
}
match lax_int(&strip_text(configured)) {
None => {
tracing::warn!(
"s3 logging: s3_max_retry_age_seconds={} is not an integer, falling back to {}",
py_repr(configured),
py_optional(fallback)
);
fallback
}
Some(bound) if bound < 0 => {
tracing::warn!(
"s3 logging: s3_max_retry_age_seconds={} must be at least 0, falling back to {}",
py_repr(configured),
py_optional(fallback)
);
fallback
}
Some(0) => None,
Some(bound) => Some(u64::try_from(bound).unwrap_or(u64::MAX)),
}
}
fn flag(
setting: &str,
configured: Option<&Value>,
unset: bool,
invalid: bool,
consequence: &str,
) -> bool {
let Some(configured) = configured.filter(|value| !value.is_null() && !is_blank(value)) else {
return unset;
};
lax_bool(&strip_text(configured)).unwrap_or_else(|| {
tracing::warn!(
"s3 logging: {setting}={} is not a boolean, {consequence}",
py_repr(configured)
);
invalid
})
}
fn sse_value(name: &str, value: Option<&Value>) -> Option<String> {
match value {
None => None,
Some(Value::String(text)) => Some(text.clone()),
Some(other) => {
tracing::warn!(
"s3 logging: ignoring {name} because it has invalid type {}; expected a string",
py_type_name(other)
);
None
}
}
}
pub fn resolve_sse(
algorithm: Option<&Value>,
key_id: Option<&Value>,
) -> (Option<String>, Option<String>) {
let valid_sse = sse_value("s3_server_side_encryption", algorithm);
let valid_key_id = sse_value("s3_sse_kms_key_id", key_id).filter(|key| !key.is_empty());
let Some(algorithm) = valid_sse
.filter(|algorithm| !algorithm.is_empty())
.or_else(|| valid_key_id.as_ref().map(|_| "aws:kms".to_owned()))
else {
return (None, None);
};
if valid_key_id.is_some() && !algorithm.starts_with("aws:kms") {
tracing::warn!(
"s3 logging: ignoring s3_sse_kms_key_id because s3_server_side_encryption is {algorithm}; set it to aws:kms to encrypt with the KMS key"
);
return (Some(algorithm), None);
}
(Some(algorithm), valid_key_id)
}
pub fn resolve_partition_granularity(raw: Option<&Value>) -> Granularity {
match raw {
Some(Value::String(text)) if text == "hour" => Granularity::Hour,
None => Granularity::Day,
Some(Value::String(text)) if text.is_empty() || text == "day" => Granularity::Day,
Some(other) => {
tracing::warn!(
"s3 logging: s3_partition_granularity={} is not one of day, hour, using day",
py_repr(other)
);
Granularity::Day
}
}
}
impl Settings {
pub fn from_params(params: &Map<String, Value>, constants: &Constants) -> Self {
let truthy_param = |name: &str| params.get(name).is_some_and(truthy);
let (server_side_encryption, sse_kms_key_id) = resolve_sse(
params
.get("s3_server_side_encryption")
.filter(|value| truthy(value)),
params
.get("s3_sse_kms_key_id")
.filter(|value| truthy(value)),
);
let max_concurrent_uploads = positive_int(
"s3_max_concurrent_uploads",
Some(
params
.get("s3_max_concurrent_uploads")
.filter(|value| !value.is_null() && !is_blank(value))
.cloned()
.unwrap_or_else(|| Value::from(constants.max_concurrent_uploads)),
)
.as_ref(),
constants.max_concurrent_uploads,
false,
);
let max_queue_size = positive_int(
"s3_max_queue_size",
params.get("s3_max_queue_size"),
DEFAULT_MAX_QUEUE_SIZE,
true,
);
let max_retry_age = match params.get("s3_max_retry_age_seconds") {
Some(value) if !value.is_null() && !is_blank(value) => {
retry_age(value, Some(DEFAULT_MAX_RETRY_AGE_SECONDS))
}
_ => Some(DEFAULT_MAX_RETRY_AGE_SECONDS),
};
Self {
bucket_name: text(params, "s3_bucket_name"),
region_name: params
.get("s3_region_name")
.filter(|value| truthy(value))
.cloned(),
endpoint_url: text(params, "s3_endpoint_url"),
path: text(params, "s3_path"),
verify: present(params, "s3_verify"),
auth: AwsAuthConfig {
access_key_id: secret(params, "s3_aws_access_key_id"),
secret_access_key: secret(params, "s3_aws_secret_access_key"),
session_token: secret(params, "s3_aws_session_token"),
region_name: text(params, "s3_region_name"),
session_name: text(params, "s3_aws_session_name"),
profile_name: text(params, "s3_aws_profile_name"),
role_name: text(params, "s3_aws_role_name"),
web_identity_token: secret(params, "s3_aws_web_identity_token"),
sts_endpoint: text(params, "s3_aws_sts_endpoint"),
external_id: None,
},
use_team_prefix: truthy_param("s3_use_team_prefix"),
use_key_prefix: truthy_param("s3_use_key_prefix"),
strip_base64_files: truthy_param("s3_strip_base64_files"),
use_virtual_hosted_style: truthy_param("s3_use_virtual_hosted_style"),
log_prompts_only: present(params, "s3_log_prompts_only"),
partition_granularity: present(params, "s3_partition_granularity"),
server_side_encryption,
sse_kms_key_id,
max_concurrent_uploads,
max_queue_size,
max_retry_age: max_retry_age.map(Duration::from_secs),
drop_on_terminal_error: flag(
"s3_drop_on_terminal_error",
params.get("s3_drop_on_terminal_error"),
true,
true,
"dropping terminal-failed uploads",
),
adaptive_concurrency: flag(
"s3_adaptive_concurrency",
params.get("s3_adaptive_concurrency"),
false,
false,
"keeping the fixed upload width",
),
max_adaptive_concurrency: positive_int(
"s3_max_adaptive_concurrency",
params.get("s3_max_adaptive_concurrency"),
constants.max_adaptive_concurrency,
true,
),
batch_file_upload: flag(
"s3_batch_file_upload",
params.get("s3_batch_file_upload"),
false,
false,
"keeping per-request objects",
),
batch_size: constants.batch_size,
attempt_timeout: Duration::from_secs_f64(
constants.upload_attempt_timeout_seconds.max(0.0),
),
download_read_timeout: Duration::from_secs_f64(
constants.download_read_timeout_seconds.max(0.0),
),
web_identity_session_policy: constants.web_identity_session_policy.clone(),
}
}
pub fn partition_granularity(&self, environment: &Environment) -> Granularity {
resolve_partition_granularity(self.partition_granularity_raw(environment).as_ref())
}
pub fn partition_granularity_raw(&self, environment: &Environment) -> Option<Value> {
match &self.partition_granularity {
Some(value) => Some(value.clone()),
None => environment(PARTITION_GRANULARITY_ENV_VAR).map(Value::String),
}
}
pub fn log_prompts_only(&self, environment: &Environment) -> bool {
let raw = match &self.log_prompts_only {
Some(value) => Some(value.clone()),
None => environment(LOG_PROMPTS_ONLY_ENV_VAR).map(Value::String),
};
let Some(raw) = raw.filter(|value| !is_blank(value)) else {
return false;
};
lax_bool(&strip_text(&raw)).unwrap_or_else(|| {
tracing::warn!(
"s3 logging: s3_log_prompts_only={} is not a boolean, logging prompts only",
py_repr(&raw)
);
true
})
}
pub fn signing_region(
&self,
from_environment: impl FnOnce() -> Result<Option<String>, String>,
) -> Result<String, String> {
let region = match &self.region_name {
Some(Value::String(region)) => region.clone(),
Some(other) => return Err(invalid_region(&py_str(other))),
None => from_environment()?.unwrap_or_else(|| DEFAULT_REGION.to_owned()),
};
if valid_region(&region) {
Ok(region)
} else {
Err(invalid_region(&region))
}
}
pub fn url_region(&self) -> String {
self.region_name
.as_ref()
.map_or_else(|| "None".to_owned(), py_str)
}
}
fn invalid_region(region: &str) -> String {
format!(
"Invalid AWS region format: {region}. Region names must contain only lowercase letters, digits, and hyphens."
)
}
fn valid_region(region: &str) -> bool {
!region.is_empty()
&& region
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
fn settings(params: Value) -> Settings {
Settings::from_params(params.as_object().unwrap(), &Constants::default())
}
fn no_env(_: &str) -> Option<String> {
None
}
#[rstest]
#[case(json!(true), Some(1))]
#[case(json!(2.0), Some(2))]
#[case(json!(2.5), None)]
#[case(json!(" 4 "), Some(4))]
#[case(json!("4.0"), Some(4))]
#[case(json!("+4"), Some(4))]
#[case(json!("1_000"), Some(1000))]
#[case(json!("0x10"), None)]
#[case(json!("1e3"), None)]
#[case(json!("4.5"), None)]
#[case(json!("007"), Some(7))]
#[case(json!(1e20), None)]
fn integers_parse_as_pydantic_parses_them(
#[case] value: Value,
#[case] expected: Option<i128>,
) {
assert_eq!(lax_int(&strip_text(&value)), expected);
}
#[rstest]
#[case(json!(0), Some(false))]
#[case(json!(2), None)]
#[case(json!(1.0), Some(true))]
#[case(json!(0.5), None)]
#[case(json!(" yes "), Some(true))]
#[case(json!("TRUE"), Some(true))]
#[case(json!("off"), Some(false))]
#[case(json!("1.0"), None)]
#[case(json!("2"), None)]
fn booleans_parse_as_pydantic_parses_them(
#[case] value: Value,
#[case] expected: Option<bool>,
) {
assert_eq!(lax_bool(&strip_text(&value)), expected);
}
#[rstest]
fn defaults_match_the_python_constructor() {
let settings = settings(json!({}));
assert_eq!(settings.bucket_name, None);
assert_eq!(settings.path, None);
assert_eq!(settings.max_concurrent_uploads, 16);
assert_eq!(settings.max_queue_size, 50_000);
assert_eq!(settings.max_retry_age, Some(Duration::from_secs(3600)));
assert!(settings.drop_on_terminal_error);
assert!(!settings.adaptive_concurrency);
assert_eq!(settings.max_adaptive_concurrency, 200);
assert!(!settings.batch_file_upload);
assert_eq!(settings.partition_granularity(&no_env), Granularity::Day);
assert!(!settings.log_prompts_only(&no_env));
}
#[rstest]
fn prefix_flags_take_python_truthiness() {
let settings = settings(json!({"s3_use_team_prefix": "false", "s3_use_key_prefix": 0}));
assert!(settings.use_team_prefix);
assert!(!settings.use_key_prefix);
}
#[rstest]
#[case(json!({"s3_max_retry_age_seconds": 0}), None)]
#[case(json!({"s3_max_retry_age_seconds": "60"}), Some(60))]
#[case(json!({"s3_max_retry_age_seconds": -1}), Some(3600))]
#[case(json!({"s3_max_retry_age_seconds": true}), Some(3600))]
#[case(json!({"s3_max_retry_age_seconds": ""}), Some(3600))]
fn retry_age_parses_as_python(#[case] params: Value, #[case] seconds: Option<u64>) {
assert_eq!(
settings(params).max_retry_age,
seconds.map(Duration::from_secs)
);
}
#[rstest]
#[case(json!({"s3_sse_kms_key_id": "k"}), Some("aws:kms"), Some("k"))]
#[case(json!({"s3_server_side_encryption": "AES256", "s3_sse_kms_key_id": "k"}), Some("AES256"), None)]
#[case(json!({"s3_server_side_encryption": 3}), None, None)]
fn encryption_combines_as_python(
#[case] params: Value,
#[case] algorithm: Option<&str>,
#[case] key: Option<&str>,
) {
let settings = settings(params);
assert_eq!(settings.server_side_encryption.as_deref(), algorithm);
assert_eq!(settings.sse_kms_key_id.as_deref(), key);
}
#[rstest]
#[case(json!({"s3_partition_granularity": "hour"}), None, Granularity::Hour)]
#[case(json!({}), Some("hour"), Granularity::Hour)]
#[case(json!({"s3_partition_granularity": "day"}), Some("hour"), Granularity::Day)]
#[case(json!({"s3_partition_granularity": "minute"}), None, Granularity::Day)]
fn granularity_reads_the_param_then_the_environment(
#[case] params: Value,
#[case] env: Option<&'static str>,
#[case] expected: Granularity,
) {
let environment = move |name: &str| {
(name == PARTITION_GRANULARITY_ENV_VAR)
.then_some(env)
.flatten()
.map(str::to_owned)
};
assert_eq!(
settings(params).partition_granularity(&environment),
expected
);
}
#[rstest]
#[case::default(json!({}), Ok(None), Ok("us-west-2"))]
#[case::param(json!({"s3_region_name": "eu-west-1"}), Ok(Some("ap-south-1")), Ok("eu-west-1"))]
#[case::bad_param(json!({"s3_region_name": "EU"}), Ok(None), Err(()))]
#[case::environment(json!({}), Ok(Some("ap-south-1")), Ok("ap-south-1"))]
#[case::bad_environment(json!({}), Ok(Some("Ap_South")), Err(()))]
#[case::empty_environment(json!({}), Ok(Some("")), Err(()))]
#[case::secret_failure(json!({}), Err(()), Err(()))]
fn the_signing_region_falls_back_like_python(
#[case] params: Value,
#[case] environment: Result<Option<&str>, ()>,
#[case] expected: Result<&str, ()>,
) {
let from_environment = || {
environment
.map(|region| region.map(str::to_owned))
.map_err(|()| "secret manager down".to_owned())
};
assert_eq!(
settings(params)
.signing_region(from_environment)
.map_err(|_| ()),
expected.map(str::to_owned)
);
}
}

View file

@ -0,0 +1,670 @@
use std::{
collections::BTreeMap,
sync::{
Arc, Mutex, MutexGuard, PoisonError,
atomic::{AtomicUsize, Ordering},
},
time::SystemTime,
};
use bytes::Bytes;
use futures_util::{
FutureExt,
future::{BoxFuture, join_all},
};
use litellm_auth_aws::AwsAuthService;
use litellm_callbacks::{
Activation, Callback, Field, FieldError, FieldSet, FieldValues, Hook, HookInput, HookOutput,
LoggedKeyIdentity, Manifest, MetadataKind, OptOutBy, PayloadRef, RenderOptions, Scalar,
Selection, Subscription, typed,
};
use litellm_tracing::Logger;
use serde_json::Value;
use tokio::time::Instant;
use crate::{
key::{Stamp, download_name, logging_id, object_key, parent},
limiter::Limiter,
settings::{Environment, Granularity, Settings, resolve_partition_granularity},
upload::{Backoff, Body, Clock, Held, Object, Outcome, Uploader, panic_message, render_bodies},
};
pub const NAME: &str = "s3_v2";
const JSON: &str = "application/json";
const NDJSON: &str = "application/x-ndjson";
const PROMPTS_ONLY_NULL_KEYS: [&str; 1] = ["response"];
pub trait Host: Send + Sync {
fn callback_failed(&self);
fn web_identity_token(&self, reference: &str) -> Result<String, String>;
}
pub struct S3LogParts {
pub settings: Settings,
pub client: litellm_http::Client,
pub download_client: litellm_http::Client,
pub environment: Arc<Environment>,
pub clock: Clock,
pub backoff: Backoff,
pub host: Arc<dyn Host>,
pub cold_storage_is_s3_v2: bool,
pub region_from_environment: Option<String>,
pub logger: Logger,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BatchUploadFailure {
pub queued: usize,
pub failed: usize,
pub total: usize,
}
#[derive(Clone, Debug)]
struct Element {
object: Object,
retrying_since: Option<Instant>,
is_file: bool,
}
#[derive(Default)]
struct Queue {
items: Vec<Element>,
requeued: usize,
}
pub struct S3Log {
manifest: Manifest,
uploader: Uploader,
limiter: Limiter,
host: Arc<dyn Host>,
cold_storage_is_s3_v2: bool,
logger: Logger,
granularity: Mutex<Option<(Option<Value>, Granularity)>>,
queue: Mutex<Queue>,
flush_lock: tokio::sync::Mutex<()>,
held: Arc<AtomicUsize>,
}
#[derive(Debug, PartialEq)]
enum Alias {
Absent,
Text(String),
NotText,
}
impl Alias {
fn of(identity: &LoggedKeyIdentity, name: &str, value: Option<&Scalar>) -> Self {
let rejected = identity
.rejected
.as_deref()
.is_some_and(|rejected| rejected.iter().any(|member| member == name));
match value {
_ if rejected => Self::NotText,
Some(value) if !value.truthy() => Self::Absent,
None => Self::Absent,
Some(Scalar::Text(text)) => Self::Text(text.clone()),
Some(_) => Self::NotText,
}
}
}
#[derive(Debug)]
struct Event {
start: Option<Stamp>,
id: Option<String>,
metadata: MetadataKind,
team_alias: Alias,
key_alias: Alias,
cold_storage_object_key: Option<String>,
cold_storage_s3_v2: bool,
}
fn manifest() -> Manifest {
Manifest {
selection: Selection {
activation: Activation::DefaultOn,
opt_out_by: vec![OptOutBy::Admin],
honors_no_log: true,
..Selection::named(NAME)
},
subscriptions: vec![Subscription {
reads: FieldSet::of(&[
Field::DeprecatedKwargs,
Field::StandardLoggingPayloadRef,
Field::LogId,
Field::ColdStorageObjectKey,
Field::LoggedKeyIdentity,
Field::AttemptTiming,
]),
awaited: true,
..Subscription::on(Hook::AttemptEnded)
}],
has_request_state: false,
}
}
impl S3Log {
pub fn new(parts: S3LogParts) -> Arc<Self> {
let settings = Arc::new(parts.settings);
let limiter = Limiter::new(
settings.max_concurrent_uploads,
settings.adaptive_concurrency,
settings.max_adaptive_concurrency,
);
let resolver_host = Arc::clone(&parts.host);
let session_policy = settings.web_identity_session_policy.clone();
Arc::new(Self {
manifest: manifest(),
uploader: Uploader {
settings,
client: parts.client,
download_client: parts.download_client,
environment_region: parts.region_from_environment,
auth: AwsAuthService::for_web_identity(
Arc::new(move |reference: &str| resolver_host.web_identity_token(reference)),
session_policy,
),
environment: parts.environment,
clock: parts.clock,
backoff: parts.backoff,
retries: Default::default(),
},
limiter,
host: parts.host,
cold_storage_is_s3_v2: parts.cold_storage_is_s3_v2,
logger: parts.logger,
granularity: Mutex::default(),
queue: Mutex::default(),
flush_lock: tokio::sync::Mutex::new(()),
held: Arc::default(),
})
}
pub fn settings(&self) -> &Settings {
&self.uploader.settings
}
pub fn partition_granularity(&self) -> Granularity {
let raw = self
.settings()
.partition_granularity_raw(self.uploader.environment.as_ref());
let mut cached = self
.granularity
.lock()
.unwrap_or_else(PoisonError::into_inner);
if let Some((seen, resolved)) = cached.as_ref()
&& *seen == raw
{
return *resolved;
}
let resolved = resolve_partition_granularity(raw.as_ref());
*cached = Some((raw, resolved));
resolved
}
pub fn log_prompts_only(&self) -> bool {
self.settings()
.log_prompts_only(self.uploader.environment.as_ref())
}
fn queue(&self) -> MutexGuard<'_, Queue> {
self.queue.lock().unwrap_or_else(PoisonError::into_inner)
}
pub fn queued_keys(&self) -> Vec<String> {
self.queue()
.items
.iter()
.map(|element| element.object.key.clone())
.collect()
}
fn push(&self, element: Element) -> usize {
let mut queue = self.queue();
queue.items.push(element);
queue.items.len()
}
pub fn held_payloads(&self) -> usize {
self.held.load(Ordering::Relaxed)
}
fn hold(&self, payload: Arc<PayloadRef>, options: RenderOptions) -> Body {
Body::Payload(Arc::new(Held::new(
payload,
options,
Arc::clone(&self.held),
)))
}
fn element(&self, event: Event, body: Body) -> Result<Element, String> {
let settings = self.settings();
let start = event
.start
.ok_or_else(|| "the payload carries no start time".to_owned())?;
let mut prefixes = Vec::new();
for (enabled, alias) in [
(settings.use_team_prefix, &event.team_alias),
(settings.use_key_prefix, &event.key_alias),
] {
if !enabled {
continue;
}
if matches!(event.metadata, MetadataKind::Null | MetadataKind::Other) {
return Err("metadata has no get".to_owned());
}
match alias {
Alias::Absent => {}
Alias::Text(alias) => prefixes.push(alias.clone()),
Alias::NotText => return Err("sequence item: expected str instance".to_owned()),
}
}
let mut prefix = prefixes.join("/");
if !prefix.is_empty() {
prefix.push('/');
}
let file_name = event
.id
.as_deref()
.map(|id| logging_id(&start, id))
.unwrap_or_default();
let path = settings.path.clone().unwrap_or_default();
let key_for = |granularity| object_key(&path, &prefix, &start, &file_name, granularity);
let cold_storage_object_key = match (event.cold_storage_s3_v2, &event.metadata) {
(true, MetadataKind::Other) => return Err("metadata has no get".to_owned()),
(true, MetadataKind::Mapping) => event.cold_storage_object_key,
_ => None,
};
let key = match cold_storage_object_key {
Some(cold)
if cold == key_for(Granularity::Day) || cold == key_for(Granularity::Hour) =>
{
cold
}
_ => key_for(self.partition_granularity()),
};
let id = event
.id
.ok_or_else(|| "the payload id is missing or not a string".to_owned())?;
Ok(Element {
object: Object {
key,
body,
content_type: JSON,
download_name: download_name(&start, &id),
},
retrying_since: None,
is_file: false,
})
}
fn event(&self, fields: &FieldValues) -> Result<Event, FieldError> {
let identity = fields
.get::<typed::LoggedKeyIdentity>()?
.unwrap_or_default();
let metadata = identity.metadata_kind.unwrap_or(MetadataKind::Missing);
let cold_storage_object_key = fields.get::<typed::ColdStorageObjectKey>()?;
let reads_cold_storage = match metadata {
MetadataKind::Other => true,
MetadataKind::Mapping => cold_storage_object_key.is_some(),
MetadataKind::Missing | MetadataKind::Null => false,
};
let cold_storage_s3_v2 = reads_cold_storage && self.cold_storage_is_s3_v2;
Ok(Event {
start: fields
.get::<typed::AttemptTiming>()?
.and_then(|timing| timing.start)
.as_ref()
.map(Stamp::from),
id: fields.get::<typed::LogId>()?,
metadata,
team_alias: Alias::of(&identity, "team_alias", identity.team_alias.as_ref()),
key_alias: Alias::of(&identity, "key_alias", identity.key_alias.as_ref()),
cold_storage_object_key,
cold_storage_s3_v2,
})
}
pub async fn submit(&self, fields: &FieldValues) -> Result<(), FieldError> {
let Some(payload) = fields.payload()? else {
return Ok(());
};
let event = self.event(fields)?;
let options = if self.log_prompts_only() {
RenderOptions {
null_keys: PROMPTS_ONLY_NULL_KEYS.map(str::to_owned).to_vec(),
}
} else {
RenderOptions::default()
};
match self.element(event, self.hold(payload, options)) {
Ok(element) => {
self.push(element);
}
Err(error) => self.failed(&error),
}
Ok(())
}
fn failed(&self, error: &str) {
tracing::error!("s3 Layer Error - {error}");
self.host.callback_failed();
}
pub async fn submit_audit(&self, payload: PayloadRef, id: &str) -> Option<BatchUploadFailure> {
let body = self.hold(Arc::new(payload), RenderOptions::default());
let now = Stamp::utc((self.uploader.clock)());
let key = object_key(
self.settings().path.as_deref().unwrap_or_default(),
"audit_logs/",
&now,
&format!("{}_{id}", now.clock()),
self.partition_granularity(),
);
let queued = self.push(Element {
object: Object {
key,
body,
content_type: JSON,
download_name: format!("audit-{id}.json"),
},
retrying_since: None,
is_file: false,
});
if queued >= self.settings().batch_size {
return self.flush().await;
}
None
}
pub async fn upload_now(&self, key: String, body: String, download_name: String) -> bool {
let body = Bytes::from(body);
let object = Object {
key,
body: Body::Ready(body.clone()),
content_type: NDJSON,
download_name,
};
let outcome = self
.uploader
.upload(&object, &body, &self.limiter, false)
.await;
if outcome != Outcome::Delivered {
self.host.callback_failed();
}
outcome == Outcome::Delivered
}
pub async fn download(&self, key: &str) -> Option<Bytes> {
self.uploader.download(key).await
}
pub async fn flush(&self) -> Option<BatchUploadFailure> {
let _flushing = self.flush_lock.lock().await;
let (batch, requeued) = {
let queue = self.queue();
if queue.items.is_empty() {
return None;
}
(queue.items.clone(), queue.requeued)
};
let length = batch.len();
let failure = match std::panic::AssertUnwindSafe(self.send_batch(batch, requeued))
.catch_unwind()
.await
{
Ok(Ok(())) => {
self.queue().items.drain(..length);
return None;
}
Ok(Err(failure)) => failure,
Err(panic) => {
tracing::error!(
"CustomLogger: async_send_batch raised; preserving {length} events in queue for retry: {}",
panic_message(panic.as_ref())
);
None
}
};
let max_queue_size = self.settings().max_queue_size;
let mut queue = self.queue();
let overflow = queue.items.len().saturating_sub(max_queue_size);
if overflow > 0 {
queue.items.drain(..overflow);
tracing::warn!(
"CustomLogger: log queue exceeded max_queue_size={max_queue_size}; dropped {overflow} oldest events."
);
}
failure
}
fn batch_file_mode(&self) -> bool {
if !self.settings().batch_file_upload {
return false;
}
if self.cold_storage_is_s3_v2 {
tracing::warn!(
"s3 logging: s3_batch_file_upload is ignored because s3_v2 is the cold storage logger; per-request objects are required for spend log lookups"
);
return false;
}
true
}
async fn batch_files(&self, batch: &[Element]) -> Result<Vec<Element>, String> {
let now = Stamp::utc((self.uploader.clock)());
let mut groups: BTreeMap<&str, Vec<&Element>> = BTreeMap::new();
for element in batch.iter().filter(|element| !element.is_file) {
groups
.entry(parent(&element.object.key))
.or_default()
.push(element);
}
let bodies: Vec<&Body> = groups
.values()
.flatten()
.map(|element| &element.object.body)
.collect();
let mut rendered = render_bodies(&bodies).await?.into_iter();
let mut files = Vec::with_capacity(groups.len());
for (folder, elements) in groups {
let name = format!("batch_{}_{}", now.clock(), uuid::Uuid::new_v4().simple());
let mut body = Vec::new();
for index in 0..elements.len() {
if index > 0 {
body.push(b'\n');
}
body.extend_from_slice(&rendered.next().unwrap_or_default());
}
files.push(Element {
object: Object {
key: if folder.is_empty() {
format!("{name}.jsonl")
} else {
format!("{folder}/{name}.jsonl")
},
body: Body::Ready(Bytes::from(body)),
content_type: NDJSON,
download_name: format!("{name}.jsonl"),
},
retrying_since: elements
.iter()
.filter_map(|element| element.retrying_since)
.min(),
is_file: true,
});
}
Ok(batch
.iter()
.filter(|element| element.is_file)
.cloned()
.chain(files)
.collect())
}
async fn upload_bounded(&self, element: &Element) -> Outcome {
let _permit = self.limiter.acquire().await;
let outcome = match element.object.body.render().await {
Ok(body) => {
self.uploader
.upload(&element.object, &body, &self.limiter, true)
.await
}
Err(error) => {
tracing::error!("Error uploading to s3: {error}");
Outcome::Retry
}
};
if outcome != Outcome::Delivered {
self.host.callback_failed();
}
outcome
}
async fn send_batch(
&self,
batch: Vec<Element>,
requeued_count: usize,
) -> Result<(), Option<BatchUploadFailure>> {
let uploads = if self.batch_file_mode() {
match self.batch_files(&batch).await {
Ok(files) => files,
Err(error) => {
tracing::error!(
"CustomLogger: async_send_batch raised; preserving {} events in queue for retry: {error}",
batch.len()
);
return Err(None);
}
}
} else {
batch.clone()
};
self.uploader.retries.store(0, Ordering::Relaxed);
let stale = if uploads.len() == batch.len() {
requeued_count.min(uploads.len())
} else {
0
};
let order: Vec<usize> = (stale..uploads.len()).chain(0..stale).collect();
let finished = join_all(
order
.iter()
.map(|index| self.upload_bounded(&uploads[*index])),
)
.await;
let mut outcomes = vec![Outcome::Retry; uploads.len()];
for (index, outcome) in order.into_iter().zip(finished) {
outcomes[index] = outcome;
}
let retries = self.uploader.retries.load(Ordering::Relaxed);
if retries > 0 {
tracing::warn!(
"s3 logging: {retries} in-call retries across {} uploads this flush",
uploads.len()
);
}
let delivered = outcomes
.iter()
.filter(|outcome| **outcome == Outcome::Delivered)
.count();
let failed: Vec<(&Element, Outcome)> = uploads
.iter()
.zip(outcomes)
.filter(|(_, outcome)| *outcome != Outcome::Delivered)
.collect();
let now = Instant::now();
let max_age = self.settings().max_retry_age;
let requeued: Vec<Element> = if delivered == 0 {
failed
.iter()
.map(|(element, _)| (*element).clone())
.collect()
} else {
failed
.iter()
.filter(|(element, outcome)| {
*outcome != Outcome::Dropped
&& !matches!(
(max_age, element.retrying_since),
(Some(max_age), Some(since)) if now.duration_since(since) > max_age
)
})
.map(|(element, _)| {
let mut element = (*element).clone();
if element.retrying_since.is_none() && max_age.is_some() {
element.retrying_since = Some(now);
}
element
})
.collect()
};
let dropped = failed.len() - requeued.len();
if dropped > 0 {
let terminal = failed
.iter()
.filter(|(_, outcome)| *outcome == Outcome::Dropped)
.count();
tracing::warn!(
"s3 logging: {dropped} uploads dropped this flush: {terminal} rejected as terminal by S3, {} retrying longer than s3_max_retry_age_seconds={}",
dropped - terminal,
max_age.map_or_else(|| "None".to_owned(), |age| age.as_secs().to_string())
);
}
let mut queue = self.queue();
if requeued.is_empty() {
queue.requeued = 0;
return Ok(());
}
let max_queue_size = self.settings().max_queue_size;
let split = batch.len().min(queue.items.len());
let arrivals = queue.items.split_off(split);
let overflow = (requeued.len() + arrivals.len()).saturating_sub(max_queue_size);
if overflow > 0 {
tracing::warn!(
"s3 logging: queue exceeded max_queue_size={max_queue_size} after a failed flush, dropped {overflow} oldest events"
);
}
let requeued_length = requeued.len();
queue.items = requeued
.into_iter()
.chain(arrivals)
.skip(overflow)
.collect();
queue.requeued = requeued_length.saturating_sub(overflow);
Err(Some(BatchUploadFailure {
queued: batch.len(),
failed: failed.len(),
total: uploads.len(),
}))
}
}
impl Callback for S3Log {
fn manifest(&self) -> &Manifest {
&self.manifest
}
type State = ();
fn handle<'a>(
&'a self,
input: &'a HookInput,
_state: Option<&'a ()>,
) -> BoxFuture<'a, HookOutput> {
Box::pin(async move {
if input.hook == Hook::AttemptEnded {
self.logger.instrument(self.submit(&input.fields)).await?;
}
Ok(None)
})
}
fn flush(&self) -> BoxFuture<'_, ()> {
Box::pin(self.logger.instrument(S3Log::flush(self)).map(drop))
}
}
pub fn system_clock() -> SystemTime {
SystemTime::now()
}

View file

@ -0,0 +1,470 @@
use std::{
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, SystemTime},
};
use base64::Engine;
use bytes::Bytes;
use futures_util::FutureExt;
use litellm_auth_aws::{AwsAuthService, Credentials};
use litellm_callbacks::{PayloadRef, RenderOptions, render_all};
use litellm_s3::signed::{Reply, SendError, SignedRequest, send};
use reqwest::Method;
use crate::{
key::{hex_sha256, object_url},
limiter::Limiter,
settings::{Environment, Settings},
};
const ATTEMPTS: u32 = 3;
const RETRYABLE: [u16; 4] = [403, 429, 500, 503];
const TERMINAL_CODES: [&str; 7] = [
"EntityTooLarge",
"InvalidArgument",
"MalformedXML",
"InvalidDigest",
"KeyTooLongError",
"BadDigest",
"InvalidRequest",
];
const CACHE_CONTROL: &str = "private, immutable, max-age=31536000, s-maxage=0";
const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Outcome {
Delivered,
Retry,
Dropped,
}
pub struct Held {
payload: Arc<PayloadRef>,
options: RenderOptions,
live: Arc<AtomicUsize>,
}
impl Held {
pub fn new(payload: Arc<PayloadRef>, options: RenderOptions, live: Arc<AtomicUsize>) -> Self {
live.fetch_add(1, Ordering::Relaxed);
Self {
payload,
options,
live,
}
}
}
impl std::fmt::Debug for Held {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("Held")
.field("payload", &self.payload)
.field("options", &self.options)
.finish()
}
}
impl Drop for Held {
fn drop(&mut self) {
self.live.fetch_sub(1, Ordering::Relaxed);
}
}
#[derive(Clone, Debug)]
pub enum Body {
Ready(Bytes),
Payload(Arc<Held>),
}
#[derive(Clone, Debug)]
pub struct Object {
pub key: String,
pub body: Body,
pub content_type: &'static str,
pub download_name: String,
}
pub fn panic_message(panic: &(dyn std::any::Any + Send)) -> String {
panic
.downcast_ref::<&str>()
.map(|message| (*message).to_owned())
.or_else(|| panic.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "a panic without a message".to_owned())
}
pub async fn render_bodies(bodies: &[&Body]) -> Result<Vec<Bytes>, String> {
let held: Vec<&Held> = bodies
.iter()
.filter_map(|body| match body {
Body::Payload(held) => Some(held.as_ref()),
Body::Ready(_) => None,
})
.collect();
let mut groups: Vec<(&RenderOptions, Vec<usize>)> = Vec::new();
for (index, item) in held.iter().enumerate() {
match groups
.iter_mut()
.find(|(options, _)| **options == item.options)
{
Some((_, indexes)) => indexes.push(index),
None => groups.push((&item.options, vec![index])),
}
}
let mut rendered: Vec<Option<Bytes>> = vec![None; held.len()];
for (options, indexes) in groups {
let payloads: Vec<&PayloadRef> = indexes
.iter()
.map(|index| held[*index].payload.as_ref())
.collect();
let encoded = std::panic::AssertUnwindSafe(render_all(&payloads, options.clone()))
.catch_unwind()
.await
.map_err(|panic| panic_message(panic.as_ref()))??;
for (index, bytes) in indexes.into_iter().zip(encoded) {
rendered[index] = Some(Bytes::from(bytes));
}
}
let mut rendered = rendered.into_iter();
bodies
.iter()
.map(|body| match body {
Body::Ready(bytes) => Ok(bytes.clone()),
Body::Payload(_) => rendered
.next()
.flatten()
.ok_or_else(|| "the host rendered too few payloads".to_owned()),
})
.collect()
}
impl Body {
pub async fn render(&self) -> Result<Bytes, String> {
render_bodies(&[self])
.await?
.pop()
.ok_or_else(|| "the host rendered no payload".to_owned())
}
}
pub type Clock = fn() -> SystemTime;
pub type Backoff = Arc<dyn Fn(u32) -> Duration + Send + Sync>;
pub struct Uploader {
pub settings: Arc<Settings>,
pub client: litellm_http::Client,
pub download_client: litellm_http::Client,
pub environment_region: Option<String>,
pub auth: AwsAuthService,
pub environment: Arc<Environment>,
pub clock: Clock,
pub backoff: Backoff,
pub retries: AtomicUsize,
}
enum Attempt {
Reply(Reply),
Failed(SendError),
}
pub fn error_code(body: &str) -> Option<&str> {
let mut rest = body;
while let Some(start) = rest.find("<Code>") {
let after = &rest[start + "<Code>".len()..];
let code_length = after.find('<').unwrap_or(after.len());
if code_length > 0 && after[code_length..].starts_with("</Code>") {
return Some(&after[..code_length]);
}
rest = after;
}
None
}
pub fn is_terminal(status: u16, body: &str) -> bool {
matches!(status, 400 | 403)
&& error_code(body).is_some_and(|code| TERMINAL_CODES.contains(&code))
}
pub fn python_backoff(attempt: u32) -> Duration {
Duration::from_secs_f64(f64::from(1u32 << attempt) * (0.5 + rand::random::<f64>()))
}
fn throttled(attempt: &Attempt) -> bool {
match attempt {
Attempt::Failed(_) => true,
Attempt::Reply(reply) => {
matches!(reply.status, 429 | 503) || error_code(&reply.text()) == Some("SlowDown")
}
}
}
impl Uploader {
pub fn headers(&self, object: &Object, body: &Bytes) -> Vec<(String, String)> {
let md5 = base64::engine::general_purpose::STANDARD
.encode(<md5::Md5 as md5::Digest>::digest(body));
let mut headers = vec![
("Content-Type".to_owned(), object.content_type.to_owned()),
("Content-MD5".to_owned(), md5),
("x-amz-content-sha256".to_owned(), hex_sha256(body)),
("Content-Language".to_owned(), "en".to_owned()),
(
"Content-Disposition".to_owned(),
format!("inline; filename=\"{}\"", object.download_name),
),
("Cache-Control".to_owned(), CACHE_CONTROL.to_owned()),
];
if let Some(algorithm) = &self.settings.server_side_encryption {
headers.push(("x-amz-server-side-encryption".to_owned(), algorithm.clone()));
}
if let Some(key_id) = &self.settings.sse_kms_key_id {
headers.push((
"x-amz-server-side-encryption-aws-kms-key-id".to_owned(),
key_id.clone(),
));
}
headers
}
async fn credentials(&self) -> Result<Credentials, String> {
let environment = Arc::clone(&self.environment);
let lookup = move |name: &str| environment(name);
self.auth
.resolve_credentials(self.settings.auth.clone(), &lookup)
.await
.map_err(|error| error.to_string())
}
async fn forget_credentials(&self) {
let environment = Arc::clone(&self.environment);
let lookup = move |name: &str| environment(name);
self.auth
.invalidate_credentials(self.settings.auth.clone(), &lookup)
.await;
}
async fn attempt(
&self,
url: &str,
headers: &[(String, String)],
body: &Bytes,
) -> Result<Attempt, String> {
let credentials = self.credentials().await?;
let region = self
.settings
.signing_region(|| Ok(self.environment_region.clone()))?;
let put = || {
send(
&self.client,
SignedRequest {
method: Method::PUT,
url,
headers,
body: body.clone(),
region: &region,
credentials: &credentials,
time: (self.clock)(),
read_timeout: Some(self.settings.attempt_timeout),
},
)
};
let sent = match put().await {
Err(error) if error.is_reconnectable() => put().await,
sent => sent,
};
match sent {
Ok(reply) => Ok(Attempt::Reply(reply)),
Err(error) if error.is_transient() => Ok(Attempt::Failed(error)),
Err(error) => Err(error.to_string()),
}
}
fn retry_reason(&self, attempt: &Attempt) -> Option<String> {
match attempt {
Attempt::Failed(error) => Some(error.to_string()),
Attempt::Reply(reply) => {
if !RETRYABLE.contains(&reply.status) {
return None;
}
if self.settings.drop_on_terminal_error && is_terminal(reply.status, &reply.text())
{
return None;
}
Some(reply.status.to_string())
}
}
}
pub async fn upload(
&self,
object: &Object,
body: &Bytes,
limiter: &Limiter,
in_flush: bool,
) -> Outcome {
let url = object_url(&self.settings, &object.key);
let headers = self.headers(object, body);
let mut attempt_number = 0;
let last = loop {
let attempt = match self.attempt(&url, &headers, body).await {
Ok(attempt) => attempt,
Err(error) => {
tracing::error!("Error uploading to s3: {error}");
return Outcome::Retry;
}
};
limiter.record(throttled(&attempt));
if attempt_number + 1 >= ATTEMPTS {
break attempt;
}
let Some(reason) = self.retry_reason(&attempt) else {
break attempt;
};
if matches!(&attempt, Attempt::Reply(reply) if reply.status == 403) {
self.forget_credentials().await;
}
let wait = (self.backoff)(attempt_number);
if in_flush {
tracing::debug!(
"S3 upload attempt {}/{ATTEMPTS} failed with {reason}, retrying in {:.2}s key={}",
attempt_number + 1,
wait.as_secs_f64(),
object.key
);
} else {
tracing::warn!(
"S3 upload attempt {}/{ATTEMPTS} failed with {reason}, retrying in {:.2}s key={}",
attempt_number + 1,
wait.as_secs_f64(),
object.key
);
}
self.retries.fetch_add(1, Ordering::Relaxed);
tokio::time::sleep(wait).await;
attempt_number += 1;
};
match last {
Attempt::Reply(reply) if (200..300).contains(&reply.status) => Outcome::Delivered,
Attempt::Reply(reply) => {
let body = reply.text();
tracing::error!(
"Error uploading to s3: status {} for {}: {}",
reply.status,
object.key,
body.chars().take(512).collect::<String>()
);
if self.settings.drop_on_terminal_error && is_terminal(reply.status, &body) {
tracing::warn!(
"s3 logging: dropping object {} after terminal status {}",
object.key,
reply.status
);
return Outcome::Dropped;
}
Outcome::Retry
}
Attempt::Failed(error) => {
tracing::error!("Error uploading to s3: {error}");
Outcome::Retry
}
}
}
pub async fn download(&self, key: &str) -> Option<Bytes> {
let credentials = match self.credentials().await {
Ok(credentials) => credentials,
Err(error) => {
tracing::error!("Error downloading from S3: {error}");
return None;
}
};
let Some(region) = self
.settings
.region_name
.as_ref()
.map(crate::pyjson::py_str)
else {
tracing::error!("Error downloading from S3: no s3_region_name to sign with");
return None;
};
let headers = [("x-amz-content-sha256".to_owned(), EMPTY_SHA256.to_owned())];
let reply = send(
&self.download_client,
SignedRequest {
method: Method::GET,
url: &object_url(&self.settings, key),
headers: &headers,
body: Bytes::new(),
region: &region,
credentials: &credentials,
time: (self.clock)(),
read_timeout: Some(self.settings.download_read_timeout),
},
)
.await;
match reply {
Ok(reply) if reply.status == 200 => Some(reply.body),
Ok(reply) => {
tracing::error!("S3 object not found, saw response={}", reply.text());
None
}
Err(error) => {
tracing::error!("Error downloading from S3: {error}");
None
}
}
}
}
#[cfg(test)]
pub fn md5_base64(body: &[u8]) -> String {
base64::engine::general_purpose::STANDARD.encode(<md5::Md5 as md5::Digest>::digest(body))
}
#[cfg(test)]
pub fn sha256_hex(body: &[u8]) -> String {
crate::key::hex(&<sha2::Sha256 as sha2::Digest>::digest(body))
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case(400, "<Error><Code>EntityTooLarge</Code></Error>", true)]
#[case(403, "<Error><Code>BadDigest</Code></Error>", true)]
#[case(403, "<Error><Code>AccessDenied</Code></Error>", false)]
#[case(503, "<Error><Code>InvalidRequest</Code></Error>", false)]
#[case(400, "bad request", false)]
#[case(400, "<Code></Code><Code>InvalidDigest</Code>", true)]
fn only_object_rejections_are_terminal(
#[case] status: u16,
#[case] body: &str,
#[case] expected: bool,
) {
assert_eq!(is_terminal(status, body), expected);
}
#[rstest]
#[case(0, 0.5, 1.5)]
#[case(1, 1.0, 3.0)]
fn backoff_doubles_with_jitter(#[case] attempt: u32, #[case] low: f64, #[case] high: f64) {
for _ in 0..50 {
let wait = python_backoff(attempt).as_secs_f64();
assert!((low..high).contains(&wait), "{wait}");
}
}
#[rstest]
fn digests_match_python() {
assert_eq!(md5_base64(b"{}"), "mZFLkyvTelC5g8XnyQrpOw==");
assert_eq!(
sha256_hex(b""),
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
);
}
}

View file

@ -0,0 +1,693 @@
use std::{
collections::HashMap,
sync::{
Arc, Mutex,
atomic::{AtomicU64, AtomicUsize, Ordering},
},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use futures_util::future::BoxFuture;
use litellm_callbacks::{
Callback, Field, FieldSet, FieldValues, Hook, HookInput, LoggedKeyIdentity, MetadataKind,
PayloadHost, PayloadRef, RenderOptions, Rendered, Scalar,
};
use litellm_callbacks_s3_log::{BatchUploadFailure, Constants, Host, S3Log, S3LogParts, Settings};
use rstest::rstest;
use serde_json::{Value, json};
use wiremock::{
Mock, MockServer, Request, ResponseTemplate,
matchers::{method, path_regex},
};
#[derive(Default)]
struct Recorder {
failures: AtomicUsize,
next: AtomicU64,
payloads: Mutex<HashMap<u64, Option<String>>>,
}
const PANICS: &str = "panics while rendering";
impl Recorder {
fn stash(self: &Arc<Self>, body: Option<&str>) -> PayloadRef {
let id = self.next.fetch_add(1, Ordering::Relaxed);
self.payloads
.lock()
.unwrap()
.insert(id, body.map(str::to_owned));
PayloadRef::new(id, Arc::clone(self) as Arc<dyn PayloadHost>)
}
fn rewrite(&self, id: u64, body: &str) {
self.payloads
.lock()
.unwrap()
.insert(id, Some(body.to_owned()));
}
fn held(&self) -> usize {
self.payloads.lock().unwrap().len()
}
fn failures(&self) -> usize {
self.failures.load(Ordering::Relaxed)
}
}
impl PayloadHost for Recorder {
fn render(&self, payloads: Vec<u64>, options: RenderOptions) -> BoxFuture<'static, Rendered> {
let bodies: Vec<Option<String>> = payloads
.iter()
.map(|payload| {
self.payloads
.lock()
.unwrap()
.get(payload)
.cloned()
.flatten()
})
.collect();
let rendered = bodies
.into_iter()
.map(|body| match body {
Some(body) if body == PANICS => panic!("dictionary changed size during iteration"),
Some(body) if options.null_keys.is_empty() => Ok(body.into_bytes()),
Some(body) => {
Ok(format!("{body} without {}", options.null_keys.join(",")).into_bytes())
}
None => Err("the payload cannot be encoded".to_owned()),
})
.collect();
Box::pin(std::future::ready(rendered))
}
fn release(&self, payload: u64) {
self.payloads.lock().unwrap().remove(&payload);
}
}
impl Host for Recorder {
fn callback_failed(&self) {
self.failures.fetch_add(1, Ordering::Relaxed);
}
fn web_identity_token(&self, reference: &str) -> Result<String, String> {
Ok(reference.to_owned())
}
}
fn fixed_clock() -> SystemTime {
UNIX_EPOCH + Duration::from_secs(1_763_024_707)
}
fn no_env(_: &str) -> Option<String> {
None
}
fn sink(server: &MockServer, extra: Value) -> (Arc<S3Log>, Arc<Recorder>) {
sink_with(server, extra, Constants::default())
}
fn sink_with(
server: &MockServer,
extra: Value,
constants: Constants,
) -> (Arc<S3Log>, Arc<Recorder>) {
sink_from(server, extra, constants, false, None)
}
fn sink_from(
server: &MockServer,
extra: Value,
constants: Constants,
cold_storage_is_s3_v2: bool,
region_from_environment: Option<&str>,
) -> (Arc<S3Log>, Arc<Recorder>) {
let mut params = json!({
"s3_bucket_name": "logs",
"s3_region_name": "us-east-1",
"s3_endpoint_url": server.uri(),
"s3_aws_access_key_id": "AKIDEXAMPLE",
"s3_aws_secret_access_key": "secret",
"s3_path": "base",
});
params
.as_object_mut()
.unwrap()
.extend(extra.as_object().unwrap().clone());
params
.as_object_mut()
.unwrap()
.retain(|_, value| !value.is_null());
let host = Arc::new(Recorder::default());
let sink = S3Log::new(S3LogParts {
settings: Settings::from_params(params.as_object().unwrap(), &constants),
client: litellm_http::Client::for_test(),
download_client: litellm_http::Client::for_test(),
environment: Arc::new(no_env),
clock: fixed_clock,
backoff: Arc::new(|_| Duration::ZERO),
host: Arc::clone(&host) as Arc<dyn Host>,
cold_storage_is_s3_v2,
region_from_environment: region_from_environment.map(str::to_owned),
logger: Default::default(),
});
(sink, host)
}
const READS: [Field; 6] = [
Field::DeprecatedKwargs,
Field::StandardLoggingPayloadRef,
Field::LogId,
Field::ColdStorageObjectKey,
Field::LoggedKeyIdentity,
Field::AttemptTiming,
];
struct Event {
payload: Option<PayloadRef>,
id: &'static str,
identity: Value,
cold_storage_object_key: Option<&'static str>,
}
impl Event {
fn input(self) -> HookInput {
let mut fields = FieldValues::new(FieldSet::of(&READS))
.with(Field::DeprecatedKwargs, Value::Null)
.with(Field::LogId, json!(self.id))
.with(Field::LoggedKeyIdentity, self.identity)
.with(
Field::AttemptTiming,
json!({"start": {"year": 2026, "month": 11, "day": 13, "hour": 9, "minute": 5, "second": 7, "microsecond": 42}}),
);
if let Some(key) = self.cold_storage_object_key {
fields = fields.with(Field::ColdStorageObjectKey, json!(key));
}
if let Some(payload) = self.payload {
fields.set_payload(payload);
}
HookInput {
hook: Hook::AttemptEnded,
fields,
call: None,
}
}
}
fn event(host: &Arc<Recorder>, id: &'static str, body: &str) -> Event {
event_with(host.stash(Some(body)), id)
}
fn event_with(payload: PayloadRef, id: &'static str) -> Event {
Event {
payload: Some(payload),
id,
identity: json!({"metadata_kind": "mapping", "team_alias": "team-a"}),
cold_storage_object_key: None,
}
}
async fn submit(sink: &S3Log, event: Event) {
sink.handle(&event.input(), None).await.unwrap();
}
fn header(request: &Request, name: &str) -> Option<String> {
request
.headers
.get(name)
.map(|value| value.to_str().unwrap().to_owned())
}
#[rstest]
#[tokio::test]
async fn a_flush_puts_one_object_per_event_with_the_python_headers() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink(
&server,
json!({"s3_use_team_prefix": true, "s3_sse_kms_key_id": "kms-1"}),
);
submit(
&sink,
event(&host, "chatcmpl-1", "{\"id\": \"chatcmpl-1\"}"),
)
.await;
let failure = sink.flush().await;
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
let put = &requests[0];
assert_eq!(
put.url.path(),
"/logs/base/team-a/2026-11-13/time-09-05-07-000042_chatcmpl-1.json"
);
assert_eq!(put.body, b"{\"id\": \"chatcmpl-1\"}");
assert_eq!(
header(put, "content-type").as_deref(),
Some("application/json")
);
assert_eq!(
header(put, "content-md5").as_deref(),
Some("Ma6yKrRUR7fU10HikmP0wA==")
);
assert_eq!(
header(put, "content-disposition").as_deref(),
Some("inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"")
);
assert_eq!(
header(put, "x-amz-server-side-encryption").as_deref(),
Some("aws:kms")
);
assert_eq!(
header(put, "x-amz-date").as_deref(),
Some("20251113T090507Z")
);
assert!(
header(put, "authorization").unwrap().starts_with(
"AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20251113/us-east-1/s3/aws4_request"
)
);
assert!(sink.queued_keys().is_empty());
assert_eq!(host.failures.load(Ordering::Relaxed), 0);
assert_eq!(failure, None);
}
#[rstest]
#[tokio::test]
async fn a_bucket_that_fails_every_upload_keeps_the_queue() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(503))
.mount(&server)
.await;
let (sink, host) = sink(&server, json!({}));
submit(&sink, event(&host, "a", "{}")).await;
submit(&sink, event(&host, "b", "{}")).await;
let failure = sink.flush().await;
assert_eq!(server.received_requests().await.unwrap().len(), 6);
assert_eq!(sink.queued_keys().len(), 2);
assert_eq!(host.failures.load(Ordering::Relaxed), 2);
assert_eq!(
failure,
Some(BatchUploadFailure {
queued: 2,
failed: 2,
total: 2
})
);
}
#[rstest]
#[tokio::test]
async fn a_terminal_rejection_is_dropped_when_another_upload_lands() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.and(path_regex("_bad\\.json$"))
.respond_with(
ResponseTemplate::new(400)
.set_body_string("<Error><Code>EntityTooLarge</Code></Error>"),
)
.mount(&server)
.await;
Mock::given(method("PUT"))
.and(path_regex("_slow\\.json$"))
.respond_with(ResponseTemplate::new(500))
.mount(&server)
.await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink(&server, json!({}));
for id in ["ok", "bad", "slow"] {
submit(&sink, event(&host, id, "{}")).await;
}
sink.flush().await;
assert_eq!(
sink.queued_keys(),
["base/2026-11-13/time-09-05-07-000042_slow.json"]
);
assert_eq!(host.held(), 1);
}
#[rstest]
#[tokio::test]
async fn batch_files_group_one_flush_by_folder() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink(&server, json!({"s3_batch_file_upload": "yes"}));
submit(&sink, event(&host, "a", "{\"n\": 1}")).await;
submit(&sink, event(&host, "b", "{\"n\": 2}")).await;
sink.flush().await;
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
assert!(
requests[0]
.url
.path()
.starts_with("/logs/base/2026-11-13/batch_09-05-07_")
);
assert_eq!(requests[0].body, b"{\"n\": 1}\n{\"n\": 2}");
assert_eq!(
header(&requests[0], "content-type").as_deref(),
Some("application/x-ndjson")
);
}
#[rstest]
#[tokio::test]
async fn an_event_python_would_reject_is_counted_and_skipped() {
let server = MockServer::start().await;
let (sink, host) = sink(&server, json!({"s3_use_key_prefix": true}));
let mut bad = event(&host, "a", "{}");
bad.identity["key_alias"] = json!(true);
submit(&sink, bad).await;
submit(
&sink,
Event {
payload: None,
id: "b",
identity: json!({"metadata_kind": "mapping"}),
cold_storage_object_key: None,
},
)
.await;
assert!(sink.queued_keys().is_empty());
assert_eq!(host.failures(), 1);
assert_eq!(host.held(), 0);
}
#[rstest]
#[tokio::test]
async fn the_body_is_encoded_at_flush_and_released_once_delivered() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink(&server, json!({}));
let payload = host.stash(Some("{\"n\": 1}"));
let handle = payload.handle();
submit(&sink, event_with(payload, "a")).await;
host.rewrite(handle, "{\"n\": 2}");
sink.flush().await;
assert_eq!(
server.received_requests().await.unwrap()[0].body,
b"{\"n\": 2}"
);
assert_eq!(host.held(), 0);
}
#[rstest]
#[tokio::test]
async fn a_body_that_cannot_be_encoded_stays_queued_and_counts_a_failure() {
let server = MockServer::start().await;
let (sink, host) = sink(&server, json!({}));
submit(&sink, event_with(host.stash(None), "a")).await;
sink.flush().await;
assert!(server.received_requests().await.unwrap().is_empty());
assert_eq!(sink.queued_keys().len(), 1);
assert_eq!(host.failures(), 1);
}
#[rstest]
#[case::environment(Some("ap-south-1"), Some("/ap-south-1/"))]
#[case::default(None, Some("/us-west-2/"))]
#[case::invalid(Some("Ap_South"), None)]
#[tokio::test]
async fn without_a_region_param_the_constructor_region_signs(
#[case] region: Option<&str>,
#[case] scope: Option<&str>,
) {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink_from(
&server,
json!({"s3_region_name": null, "s3_aws_session_token": "session"}),
Constants::default(),
false,
region,
);
submit(&sink, event(&host, "a", "{}")).await;
sink.flush().await;
let requests = server.received_requests().await.unwrap();
let signed = requests
.first()
.and_then(|put| header(put, "authorization"))
.map(|authorization| scope.is_some_and(|scope| authorization.contains(scope)));
assert_eq!(signed, scope.map(|_| true));
assert_eq!(sink.queued_keys().len(), usize::from(scope.is_none()));
}
#[rstest]
#[tokio::test]
async fn a_reply_slower_than_the_attempt_timeout_is_retried_then_kept() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200).set_delay(Duration::from_millis(600)))
.mount(&server)
.await;
let (sink, host) = sink_with(
&server,
json!({}),
Constants {
upload_attempt_timeout_seconds: 0.1,
..Constants::default()
},
);
submit(&sink, event(&host, "a", "{}")).await;
sink.flush().await;
assert_eq!(server.received_requests().await.unwrap().len(), 3);
assert_eq!(sink.queued_keys().len(), 1);
assert_eq!(host.failures(), 1);
}
#[rstest]
#[tokio::test]
async fn a_panic_while_rendering_keeps_that_object_and_the_flush_goes_on() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink(&server, json!({}));
let racing = host.stash(Some(PANICS));
let handle = racing.handle();
submit(&sink, event_with(racing, "a")).await;
submit(&sink, event(&host, "b", "{}")).await;
sink.flush().await;
let paths: Vec<String> = server
.received_requests()
.await
.unwrap()
.iter()
.map(|put| put.url.path().to_owned())
.collect();
assert_eq!(paths, ["/logs/base/2026-11-13/time-09-05-07-000042_b.json"]);
assert_eq!(
sink.queued_keys(),
["base/2026-11-13/time-09-05-07-000042_a.json"]
);
assert_eq!(host.failures(), 1);
host.rewrite(handle, "{}");
sink.flush().await;
assert!(sink.queued_keys().is_empty());
assert_eq!(server.received_requests().await.unwrap().len(), 2);
}
#[rstest]
#[tokio::test]
async fn a_panic_while_rendering_a_batch_file_keeps_the_whole_queue() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink(&server, json!({"s3_batch_file_upload": true}));
let racing = host.stash(Some(PANICS));
let handle = racing.handle();
submit(&sink, event_with(racing, "a")).await;
submit(&sink, event(&host, "b", "{}")).await;
sink.flush().await;
assert!(server.received_requests().await.unwrap().is_empty());
assert_eq!(sink.queued_keys().len(), 2);
host.rewrite(handle, "{}");
sink.flush().await;
assert!(sink.queued_keys().is_empty());
assert_eq!(server.received_requests().await.unwrap().len(), 1);
}
#[rstest]
#[case::bool_alias(json!({"metadata_kind": "mapping", "team_alias": true}))]
#[case::int_alias(json!({"metadata_kind": "mapping", "team_alias": 7}))]
#[case::nan_alias(serde_json::to_value(LoggedKeyIdentity {
metadata_kind: Some(MetadataKind::Mapping),
team_alias: Some(Scalar::Float(f64::NAN)),
..LoggedKeyIdentity::default()
}).unwrap())]
#[case::list_alias(json!({"metadata_kind": "mapping", "team_alias": {"text": "['team-a']", "truthy": true, "hashable": false, "integer": false, "type_name": "list", "hash_error": "unhashable type: 'list'"}}))]
#[case::alias_whose_str_raises(json!({"metadata_kind": "mapping", "rejected": ["team_alias"]}))]
#[case::null_metadata(json!({"metadata_kind": "null"}))]
#[case::metadata_without_get(json!({"metadata_kind": "other"}))]
#[tokio::test]
async fn a_team_prefix_python_cannot_build_is_counted_and_released(#[case] identity: Value) {
let server = MockServer::start().await;
let (sink, host) = sink(&server, json!({"s3_use_team_prefix": true}));
let mut bad = event(&host, "a", "{}");
bad.identity = identity;
submit(&sink, bad).await;
assert!(sink.queued_keys().is_empty());
assert_eq!(host.failures(), 1);
assert_eq!(host.held(), 0);
}
#[rstest]
#[case::empty_text(json!(""))]
#[case::zero(json!(0))]
#[case::zero_float(json!(0.0))]
#[case::false_flag(json!(false))]
#[case::null(Value::Null)]
#[case::empty_list(json!({"text": "[]", "truthy": false, "hashable": false, "integer": false, "type_name": "list", "hash_error": "unhashable type: 'list'"}))]
#[tokio::test]
async fn a_falsy_team_alias_adds_no_prefix(#[case] alias: Value) {
let server = MockServer::start().await;
let (sink, host) = sink(&server, json!({"s3_use_team_prefix": true}));
let mut falsy = event(&host, "a", "{}");
falsy.identity = json!({"metadata_kind": "mapping", "team_alias": alias});
submit(&sink, falsy).await;
assert_eq!(
sink.queued_keys(),
["base/2026-11-13/time-09-05-07-000042_a.json"]
);
assert_eq!(host.failures(), 0);
}
#[rstest]
#[case::prompts_only(json!({"s3_log_prompts_only": true}), "{} without response")]
#[case::whole_payload(json!({}), "{}")]
#[tokio::test]
async fn prompts_only_renders_the_response_as_null(#[case] extra: Value, #[case] body: &str) {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink(&server, extra);
submit(&sink, event(&host, "a", "{}")).await;
sink.flush().await;
assert_eq!(
server.received_requests().await.unwrap()[0].body,
body.as_bytes()
);
}
#[rstest]
#[tokio::test]
async fn a_region_param_wins_over_the_constructor_region() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink_from(
&server,
json!({}),
Constants::default(),
false,
Some("ap-south-1"),
);
submit(&sink, event(&host, "a", "{}")).await;
sink.flush().await;
let requests = server.received_requests().await.unwrap();
assert!(
header(&requests[0], "authorization")
.unwrap()
.contains("/us-east-1/")
);
}
const HOUR_KEY: &str = "base/2026-11-13/09/time-09-05-07-000042_a.json";
const DAY_KEY: &str = "base/2026-11-13/time-09-05-07-000042_a.json";
#[rstest]
#[case::s3_v2_reuses_its_hour_key(true, json!({"metadata_kind": "mapping"}), Some(HOUR_KEY))]
#[case::another_owner_keeps_the_day_key(false, json!({"metadata_kind": "mapping"}), Some(DAY_KEY))]
#[case::s3_v2_rejects_metadata_without_get(true, json!({"metadata_kind": "other"}), None)]
#[case::another_owner_ignores_metadata_without_get(false, json!({"metadata_kind": "other"}), Some(DAY_KEY))]
#[tokio::test]
async fn the_constructor_cold_storage_owner_decides_the_cold_key(
#[case] cold_storage_is_s3_v2: bool,
#[case] identity: Value,
#[case] key: Option<&str>,
) {
let server = MockServer::start().await;
let (sink, host) = sink_from(
&server,
json!({}),
Constants::default(),
cold_storage_is_s3_v2,
None,
);
let mut cold = event(&host, "a", "{}");
cold.identity = identity;
cold.cold_storage_object_key = Some(HOUR_KEY);
submit(&sink, cold).await;
assert_eq!(sink.queued_keys(), key.into_iter().collect::<Vec<_>>());
assert_eq!(host.failures(), usize::from(key.is_none()));
}
#[rstest]
#[case::s3_v2_owner(true, "/logs/base/2026-11-13/time-09-05-07-000042_a.json")]
#[case::another_owner(false, "/logs/base/2026-11-13/batch_09-05-07_")]
#[tokio::test]
async fn the_constructor_cold_storage_owner_turns_batch_files_off(
#[case] cold_storage_is_s3_v2: bool,
#[case] path: &str,
) {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let (sink, host) = sink_from(
&server,
json!({"s3_batch_file_upload": true}),
Constants::default(),
cold_storage_is_s3_v2,
None,
);
submit(&sink, event(&host, "a", "{}")).await;
sink.flush().await;
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
assert!(requests[0].url.path().starts_with(path));
}

View file

@ -46,6 +46,9 @@ litellm-auth-aws.workspace = true
litellm-auth-azure.workspace = true
azure_core = "1.0.0"
litellm-callbacks-legacy-python.workspace = true
litellm-callbacks.workspace = true
litellm-callbacks-python.workspace = true
litellm-callbacks-s3-log.workspace = true
litellm-inference.workspace = true
litellm-inference-transcription.workspace = true
litellm-inference-responses.workspace = true

View file

@ -16,6 +16,7 @@ mod python_settings;
mod redis;
mod router;
mod routes;
mod s3_log;
mod secrets;
mod tokenizer;
@ -97,6 +98,8 @@ mod _native {
use crate::routes::token_counter::TokenCounter;
#[pymodule_export]
use crate::routes::traces::trace_encode_error;
#[pymodule_export]
use crate::s3_log::NativeS3Log;
#[cfg(feature = "huggingface")]
#[pymodule_export]
use crate::tokenizer::HuggingFaceEncoding;
@ -202,6 +205,7 @@ mod tests {
"process_state_started",
"reserve_process_for_forking",
"unsupported_tls_settings",
"NativeS3Log",
];
#[cfg(feature = "huggingface")]
expected.push("HuggingFaceEncoding");

View file

@ -0,0 +1,225 @@
use std::{sync::Arc, time::Duration};
use litellm_callbacks::ErasedCallback;
use litellm_callbacks_python::{RustCallback, payload};
use litellm_callbacks_s3_log::{
BatchUploadFailure, Constants, Host, S3Log, S3LogParts, Settings, python_backoff, system_clock,
};
use litellm_host_python::{from_py, json_from_py};
use litellm_http::ClientVariant;
use pyo3::{
prelude::*,
types::{PyBool, PyDict, PyInt, PyString},
};
use serde_json::{Map, Value};
struct PythonHost {
host: Arc<Py<PyAny>>,
}
impl PythonHost {
fn ask<T>(
&self,
method: &str,
argument: Option<&str>,
read: impl for<'py> FnOnce(&Bound<'py, PyAny>) -> PyResult<T>,
) -> Result<T, String> {
Python::attach(|py| {
let answer = match argument {
Some(argument) => self.host.call_method1(py, method, (argument,)),
None => self.host.call_method0(py, method),
};
answer
.and_then(|answer| read(answer.bind(py)))
.map_err(|error| error.to_string())
})
}
}
impl Host for PythonHost {
fn callback_failed(&self) {
let host = Arc::clone(&self.host);
let report = move || {
Python::attach(|py| {
if let Err(error) = host.call_method0(py, "callback_failed") {
error.print(py);
}
});
};
match tokio::runtime::Handle::try_current() {
Ok(runtime) => drop(runtime.spawn_blocking(report)),
Err(_) => report(),
}
}
fn web_identity_token(&self, reference: &str) -> Result<String, String> {
self.ask("web_identity_token", Some(reference), |answer| {
answer.extract::<String>()
})
}
}
type Failure = Option<(usize, usize, usize)>;
fn failure(failure: Option<BatchUploadFailure>) -> Failure {
failure.map(|failure| (failure.queued, failure.failed, failure.total))
}
#[pyclass(frozen, name = "NativeS3Log")]
pub struct NativeS3Log {
sink: Arc<S3Log>,
}
fn client(
py: Python<'_>,
params: &Bound<'_, PyDict>,
connect_timeout_seconds: f64,
) -> PyResult<litellm_http::Client> {
let kwargs = PyDict::new(py);
kwargs.set_item("ssl_verify", params.get_item("s3_verify")?)?;
let mut config = crate::http::call_config(py, &kwargs, true)?;
config.connect_timeout = Duration::from_secs_f64(connect_timeout_seconds.max(0.0));
crate::http::pool()
.client(&config, ClientVariant::Provider)
.map_err(crate::http::client_error)
}
fn settings_param(value: &Bound<'_, PyAny>) -> Value {
if let Ok(integer) = value.cast::<PyInt>()
&& !value.is_instance_of::<PyBool>()
&& integer.extract::<i64>().is_err()
&& integer.extract::<u64>().is_err()
{
return value
.str()
.map_or(Value::Null, |text| Value::String(text.to_string()));
}
json_from_py(value).unwrap_or(Value::Null)
}
fn settings_params(params: &Bound<'_, PyDict>) -> Map<String, Value> {
params
.iter()
.filter_map(|(key, value)| {
let key = key.cast_into::<PyString>().ok()?;
Some((key.to_str().ok()?.to_owned(), settings_param(&value)))
})
.collect()
}
#[pymethods]
impl NativeS3Log {
#[new]
#[pyo3(signature = (params, constants, host, cold_storage_is_s3_v2, region_from_environment))]
fn new(
py: Python<'_>,
params: &Bound<'_, PyDict>,
constants: &Bound<'_, PyAny>,
host: Py<PyAny>,
cold_storage_is_s3_v2: bool,
region_from_environment: Option<String>,
) -> PyResult<Self> {
let resolved = settings_params(params);
let constants: Constants = from_py(constants)?;
let logger = crate::logger::capture(py);
let settings = logger.scope(|| Settings::from_params(&resolved, &constants));
let host = Arc::new(PythonHost {
host: Arc::new(host),
});
let sink = S3Log::new(S3LogParts {
settings,
client: client(py, params, constants.upload_attempt_timeout_seconds)?,
download_client: client(py, params, constants.download_connect_timeout_seconds)?,
environment: Arc::new(|name: &str| std::env::var(name).ok()),
clock: system_clock,
backoff: Arc::new(python_backoff),
host: host as Arc<dyn Host>,
cold_storage_is_s3_v2,
region_from_environment,
logger,
});
Ok(Self { sink })
}
fn callback(&self) -> PyResult<RustCallback> {
Ok(RustCallback::new(
Arc::clone(&self.sink) as Arc<dyn ErasedCallback>
)?)
}
fn stashed(&self) -> usize {
self.sink.held_payloads()
}
fn partition_granularity(&self) -> &'static str {
self.sink.partition_granularity().as_str()
}
#[getter]
fn s3_path(&self) -> Option<String> {
self.sink.settings().path.clone()
}
#[getter]
fn s3_bucket_name(&self) -> Option<String> {
self.sink.settings().bucket_name.clone()
}
#[getter]
fn strip_base64_files(&self) -> bool {
self.sink.settings().strip_base64_files
}
fn queued_keys(&self) -> Vec<String> {
self.sink.queued_keys()
}
fn submit_audit<'py>(
&self,
py: Python<'py>,
audit_log: Py<PyAny>,
id: String,
) -> PyResult<Bound<'py, PyAny>> {
let sink = Arc::clone(&self.sink);
let stashed = payload::stash(audit_log);
let event_loop = payload::running_loop(py);
crate::execution::run_async_value(
py,
payload::on_render_loop(event_loop, async move {
Ok(failure(sink.submit_audit(stashed, &id).await))
}),
)
}
fn upload_line_items<'py>(
&self,
py: Python<'py>,
key: String,
body: String,
download_name: String,
) -> PyResult<Bound<'py, PyAny>> {
let sink = Arc::clone(&self.sink);
crate::execution::run_async_value(py, async move {
Ok(sink.upload_now(key, body, download_name).await)
})
}
fn download<'py>(&self, py: Python<'py>, key: String) -> PyResult<Bound<'py, PyAny>> {
let sink = Arc::clone(&self.sink);
crate::execution::run_async_value(py, async move {
Ok(sink
.download(&key)
.await
.map(|body| String::from_utf8_lossy(&body).into_owned()))
})
}
fn flush<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let sink = Arc::clone(&self.sink);
let event_loop = payload::running_loop(py);
crate::execution::run_async_value(
py,
payload::on_render_loop(event_loop, async move { Ok(failure(sink.flush().await)) }),
)
}
}

View file

@ -1,5 +1,5 @@
{
"python": "3.14.7",
"python": "3.14.6",
"rows": [
{
"name": "None",
@ -495,6 +495,25 @@
},
"view": "1e+22"
},
{
"name": "0.07872390747070312",
"source": "0.07872390747070312",
"literal": true,
"plain": true,
"repr": "0.07872390747070312",
"str": "0.07872390747070312",
"truthy": true,
"json": "0.07872390747070312",
"pickle": {
"0": "46302e30373837323339303734373037303331320a2e",
"1": "473fb42740000000002e",
"2": "8002473fb42740000000002e",
"3": "8003473fb42740000000002e",
"4": "8004950a00000000000000473fb42740000000002e",
"5": "8005950a00000000000000473fb42740000000002e"
},
"view": "0.07872390747070312"
},
{
"name": "float('inf')",
"source": "float('inf')",

View file

@ -47,6 +47,7 @@ CORPUS = [
"5e-324",
"1.7976931348623157e308",
"1e22",
"0.07872390747070312",
"float('inf')",
"float('-inf')",
"float('nan')",

View file

@ -113,8 +113,14 @@ fn format_float(value: f64, add_dot_0: bool) -> String {
if value.is_infinite() {
return if value > 0.0 { "inf" } else { "-inf" }.to_owned();
}
// Rust's `{:e}` prints the shortest round-trip digits, like CPython's 'r' mode.
let scientific = format!("{value:e}");
let shortest = format!("{value:e}");
let length = shortest
.split_once('e')
.map_or(shortest.as_str(), |(mantissa, _)| mantissa)
.chars()
.filter(char::is_ascii_digit)
.count();
let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1));
let (mantissa, exponent) = scientific
.split_once('e')
.expect("`{:e}` always prints an exponent");

View file

@ -6,7 +6,10 @@ license.workspace = true
repository.workspace = true
[dependencies]
litellm-auth-aws.workspace = true
litellm-http.workspace = true
hyper = "1.12.0"
bytes.workspace = true
aws-sdk-s3.workspace = true
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] }
aws-types = "1.4.0"
@ -16,6 +19,7 @@ aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] }
http.workspace = true
reqwest.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["time"] }
[dev-dependencies]
rstest.workspace = true

View file

@ -1,4 +1,5 @@
mod error;
pub mod signed;
mod transport;
mod uri;

View file

@ -0,0 +1,137 @@
use std::time::{Duration, SystemTime};
use bytes::Bytes;
use litellm_auth_aws::{Credentials, SigningScope, sign_s3_headers};
use reqwest::Method;
pub struct SignedRequest<'a> {
pub method: Method,
pub url: &'a str,
pub headers: &'a [(String, String)],
pub body: Bytes,
pub region: &'a str,
pub credentials: &'a Credentials,
pub time: SystemTime,
pub read_timeout: Option<Duration>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Reply {
pub status: u16,
pub body: Bytes,
}
impl Reply {
pub fn text(&self) -> String {
String::from_utf8_lossy(&self.body).into_owned()
}
}
#[derive(Debug, thiserror::Error)]
pub enum SendError {
#[error("the request cannot be signed: {0}")]
Signing(#[from] litellm_auth_aws::Error),
#[error("the request was not sent: {0}")]
Request(String),
#[error("{kind}: {message}")]
Transport { kind: &'static str, message: String },
}
impl SendError {
pub fn is_transient(&self) -> bool {
matches!(self, Self::Transport { .. })
}
pub fn is_reconnectable(&self) -> bool {
matches!(
self,
Self::Transport {
kind: "ConnectError" | "RemoteProtocolError",
..
}
)
}
}
pub fn signature_headers(request: &SignedRequest<'_>) -> Result<Vec<(String, String)>, SendError> {
Ok(sign_s3_headers(
request.method.as_str(),
request.url,
&request.body,
request.headers,
SigningScope {
region: request.region,
service: "s3",
credentials: request.credentials,
time: request.time,
},
)?)
}
pub async fn send(
client: &litellm_http::Client,
request: SignedRequest<'_>,
) -> Result<Reply, SendError> {
let signature = signature_headers(&request)?;
let mut builder = client.request(request.method.clone(), request.url);
for (name, value) in request.headers.iter().chain(&signature) {
builder = builder.header(name, value);
}
if !request.body.is_empty() || request.method == Method::PUT {
builder = builder.body(request.body.clone());
}
let mut response = timed(request.read_timeout, builder.send()).await?;
let status = response.status().as_u16();
let mut body = Vec::new();
while let Some(chunk) = timed(request.read_timeout, response.chunk()).await? {
body.extend_from_slice(&chunk);
}
Ok(Reply {
status,
body: Bytes::from(body),
})
}
async fn timed<T>(
limit: Option<Duration>,
pending: impl Future<Output = Result<T, reqwest::Error>>,
) -> Result<T, SendError> {
let finished = match limit {
Some(limit) => tokio::time::timeout(limit, pending)
.await
.map_err(|elapsed| SendError::Transport {
kind: "ReadTimeout",
message: elapsed.to_string(),
})?,
None => pending.await,
};
finished.map_err(transport)
}
fn transport(error: reqwest::Error) -> SendError {
let message = error.to_string();
if error.is_builder() || error.is_redirect() {
return SendError::Request(message);
}
let kind = if error.is_timeout() {
"Timeout"
} else if error.is_connect() {
"ConnectError"
} else if remote_protocol(&error) {
"RemoteProtocolError"
} else {
"TransportError"
};
SendError::Transport { kind, message }
}
fn remote_protocol(error: &reqwest::Error) -> bool {
let mut cause = std::error::Error::source(error);
while let Some(current) = cause {
if let Some(hyper) = current.downcast_ref::<hyper::Error>() {
return hyper.is_incomplete_message() || hyper.is_parse();
}
cause = current.source();
}
false
}

View file

@ -6,6 +6,7 @@ import httpx
from pydantic import JsonValue
from typing_extensions import disjoint_base
from litellm.integrations.custom_logger_v2.rust import RustHandle
from litellm.redis.store_settings import RedisBreakerSettings, RedisStoreSettings
from litellm.redis.stores.router_session_pins.keys import PinKey
from litellm.rust_bridge.execution import Execution
@ -119,6 +120,35 @@ class NativeModelCostMap:
@final
class NativeModelCostMapState: ...
class _S3LogHost(Protocol):
def callback_failed(self) -> None: ...
def web_identity_token(self, reference: str) -> str: ...
@final
class NativeS3Log:
def __new__(
cls,
params: dict[str, object],
constants: Mapping[str, object],
host: _S3LogHost,
cold_storage_is_s3_v2: bool,
region_from_environment: str | None,
) -> NativeS3Log: ...
@property
def s3_path(self) -> str | None: ...
@property
def s3_bucket_name(self) -> str | None: ...
@property
def strip_base64_files(self) -> bool: ...
def callback(self) -> RustHandle: ...
def stashed(self) -> int: ...
def partition_granularity(self) -> Literal["day", "hour"]: ...
def queued_keys(self) -> list[str]: ...
def submit_audit(self, audit_log: dict[str, object], id: str) -> Future[tuple[int, int, int] | None]: ...
def upload_line_items(self, key: str, body: str, download_name: str) -> Future[bool]: ...
def download(self, key: str) -> Future[str | None]: ...
def flush(self) -> Future[tuple[int, int, int] | None]: ...
@final
class NativeClickHouseSpendConfig:
def __new__(cls, database: str, url: str, retention_days: int) -> NativeClickHouseSpendConfig: ...
@ -786,6 +816,7 @@ __all__ = [
"NativeRouterCooldowns",
"NativeRouterSessionPins",
"NativeRouterUsage",
"NativeS3Log",
"NativeSend",
"NativeSensitiveRoutePins",
"NativeSpendCounters",

View file

@ -1,18 +1,41 @@
from typing import Literal
from litellm.types.llms.base import LiteLLMBaseModel
from typing_extensions import ReadOnly, TypedDict
S3PartitionGranularity = Literal["day", "hour"]
class s3BatchLoggingElement(LiteLLMBaseModel):
"""
Type of element stored in self.log_queue in S3Logger
"""
payload: dict
s3_object_key: str
s3_object_download_filename: str
body: str | None = None
content_type: str = "application/json"
retrying_since: float | None = None
class S3LoggerSettings(TypedDict, total=False):
s3_bucket_name: ReadOnly[str | None]
s3_path: ReadOnly[str | None]
s3_region_name: ReadOnly[str | None]
s3_api_version: ReadOnly[str | None]
s3_use_ssl: ReadOnly[bool]
s3_verify: ReadOnly[bool | str | None]
s3_endpoint_url: ReadOnly[str | None]
s3_aws_access_key_id: ReadOnly[str | None]
s3_aws_secret_access_key: ReadOnly[str | None]
s3_aws_session_token: ReadOnly[str | None]
s3_aws_session_name: ReadOnly[str | None]
s3_aws_profile_name: ReadOnly[str | None]
s3_aws_role_name: ReadOnly[str | None]
s3_aws_web_identity_token: ReadOnly[str | None]
s3_aws_sts_endpoint: ReadOnly[str | None]
s3_flush_interval: ReadOnly[int | None]
s3_batch_size: ReadOnly[int | None]
s3_config: ReadOnly[object]
s3_use_team_prefix: ReadOnly[bool]
s3_strip_base64_files: ReadOnly[bool]
s3_use_key_prefix: ReadOnly[bool]
s3_use_virtual_hosted_style: ReadOnly[bool]
s3_server_side_encryption: ReadOnly[str | None]
s3_sse_kms_key_id: ReadOnly[str | None]
s3_log_prompts_only: ReadOnly[bool | None]
s3_partition_granularity: ReadOnly[str | None]
s3_max_concurrent_uploads: ReadOnly[int]
s3_max_queue_size: ReadOnly[int | None]
s3_max_retry_age_seconds: ReadOnly[int | None]
s3_drop_on_terminal_error: ReadOnly[bool]
s3_adaptive_concurrency: ReadOnly[bool]
s3_max_adaptive_concurrency: ReadOnly[int | None]
s3_batch_file_upload: ReadOnly[bool]

View file

@ -1,7 +1,7 @@
# Logging integration delivery (behavior features). Grounded in litellm/integrations/.
- {id: logging.s3.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/s3_v2.py", rationale: "Primary audit trail; batch flush no-drop"}
- {id: logging.s3.failure.writes_object, module: logging, tier: P0, event: failure, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/s3_v2.py", rationale: "Failed calls persisted for compliance"}
- {id: logging.s3.success.partition_layout, module: logging, tier: P1, event: success, assertions: [object_key_layout], exercised_on: [chat_completions], source: "integrations/s3_v2.py / LIT-8985", rationale: "s3_partition_granularity picks the date or date/hour folder every downstream query and lifecycle rule reads"}
- {id: logging.s3.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/s3_v3.py", rationale: "Primary audit trail; batch flush no-drop"}
- {id: logging.s3.failure.writes_object, module: logging, tier: P0, event: failure, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/s3_v3.py", rationale: "Failed calls persisted for compliance"}
- {id: logging.s3.success.partition_layout, module: logging, tier: P1, event: success, assertions: [object_key_layout], exercised_on: [chat_completions], source: "integrations/s3_v3.py / LIT-8985", rationale: "s3_partition_granularity picks the date or date/hour folder every downstream query and lifecycle rule reads"}
- {id: logging.gcs_bucket.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/gcs_bucket/gcs_bucket.py", rationale: "GCS parallel to S3"}
- {id: logging.datadog.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/datadog/datadog.py", rationale: "Powers dashboards/alerts; cardinality regressions common"}
- {id: logging.datadog.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/datadog/datadog.py", rationale: "Streaming aggregates usage after the last chunk; delivery and cost must survive that path"}

View file

@ -21,6 +21,7 @@ from integration._support.process import owned_proxy
from integration._support.wire import wire_server
from integration.caching.response_cache_case import entry_keys, redis_store
pytestmark: Final = pytest.mark.shares_worker_state
PER_REQUEST_KEY: Final = re.compile(rf"^/{BUCKET}/{PREFIX}/\d{{4}}-\d{{2}}-\d{{2}}/.+\.json$")
@ -29,7 +30,6 @@ BATCH_KEY: Final = re.compile(
)
@pytest.mark.covers("other.observability.s3_v2.mixed_surface_burst_bounds_puts_one_object_per_response_id")
def test_s3_v2_mixed_surface_burst_bounds_puts_one_object_per_response_id(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3mix" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink()
@ -54,7 +54,6 @@ def test_s3_v2_mixed_surface_burst_bounds_puts_one_object_per_response_id(gatewa
assert matched_ids(payloads, answered)
@pytest.mark.covers("other.observability.s3_v2.mixed_surface_batch_writes_ndjson_lines_per_response_id")
def test_s3_v2_mixed_surface_batch_writes_ndjson_lines_per_response_id(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3mixb" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink()
@ -80,7 +79,6 @@ def test_s3_v2_mixed_surface_batch_writes_ndjson_lines_per_response_id(gateway:
assert len(payloads) == 48
@pytest.mark.covers("other.observability.s3_v2.sink_outage_mid_mixed_burst_recovers_every_response_id")
def test_s3_v2_sink_outage_mid_mixed_burst_recovers_every_response_id(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3mixo" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink(fail_attempts=30, fail_status=503, delay_seconds=0.2)

View file

@ -33,6 +33,7 @@ from integration._support.database_relay import database_relay
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, wire_server
pytestmark: Final = pytest.mark.shares_worker_state
FLUSH: Final = {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "1"}

View file

@ -14,6 +14,7 @@ from integration._support.client import Gateway, JsonValue, Scenario, eventually
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, wire_server
Surface = Literal["chat", "messages", "responses"]
SURFACES: Final[tuple[Surface, ...]] = ("chat", "messages", "responses")
FAILURE_BODY: Final[bytes] = json.dumps(
@ -77,9 +78,7 @@ def _responses_stream_id(response: httpx.Response) -> str:
if line.startswith("data: {")
)
completed: Final = tuple(
object_value(event["response"])
for event in events
if event.get("type") == "response.completed"
object_value(event["response"]) for event in events if event.get("type") == "response.completed"
)
assert len(completed) == 1, events
response_id: Final = completed[0]["id"]
@ -137,13 +136,11 @@ def _assert_one_payload(
return_last_on_timeout=True,
)
assert len(payloads) == 1, (
f"expected one {status} payload after {upstream_posts} upstream POSTs, "
f"observed {len(payloads)} payloads"
f"expected one {status} payload after {upstream_posts} upstream POSTs, observed {len(payloads)} payloads"
)
assert (
_request_key(payloads[0], response, marker) == response.headers.get("x-litellm-call-id")
or marker in json.dumps(payloads[0])
), f"payload did not match request id or marker: {payloads[0]!r}"
assert _request_key(payloads[0], response, marker) == response.headers.get(
"x-litellm-call-id"
) or marker in json.dumps(payloads[0]), f"payload did not match request id or marker: {payloads[0]!r}"
return payloads
@ -337,9 +334,7 @@ def test_failure_burst_through_sink_outage_lands_each_request_once(
with ThreadPoolExecutor(max_workers=len(jobs)) as pool:
responses: Final = tuple(pool.map(call, jobs))
upstream_requests: Final = tuple(
request for request in upstream.drain() if request.method == "POST"
)
upstream_requests: Final = tuple(request for request in upstream.drain() if request.method == "POST")
route_counts: Final = Counter(request.target for request in upstream_requests)
expected_route_counts: Final = {
_surface_target(surface): 2 * requests_per_variant * 3 for surface in SURFACES
@ -358,8 +353,7 @@ def test_failure_burst_through_sink_outage_lands_each_request_once(
rejected: Final = sink.attempts - len(sink.store)
assert rejected >= 1, f"expected at least one rejected sink upload, rejected={rejected}"
assert len(sink.store) == 0, (
f"expected all sink uploads to fail before recovery, rejected={rejected}, "
f"attempts={sink.attempts}"
f"expected all sink uploads to fail before recovery, rejected={rejected}, attempts={sink.attempts}"
)
sink.fail_until = 0.0
payloads: Final = eventually(

View file

@ -20,6 +20,7 @@ from integration._support.process import owned_proxy, owned_proxy_process
from integration._support.timing import assert_enforced_within
from integration._support.wire import Reply, Request, Wire, wire_server
BUCKET: Final = "integration-bucket"
PREFIX: Final = "integration-logs"
REQUESTS: Final = 64
@ -121,7 +122,6 @@ BATCH_KEY: Final = re.compile(
)
@pytest.mark.covers("other.observability.s3_v2.flush_bounds_concurrent_puts_to_default_ceiling_and_keeps_every_log")
def test_s3_v2_flush_bounds_concurrent_puts_to_the_default_ceiling(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3fan" + uuid.uuid4().hex[:8]
sink: Final = S3Sink()
@ -144,7 +144,6 @@ def test_s3_v2_flush_bounds_concurrent_puts_to_the_default_ceiling(gateway: Gate
assert len({put.target for put in puts}) == REQUESTS
@pytest.mark.covers("other.observability.s3_v2.configured_bound_and_env_backed_false_keeps_per_request_objects")
def test_s3_v2_honors_configured_bound_and_env_backed_false_batch_flag(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3cap" + uuid.uuid4().hex[:8]
sink: Final = S3Sink()
@ -173,7 +172,6 @@ def test_s3_v2_honors_configured_bound_and_env_backed_false_batch_flag(gateway:
assert frozenset(json.loads(put.body)["id"] for put in puts) == ids
@pytest.mark.covers("other.observability.s3_v2.batch_file_upload_writes_one_ndjson_object_per_flush")
def test_s3_v2_batch_file_upload_writes_one_jsonl_object_per_flush(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3jsonl" + uuid.uuid4().hex[:8]
sink: Final = S3Sink()
@ -196,7 +194,6 @@ def test_s3_v2_batch_file_upload_writes_one_jsonl_object_per_flush(gateway: Gate
assert len(lines) == REQUESTS
@pytest.mark.covers("other.observability.s3_v2.batch_file_upload_keeps_team_prefix_in_object_key")
def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3team" + uuid.uuid4().hex[:8]
team_alias: Final = f"alpha-{uuid.uuid4().hex[:8]}"
@ -223,7 +220,6 @@ def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_p
assert len(lines) == REQUESTS
@pytest.mark.covers("other.observability.s3_v2.upstream_failure_events_land_alongside_successes")
def test_s3_v2_upstream_failure_events_land_alongside_successes(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3fail" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink()
@ -276,7 +272,6 @@ def test_s3_v2_upstream_failure_events_land_alongside_successes(gateway: Gateway
assert all("synthetic upstream rejection" in json.dumps(payload["error_information"]) for payload in failures)
@pytest.mark.covers("other.observability.s3_v2.invalid_or_empty_bound_falls_back_to_default_ceiling")
@pytest.mark.parametrize(
("bad", "warns"),
[
@ -313,7 +308,6 @@ def test_s3_v2_invalid_or_empty_bound_falls_back_to_default_ceiling(
assert frozenset(payload["id"] for payload in payloads) == ids
@pytest.mark.covers("other.observability.s3_v2.sink_rejection_requeues_and_delivers_every_id_once")
def test_s3_v2_sink_rejection_requeues_and_delivers_every_id_once(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3deny" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink(fail_status=503, delay_seconds=0.2)
@ -632,7 +626,6 @@ def test_s3_v2_default_config_retries_access_denied_and_every_event_lands(gatewa
)
@pytest.mark.covers("other.observability.s3_v2.batch_retry_resends_identical_key_and_body")
def test_s3_v2_batch_retry_resends_identical_key_and_body(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3retry" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink(fail_status=500, delay_seconds=0.2)
@ -658,7 +651,6 @@ def test_s3_v2_batch_retry_resends_identical_key_and_body(gateway: Gateway, tmp_
assert len(payloads) == REQUESTS
@pytest.mark.covers("other.observability.s3_v2.unknown_model_rejection_keeps_other_requests_logging")
def test_s3_v2_unknown_model_rejection_keeps_other_requests_logging(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3ghost" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink()
@ -690,7 +682,6 @@ def test_s3_v2_unknown_model_rejection_keeps_other_requests_logging(gateway: Gat
assert all(payload["status"] == "failure" for payload in extras), extras
@pytest.mark.covers("other.observability.s3_v2.batch_flag_ignored_when_s3_v2_is_cold_storage_logger")
def test_s3_v2_batch_flag_ignored_when_s3_v2_is_cold_storage_logger(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3cold" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink()
@ -734,7 +725,6 @@ def test_s3_v2_batch_flag_ignored_when_s3_v2_is_cold_storage_logger(gateway: Gat
assert payloads[0]["id"] == request_id
@pytest.mark.covers("other.observability.s3_v2.identical_requests_land_distinct_objects")
def test_s3_v2_identical_requests_land_distinct_objects(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3same" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink()
@ -766,7 +756,6 @@ def test_s3_v2_identical_requests_land_distinct_objects(gateway: Gateway, tmp_pa
assert all(payload["id"] == marker for payload in payloads)
@pytest.mark.covers("other.observability.s3_v2.two_workers_bound_and_deliver_every_id")
def test_s3_v2_two_workers_bound_and_deliver_every_id(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3work" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink()
@ -788,7 +777,6 @@ def test_s3_v2_two_workers_bound_and_deliver_every_id(gateway: Gateway, tmp_path
assert frozenset(payload["id"] for payload in payloads) == ids
@pytest.mark.covers("other.observability.s3_v2.slow_sink_never_duplicates_or_stalls_readiness")
def test_s3_v2_slow_sink_never_duplicates_or_stalls_readiness(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3slow" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink(delay_seconds=1.5)
@ -817,7 +805,6 @@ def test_s3_v2_slow_sink_never_duplicates_or_stalls_readiness(gateway: Gateway,
assert len(payloads) == REQUESTS
@pytest.mark.covers("other.observability.s3_v2.worker_kill_mid_burst_keeps_surviving_deliveries")
def test_s3_v2_worker_kill_mid_burst_keeps_surviving_deliveries(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3kill" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink()
@ -874,7 +861,6 @@ def test_s3_v2_worker_kill_mid_burst_keeps_surviving_deliveries(gateway: Gateway
assert landed <= frozenset(burst) | frozenset(after), "an id that was never sent landed"
@pytest.mark.covers("other.observability.s3_v2.sigterm_mid_burst_loses_only_inflight_without_duplicates")
def test_s3_v2_sigterm_mid_burst_loses_only_inflight_without_duplicates(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3term" + uuid.uuid4().hex[:8]
sink: Final = RecordingS3Sink()

View file

@ -37,14 +37,26 @@ def test_sigv4_verifier_matches_published_put_and_rejects_corruption() -> None:
@pytest.mark.covers("other.provider_wire.s3.sync_async_reserved_keys_are_signed_and_accepted")
async def test_s3_sync_and_async_uploads_pass_independent_wire_verification(monkeypatch: pytest.MonkeyPatch) -> None:
async def test_s3_sync_and_async_uploads_pass_independent_wire_verification() -> None:
from litellm.integrations.s3_v2 import S3Logger
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
from litellm.types.utils import StandardBatchLineItem, StandardBatchLineItemMetadata, StandardBatchLineItemsPayload
monkeypatch.setattr("botocore.auth.get_current_datetime", lambda: datetime(2026, 9, 14))
payload: Final = {"id": "synthetic-event", "content": "synthetic snow 雪"}
metadata: Final = StandardBatchLineItemMetadata(
user_api_key_hash=None, user_api_key_alias=None, user_api_key_team_id=None,
user_api_key_team_alias=None, user_api_key_user_id=None, user_api_key_org_id=None,
)
lines: Final = (
StandardBatchLineItem(
batch_id="synthetic-batch", custom_id="synthetic-event", status="success",
request={"content": "synthetic snow 雪"}, response=None, usage=None, response_cost=0.0,
model="synthetic-model", custom_llm_provider=None, metadata=metadata,
),
)
expected_path = ""
def scope_of(request: Request) -> str:
return f"{request.headers.get('x-amz-date', '')[:8]}/us-east-1/s3/aws4_request"
def verify(request: Request) -> Reply:
if request.method != "PUT" or request.target != expected_path:
return Reply(status=403)
@ -53,8 +65,8 @@ async def test_s3_sync_and_async_uploads_pass_independent_wire_verification(monk
assert authorization.startswith("AWS4-HMAC-SHA256 ")
fields: Final = dict(part.split("=", 1) for part in authorization.removeprefix("AWS4-HMAC-SHA256 ").split(", "))
access, scope = fields["Credential"].split("/", 1)
assert access == ACCESS and scope == "20260914/us-east-1/s3/aws4_request"
assert request.headers["x-amz-date"] == "20260914T000000Z"
assert access == ACCESS and scope == scope_of(request)
assert datetime.strptime(request.headers["x-amz-date"], "%Y%m%dT%H%M%SZ")
signed: Final = fields["SignedHeaders"].split(";")
assert signed == sorted(set(signed))
assert {"host", "content-md5", "x-amz-date"}.issubset(signed)
@ -69,40 +81,37 @@ async def test_s3_sync_and_async_uploads_pass_independent_wire_verification(monk
with wire_server(verify) as wire:
prior: Final = asyncio.all_tasks()
logger: Final = S3Logger(s3_bucket_name="integration-bucket", s3_region_name="us-east-1", s3_endpoint_url=wire.url,
s3_aws_access_key_id=ACCESS, s3_aws_secret_access_key=SECRET, s3_callback_params_override={})
owned: Final = {task for task in asyncio.all_tasks() - prior if task.get_coro().__qualname__ == "CustomBatchLogger.periodic_flush"}
assert len(owned) == 1
s3_aws_access_key_id=ACCESS, s3_aws_secret_access_key=SECRET, s3_callback_params_override={},
s3_path="a=b+c/d e/雪")
owned: Final = asyncio.all_tasks() - prior
assert owned == {logger.periodic_flush_task}
try:
for mode in ("sync", "async"):
for key in ("plain.json", "a=b+c/d e/雪.json", "percent%2Fplus+.json"):
expected_path = encoded_path(f"/integration-bucket/{key}")
element: Final = s3BatchLoggingElement(payload=payload, s3_object_key=key, s3_object_download_filename="event.json")
if mode == "sync":
await asyncio.to_thread(logger.upload_data_to_s3, element)
else:
await logger.async_upload_data_to_s3(element)
requests: Final = wire.drain()
assert len(requests) == 1, "Upload must be accepted on its first actual PUT"
request: Final = requests[0]
assert request.target == expected_path
assert json.loads(request.body) == payload
assert verify(request).status == 200
with httpx.Client(timeout=5, trust_env=False) as client:
corrupt: Final = {**request.headers, "authorization": request.headers["authorization"][:-1] + ("0" if request.headers["authorization"][-1] != "0" else "1")}
assert client.put(wire.url + expected_path, content=request.body, headers=corrupt).status_code == 403
assert client.put(wire.url + expected_path + "-wrong", content=request.body, headers=request.headers).status_code == 403
assert client.put(wire.url + expected_path, content=request.body + b" ", headers={name: value for name, value in request.headers.items() if name != "content-length"}).status_code == 403
fields: Final = dict(part.split("=", 1) for part in request.headers["authorization"].removeprefix("AWS4-HMAC-SHA256 ").split(", "))
for signed, scope, md5 in (
(fields["SignedHeaders"].replace("host;", ""), "20260914/us-east-1/s3/aws4_request", request.headers["content-md5"]),
(fields["SignedHeaders"], "20260914/us-west-2/s3/aws4_request", request.headers["content-md5"]),
(fields["SignedHeaders"], "20260914/us-east-1/s3/aws4_request", "AAAAAAAAAAAAAAAAAAAAAA=="),
):
candidate_headers: Final = {**request.headers, "content-md5": md5}
digest: Final = signature("PUT", request.target, candidate_headers, signed, request.body, SECRET, scope)[1]
candidate_headers["authorization"] = f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, SignedHeaders={signed}, Signature={digest}"
assert client.put(wire.url + expected_path, content=request.body, headers=candidate_headers).status_code == 403
assert len(wire.drain()) == 6
for batch_id in ("plain", "a=b+c d e", "percent%2Fplus+"):
expected_path = encoded_path(f"/integration-bucket/a=b+c/d e/雪/batch_line_items/2026-09-14/{batch_id}/part-00001.jsonl")
await logger.async_log_batch_line_items(StandardBatchLineItemsPayload(batch_id=batch_id, finished_at=1789344000, part=1, lines=lines))
requests: Final = wire.drain()
assert len(requests) == 1, "Upload must be accepted on its first actual PUT"
request: Final = requests[0]
assert request.target == expected_path
assert tuple(json.loads(line) for line in request.body.decode().splitlines()) == lines
assert verify(request).status == 200
day: Final = scope_of(request).split("/", 1)[0]
with httpx.Client(timeout=5, trust_env=False) as client:
corrupt: Final = {**request.headers, "authorization": request.headers["authorization"][:-1] + ("0" if request.headers["authorization"][-1] != "0" else "1")}
assert client.put(wire.url + expected_path, content=request.body, headers=corrupt).status_code == 403
assert client.put(wire.url + expected_path + "-wrong", content=request.body, headers=request.headers).status_code == 403
assert client.put(wire.url + expected_path, content=request.body + b" ", headers={name: value for name, value in request.headers.items() if name != "content-length"}).status_code == 403
fields: Final = dict(part.split("=", 1) for part in request.headers["authorization"].removeprefix("AWS4-HMAC-SHA256 ").split(", "))
for signed, scope, md5 in (
(fields["SignedHeaders"].replace("host;", ""), f"{day}/us-east-1/s3/aws4_request", request.headers["content-md5"]),
(fields["SignedHeaders"], f"{day}/us-west-2/s3/aws4_request", request.headers["content-md5"]),
(fields["SignedHeaders"], f"{day}/us-east-1/s3/aws4_request", "AAAAAAAAAAAAAAAAAAAAAA=="),
):
candidate_headers: Final = {**request.headers, "content-md5": md5}
digest: Final = signature("PUT", request.target, candidate_headers, signed, request.body, SECRET, scope)[1]
candidate_headers["authorization"] = f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, SignedHeaders={signed}, Signature={digest}"
assert client.put(wire.url + expected_path, content=request.body, headers=candidate_headers).status_code == 403
assert len(wire.drain()) == 6
finally:
for task in owned:

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,145 @@
import threading
import time
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field
from datetime import datetime
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Final
import pytest
from litellm.integrations.custom_logger_v2.call import CallContext
from litellm.integrations.custom_logger_v2.derivations import DERIVED_ATTEMPT_FIELDS
from litellm.integrations.custom_logger_v2.enums import Hook
from litellm.integrations.custom_logger_v2.enforcement import FieldView
from litellm.integrations.custom_logger_v2.field_sources import HookArgs, with_sources
from litellm.integrations.custom_logger_v2.legacy import KwargsFields
from litellm.integrations.custom_logger_v2.rust import WireManifest
from litellm.integrations.s3_v3 import S3Logger
from litellm.litellm_core_utils.litellm_logging import LOGGING_KEYS, V2_ASYNC_LOGGING_LINE
from litellm.types.integrations.custom_logger_v2 import Outcome
DROP: Final = -1
@dataclass(frozen=True)
class Received:
method: str
path: str
headers: Mapping[str, str]
body: bytes
at: float
@dataclass
class ScriptedBucket:
port: int = 0
received: list[Received] = field(default_factory=list) # mutable-ok: appended by the server thread
objects: dict[str, bytes] = field(default_factory=dict) # mutable-ok: GET serves what PUT stored
script: list[tuple[str, list[tuple[int, str]]]] = field(default_factory=list) # mutable-ok: replies are consumed
delay: float = 0.0
in_flight: int = 0
peak: int = 0
lock: threading.Lock = field(default_factory=threading.Lock)
@property
def url(self) -> str:
return f"http://127.0.0.1:{self.port}"
def reply(self, fragment: str, *replies: tuple[int, str]) -> None:
self.script.append((fragment, list(replies)))
def puts(self) -> list[Received]:
return [request for request in self.received if request.method == "PUT"]
def next_reply(self, path: str) -> tuple[int, str]:
with self.lock:
for fragment, replies in self.script:
if fragment in path and replies:
return replies.pop(0) if len(replies) > 1 else replies[0]
return 200, ""
def _handler(bucket: ScriptedBucket) -> type[BaseHTTPRequestHandler]:
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *_: object) -> None:
return
def _send(self, status: int, body: bytes) -> None:
self.send_response(status)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_PUT(self) -> None:
body: Final = self.rfile.read(int(self.headers.get("Content-Length", "0")))
headers: Final = {name.lower(): value for name, value in self.headers.items()}
with bucket.lock:
bucket.received.append(Received("PUT", self.path, headers, body, time.monotonic()))
bucket.in_flight += 1
bucket.peak = max(bucket.peak, bucket.in_flight)
if bucket.delay:
time.sleep(bucket.delay)
with bucket.lock:
bucket.in_flight -= 1
status, text = bucket.next_reply(self.path)
if status == DROP:
self.close_connection = True
return
if status == 200:
with bucket.lock:
bucket.objects[self.path] = body
self._send(status, text.encode())
def do_GET(self) -> None:
headers: Final = {name.lower(): value for name, value in self.headers.items()}
with bucket.lock:
bucket.received.append(Received("GET", self.path, headers, b"", time.monotonic()))
stored: Final = bucket.objects.get(self.path)
if stored is None:
self._send(404, b"<Error><Code>NoSuchKey</Code></Error>")
return
self._send(200, stored)
return Handler
class _BucketServer(ThreadingHTTPServer):
request_queue_size = 256
@contextmanager
def scripted_bucket() -> Iterator[ScriptedBucket]:
bucket: Final = ScriptedBucket()
server: Final = _BucketServer(("127.0.0.1", 0), _handler(bucket))
bucket.port = server.server_address[1]
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
yield bucket
finally:
server.shutdown()
def rust_s3_logger(monkeypatch: pytest.MonkeyPatch, **params: object) -> S3Logger: # kwargs-ok: the S3Logger keywords
return S3Logger(params) if params else S3Logger()
async def log_attempt(
logger: S3Logger, kwargs: Mapping[str, object], start_time: datetime, outcome: Outcome = Outcome.success
) -> None:
args: Final = HookArgs(request=kwargs, start_time=start_time, end_time=start_time, outcome=outcome)
view: Final = FieldView(with_sources(KwargsFields(kwargs, LOGGING_KEYS), DERIVED_ATTEMPT_FIELDS, args))
manifest: Final = WireManifest.model_validate_json(logger.rust.manifest())
reads: Final = next(s.reads for s in manifest.subscriptions if s.hook is Hook.attempt_ended)
line: Final = V2_ASYNC_LOGGING_LINE.set(True)
try:
_ = await logger.run_rust(Hook.attempt_ended, reads, view, CallContext(Hook.attempt_ended, None, None))
finally:
V2_ASYNC_LOGGING_LINE.reset(line)
async def log_payload(logger: S3Logger, payload: Mapping[str, object], start_time: datetime) -> None:
await log_attempt(logger, {"standard_logging_object": payload}, start_time)

View file

@ -1,179 +0,0 @@
import asyncio
from typing import Final
import pytest
from litellm.integrations.adaptive_concurrency import AdaptiveConcurrencyLimiter, PutSample
_real_sleep: Final = asyncio.sleep
def _limiter(initial: int = 4, floor: int = 1, ceiling: int = 16) -> AdaptiveConcurrencyLimiter:
return AdaptiveConcurrencyLimiter(initial=initial, floor=floor, ceiling=ceiling)
@pytest.mark.asyncio
async def test_limit_grows_after_limit_clean_samples() -> None:
limiter: Final = _limiter(initial=4)
for _ in range(4):
limiter.record(PutSample(throttled=False))
assert limiter.limit == 5
@pytest.mark.asyncio
async def test_limit_does_not_grow_before_the_streak_completes() -> None:
limiter: Final = _limiter(initial=4)
for _ in range(3):
limiter.record(PutSample(throttled=False))
assert limiter.limit == 4
@pytest.mark.asyncio
async def test_throttled_sample_halves_the_limit() -> None:
limiter: Final = _limiter(initial=16)
limiter.record(PutSample(throttled=True))
assert limiter.limit == 8
@pytest.mark.asyncio
async def test_limit_clamps_at_the_floor() -> None:
limiter: Final = _limiter(initial=4, floor=4)
limiter.record(PutSample(throttled=True))
assert limiter.limit == 4
@pytest.mark.asyncio
async def test_limit_clamps_at_the_ceiling() -> None:
limiter: Final = _limiter(initial=15, ceiling=16)
for _ in range(1000):
limiter.record(PutSample(throttled=False))
assert limiter.limit == 16
@pytest.mark.asyncio
async def test_throttled_sample_resets_the_clean_streak() -> None:
limiter: Final = _limiter(initial=4, ceiling=32)
for _ in range(3):
limiter.record(PutSample(throttled=False))
limiter.record(PutSample(throttled=True))
limiter.record(PutSample(throttled=False))
assert limiter.limit == 2
@pytest.mark.asyncio
async def test_growing_the_limit_wakes_a_waiting_acquirer() -> None:
limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=4)
acquired: Final[list[str]] = [] # mutable-ok: the waiter task appends to it across the await boundary
released: Final = asyncio.Event()
async def hold() -> None:
async with limiter:
await released.wait()
holder: Final = asyncio.create_task(hold())
async def waiter() -> None:
async with limiter:
acquired.append("waiter")
pending: Final = asyncio.create_task(waiter())
await _real_sleep(0)
assert not acquired
limiter.record(PutSample(throttled=False))
await asyncio.wait_for(asyncio.shield(pending), timeout=5)
released.set()
await asyncio.wait_for(holder, timeout=5)
assert tuple(acquired) == ("waiter",)
@pytest.mark.asyncio
async def test_releasing_a_slot_wakes_exactly_one_waiter() -> None:
limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=4)
acquired: Final[list[str]] = [] # mutable-ok: the waiter tasks append to it across the await boundary
release: Final = asyncio.Event()
async def hold() -> None:
async with limiter:
await _real_sleep(0)
async def waiter(name: str) -> None:
async with limiter:
acquired.append(name)
await release.wait()
holder: Final = asyncio.create_task(hold())
waiters: Final = tuple(asyncio.create_task(waiter(f"w{i}")) for i in range(3))
await _real_sleep(0)
await asyncio.wait_for(holder, timeout=5)
await _real_sleep(0)
assert len(acquired) == 1
for _ in range(3):
limiter.record(PutSample(throttled=False))
await _real_sleep(0)
assert len(acquired) == 3
release.set()
await asyncio.gather(*waiters)
@pytest.mark.asyncio
async def test_double_cancel_during_release_leaves_in_flight_at_zero() -> None:
limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=1)
entered: Final = asyncio.Event()
release: Final = asyncio.Event()
async def hold() -> None:
async with limiter:
entered.set()
await release.wait()
holder: Final = asyncio.create_task(hold())
await entered.wait()
waiter: Final = asyncio.create_task(hold())
await _real_sleep(0)
waiter.cancel()
with pytest.raises(asyncio.CancelledError):
await waiter
release.set()
await asyncio.wait_for(holder, timeout=5)
holder.cancel()
try:
await holder
except asyncio.CancelledError:
pass
assert limiter._in_flight == 0
@pytest.mark.asyncio
async def test_a_cancelled_waiter_is_skipped_when_a_slot_frees() -> None:
limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=1)
acquired: Final[list[str]] = [] # mutable-ok: waiter tasks append across the await boundary
first_entered: Final = asyncio.Event()
release: Final = asyncio.Event()
async def hold(name: str, entered: asyncio.Event | None = None) -> None:
async with limiter:
acquired.append(name)
if entered is not None:
entered.set()
await release.wait()
holder: Final = asyncio.create_task(hold("holder", first_entered))
await first_entered.wait()
doomed: Final = asyncio.create_task(hold("doomed"))
next_waiter: Final = asyncio.create_task(hold("next"))
await _real_sleep(0)
doomed.cancel()
with pytest.raises(asyncio.CancelledError):
await doomed
release.set()
await asyncio.wait_for(holder, timeout=5)
await asyncio.wait_for(next_waiter, timeout=5)
assert "doomed" not in acquired
assert "next" in acquired
assert limiter._in_flight == 0

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,445 @@
import asyncio
import copy
import datetime
import hashlib
import json
from collections.abc import Iterator, Mapping, Sequence
from typing import Final
from unittest.mock import MagicMock
import pytest
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_logger_v2.enums import Field, Hook
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
from litellm.integrations.custom_logger_v2.manifest import build_manifest
from litellm.integrations.s3_v3 import STRIP_CALLBACK_NAME, S3Logger, S3StripBase64Files
from litellm.litellm_core_utils import litellm_logging
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.rust_bridge import _native
from litellm.types.integrations.custom_logger_v2 import Outcome
from litellm.types.utils import ModelResponse
from tests.unit.integrations.s3_v3_rust_support import ScriptedBucket, scripted_bucket
pytestmark = pytest.mark.skipif(not hasattr(_native, "NativeS3Log"), reason="the built _native has no NativeS3Log")
START: Final = datetime.datetime(2025, 9, 14, 9, 5, 7, 42)
RECORDED_AT_86B70FEF16: Final = {
"stripped": (
"/test-bucket/2025-09-14/time-09-05-07-000042_stripped.json",
"31e903035e2185abea6cac867139417cb42a37a0da6b7d4efea47b9d1c24f1e1",
),
"excluded": (
"/test-bucket/2025-09-14/time-09-05-07-000042_excluded.json",
"5958206c8eee87b3221e6ee760b48a0ead92343768adbf8f6f80295bd4d51329",
),
"failed": (
"/test-bucket/2025-09-14/time-09-05-07-000042_failed.json",
"04dcb95630cb5b61fbf96560be30411b5fcacebbb4f4ed03aeaccc2e0687c95f",
),
}
STRIPPED_MESSAGES: Final = [
{"role": "user", "content": [{"type": "text", "text": "see [BASE64_REDACTED]"}]},
{"role": "assistant", "content": "plain [BASE64_REDACTED]", "name": "[BASE64_REDACTED]"},
]
def _messages() -> list[object]:
return [
{
"role": "user",
"content": [
{"type": "text", "text": "see data:image/png;base64,AAAA== inline"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,BBBB"}},
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBER"}},
],
},
{"role": "assistant", "content": "plain data:audio/wav;base64,CCCC tail", "name": "data:image/gif;base64,DDDD"},
"not a message",
]
def _payload(response_id: str, status: str = "success") -> dict[str, object]:
return {
"id": response_id,
"status": status,
"metadata": {},
"response": {"text": "ok"},
"messages": _messages(),
}
class _Reader(CustomLogger):
def __init__(self, name: str) -> None:
super().__init__()
self.name: Final = name
self.seen: list[object] = []
def saw(self, kwargs: object) -> None:
payload: Final = kwargs.get("standard_logging_object") if isinstance(kwargs, Mapping) else None
self.seen.append(copy.deepcopy(payload.get("messages")) if isinstance(payload, Mapping) else None)
async def async_log_success_event(
self, kwargs: object, response_obj: object, start_time: object, end_time: object
) -> None:
self.saw(kwargs)
class _SyncReader(_Reader):
def log_success_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
self.saw(kwargs)
@pytest.fixture(autouse=True)
def _isolated(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
for name in ("S3_PARTITION_GRANULARITY", "S3_LOG_PROMPTS_ONLY", "AWS_REGION", "AWS_REGION_NAME"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setattr(litellm, "cold_storage_custom_logger", None)
monkeypatch.setattr(litellm, "s3_callback_params", None)
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", None)
for name in (
"callbacks",
"input_callback",
"success_callback",
"failure_callback",
"_async_success_callback",
"_async_failure_callback",
):
monkeypatch.setattr(litellm, name, [])
monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [])
yield
@pytest.fixture
def bucket() -> Iterator[ScriptedBucket]:
with scripted_bucket() as scripted:
yield scripted
def _params(bucket: ScriptedBucket, strip: bool, region: str = "us-east-1") -> dict[str, object]:
return {
"s3_bucket_name": "test-bucket",
"s3_endpoint_url": bucket.url,
"s3_aws_access_key_id": "test-key",
"s3_aws_secret_access_key": "test-secret",
"s3_region_name": region,
**({"s3_strip_base64_files": True} if strip else {}),
}
def _s3(bucket: ScriptedBucket, strip: bool = True, region: str = "us-east-1") -> S3Logger:
logger: Final = S3Logger(_params(bucket, strip, region))
_ = logger.periodic_flush_task.cancel()
return logger
def _registered(*callbacks: object) -> list[object]:
for callback in callbacks:
assert isinstance(callback, CustomLogger | S3Logger)
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
return list(litellm._async_success_callback)
def _deliver(callback: object, kwargs: object) -> None:
if isinstance(callback, _Reader):
callback.saw(kwargs)
def _prepared(_callback: object, _kwargs: object) -> None:
return None
async def _run_line(callbacks: Sequence[object], kwargs: dict[str, object], outcome: Outcome = Outcome.success) -> None:
if outcome is Outcome.success:
await litellm_logging._run_callbacks(
Hook.log_prepare,
callbacks,
kwargs,
_prepared,
args=HookArgs(request=kwargs, start_time=START, end_time=START),
)
await litellm_logging._run_callbacks(
Hook.attempt_ended,
callbacks,
kwargs,
_deliver,
args=HookArgs(request=kwargs, start_time=START, end_time=START, outcome=outcome),
)
async def _flushed(logger: S3Logger) -> None:
for _ in range(100):
if logger.sink.queued_keys():
break
await asyncio.sleep(0.02)
await logger.flush_queue()
def _recorded(bucket: ScriptedBucket, case: str) -> bytes:
(put,) = bucket.puts()
assert (put.path, hashlib.sha256(put.body).hexdigest()) == RECORDED_AT_86B70FEF16[case]
return put.body
@pytest.mark.asyncio
async def test_strip_on_success_patches_readers_before_and_after_s3_and_uploads_the_recorded_object(
bucket: ScriptedBucket,
) -> None:
before: Final = _Reader("before")
after: Final = _Reader("after")
s3: Final = _s3(bucket)
callbacks: Final = _registered(before, s3, after)
assert callbacks == [before, s3.v2_companion, s3, after]
kwargs: Final[dict[str, object]] = {"standard_logging_object": _payload("stripped"), "litellm_params": {}}
await _run_line(callbacks, kwargs)
await _flushed(s3)
assert before.seen == after.seen == [STRIPPED_MESSAGES]
body: Final = _recorded(bucket, "stripped")
assert json.loads(body)["messages"] == STRIPPED_MESSAGES
@pytest.mark.asyncio
async def test_the_object_matches_the_attempt_ended_strip_alone(bucket: ScriptedBucket) -> None:
patched: Final = _s3(bucket)
await _run_line(_registered(patched), {"standard_logging_object": _payload("same"), "litellm_params": {}})
await _flushed(patched)
unpatched: Final = _s3(bucket)
await _run_line([unpatched], {"standard_logging_object": _payload("same"), "litellm_params": {}})
await _flushed(unpatched)
first, second = bucket.puts()
assert (first.path, first.body) == (second.path, second.body)
@pytest.mark.asyncio
async def test_strip_on_with_excluded_fields_uploads_the_recorded_object(
bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["response"])
after: Final = _Reader("after")
s3: Final = _s3(bucket)
kwargs: Final[dict[str, object]] = {"standard_logging_object": _payload("excluded"), "litellm_params": {}}
await _run_line(_registered(s3, after), kwargs)
await _flushed(s3)
assert after.seen == [STRIPPED_MESSAGES]
body: Final = json.loads(_recorded(bucket, "excluded"))
assert "response" not in body
assert body["messages"] == STRIPPED_MESSAGES
@pytest.mark.asyncio
async def test_strip_on_failure_still_uploads_the_recorded_stripped_object(bucket: ScriptedBucket) -> None:
s3: Final = _s3(bucket)
kwargs: Final[dict[str, object]] = {
"standard_logging_object": _payload("failed", status="failure"),
"litellm_params": {},
}
await _run_line(_registered(s3), kwargs, Outcome.failure)
await _flushed(s3)
assert json.loads(_recorded(bucket, "failed"))["messages"] == STRIPPED_MESSAGES
@pytest.mark.asyncio
async def test_strip_off_registers_no_writer_and_readers_see_the_original_messages(bucket: ScriptedBucket) -> None:
before: Final = _Reader("before")
s3: Final = _s3(bucket, strip=False)
callbacks: Final = _registered(before, s3)
kwargs: Final[dict[str, object]] = {"standard_logging_object": _payload("kept"), "litellm_params": {}}
await _run_line(callbacks, kwargs)
await _flushed(s3)
assert s3.v2_companion is None
assert callbacks == [before, s3]
assert before.seen == [_messages()]
assert json.loads(bucket.puts()[0].body)["messages"] == _messages()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"litellm_params",
[
pytest.param({"no-log": True}, id="no-log"),
pytest.param({"proxy_server_request": {"headers": {"x-litellm-disable-callbacks": "s3_v2"}}}, id="opt-out"),
],
)
async def test_a_request_s3_skips_gets_no_patch_and_nothing_is_uploaded(
bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, object]
) -> None:
from litellm.proxy import proxy_server
monkeypatch.setattr(litellm, "allow_dynamic_callback_disabling", True)
monkeypatch.setattr(proxy_server, "premium_user", True)
before: Final = _Reader("before")
after: Final = _Reader("after")
s3: Final = _s3(bucket)
kwargs: Final[dict[str, object]] = {
"standard_logging_object": _payload("skipped"),
"litellm_params": litellm_params,
}
await _run_line(_registered(before, s3, after), kwargs)
await s3.flush_queue()
assert before.seen == after.seen == [_messages()]
assert bucket.puts() == []
@pytest.mark.asyncio
async def test_the_writer_patches_nothing_on_the_sync_line(bucket: ScriptedBucket) -> None:
s3: Final = _s3(bucket)
writer: Final = s3.v2_companion
assert isinstance(writer, S3StripBase64Files)
payload: Final = _payload("sync")
kwargs: Final = {"standard_logging_object": payload, "litellm_params": {}}
await asyncio.to_thread(
litellm_logging._run_callbacks_sync,
Hook.log_prepare,
[writer, s3],
kwargs,
lambda _callback, _kwargs: None,
args=HookArgs(request=kwargs, start_time=START, end_time=START),
)
assert payload["messages"] == _messages()
@pytest.mark.asyncio
async def test_an_acompletion_shows_every_reader_the_messages_s3_stores(
bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch
) -> None:
before: Final = _Reader("before")
after: Final = _Reader("after")
s3: Final = _s3(bucket)
monkeypatch.setattr(litellm, "callbacks", [before, s3, after])
await litellm.acompletion(
model="gpt-4o",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "see data:image/png;base64,AAAA== inline"},
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBER"}},
],
}
],
mock_response="hi there",
)
for _ in range(100):
if after.seen:
break
await asyncio.sleep(0.02)
await _flushed(s3)
assert litellm._async_success_callback == [before, s3.v2_companion, s3, after]
expected: Final = [{"role": "user", "content": [{"type": "text", "text": "see [BASE64_REDACTED]"}]}]
assert before.seen == after.seen == [expected]
assert json.loads(bucket.puts()[0].body)["messages"] == expected
def test_a_sync_success_leaves_the_messages_for_readers_on_the_sync_line(bucket: ScriptedBucket) -> None:
async def built() -> S3Logger:
return _s3(bucket)
s3: Final = asyncio.run(built())
before: Final = _SyncReader("before")
litellm.logging_callback_manager.add_litellm_success_callback(before)
litellm.logging_callback_manager.add_litellm_success_callback(s3)
logging_obj: Final = Logging(
model="gpt-4o",
messages=[{"role": "user", "content": "see data:image/png;base64,AAAA== inline"}],
stream=False,
call_type="completion",
start_time=START,
litellm_call_id="call-sync",
function_id="fn-sync",
)
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
logging_obj.success_handler(result=ModelResponse(id="served"), start_time=START, end_time=START)
assert litellm.success_callback == [before, s3.v2_companion, s3]
assert before.seen == [[{"role": "user", "content": "see data:image/png;base64,AAAA== inline"}]]
assert bucket.puts() == []
@pytest.mark.asyncio
async def test_registering_s3_again_adds_no_second_writer(bucket: ScriptedBucket) -> None:
s3: Final = _s3(bucket)
_ = _registered(s3, s3)
litellm.logging_callback_manager.add_litellm_success_callback(s3)
litellm.logging_callback_manager.add_litellm_success_callback(s3)
assert litellm._async_success_callback == litellm.success_callback == [s3.v2_companion, s3]
@pytest.mark.asyncio
async def test_reinitialising_proxy_callbacks_keeps_one_writer_right_before_s3(
bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.proxy.utils import ProxyLogging
monkeypatch.setattr(ProxyLogging, "_add_proxy_hooks", lambda _self, _router=None: None)
monkeypatch.setattr(litellm, "s3_callback_params", _params(bucket, strip=True))
reader: Final = _Reader("reader")
monkeypatch.setattr(litellm, "callbacks", [reader, "s3_v2"])
proxy_logging: Final = ProxyLogging(user_api_key_cache=MagicMock())
proxy_logging._init_litellm_callbacks(llm_router=None)
proxy_logging._init_litellm_callbacks(llm_router=None)
(s3,) = (entry for entry in litellm.callbacks if isinstance(entry, S3Logger))
_ = s3.periodic_flush_task.cancel()
lists: Final = (
litellm.callbacks,
litellm.success_callback,
litellm.failure_callback,
litellm._async_success_callback,
litellm._async_failure_callback,
)
assert [[entry for entry in listed if entry is not proxy_logging.service_logging_obj] for listed in lists] == [
[reader, s3.v2_companion, s3]
] * len(lists)
@pytest.mark.asyncio
async def test_removing_s3_removes_its_writer(bucket: ScriptedBucket) -> None:
dropped: Final = _s3(bucket)
removed: Final = _s3(bucket, region="eu-west-1")
reader: Final = _Reader("reader")
registered: Final = _registered(dropped, reader, removed)
assert registered == [dropped.v2_companion, dropped, reader, removed.v2_companion, removed]
_ = litellm.logging_callback_manager.drop_callbacks((dropped,))
litellm.logging_callback_manager.remove_callback_from_all_lists(removed)
assert litellm._async_success_callback == [reader]
@pytest.mark.asyncio
async def test_the_writer_manifest_builds_as_a_blocking_log_prepare_writer(bucket: ScriptedBucket) -> None:
s3: Final = _s3(bucket)
writer: Final = s3.v2_companion
assert isinstance(writer, S3StripBase64Files)
manifest: Final = build_manifest(writer)
subscription: Final = manifest.subscriptions[Hook.log_prepare]
assert manifest.selection.name == STRIP_CALLBACK_NAME
assert manifest.selection.honors_no_log is False
assert set(manifest.subscriptions) == {Hook.log_prepare}
assert subscription.writes == frozenset({Field.standard_logging_payload})
assert subscription.reads == frozenset({Field.deprecated_kwargs, Field.standard_logging_payload})
assert build_manifest(s3).selection.honors_no_log is True
assert [entry["hook"] for entry in json.loads(s3.sink.callback().manifest())["subscriptions"]] == ["attempt_ended"]

View file

@ -7,7 +7,6 @@ from urllib.parse import urlparse
import pytest
import litellm
from litellm.integrations.s3_v2 import S3Logger
from litellm.litellm_core_utils.aws_partition import (
AwsPartition,
contains_aws_arn,
@ -100,14 +99,6 @@ def _agentcore_model(region: str) -> str:
return f"agentcore/{get_aws_arn_prefix(region)}bedrock-agentcore:{region}:111122223333:runtime/my-agent"
def _s3_object_url(region: str) -> str:
logger = S3Logger.__new__(S3Logger)
logger.s3_endpoint_url = None
logger.s3_bucket_name = "audit-bucket"
logger.s3_region_name = region
return logger._build_object_url("2025-01-01/key.json")
ENDPOINT_BUILDERS: Final = {
"bedrock_runtime_default": lambda region: BaseAWSLLM()._select_default_endpoint_url("runtime", region),
"bedrock_agent_default": lambda region: BaseAWSLLM()._select_default_endpoint_url("agent", region),
@ -158,7 +149,6 @@ ENDPOINT_BUILDERS: Final = {
api_base=None,
litellm_params={"aws_region_name": region},
),
"s3_object_url": _s3_object_url,
}

View file

@ -43,7 +43,6 @@ import pytest
from botocore.session import get_session
from pydantic import TypeAdapter
from litellm.integrations.s3_v2 import S3Logger
from litellm.integrations.sqs import SQSLogger
from litellm.llms.aws_polly.text_to_speech.transformation import AWSPollyTextToSpeechConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@ -55,7 +54,11 @@ from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
from litellm.llms.bedrock.model_listing import BedrockModelLister
from litellm.llms.bedrock.passthrough.transformation import BedrockAgentRuntimeAWS
from litellm.llms.bedrock.realtime.handler import BedrockRealtime
from litellm.llms.bedrock.web_identity_session_policy import WebIdentitySessionPolicy, _SessionPolicyStatement
from litellm.llms.bedrock.web_identity_session_policy import (
S3_LOGGING_SESSION_POLICY,
WebIdentitySessionPolicy,
_SessionPolicyStatement,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.opensearch.vector_stores.transformation import OpenSearchVectorStoreConfig
from litellm.llms.s3_vectors.vector_stores.transformation import S3VectorsVectorStoreConfig
@ -433,9 +436,12 @@ class TestSessionPolicyGrantsEveryBedrockRoute:
assert len(_captured_policy_document()) <= _STS_SESSION_POLICY_PLAINTEXT_LIMIT
def _s3_logger() -> S3Logger:
with patch("asyncio.create_task"):
return S3Logger(s3_bucket_name="litellm-logs", s3_region_name="us-east-1")
class _S3LoggingSession(BaseAWSLLM):
_web_identity_session_policy = S3_LOGGING_SESSION_POLICY
def _s3_logger() -> _S3LoggingSession:
return _S3LoggingSession()
def _sqs_logger() -> SQSLogger:

View file

@ -22,6 +22,7 @@ from litellm.proxy.management_helpers.audit_logs import (
is_audit_logging_enabled,
)
from litellm.types.utils import StandardAuditLogPayload
from tests.unit.integrations.s3_v3_rust_support import scripted_bucket
@pytest.fixture(autouse=True)
@ -313,16 +314,25 @@ class TestAuditLogTaskDoneCallback:
class TestS3LoggerAuditLogEvent:
@pytest.mark.asyncio
async def test_queues_audit_log_with_correct_s3_key(self):
with patch("litellm.integrations.s3_v2.S3Logger.__init__", return_value=None):
from litellm.integrations.s3_v2 import S3Logger
@staticmethod
def _logger(monkeypatch: pytest.MonkeyPatch, endpoint: str, s3_path: str | None):
from litellm.integrations.s3_v2 import S3Logger
logger = S3Logger()
logger.s3_path = "my-prefix"
logger.log_queue = []
logger.batch_size = 100
logger.max_queue_size = 100
monkeypatch.setattr(litellm, "s3_callback_params", None)
return S3Logger(
s3_bucket_name="audit-bucket",
s3_region_name="us-east-1",
s3_endpoint_url=endpoint,
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_path=s3_path,
s3_batch_size=100,
)
@pytest.mark.asyncio
async def test_queues_audit_log_with_correct_s3_key(self, monkeypatch: pytest.MonkeyPatch):
with scripted_bucket() as bucket:
logger = self._logger(monkeypatch, bucket.url, "my-prefix")
audit_log = StandardAuditLogPayload(
id="audit-123",
@ -338,25 +348,26 @@ class TestS3LoggerAuditLogEvent:
await logger.async_log_audit_log_event(audit_log)
assert len(logger.log_queue) == 1
element = logger.log_queue[0]
assert element.s3_object_key.startswith("my-prefix/audit_logs/")
assert "audit-123" in element.s3_object_key
assert element.s3_object_key.endswith(".json")
assert element.s3_object_download_filename == "audit-audit-123.json"
assert element.payload["id"] == "audit-123"
assert element.payload["action"] == "created"
queued = logger.sink.queued_keys()
assert len(queued) == 1
s3_object_key = queued[0]
assert s3_object_key.startswith("my-prefix/audit_logs/")
assert "audit-123" in s3_object_key
assert s3_object_key.endswith(".json")
await logger.flush_queue()
put = bucket.puts()[0]
assert put.path == f"/audit-bucket/{s3_object_key}"
assert put.headers["content-disposition"] == 'inline; filename="audit-audit-123.json"'
payload = json.loads(put.body)
assert payload["id"] == "audit-123"
assert payload["action"] == "created"
@pytest.mark.asyncio
async def test_s3_key_format_no_path(self):
with patch("litellm.integrations.s3_v2.S3Logger.__init__", return_value=None):
from litellm.integrations.s3_v2 import S3Logger
logger = S3Logger()
logger.s3_path = None
logger.log_queue = []
logger.batch_size = 100
logger.max_queue_size = 100
async def test_s3_key_format_no_path(self, monkeypatch: pytest.MonkeyPatch):
with scripted_bucket() as bucket:
logger = self._logger(monkeypatch, bucket.url, None)
audit_log = StandardAuditLogPayload(
id="audit-456",
@ -372,10 +383,10 @@ class TestS3LoggerAuditLogEvent:
await logger.async_log_audit_log_event(audit_log)
assert len(logger.log_queue) == 1
element = logger.log_queue[0]
assert element.s3_object_key.startswith("audit_logs/")
assert "audit-456" in element.s3_object_key
queued = logger.sink.queued_keys()
assert len(queued) == 1
assert queued[0].startswith("audit_logs/")
assert "audit-456" in queued[0]
class TestS3AuditCallbackParamsDecoupling:

View file

@ -14,6 +14,8 @@ from unittest.mock import AsyncMock, patch
import pytest
import litellm
from litellm.integrations.s3_v2 import S3Logger
from litellm.proxy._types import SpendLogsMetadata
from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
@ -22,6 +24,7 @@ from litellm.responses.litellm_completion_transformation.session_handler import
ResponsesSessionHandler,
)
from litellm.types.utils import StandardLoggingMetadata
from tests.unit.integrations.s3_v3_rust_support import scripted_bucket
class TestColdStorageObjectKeyIntegration:
@ -172,24 +175,29 @@ class TestColdStorageObjectKeyIntegration:
)
@pytest.mark.asyncio
@patch("asyncio.create_task") # Mock asyncio.create_task to avoid event loop issues
async def test_s3_logger_supports_object_key_retrieval(self, mock_create_task):
async def test_s3_logger_supports_object_key_retrieval(self, monkeypatch: pytest.MonkeyPatch):
"""
Test: S3Logger supports retrieval using provided object key.
This test verifies that the S3Logger can retrieve objects using
the object key directly without generating it from request_id and start_time.
"""
# Create S3Logger instance
s3_logger = S3Logger(s3_bucket_name="test-bucket")
monkeypatch.setattr(litellm, "s3_callback_params", None)
with scripted_bucket() as bucket:
bucket.objects["/test-bucket/test/path/to/object.json"] = b'{"test": "data"}'
s3_logger = S3Logger(
s3_bucket_name="test-bucket",
s3_region_name="us-east-1",
s3_endpoint_url=bucket.url,
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
)
# Mock the _download_object_from_s3 method
with patch.object(
s3_logger, "_download_object_from_s3", return_value={"test": "data"}
) as mock_download:
result = await s3_logger.get_proxy_server_request_from_cold_storage_with_object_key(
object_key="test/path/to/object.json"
)
assert result == {"test": "data"}
mock_download.assert_called_once_with("test/path/to/object.json")
assert [(request.method, request.path) for request in bucket.received] == [
("GET", "/test-bucket/test/path/to/object.json")
]