From 8e92894ff2585770a9b5d0bad1a30173f10d09ca Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Sat, 14 Nov 2026 15:03:58 -0800 Subject: [PATCH] feat(s3-log): replace the python s3_v2 logger with a rust callback (#51539) The s3_v2 S3Logger is ported to a Rust callback, callbacks-s3-log, behind the CustomLoggerV2 contract, with the bridge constructor NativeS3Log and the Python glue in litellm/integrations/s3_v3.py. It is now the only implementation: the Python logger is deleted, s3_v2.py re-exports the Rust-backed S3Logger, and LITELLM_NATIVE_S3_V2 is removed. The callback keeps its name, list position and settings, so no config changes. The original s3_v2 tests are replicated against the Rust callback as _rust.py, and Python's outputs for the differential suite are recorded in tests/unit/integrations/s3_v2_golden/python_s3_v2.json. --- litellm/batches/batch_line_items.py | 13 +- litellm/constants.py | 1 - .../enterprise_callbacks/callback_controls.py | 3 +- litellm/integrations/adaptive_concurrency.py | 78 - litellm/integrations/s3.py | 59 - litellm/integrations/s3_v2.py | 1074 +-- litellm/integrations/s3_v3.py | 481 ++ .../custom_logger_registry.py | 2 +- litellm/litellm_core_utils/litellm_logging.py | 45 +- .../logging_callback_manager.py | 63 +- .../proxy/management_helpers/audit_logs.py | 24 +- litellm/proxy/proxy_server.py | 5 +- litellm/proxy/utils.py | 1 + litellm/rust/Cargo.lock | 35 + litellm/rust/Cargo.toml | 1 + litellm/rust/crates/auth-aws/Cargo.toml | 1 + litellm/rust/crates/auth-aws/src/aws.rs | 111 +- .../rust/crates/callbacks-s3-log/Cargo.toml | 35 + .../rust/crates/callbacks-s3-log/src/key.rs | 367 + .../rust/crates/callbacks-s3-log/src/lib.rs | 11 + .../crates/callbacks-s3-log/src/limiter.rs | 304 + .../crates/callbacks-s3-log/src/pyjson.rs | 214 + .../crates/callbacks-s3-log/src/settings.rs | 656 ++ .../rust/crates/callbacks-s3-log/src/sink.rs | 670 ++ .../crates/callbacks-s3-log/src/upload.rs | 470 ++ .../crates/callbacks-s3-log/tests/sink.rs | 693 ++ litellm/rust/crates/python-bridge/Cargo.toml | 3 + litellm/rust/crates/python-bridge/src/lib.rs | 4 + .../rust/crates/python-bridge/src/s3_log.rs | 225 + .../python-compat/generated/values.json | 21 +- .../scripts/generate_fixtures.py | 1 + litellm/rust/crates/python-compat/src/repr.rs | 10 +- litellm/rust/crates/s3/Cargo.toml | 4 + litellm/rust/crates/s3/src/lib.rs | 1 + litellm/rust/crates/s3/src/signed.rs | 137 + litellm/rust_bridge/_native.pyi | 31 + litellm/types/integrations/s3_v2.py | 47 +- tests/e2e/coverage_registry/logging.yaml | 6 +- ...s.py => test_s3_v2_flush_surfaces_rust.py} | 4 +- ... test_s3_v2_partition_granularity_rust.py} | 1 + ...st_s3_v2_streaming_failure_dedupe_rust.py} | 22 +- ...ut.py => test_s3_v2_upload_fanout_rust.py} | 16 +- tests/integration/providers/test_s3_wire.py | 87 +- .../integrations/s3_v2_golden/__init__.py | 0 .../s3_v2_golden/python_s3_v2.json | 5945 ++++++++++++++++ tests/unit/integrations/s3_v3_rust_support.py | 145 + .../integrations/test_adaptive_concurrency.py | 179 - tests/unit/integrations/test_s3_v2.py | 6044 ----------------- tests/unit/integrations/test_s3_v2_rust.py | 3105 +++++++++ tests/unit/integrations/test_s3_v3_native.py | 1525 +++++ .../integrations/test_s3_v3_strip_writer.py | 445 ++ .../litellm_core_utils/test_aws_partition.py | 10 - .../test_web_identity_session_policy.py | 16 +- .../test_audit_log_callbacks.py | 71 +- .../test_session_handler_with_cold_storage.py | 26 +- 55 files changed, 15926 insertions(+), 7622 deletions(-) delete mode 100644 litellm/integrations/adaptive_concurrency.py create mode 100644 litellm/integrations/s3_v3.py create mode 100644 litellm/rust/crates/callbacks-s3-log/Cargo.toml create mode 100644 litellm/rust/crates/callbacks-s3-log/src/key.rs create mode 100644 litellm/rust/crates/callbacks-s3-log/src/lib.rs create mode 100644 litellm/rust/crates/callbacks-s3-log/src/limiter.rs create mode 100644 litellm/rust/crates/callbacks-s3-log/src/pyjson.rs create mode 100644 litellm/rust/crates/callbacks-s3-log/src/settings.rs create mode 100644 litellm/rust/crates/callbacks-s3-log/src/sink.rs create mode 100644 litellm/rust/crates/callbacks-s3-log/src/upload.rs create mode 100644 litellm/rust/crates/callbacks-s3-log/tests/sink.rs create mode 100644 litellm/rust/crates/python-bridge/src/s3_log.rs create mode 100644 litellm/rust/crates/s3/src/signed.rs rename tests/integration/observability/{test_s3_v2_flush_surfaces.py => test_s3_v2_flush_surfaces_rust.py} (95%) rename tests/integration/observability/{test_s3_v2_partition_granularity.py => test_s3_v2_partition_granularity_rust.py} (99%) rename tests/integration/observability/{test_s3_v2_streaming_failure_dedupe.py => test_s3_v2_streaming_failure_dedupe_rust.py} (96%) rename tests/integration/observability/{test_s3_v2_upload_fanout.py => test_s3_v2_upload_fanout_rust.py} (97%) create mode 100644 tests/unit/integrations/s3_v2_golden/__init__.py create mode 100644 tests/unit/integrations/s3_v2_golden/python_s3_v2.json create mode 100644 tests/unit/integrations/s3_v3_rust_support.py delete mode 100644 tests/unit/integrations/test_adaptive_concurrency.py delete mode 100644 tests/unit/integrations/test_s3_v2.py create mode 100644 tests/unit/integrations/test_s3_v2_rust.py create mode 100644 tests/unit/integrations/test_s3_v3_native.py create mode 100644 tests/unit/integrations/test_s3_v3_strip_writer.py diff --git a/litellm/batches/batch_line_items.py b/litellm/batches/batch_line_items.py index 120c1d958e1..8f1e3efaaf3 100644 --- a/litellm/batches/batch_line_items.py +++ b/litellm/batches/batch_line_items.py @@ -7,7 +7,7 @@ from collections import Counter from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass from itertools import accumulate, chain, count -from typing import Final, Literal, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, TypeAlias import litellm from litellm._logging import verbose_logger @@ -23,6 +23,9 @@ from litellm.types.utils import ( StandardBatchLineItemUsage, ) +if TYPE_CHECKING: + from litellm.integrations.s3_v3 import S3Logger + BatchFileProvider: TypeAlias = Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] _VERTEX_AI_UNKNOWN_CUSTOM_ID: Final = "unknown" # what a Vertex AI batch row that carries no custom_id is read as @@ -238,9 +241,11 @@ async def _deliver( ) -def _resolve_logger(callback: litellm.CALLBACK_TYPES) -> CustomLogger | None: +def _resolve_logger(callback: "litellm.CALLBACK_TYPES | S3Logger") -> "CustomLogger | S3Logger | None": + from litellm.integrations.s3_v3 import S3Logger + match callback: - case CustomLogger(): + case CustomLogger() | S3Logger(): return callback case "s3_v2" | "gcs_bucket": return init_custom_logger_compatible_class(callback, internal_usage_cache=None, llm_router=None) @@ -248,7 +253,7 @@ def _resolve_logger(callback: litellm.CALLBACK_TYPES) -> CustomLogger | None: return None -def _logger_turned_off_message_logging(logger: CustomLogger) -> bool: +def _logger_turned_off_message_logging(logger: object) -> bool: return ( getattr(logger, "turn_off_message_logging", False) is True or getattr(logger, "message_logging", True) is not True diff --git a/litellm/constants.py b/litellm/constants.py index 166300b8145..68331e3199e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -130,7 +130,6 @@ S3_PREFIX_DIGEST_CHARS: Final = 16 # s3 allows 2048 bytes of combined metadata headers, which Content-Disposition counts against MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024 S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY" -S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY" MAX_FILE_LIST_LIMIT: Final = 10000 ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) diff --git a/litellm/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py b/litellm/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py index 6a97a9de414..c28fa74cd98 100644 --- a/litellm/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py +++ b/litellm/enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py @@ -4,6 +4,7 @@ import litellm from litellm._logging import verbose_logger from litellm.constants import X_LITELLM_DISABLE_CALLBACKS from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 from litellm.litellm_core_utils.llm_request_utils import ( get_proxy_server_request_headers, ) @@ -45,7 +46,7 @@ class EnterpriseCallbackControls: if callback.lower() in disabled_callbacks: verbose_logger.debug(f"Not logging to {callback} because it is disabled via {X_LITELLM_DISABLE_CALLBACKS}") return True - elif isinstance(callback, CustomLogger): + elif isinstance(callback, CustomLogger | CustomLoggerV2): # get the string name of the callback callback_str = CustomLoggerRegistry.get_callback_str_from_class_type(callback.__class__) if callback_str is not None and callback_str.lower() in disabled_callbacks: diff --git a/litellm/integrations/adaptive_concurrency.py b/litellm/integrations/adaptive_concurrency.py deleted file mode 100644 index e0c6b730dda..00000000000 --- a/litellm/integrations/adaptive_concurrency.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Adaptive in-flight concurrency limiter (AIMD, Vector ARC style). - -Grows the limit additively after `limit` consecutive clean completions and -halves it only on an explicit throttle signal (429, 503, SlowDown, or a -transport error out of the PUT). With floor == ceiling it degenerates to a -fixed-width semaphore. -""" - -import asyncio -from collections import deque -from contextlib import suppress -from dataclasses import dataclass -from typing import Final - - -@dataclass(frozen=True, slots=True) -class PutSample: - throttled: bool - - -class AdaptiveConcurrencyLimiter: - """AIMD in-flight limiter used as `async with limiter:`.""" - - def __init__(self, initial: int, floor: int, ceiling: int) -> None: - if not 1 <= floor <= ceiling: - raise ValueError(f"adaptive limiter bounds must satisfy 1 <= floor <= ceiling, got {floor}..{ceiling}") - self._limit: int = min(max(initial, floor), ceiling) - self._floor: Final[int] = floor - self._ceiling: Final[int] = ceiling - self._clean_streak: int = 0 - self._in_flight: int = 0 - self._waiters: deque[asyncio.Future[None]] = deque() # mutable-ok: waiters queue up behind a full limit - - @property - def limit(self) -> int: - return self._limit - - async def __aenter__(self) -> "AdaptiveConcurrencyLimiter": - if self._in_flight < self._limit: - self._in_flight += 1 - return self - waiter: Final = asyncio.get_running_loop().create_future() - self._waiters.append(waiter) - try: - await waiter - except asyncio.CancelledError: - if waiter.done() and not waiter.cancelled(): - self._in_flight -= 1 - self._grant() - else: - with suppress(ValueError): - self._waiters.remove(waiter) - raise - return self - - def _grant(self) -> None: - while self._in_flight < self._limit and self._waiters: - waiter = self._waiters.popleft() - if waiter.done(): - continue - self._in_flight += 1 - waiter.set_result(None) - - async def __aexit__(self, *_: object) -> None: - self._in_flight -= 1 - self._grant() - - def record(self, sample: PutSample) -> None: - if sample.throttled: - self._limit = max(self._floor, self._limit // 2) - self._clean_streak = 0 - return - self._clean_streak += 1 - if self._clean_streak >= self._limit and self._limit < self._ceiling: - self._limit += 1 - self._clean_streak = 0 - self._grant() diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index 962064127fa..b55bd8b30c2 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -17,7 +17,6 @@ from litellm.constants import ( MAX_S3_OBJECT_KEY_FILENAME_BYTES, S3_BOUNDED_OBJECT_KEY_HEAD_BYTES, S3_LOG_PROMPTS_ONLY_ENV_VAR, - S3_PARTITION_GRANULARITY_ENV_VAR, S3_PREFIX_DIGEST_CHARS, ) from litellm.types.integrations.s3_v2 import S3PartitionGranularity @@ -39,18 +38,6 @@ def resolve_s3_log_prompts_only(configured: object, environ: Mapping[str, str] | return True -def resolve_s3_partition_granularity( - configured: object, environ: Mapping[str, str] | None = None -) -> S3PartitionGranularity: - env: Final = os.environ if environ is None else environ - raw: Final = env.get(S3_PARTITION_GRANULARITY_ENV_VAR) if configured is None else configured - if raw == "hour": - return "hour" - if raw is not None and raw not in ("", "day"): - verbose_logger.warning("s3 logging: s3_partition_granularity=%r is not one of day, hour, using day", raw) - return "day" - - def _resolve_positive_int(setting: str, configured: object, fallback: int, *, reject_bool: bool) -> int: if configured is None or configured == "": return fallback @@ -70,10 +57,6 @@ def _resolve_positive_int(setting: str, configured: object, fallback: int, *, re return bound -def resolve_s3_max_concurrent_uploads(configured: object, fallback: int) -> int: - return _resolve_positive_int("s3_max_concurrent_uploads", configured, fallback, reject_bool=False) - - def resolve_s3_max_queue_size(configured: object, fallback: int) -> int: return _resolve_positive_int("s3_max_queue_size", configured, fallback, reject_bool=True) @@ -103,48 +86,6 @@ def resolve_s3_max_retry_age_seconds(configured: object, fallback: int | None) - return bound or None -def resolve_s3_max_adaptive_concurrency(configured: object, fallback: int) -> int: - return _resolve_positive_int("s3_max_adaptive_concurrency", configured, fallback, reject_bool=True) - - -def resolve_s3_drop_on_terminal_error(configured: object) -> bool: - if configured is None or configured == "": - return True - try: - return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured) - except ValidationError: - verbose_logger.warning( - "s3 logging: s3_drop_on_terminal_error=%r is not a boolean, dropping terminal-failed uploads", - configured, - ) - return True - - -def resolve_s3_adaptive_concurrency(configured: object) -> bool: - if configured is None or configured == "": - return False - try: - return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured) - except ValidationError: - verbose_logger.warning( - "s3 logging: s3_adaptive_concurrency=%r is not a boolean, keeping the fixed upload width", - configured, - ) - return False - - -def resolve_s3_batch_file_upload(configured: object) -> bool: - if configured is None or configured == "": - return False - try: - return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured) - except ValidationError: - verbose_logger.warning( - "s3 logging: s3_batch_file_upload=%r is not a boolean, keeping per-request objects", configured - ) - return False - - def prompts_only_payload(payload: StandardLoggingPayload) -> StandardLoggingPayload: return {**payload, "response": None} diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 6c35b53c92d..98892a7fdf4 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -1,1073 +1,3 @@ -""" -s3 Bucket Logging Integration +from litellm.integrations.s3_v3 import S3BatchUploadError, S3Logger -async_log_success_event: Processes the event, stores it in memory for DEFAULT_S3_FLUSH_INTERVAL_SECONDS seconds or until DEFAULT_S3_BATCH_SIZE and then flushes to s3 -async_log_failure_event: Processes the event, stores it in memory for DEFAULT_S3_FLUSH_INTERVAL_SECONDS seconds or until DEFAULT_S3_BATCH_SIZE and then flushes to s3 -NOTE 1: S3 does not provide a BATCH PUT API endpoint; by default each element is uploaded concurrently with the fixed s3_max_concurrent_uploads bound (or an adaptive bound when s3_adaptive_concurrency is on, backing off only on throttling), or with s3_batch_file_upload the whole flush is written as one .jsonl file -""" - -import asyncio -import contextvars -import logging -import os -import random -import re -import time -from collections.abc import Awaitable, Callable, Mapping -from dataclasses import dataclass -from datetime import datetime, timezone -from functools import partial -from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, cast -from urllib.parse import quote -from uuid import uuid4 - -import httpx - -import litellm -from litellm._logging import print_verbose, verbose_logger -from litellm.constants import ( - DEFAULT_S3_BATCH_SIZE, - DEFAULT_S3_FLUSH_INTERVAL_SECONDS, - DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY, - DEFAULT_S3_MAX_CONCURRENT_UPLOADS, - S3_PARTITION_GRANULARITY_ENV_VAR, - S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS, -) -from litellm.exceptions import Timeout as LiteLLMTimeout -from litellm.integrations.adaptive_concurrency import AdaptiveConcurrencyLimiter, PutSample -from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name -from litellm.integrations.s3 import ( - get_s3_object_download_filename, - get_s3_object_key, - prompts_only_payload, - resolve_s3_adaptive_concurrency, - resolve_s3_batch_file_upload, - resolve_s3_drop_on_terminal_error, - resolve_s3_log_prompts_only, - resolve_s3_max_adaptive_concurrency, - resolve_s3_max_concurrent_uploads, - resolve_s3_max_queue_size, - resolve_s3_max_retry_age_seconds, - resolve_s3_partition_granularity, - resolve_sse_params, -) -from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix -from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker -from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing -from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, WebIdentitySessionPolicy -from litellm.llms.custom_httpx.http_handler import ( - get_async_httpx_client, - get_httpx_client, - httpxSpecialProvider, -) -from litellm.types.integrations.s3_v2 import S3PartitionGranularity, s3BatchLoggingElement -from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload - -from .custom_batch_logger import CustomBatchLogger - -if TYPE_CHECKING: - from botocore.credentials import Credentials - - -UploadOutcome = Literal["delivered", "retry", "dropped"] - -_TERMINAL_ERROR_CODES: Final = frozenset( - { - "EntityTooLarge", - "InvalidArgument", - "MalformedXML", - "InvalidDigest", - "KeyTooLongError", - "BadDigest", - "InvalidRequest", - } -) -_BODY_CODED_STATUSES: Final = frozenset({400, 403}) -_RETRYABLE_STATUSES: Final = frozenset({403, 429, 500, 503}) -_S3_ERROR_CODE: Final = re.compile(r"([^<]+)") -_TRANSIENT_ERRORS: Final = (httpx.TransportError, LiteLLMTimeout) -_UPLOAD_ATTEMPTS: Final = 3 -_UPLOAD_ATTEMPT_TIMEOUT: Final = httpx.Timeout(S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS) - -PutOutcome: TypeAlias = httpx.Response | Exception - - -@dataclass(frozen=True, slots=True) -class _PreparedPut: - json_string: str - headers: Mapping[str, str] - - -def _s3_error_code(response: httpx.Response) -> str | None: - text: Final = response.text - match: Final = _S3_ERROR_CODE.search(text) if isinstance(text, str) else None - return match.group(1) if match else None - - -def _is_terminal(response: httpx.Response) -> bool: - """True only for object-specific, unrecoverable rejections (400/403 with a terminal XML code). - Unknown codes, empty or non-XML bodies, and every other status fail safe toward retry.""" - return response.status_code in _BODY_CODED_STATUSES and _s3_error_code(response) in _TERMINAL_ERROR_CODES - - -def _s3_key_parent(s3_object_key: str) -> str: - return s3_object_key.rsplit("/", 1)[0] if "/" in s3_object_key else "" - - -def _backoff_seconds(attempt: int) -> float: - return (1 << attempt) * (0.5 + random.random()) - - -def _log_retry(level: int, attempt: int, reason: str, wait: float, s3_object_key: str) -> None: - verbose_logger.log( - level, - "S3 upload attempt %s/%s failed with %s, retrying in %.2fs key=%s", - attempt + 1, - _UPLOAD_ATTEMPTS, - reason, - wait, - s3_object_key, - ) - - -def _response_or_raise(outcome: PutOutcome) -> httpx.Response: - if isinstance(outcome, Exception): - raise outcome - outcome.raise_for_status() - return outcome - - -def _throttled(outcome: PutOutcome) -> bool: - if isinstance(outcome, Exception): - return True - return outcome.status_code in (429, 503) or _s3_error_code(outcome) == "SlowDown" - - -class S3BatchUploadError(Exception): - def __init__(self, failed: int, total: int) -> None: - self.failed = failed - self.total = total - super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush") - - -_in_flush: Final[contextvars.ContextVar[bool]] = contextvars.ContextVar("s3_v2_in_flush", default=False) - - -class S3Logger(CustomBatchLogger, BaseAWSLLM): - _web_identity_session_policy: ClassVar[WebIdentitySessionPolicy] = S3_LOGGING_SESSION_POLICY - preserve_events_added_during_flush = True - _flush_retries: int = 0 - _requeued_count: int = 0 - _upload_limiter: asyncio.Semaphore | AdaptiveConcurrencyLimiter | None = None - s3_drop_on_terminal_error: bool = True - s3_max_retry_age_seconds: int | None = 3600 - s3_partition_granularity: object = None - _partition_granularity_cache: tuple[object, S3PartitionGranularity] | None = None - - def __init__( - self, - s3_bucket_name: str | None = None, - s3_path: str | None = None, - s3_region_name: str | None = None, - s3_api_version: str | None = None, - s3_use_ssl: bool = True, - s3_verify: bool | None = None, - s3_endpoint_url: str | None = None, - s3_aws_access_key_id: str | None = None, - s3_aws_secret_access_key: str | None = None, - s3_aws_session_token: str | None = None, - s3_aws_session_name: str | None = None, - s3_aws_profile_name: str | None = None, - s3_aws_role_name: str | None = None, - s3_aws_web_identity_token: str | None = None, - s3_aws_sts_endpoint: str | None = None, - s3_flush_interval: int | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS, - s3_batch_size: int | None = DEFAULT_S3_BATCH_SIZE, - s3_config=None, - s3_use_team_prefix: bool = False, - s3_strip_base64_files: bool = False, - s3_use_key_prefix: bool = False, - s3_use_virtual_hosted_style: bool = False, - s3_server_side_encryption: str | None = None, - s3_sse_kms_key_id: str | None = None, - s3_log_prompts_only: bool | None = None, - s3_partition_granularity: str | None = None, - s3_max_concurrent_uploads: int = DEFAULT_S3_MAX_CONCURRENT_UPLOADS, - s3_max_queue_size: int | None = None, - s3_max_retry_age_seconds: int | None = 3600, - s3_drop_on_terminal_error: bool = True, - s3_adaptive_concurrency: bool = False, - s3_max_adaptive_concurrency: int | None = None, - s3_batch_file_upload: bool = False, - s3_callback_params_override: dict | None = None, - **kwargs, - ): - try: - _masker: Final = SensitiveDataMasker() - if s3_callback_params_override is not None: - verbose_logger.debug( - "in init s3 logger (audit override) - %s", _masker.mask_dict(dict(s3_callback_params_override)) - ) - else: - verbose_logger.debug( - "in init s3 logger - s3_callback_params %s", - _masker.mask_dict(dict(litellm.s3_callback_params or {})), - ) - - # Initialize S3 params first to get the correct s3_verify value - self._init_s3_params( - params_source=s3_callback_params_override, - s3_bucket_name=s3_bucket_name, - s3_region_name=s3_region_name, - s3_api_version=s3_api_version, - s3_use_ssl=s3_use_ssl, - s3_verify=s3_verify, - s3_endpoint_url=s3_endpoint_url, - s3_aws_access_key_id=s3_aws_access_key_id, - s3_aws_secret_access_key=s3_aws_secret_access_key, - s3_aws_session_token=s3_aws_session_token, - s3_aws_session_name=s3_aws_session_name, - s3_aws_profile_name=s3_aws_profile_name, - s3_aws_role_name=s3_aws_role_name, - s3_aws_web_identity_token=s3_aws_web_identity_token, - s3_aws_sts_endpoint=s3_aws_sts_endpoint, - s3_config=s3_config, - s3_path=s3_path, - s3_use_team_prefix=s3_use_team_prefix, - s3_strip_base64_files=s3_strip_base64_files, - s3_use_key_prefix=s3_use_key_prefix, - s3_use_virtual_hosted_style=s3_use_virtual_hosted_style, - s3_server_side_encryption=s3_server_side_encryption, - s3_sse_kms_key_id=s3_sse_kms_key_id, - s3_log_prompts_only=s3_log_prompts_only, - s3_partition_granularity=s3_partition_granularity, - s3_max_concurrent_uploads=s3_max_concurrent_uploads, - s3_max_queue_size=s3_max_queue_size, - s3_max_retry_age_seconds=s3_max_retry_age_seconds, - s3_drop_on_terminal_error=s3_drop_on_terminal_error, - s3_adaptive_concurrency=s3_adaptive_concurrency, - s3_max_adaptive_concurrency=s3_max_adaptive_concurrency, - s3_batch_file_upload=s3_batch_file_upload, - ) - self._upload_limiter = ( - AdaptiveConcurrencyLimiter( - initial=self.s3_max_concurrent_uploads, - floor=self.s3_max_concurrent_uploads, - ceiling=max(self.s3_max_concurrent_uploads, self.s3_max_adaptive_concurrency), - ) - if self.s3_adaptive_concurrency - else asyncio.Semaphore(self.s3_max_concurrent_uploads) - ) - verbose_logger.debug("s3 logger using endpoint url %s", s3_endpoint_url) - - # IMPORTANT - # Create httpx client AFTER _init_s3_params so we have the correct s3_verify value - verbose_logger.debug("s3_v2 logger creating async httpx client with s3_verify=%s", self.s3_verify) - self.async_httpx_client = get_async_httpx_client( - llm_provider=httpxSpecialProvider.LoggingCallback, - params={"ssl_verify": self.s3_verify}, - ) - - asyncio.create_task(self.periodic_flush()) - self.flush_lock = asyncio.Lock() - - verbose_logger.debug("s3 flush interval: %s, s3 batch size: %s", s3_flush_interval, s3_batch_size) - # Call CustomLogger's __init__ - CustomBatchLogger.__init__( - self, - flush_lock=self.flush_lock, - flush_interval=s3_flush_interval, - batch_size=s3_batch_size, - max_queue_size=self.s3_max_queue_size, - ) - self.log_queue: list[s3BatchLoggingElement] = [] - self._requeued_count = 0 - self._flush_retries = 0 - self._flush_dropped: dict[int, s3BatchLoggingElement] = {} - - # Call BaseAWSLLM's __init__ - BaseAWSLLM.__init__(self) - - except Exception as e: - print_verbose(f"Got exception on init s3 client {e}") - raise e - - def _init_s3_params( - self, - s3_bucket_name: str | None = None, - s3_region_name: str | None = None, - s3_api_version: str | None = None, - s3_use_ssl: bool = True, - s3_verify: bool | None = None, - s3_endpoint_url: str | None = None, - s3_aws_access_key_id: str | None = None, - s3_aws_secret_access_key: str | None = None, - s3_aws_session_token: str | None = None, - s3_aws_session_name: str | None = None, - s3_aws_profile_name: str | None = None, - s3_aws_role_name: str | None = None, - s3_aws_web_identity_token: str | None = None, - s3_aws_sts_endpoint: str | None = None, - s3_config=None, - s3_path: str | None = None, - s3_use_team_prefix: bool = False, - s3_strip_base64_files: bool = False, - s3_use_key_prefix: bool = False, - s3_use_virtual_hosted_style: bool = False, - s3_server_side_encryption: str | None = None, - s3_sse_kms_key_id: str | None = None, - s3_log_prompts_only: bool | None = None, - s3_partition_granularity: str | None = None, - s3_max_concurrent_uploads: int = DEFAULT_S3_MAX_CONCURRENT_UPLOADS, - s3_max_queue_size: int | None = None, - s3_max_retry_age_seconds: int | None = 3600, - s3_drop_on_terminal_error: bool = True, - s3_adaptive_concurrency: bool = False, - s3_max_adaptive_concurrency: int | None = None, - s3_batch_file_upload: bool = False, - params_source: dict | None = None, - ): - """ - Initialize the s3 params for this logging callback. Reads from - `params_source` if given (e.g. `s3_audit_callback_params` for the - audit-log instance), otherwise falls back to `litellm.s3_callback_params`. - Resolves `os.environ/X` markers into a local dict; never mutates the source. - """ - if params_source is None: - params_source = litellm.s3_callback_params or {} - params: Final[dict] = { - key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value) - for key, value in params_source.items() - } - - self.s3_bucket_name = params.get("s3_bucket_name") or s3_bucket_name - self.s3_region_name = params.get("s3_region_name") or s3_region_name - self.s3_api_version = params.get("s3_api_version") or s3_api_version - self.s3_use_ssl = params.get("s3_use_ssl", True) if params.get("s3_use_ssl") is not None else s3_use_ssl - self.s3_verify = params.get("s3_verify") if params.get("s3_verify") is not None else s3_verify - self.s3_endpoint_url = params.get("s3_endpoint_url") or s3_endpoint_url - self.s3_aws_access_key_id = params.get("s3_aws_access_key_id") or s3_aws_access_key_id - - self.s3_aws_secret_access_key = params.get("s3_aws_secret_access_key") or s3_aws_secret_access_key - - self.s3_aws_session_token = params.get("s3_aws_session_token") or s3_aws_session_token - - self.s3_aws_session_name = params.get("s3_aws_session_name") or s3_aws_session_name - - self.s3_aws_profile_name = params.get("s3_aws_profile_name") or s3_aws_profile_name - - self.s3_aws_role_name = params.get("s3_aws_role_name") or s3_aws_role_name - - self.s3_aws_web_identity_token = params.get("s3_aws_web_identity_token") or s3_aws_web_identity_token - - self.s3_aws_sts_endpoint = params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint - - self.s3_config = params.get("s3_config") or s3_config - self.s3_path = params.get("s3_path") or s3_path - self.s3_use_team_prefix = bool(params.get("s3_use_team_prefix", False)) or s3_use_team_prefix - - self.s3_use_key_prefix = bool(params.get("s3_use_key_prefix", False)) or s3_use_key_prefix - - self.s3_strip_base64_files = bool(params.get("s3_strip_base64_files", False)) or s3_strip_base64_files - - self.s3_use_virtual_hosted_style = ( - bool(params.get("s3_use_virtual_hosted_style", False)) or s3_use_virtual_hosted_style - ) - - self.s3_log_prompts_only: object = ( - params.get("s3_log_prompts_only") if s3_log_prompts_only is None else s3_log_prompts_only - ) - - self.s3_partition_granularity = ( - params.get("s3_partition_granularity") if s3_partition_granularity is None else s3_partition_granularity - ) - self._partition_granularity_cache = None - - self.s3_server_side_encryption, self.s3_sse_kms_key_id = resolve_sse_params( - params.get("s3_server_side_encryption") or s3_server_side_encryption, - params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id, - ) - - configured_bound: Final = params.get("s3_max_concurrent_uploads") - self.s3_max_concurrent_uploads = resolve_s3_max_concurrent_uploads( - s3_max_concurrent_uploads if configured_bound is None or configured_bound == "" else configured_bound, - DEFAULT_S3_MAX_CONCURRENT_UPLOADS, - ) - - configured_queue_size: Final = params.get("s3_max_queue_size") - constructor_queue_size: Final = resolve_s3_max_queue_size( - s3_max_queue_size, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE - ) - self.s3_max_queue_size = resolve_s3_max_queue_size(configured_queue_size, constructor_queue_size) - - configured_retry_age: Final = params.get("s3_max_retry_age_seconds") - constructor_retry_age: Final = resolve_s3_max_retry_age_seconds(s3_max_retry_age_seconds, 3600) - self.s3_max_retry_age_seconds = ( - constructor_retry_age - if configured_retry_age is None or configured_retry_age == "" - else resolve_s3_max_retry_age_seconds(configured_retry_age, constructor_retry_age) - ) - - configured_drop: Final = params.get("s3_drop_on_terminal_error") - self.s3_drop_on_terminal_error = resolve_s3_drop_on_terminal_error( - configured_drop if configured_drop is not None else s3_drop_on_terminal_error - ) - - self.s3_adaptive_concurrency = s3_adaptive_concurrency or resolve_s3_adaptive_concurrency( - params.get("s3_adaptive_concurrency") - ) - - configured_adaptive_ceiling: Final = params.get("s3_max_adaptive_concurrency") - self.s3_max_adaptive_concurrency = resolve_s3_max_adaptive_concurrency( - s3_max_adaptive_concurrency - if configured_adaptive_ceiling is None or configured_adaptive_ceiling == "" - else configured_adaptive_ceiling, - DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY, - ) - - self.s3_batch_file_upload = s3_batch_file_upload or resolve_s3_batch_file_upload( - params.get("s3_batch_file_upload") - ) - - def _build_object_url(self, s3_object_key: str) -> str: - """ - Build the exact URL that is both signed and sent, with the key percent-encoded once. - - S3SigV4Auth signs the path verbatim while S3 canonicalizes the received path with reserved - characters encoded, so an unencoded `=`, `+`, `&`, `#`, `?`, `%` or space in the key makes - the two signatures disagree (403 SignatureDoesNotMatch). - """ - encoded_key: Final = quote(s3_object_key, safe="/") - if self.s3_endpoint_url and self.s3_bucket_name: - if self.s3_use_virtual_hosted_style: - endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "") - protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://" - return f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{encoded_key}" - return f"{self.s3_endpoint_url}/{self.s3_bucket_name}/{encoded_key}" - return ( - f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}." - f"{get_aws_dns_suffix(self.s3_region_name)}/{encoded_key}" - ) - - def _sign_put( - self, credentials: "Credentials", url: str, json_string: str, headers: Mapping[str, str] - ) -> dict[str, str]: # mutable-ok: [LIT001] AsyncHTTPHandler.put/HTTPHandler.put only accept dict headers - """ - ``RefreshableCredentials`` (IMDS roles) may refresh between the access key, secret and token - reads SigV4 performs, producing a mixed-generation signature that S3 rejects with 403. - Freezing first makes the three values one atomic snapshot. - """ - from botocore.auth import S3SigV4Auth - from botocore.awsrequest import AWSRequest - from botocore.credentials import RefreshableCredentials - - frozen: Final = ( - credentials.get_frozen_credentials() if isinstance(credentials, RefreshableCredentials) else credentials - ) - aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=dict(headers)) - aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name) - S3SigV4Auth(frozen, "s3", aws_region_name).add_auth(aws_request) - return dict(aws_request.headers.items()) - - def _sse_headers(self) -> Mapping[str, str]: - candidates: Final = { - "x-amz-server-side-encryption": self.s3_server_side_encryption, - "x-amz-server-side-encryption-aws-kms-key-id": self.s3_sse_kms_key_id, - } - return {key: value for key, value in candidates.items() if value} - - def _prepare_put(self, batch_logging_element: s3BatchLoggingElement) -> _PreparedPut: - try: - import base64 - import hashlib - except ImportError: - raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") - - json_string: Final = ( - batch_logging_element.body - if batch_logging_element.body is not None - else safe_dumps(batch_logging_element.payload) - ) - content_hash: Final = hashlib.sha256(json_string.encode("utf-8")).hexdigest() - content_md5: Final = base64.b64encode( - hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest() - ).decode() - return _PreparedPut( - json_string=json_string, - headers={ - "Content-Type": batch_logging_element.content_type, - "Content-MD5": content_md5, - "x-amz-content-sha256": content_hash, - "Content-Language": "en", - "Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"', - "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0", - **self._sse_headers(), - }, - ) - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - await self._async_log_event_base( - kwargs=kwargs, - response_obj=response_obj, - start_time=start_time, - end_time=end_time, - ) - - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): - await self._async_log_event_base( - kwargs=kwargs, - response_obj=response_obj, - start_time=start_time, - end_time=end_time, - ) - - async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None: - """Batch audit logs and upload to S3 under audit_logs/ prefix.""" - try: - from datetime import timezone - - now: Final = datetime.now(timezone.utc) - audit_log_id: Final = audit_log.get("id", "unknown") - - s3_object_key: Final = get_s3_object_key( - cast(str | None, self.s3_path) or "", - "audit_logs/", - now, - f"{now.strftime('%H-%M-%S')}_{audit_log_id}", - partition_granularity=self.resolve_partition_granularity(), - ) - - element: Final = s3BatchLoggingElement( - payload=dict(audit_log), - s3_object_key=s3_object_key, - s3_object_download_filename=f"audit-{audit_log_id}.json", - ) - - self.log_queue.append(element) - - if len(self.log_queue) >= self.batch_size: - await self.flush_queue() - except Exception as e: - verbose_logger.exception("S3 audit log error: %s", e) - - async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None: - """Upload one part of a batch's lines as an NDJSON object under /batch_line_items/.""" - object_name: Final = batch_line_items_object_name(payload) - s3_path: Final = cast(str | None, self.s3_path) # cast-ok: s3_path is set from untyped callback params - s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name - element: Final = s3BatchLoggingElement( - payload={}, - body=await asyncio.to_thread(batch_line_items_ndjson, payload), - content_type="application/x-ndjson", - s3_object_key=s3_object_key, - s3_object_download_filename="-".join(object_name.rsplit("/", 2)[-2:]), - ) - # hashing a large object is CPU-bound too, so it is also prepared off the event loop - prepared_put: Final = await asyncio.to_thread(self._prepare_put, element) - delivered: Final = await self.async_upload_data_to_s3(element, prepared_put) - if not delivered: - # only a log-queue flush reads and clears the drop marks, and this upload is not part of one - self._flush_dropped.pop(id(element), None) - verbose_logger.error( - "s3 logging: the %d line items of part %d of batch %s were not stored at %s", - len(payload["lines"]), - payload["part"], - payload["batch_id"], - s3_object_key, - ) - - async def _async_log_event_base(self, kwargs, response_obj, start_time, end_time): - try: - verbose_logger.debug("s3 Logging - Enters logging function for model %s", kwargs) - - s3_batch_logging_element: Final = self.create_s3_batch_logging_element( - start_time=start_time, - standard_logging_payload=kwargs.get("standard_logging_object", None), - ) - - # afile_delete and other non-model call types never produce a standard_logging_object, - # so s3_batch_logging_element is None. Skip gracefully instead of raising ValueError. - if s3_batch_logging_element is None: - verbose_logger.debug( - "s3 Logging - skipping event, no standard_logging_object for call_type=%s", - kwargs.get("call_type", "unknown"), - ) - return - - verbose_logger.debug("\ns3 Logger - Logging payload = %s", s3_batch_logging_element) - - self.log_queue.append(s3_batch_logging_element) - verbose_logger.debug( - "s3 logging: queue length %s, batch size %s", - len(self.log_queue), - self.batch_size, - ) - except Exception as e: - verbose_logger.exception("s3 Layer Error - %s", e) - self.handle_callback_failure(callback_name="S3Logger") - - @property - def _upload_semaphore(self) -> asyncio.Semaphore | AdaptiveConcurrencyLimiter: - limiter: Final = self._upload_limiter - if limiter is None: - raise AttributeError("_upload_semaphore") - return limiter - - @_upload_semaphore.setter - def _upload_semaphore(self, value: asyncio.Semaphore | AdaptiveConcurrencyLimiter) -> None: - self._upload_limiter = value - - async def async_upload_data_to_s3( - self, - batch_logging_element: s3BatchLoggingElement, - prepared_put: _PreparedPut | None = None, - ) -> bool: - try: - from litellm.litellm_core_utils.asyncify import asyncify - - asyncified_get_credentials: Final = asyncify(self.get_credentials) - - verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key) - verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify) - - url: Final = self._build_object_url(batch_logging_element.s3_object_key) - - async def signed_put(prepared: _PreparedPut) -> PutOutcome: - credentials: Final = await asyncified_get_credentials( - aws_access_key_id=self.s3_aws_access_key_id, - aws_secret_access_key=self.s3_aws_secret_access_key, - aws_session_token=self.s3_aws_session_token, - aws_region_name=self.s3_region_name, - aws_session_name=self.s3_aws_session_name, - aws_profile_name=self.s3_aws_profile_name, - aws_role_name=self.s3_aws_role_name, - aws_web_identity_token=self.s3_aws_web_identity_token, - aws_sts_endpoint=self.s3_aws_sts_endpoint, - ) - signed_headers: Final = await run_aws_signing( - self._sign_put, credentials, url, prepared.json_string, prepared.headers - ) - try: - return await self.async_httpx_client.put( - url, data=prepared.json_string, headers=signed_headers, timeout=_UPLOAD_ATTEMPT_TIMEOUT - ) - except httpx.HTTPStatusError as error: - return error.response - except _TRANSIENT_ERRORS as error: - return error - - prepared: Final = prepared_put if prepared_put is not None else self._prepare_put(batch_logging_element) - await self._put_with_retries(partial(signed_put, prepared), batch_logging_element.s3_object_key) - except Exception as e: - verbose_logger.exception("Error uploading to s3: %s", e) - self.handle_callback_failure(callback_name="S3Logger") - if isinstance(e, httpx.HTTPStatusError) and self.s3_drop_on_terminal_error and _is_terminal(e.response): - verbose_logger.warning( - "s3 logging: dropping object %s after terminal status %s", - batch_logging_element.s3_object_key, - e.response.status_code, - ) - self._flush_dropped[id(batch_logging_element)] = batch_logging_element - return False - return True - - async def async_send_batch(self) -> None: - """ - Sends runs from self.log_queue. - - Raises S3BatchUploadError when any upload failed; CustomBatchLogger.flush_queue - keeps the surviving queue entries for the next flush. - """ - batch: Final = tuple(self.log_queue) - if not batch: - return - verbose_logger.debug("s3_v2 logger - sending batch of %s", len(batch)) - - ######################################################### - # Flush the log queue to s3 - # the log queue can be bounded by DEFAULT_S3_BATCH_SIZE - # see custom_batch_logger.py which triggers the flush - ######################################################### - uploads: Final = self._batch_file_elements(batch) if self._batch_file_mode_active() else batch - self._flush_retries = 0 - self._flush_dropped = {} - stale: Final = min(self._requeued_count, len(uploads)) if len(uploads) == len(batch) else 0 - order: Final = (*range(stale, len(uploads)), *range(stale)) - ordered: Final = await asyncio.gather(*(self._upload_outcome(uploads[i]) for i in order)) - outcomes: Final = dict(zip(order, ordered, strict=True)) - results: Final = tuple(outcomes[i] for i in range(len(uploads))) - if self._flush_retries: - verbose_logger.warning( - "s3 logging: %s in-call retries across %s uploads this flush", - self._flush_retries, - len(uploads), - ) - delivered: Final = sum(1 for outcome in results if outcome == "delivered") - bucket_wide: Final = delivered == 0 - failed: Final = tuple( - (element, outcome) for element, outcome in zip(uploads, results, strict=True) if outcome != "delivered" - ) - now: Final = time.monotonic() - requeued: Final = ( - tuple(element for element, _ in failed) - if bucket_wide - else tuple( - element - if element.retrying_since is not None or self.s3_max_retry_age_seconds is None - else element.model_copy(update={"retrying_since": now}) - for element, outcome in failed - if outcome != "dropped" - and not ( - self.s3_max_retry_age_seconds is not None - and element.retrying_since is not None - and now - element.retrying_since > self.s3_max_retry_age_seconds - ) - ) - ) - dropped: Final = len(failed) - len(requeued) - if dropped: - terminal: Final = sum(1 for _, outcome in failed if outcome == "dropped") - verbose_logger.warning( - "s3 logging: %s uploads dropped this flush: %s rejected as terminal by S3, " - "%s retrying longer than s3_max_retry_age_seconds=%s", - dropped, - terminal, - dropped - terminal, - self.s3_max_retry_age_seconds, - ) - if not requeued: - self._requeued_count = 0 - return - arrivals: Final = self.log_queue[len(batch) :] - overflow: Final = max(0, len(requeued) + len(arrivals) - self.max_queue_size) - if overflow: - verbose_logger.warning( - "s3 logging: queue exceeded max_queue_size=%s after a failed flush, dropped %s oldest events", - self.max_queue_size, - overflow, - ) - self.log_queue = [ - *requeued, - *arrivals, - ][overflow:] - self._requeued_count = max(0, len(requeued) - overflow) - raise S3BatchUploadError(failed=len(failed), total=len(uploads)) - - def _batch_file_mode_active(self) -> bool: - if not self.s3_batch_file_upload: - return False - if litellm.cold_storage_custom_logger == "s3_v2": - verbose_logger.warning( - "s3 logging: s3_batch_file_upload is ignored because s3_v2 is the cold storage logger; " - "per-request objects are required for spend log lookups" - ) - return False - return True - - async def _upload_bounded(self, element: s3BatchLoggingElement) -> bool: - token: Final = _in_flush.set(True) - try: - async with self._upload_semaphore: - return await self.async_upload_data_to_s3(element) - finally: - _in_flush.reset(token) - - async def _upload_outcome(self, element: s3BatchLoggingElement) -> UploadOutcome: - delivered: Final = await self._upload_bounded(element) - if delivered: - return "delivered" - if id(element) in self._flush_dropped: - return "dropped" - return "retry" - - async def _recorded_put(self, signed_put: Callable[[], Awaitable[PutOutcome]]) -> PutOutcome: - outcome: Final = await signed_put() - limiter: Final = self._upload_limiter - if isinstance(limiter, AdaptiveConcurrencyLimiter): - limiter.record(PutSample(throttled=_throttled(outcome))) - return outcome - - def _retry_reason(self, outcome: PutOutcome) -> str | None: - if isinstance(outcome, Exception): - return type(outcome).__name__ - if outcome.status_code not in _RETRYABLE_STATUSES: - return None - if self.s3_drop_on_terminal_error and _is_terminal(outcome): - return None - return str(outcome.status_code) - - async def _put_with_retries( - self, signed_put: Callable[[], Awaitable[PutOutcome]], s3_object_key: str - ) -> httpx.Response: - for attempt in range(_UPLOAD_ATTEMPTS - 1): - outcome = await self._recorded_put(signed_put) - reason = self._retry_reason(outcome) - if reason is None: - return _response_or_raise(outcome) - wait = _backoff_seconds(attempt) - _log_retry(logging.DEBUG if _in_flush.get() else logging.WARNING, attempt, reason, wait, s3_object_key) - self._flush_retries += 1 - await asyncio.sleep(wait) - return _response_or_raise(await self._recorded_put(signed_put)) - - def _sync_put_with_retries(self, signed_put: Callable[[], PutOutcome], s3_object_key: str) -> httpx.Response: - for attempt in range(_UPLOAD_ATTEMPTS - 1): - outcome = signed_put() - reason = self._retry_reason(outcome) - if reason is None: - return _response_or_raise(outcome) - wait = _backoff_seconds(attempt) - _log_retry(logging.WARNING, attempt, reason, wait, s3_object_key) - time.sleep(wait) - return _response_or_raise(signed_put()) - - def _batch_file_elements(self, batch: tuple[s3BatchLoggingElement, ...]) -> tuple[s3BatchLoggingElement, ...]: - now: Final = datetime.now(timezone.utc) - groups: Final = { - parent: tuple( - element for element in batch if element.body is None and _s3_key_parent(element.s3_object_key) == parent - ) - for parent in sorted({_s3_key_parent(element.s3_object_key) for element in batch if element.body is None}) - } - return tuple(element for element in batch if element.body is not None) + tuple( - self._build_batch_file_element(elements, parent, now) for parent, elements in groups.items() - ) - - def _build_batch_file_element( - self, elements: tuple[s3BatchLoggingElement, ...], parent: str, now: datetime - ) -> s3BatchLoggingElement: - batch_name: Final = f"batch_{now.strftime('%H-%M-%S')}_{uuid4().hex}" - return s3BatchLoggingElement( - payload={}, - body="\n".join(safe_dumps(element.payload) for element in elements), - content_type="application/x-ndjson", - s3_object_key=f"{parent}/{batch_name}.jsonl" if parent else f"{batch_name}.jsonl", - s3_object_download_filename=f"{batch_name}.jsonl", - retrying_since=min( - (element.retrying_since for element in elements if element.retrying_since is not None), default=None - ), - ) - - def resolve_partition_granularity(self) -> S3PartitionGranularity: - raw: Final = ( - os.environ.get(S3_PARTITION_GRANULARITY_ENV_VAR) - if self.s3_partition_granularity is None - else self.s3_partition_granularity - ) - cached: Final = self._partition_granularity_cache - if cached is not None and cached[0] == raw: - return cached[1] - resolved: Final = resolve_s3_partition_granularity(raw) - self._partition_granularity_cache = (raw, resolved) - return resolved - - def create_s3_batch_logging_element( - self, - start_time: datetime, - standard_logging_payload: StandardLoggingPayload | None, - ) -> s3BatchLoggingElement | None: - """ - Helper function to create an s3BatchLoggingElement. - - Args: - start_time (datetime): The start time of the logging event. - standard_logging_payload (Optional[StandardLoggingPayload]): The payload to be logged. - s3_path (Optional[str]): The S3 path prefix. - - Returns: - Optional[s3BatchLoggingElement]: The created s3BatchLoggingElement, or None if payload is None. - """ - if standard_logging_payload is None: - return None - - if self.s3_strip_base64_files: - standard_logging_payload = self._strip_base64_from_messages_sync(standard_logging_payload) - - # Base prefix (default empty) - prefix_components: Final = [] - if self.s3_use_team_prefix: - team_alias: Final = standard_logging_payload.get("metadata", {}).get("user_api_key_team_alias", None) - if team_alias: - prefix_components.append(team_alias) - if self.s3_use_key_prefix: - user_api_key_alias: Final = standard_logging_payload.get("metadata", {}).get("user_api_key_alias", None) - if user_api_key_alias: - prefix_components.append(user_api_key_alias) - - # Construct full prefix path - prefix_path = "/".join(prefix_components) - if prefix_path: - prefix_path += "/" - - s3_file_name: Final = litellm.utils.get_logging_id(start_time, standard_logging_payload) or "" - verbose_logger.debug( - "Creating s3 file with prefix_components=%s,prefix_path=%s and %s", - prefix_components, - prefix_path, - s3_file_name, - ) - - def object_key(partition_granularity: S3PartitionGranularity) -> str: - return get_s3_object_key( - s3_path=cast(str | None, self.s3_path) or "", - prefix=prefix_path, - start_time=start_time, - s3_file_name=s3_file_name, - partition_granularity=partition_granularity, - ) - - metadata: Final = standard_logging_payload.get("metadata") - cold_storage_object_key: Final = ( - metadata.get("cold_storage_object_key") - if metadata is not None and litellm.cold_storage_custom_logger == "s3_v2" - else None - ) - s3_object_key: Final = ( - cold_storage_object_key - if cold_storage_object_key is not None - and cold_storage_object_key in (object_key("day"), object_key("hour")) - else object_key(self.resolve_partition_granularity()) - ) - verbose_logger.debug("s3_object_key=%s", s3_object_key) - - s3_object_download_filename: Final = get_s3_object_download_filename(start_time, standard_logging_payload["id"]) - - payload: Final = ( - prompts_only_payload(standard_logging_payload) - if resolve_s3_log_prompts_only(self.s3_log_prompts_only) - else standard_logging_payload - ) - return s3BatchLoggingElement( - payload=dict(payload), - s3_object_key=s3_object_key, - s3_object_download_filename=s3_object_download_filename, - ) - - def upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement): - try: - verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key) - - url: Final = self._build_object_url(batch_logging_element.s3_object_key) - - prepared: Final = self._prepare_put(batch_logging_element) - - httpx_client: Final = get_httpx_client( - params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None) - ) - - def signed_put(prepared_put: _PreparedPut) -> PutOutcome: - credentials: Final = self.get_credentials( - aws_access_key_id=self.s3_aws_access_key_id, - aws_secret_access_key=self.s3_aws_secret_access_key, - aws_session_token=self.s3_aws_session_token, - aws_region_name=self.s3_region_name, - ) - signed_headers: Final = self._sign_put(credentials, url, prepared_put.json_string, prepared_put.headers) - try: - return httpx_client.put( - url, data=prepared_put.json_string, headers=signed_headers, timeout=_UPLOAD_ATTEMPT_TIMEOUT - ) - except _TRANSIENT_ERRORS as error: - return error - - self._sync_put_with_retries(partial(signed_put, prepared), batch_logging_element.s3_object_key) - except Exception as e: - verbose_logger.exception("Error uploading to s3: %s", e) - self.handle_callback_failure(callback_name="S3Logger") - if isinstance(e, httpx.HTTPStatusError) and self.s3_drop_on_terminal_error and _is_terminal(e.response): - verbose_logger.warning( - "s3 logging: dropping object %s after terminal status %s", - batch_logging_element.s3_object_key, - e.response.status_code, - ) - - async def _download_object_from_s3(self, s3_object_key: str) -> dict | None: - """ - Download and parse JSON object from S3. - - Args: - s3_object_key: The S3 object key to download - - Returns: - Optional[dict]: The parsed JSON object or None if not found/error - """ - try: - import hashlib - - from botocore.auth import S3SigV4Auth - from botocore.awsrequest import AWSRequest - except ImportError: - raise ImportError("Missing boto3 to call S3. Run 'pip install boto3'.") - - try: - from litellm.litellm_core_utils.asyncify import asyncify - - # Get AWS credentials - asyncified_get_credentials: Final = asyncify(self.get_credentials) - credentials: Final = await asyncified_get_credentials( - aws_access_key_id=self.s3_aws_access_key_id, - aws_secret_access_key=self.s3_aws_secret_access_key, - aws_session_token=self.s3_aws_session_token, - aws_region_name=self.s3_region_name, - aws_session_name=self.s3_aws_session_name, - aws_profile_name=self.s3_aws_profile_name, - aws_role_name=self.s3_aws_role_name, - aws_web_identity_token=self.s3_aws_web_identity_token, - aws_sts_endpoint=self.s3_aws_sts_endpoint, - ) - - verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key) - - url: Final = self._build_object_url(s3_object_key) - - # Prepare the request for GET operation - # For GET requests, we need x-amz-content-sha256 with hash of empty string - empty_string_hash: Final = hashlib.sha256(b"").hexdigest() - headers: Final = { - "x-amz-content-sha256": empty_string_hash, - } - - # Sign the request - aws_request: Final = AWSRequest(method="GET", url=url, headers=headers) - await run_aws_signing(S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth, aws_request) - - # Prepare the signed headers - signed_headers: Final = dict(aws_request.headers.items()) - - response: Final = await self.async_httpx_client.get(url, headers=signed_headers) - - if response.status_code != 200: - verbose_logger.exception("S3 object not found, saw response=", response.text) - return None - - # Parse JSON response - return response.json() - - except Exception as e: - verbose_logger.exception("Error downloading from S3: %s", e) - return None - - async def get_proxy_server_request_from_cold_storage_with_object_key( - self, - object_key: str, - ) -> dict | None: - """ - Get the proxy server request from cold storage - - Allows fetching a dict of the proxy server request from s3 or GCS bucket. - - Args: - request_id: The unique request ID to search for - start_time: The start time of the request (datetime or ISO string) - - Returns: - Optional[dict]: The request data dictionary or None if not found - """ - try: - # Download and return the object from S3 - downloaded_object: Final = await self._download_object_from_s3(object_key) - return downloaded_object - except Exception as e: - verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e) - return None +__all__ = ["S3BatchUploadError", "S3Logger"] diff --git a/litellm/integrations/s3_v3.py b/litellm/integrations/s3_v3.py new file mode 100644 index 00000000000..45a6f887650 --- /dev/null +++ b/litellm/integrations/s3_v3.py @@ -0,0 +1,481 @@ +import asyncio +import copy +import json +import os +from collections.abc import Callable, Collection, Mapping +from typing import TYPE_CHECKING, Final + +from typing_extensions import Unpack + +import litellm +from litellm._logging import verbose_logger +from litellm.constants import ( + COMPLETION_HTTP_FALLBACK_SECONDS, + DEFAULT_S3_BATCH_SIZE, + DEFAULT_S3_FLUSH_INTERVAL_SECONDS, + DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY, + DEFAULT_S3_MAX_CONCURRENT_UPLOADS, + HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, + S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS, +) +from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger_v2.call import CallContext +from litellm.integrations.custom_logger_v2.derivations import ( + cold_storage_object_key, + log_id, + logged_key_identity, + mapping, + payload_copy, +) +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.field_sources import HookArgs +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on +from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch +from litellm.integrations.custom_logger_v2.rust import RustLogger +from litellm.integrations.custom_logger_v2.selection import Selection +from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds +from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document +from litellm.secret_managers.main import get_secret, get_secret_str +from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity +from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload + +if TYPE_CHECKING: + from litellm.rust_bridge._native import NativeS3Log + +CALLBACK_NAME: Final = "s3_v2" +STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files" +_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4 +_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600 + + +class S3BatchUploadError(Exception): + def __init__(self, failed: int, total: int) -> None: + self.failed: Final = failed + self.total: Final = total + super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush") + + +def _raise_batch_upload_error(failed: int, total: int) -> None: + raise S3BatchUploadError(failed=failed, total=total) + + +def batch_upload_failed(failure: tuple[int, int, int] | None) -> None: + if failure is None: + return + queued, failed, total = failure + try: + _raise_batch_upload_error(failed, total) + except S3BatchUploadError: + verbose_logger.exception( + "CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued + ) + + +def _cold_storage_is_s3_v2() -> bool: + return litellm.cold_storage_custom_logger == "s3_v2" + + +def _region_from_environment() -> str | None: + region_name: Final = get_secret_str("AWS_REGION_NAME", None) + region: Final = get_secret_str("AWS_REGION", None) + return region if region is not None else region_name + + +class _Host: + def __init__(self) -> None: + self.failures: Final = CustomLogger() + + def callback_failed(self) -> None: + self.failures.handle_callback_failure(callback_name="S3Logger") + + def web_identity_token(self, reference: str) -> str: + if reference.startswith("os.environ/") or reference in os.environ: + raise ValueError("Invalid web identity token reference.") + match _read_secret(get_secret, reference): + case None: + raise ValueError("OIDC token could not be retrieved from secret manager.") + case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH: + raise ValueError( + "Parameter validation failed:\nInvalid length for parameter WebIdentityToken, " + f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}" + ) + case str() as token: + return token + case other: + raise ValueError( + "Parameter validation failed:\nInvalid type for parameter WebIdentityToken, " + f"value: {other}, type: {type(other)}, valid types: {str}" + ) + + +def _read_secret(read: Callable[[str], object], reference: str) -> object: + return read(reference) + + +def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust + configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None) + source: Final[Mapping[str, object] | Mapping[object, object]] = ( + params_source if params_source is not None else (configured or {}) + ) + return { + key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value) + for key, value in source.items() + if isinstance(key, str) + } + + +def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]: + return { + "batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE, + "max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS, + "max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY, + "upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS, + "download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS, + "download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, + "web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY), + } + + +def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float: + return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS + + +_CONFIGURED_FIRST: Final = frozenset( + { + "s3_bucket_name", + "s3_path", + "s3_region_name", + "s3_api_version", + "s3_endpoint_url", + "s3_aws_access_key_id", + "s3_aws_secret_access_key", + "s3_aws_session_token", + "s3_aws_session_name", + "s3_aws_profile_name", + "s3_aws_role_name", + "s3_aws_web_identity_token", + "s3_aws_sts_endpoint", + "s3_config", + "s3_server_side_encryption", + "s3_sse_kms_key_id", + } +) +_CONSTRUCTOR_ENABLES: Final = frozenset( + { + "s3_use_team_prefix", + "s3_strip_base64_files", + "s3_use_key_prefix", + "s3_use_virtual_hosted_style", + "s3_adaptive_concurrency", + "s3_batch_file_upload", + } +) +_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"}) +_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"}) +_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"}) +_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = { + "s3_use_ssl": True, + "s3_drop_on_terminal_error": True, + "s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS, + "s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS, + "s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS, + "s3_batch_size": DEFAULT_S3_BATCH_SIZE, +} + + +def _blank(value: object) -> bool: + return value is None or value == "" + + +def _constructor_wins(key: str, configured: object, value: object) -> bool: + return ( + (key in _CONFIGURED_FIRST and not configured) + or (key in _CONFIGURED_UNLESS_NONE and configured is None) + or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured)) + or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None) + ) + + +_KEEP_CONFIGURED: Final = object() + + +def _constructor_value(key: str, configured: object, value: object) -> object: + if _constructor_wins(key, configured, value): + return value + if key in _CONSTRUCTOR_ENABLES and value: + return True + if key == "s3_max_queue_size" and value is not None: + return resolve_s3_max_queue_size( + configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE) + ) + if key == "s3_max_retry_age_seconds": + fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS) + resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback) + return 0 if resolved is None else resolved + return _KEEP_CONFIGURED + + +def _constructed_params( + params_source: Mapping[str, object] | None, settings: S3LoggerSettings +) -> dict[str, object]: # mutable-ok: handed to Rust + params: Final = _resolved_params(params_source) + overrides: Final = { + key: replaced + for key, value in settings.items() + if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None) + and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED + } + return {**params, **overrides} + + +class _Base64Stripper(CustomLogger): + def strip(self, payload: StandardLoggingPayload) -> None: + _ = self._strip_base64_from_messages_sync(payload) + + def stripped(self, messages: list[object]) -> list[dict[str, object]]: + return self._process_messages(messages=[_shallow_copy(message) for message in messages]) + + +def _shallow_copy(message: object) -> object: + entries: Final = mapping(message) + return copy.copy(entries) if entries is not None and isinstance(message, dict) else message + + +_STRIPPER: Final = _Base64Stripper() + + +def legacy_should_log(kwargs: Mapping[str, object]) -> bool: + from litellm.litellm_core_utils import litellm_logging + + if litellm.global_disable_no_log_param: + return True + litellm_params: Final = kwargs.get("litellm_params", {}) + lookup: Final = getattr(litellm_params, "get", None) + if callable(lookup) and lookup("no-log", False) is True: + return False + return not litellm_logging.callback_disabled_dynamically( + callback=CALLBACK_NAME, + litellm_params=litellm_params, + standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check + ) + + +def _payload_messages(payload: Mapping[str, object]) -> list[object]: + raw: Final = payload.get("messages") + return [*raw] if isinstance(raw, list) else [] + + +class S3StripBase64Files(CustomLoggerV2): + selection = Selection(name=STRIP_CALLBACK_NAME) + + def __init__(self, s3_logger_key: str) -> None: + self.s3_logger_key: Final = s3_logger_key + + @on( + Hook.log_prepare, + reads={Field.deprecated_kwargs, Field.standard_logging_payload}, + writes={Field.standard_logging_payload}, + ) + async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None: + from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE + + kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None)) + payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None)) + if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs): + return None + messages: Final = _payload_messages(payload) + if not messages: + return None + return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)}) + + +async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None: + while True: + await asyncio.sleep(interval) + try: + batch_upload_failed(await sink.flush()) + except (asyncio.CancelledError, KeyboardInterrupt, SystemExit): + raise + except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does + verbose_logger.exception("CustomLogger: periodic flush failed: %s", e) + + +def _excluding_fields(kwargs: object) -> Mapping[str, object] | None: + excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None) + details: Final = mapping(kwargs) + logged: Final = None if details is None else mapping(details.get("standard_logging_object")) + if details is None or logged is None or not isinstance(excluded, Collection) or not excluded: + return None + if logged.get("status") == "failure": + return None + return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}} + + +def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object: + source: Final = mapping(kwargs) if details is None else details + return None if source is None else source.get("standard_logging_object") + + +def _legacy_kwargs(view: object) -> object: + return getattr(view, Field.deprecated_kwargs.value, None) + + +_read_attribute: Final[Callable[[object, str], object]] = getattr + + +class _ExcludedView: + def __init__(self, view: object, details: Mapping[str, object]) -> None: + args: Final = HookArgs(request=details) + self.view: Final = view + self.fields: Final[Mapping[str, object]] = { + Field.deprecated_kwargs.value: details, + Field.standard_logging_payload_ref.value: payload_copy(args), + Field.log_id.value: log_id(args), + Field.cold_storage_object_key.value: cold_storage_object_key(args), + Field.logged_key_identity.value: logged_key_identity(args), + } + + def __getattr__(self, name: str) -> object: + if name not in self.fields: + return _read_attribute(self.view, name) + value: Final = self.fields[name] + if value is None: + raise AttributeError(name) + return value + + +_ATTEMPT_ENDED_READS: Final = frozenset( + { + Field.deprecated_kwargs, + Field.attempt_timing, + Field.log_id, + Field.cold_storage_object_key, + Field.standard_logging_payload_ref, + Field.logged_key_identity, + } +) + + +class S3Logger(RustLogger): + selection = Selection(name=CALLBACK_NAME, honors_no_log=True) + + def __init__( + self, + s3_callback_params_override: Mapping[str, object] | None = None, + **settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords + ) -> None: + from litellm.rust_bridge._native import NativeS3Log + + host: Final = _Host() + params: Final = _constructed_params(s3_callback_params_override, settings) + sink: Final = NativeS3Log( + params, + _constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)), + host, + _cold_storage_is_s3_v2(), + None if params.get("s3_region_name") else _region_from_environment(), + ) + self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name") + self.sink: Final = sink + self.s3_host: Final = host + self.periodic_flush_task: Final = asyncio.create_task( + _periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS))) + ) + super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None}) + self.v2_companion: Final = ( + S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self)) + if sink.strip_base64_files + else None + ) + + @on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True) + async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None: + return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx) + + async def run_rust( + self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object] + ) -> Patch | Abort | None: + from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE + + kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None) + if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict + return None + payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict + if payload is not None and not isinstance(payload, Mapping): + return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload + try: + raw: Final = _legacy_kwargs(view) + details: Final = _excluding_fields(raw) + logged: Final = _logged_payload(raw, details) + if self.sink.strip_base64_files and isinstance(logged, Mapping): + _STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does + return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx) + except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on + return self._failed(e) + + def _failed(self, error: Exception) -> None: + verbose_logger.error("s3 Layer Error - %s", error, exc_info=error) + self.s3_host.callback_failed() + + @property + def s3_path(self) -> str | None: + return self.sink.s3_path + + @property + def s3_bucket_name(self) -> str | None: + return self.sink.s3_bucket_name + + def resolve_partition_granularity(self) -> S3PartitionGranularity: + return "hour" if self.sink.partition_granularity() == "hour" else "day" + + async def flush_queue(self) -> None: + batch_upload_failed(await self.sink.flush()) + + async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None: + try: + audit_log_id: Final = audit_log.get("id", "unknown") + batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}")) + except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger + verbose_logger.exception("S3 audit log error: %s", e) + + async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None: + object_name: Final = batch_line_items_object_name(payload) + s3_path: Final = self.s3_path + s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name + body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload) + delivered: Final = await self.sink.upload_line_items( + s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:]) + ) + if not delivered: + verbose_logger.error( + "s3 logging: the %d line items of part %d of batch %s were not stored at %s", + len(payload["lines"]), + payload["part"], + payload["batch_id"], + s3_object_key, + ) + + async def _download_object_from_s3(self, s3_object_key: str) -> object: + try: + downloaded: Final = await self.sink.download(s3_object_key) + return None if downloaded is None else json.loads(downloaded) + except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger + verbose_logger.exception("Error downloading from S3: %s", e) + return None + + async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object: + try: + return await self._download_object_from_s3(object_key) + except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger + verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e) + return None + + +__all__ = [ + "S3BatchUploadError", + "S3Logger", + "S3StripBase64Files", + "legacy_should_log", +] diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index 74badc2c8c1..535fef8fcec 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -46,7 +46,7 @@ from litellm.integrations.opik.opik import OpikLogger from litellm.integrations.pointfive import PointFiveLogger from litellm.integrations.posthog import PostHogLogger from litellm.integrations.prometheus import PrometheusLogger -from litellm.integrations.s3_v2 import S3Logger +from litellm.integrations.s3_v3 import S3Logger from litellm.integrations.sqs import SQSLogger from litellm.integrations.ternary.ternary_logger import TernaryLogger from litellm.integrations.vantage.vantage_logger import VantageLogger diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 1b682d078f0..15a95e91c52 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -15,6 +15,7 @@ import time import traceback from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from contextlib import AbstractContextManager +from contextvars import ContextVar from dataclasses import dataclass from datetime import datetime as dt_object from enum import Enum @@ -305,7 +306,6 @@ from ..integrations.pointfive import PointFiveLogger from ..integrations.posthog import PostHogLogger from ..integrations.prompt_layer import PromptLayerLogger from ..integrations.s3 import S3Logger -from ..integrations.s3_v2 import S3Logger as S3V2Logger from ..integrations.supabase import Supabase from ..integrations.traceloop import TraceloopLogger from ..integrations.zerobus import ZerobusLogger @@ -768,6 +768,16 @@ def _callback_control_params(litellm_params: object) -> dict[str, object]: return litellm_params.model_dump() if isinstance(litellm_params, BaseModel) else {} +def callback_disabled_dynamically( + callback: object, litellm_params: object, standard_callback_dynamic_params: StandardCallbackDynamicParams +) -> bool: + return EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically( + callback=callback, + litellm_params=_callback_control_params(litellm_params), + standard_callback_dynamic_params=standard_callback_dynamic_params, + ) + + LOGGING_KEYS: Final[Mapping[Field, str]] = MappingProxyType( { Field.model: "model", @@ -783,6 +793,9 @@ def _holds_v2(callbacks: Iterable[object]) -> bool: return any(isinstance(callback, CustomLoggerV2) for callback in callbacks) +V2_ASYNC_LOGGING_LINE: Final[ContextVar[bool]] = ContextVar("v2_async_logging_line", default=False) + + _HOOK_DERIVED_FIELDS: Final[Mapping[Hook, Mapping[Field, FieldSource]]] = MappingProxyType( {Hook.log_prepare: v2_derive.DERIVED_LOGGING_FIELDS, Hook.attempt_ended: v2_derive.DERIVED_ATTEMPT_FIELDS} ) @@ -839,15 +852,19 @@ async def _run_callbacks( args: HookArgs | None = None, ) -> None: if include_v2 and _holds_v2(callbacks): - _ = kwargs_after( - await run_subscribers( - hook_subscribers(hook, callbacks, admit=Admit.every_entry), - _logging_fields(hook, kwargs, args, owner), - binding, - call=_v2_call(owner, args), - scope=None if owner is None else owner.v2_logging_scope(), + line: Final = V2_ASYNC_LOGGING_LINE.set(True) + try: + _ = kwargs_after( + await run_subscribers( + hook_subscribers(hook, callbacks, admit=Admit.every_entry), + _logging_fields(hook, kwargs, args, owner), + binding, + call=_v2_call(owner, args), + scope=None if owner is None else owner.v2_logging_scope(), + ) ) - ) + finally: + V2_ASYNC_LOGGING_LINE.reset(line) return for callback in callbacks: if isinstance(callback, CustomLoggerV2): @@ -3283,9 +3300,9 @@ class Logging(LiteLLMLoggingBaseClass): return False # Check for dynamically disabled callbacks via headers - if EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically( + if callback_disabled_dynamically( callback=callback, - litellm_params=_callback_control_params(litellm_params), + litellm_params=litellm_params, standard_callback_dynamic_params=self.standard_callback_dynamic_params, ): verbose_logger.debug( @@ -6519,6 +6536,8 @@ def build_custom_logger_compatible_class( _in_memory_loggers.append(_gcs_bucket_logger) return _gcs_bucket_logger elif logging_integration == "s3_v2": + from litellm.integrations.s3_v3 import S3Logger as S3V2Logger + for callback in _in_memory_loggers: if isinstance(callback, S3V2Logger): return callback @@ -7341,6 +7360,8 @@ def get_custom_logger_compatible_class( if isinstance(callback, GCSBucketLogger): return callback elif logging_integration == "s3_v2": + from litellm.integrations.s3_v3 import S3Logger as S3V2Logger + for callback in _in_memory_loggers: if isinstance(callback, S3V2Logger): return callback @@ -8046,6 +8067,8 @@ class StandardLoggingPayloadSetup: ) if custom_logger and hasattr(custom_logger, "s3_path") and getattr(custom_logger, "s3_path"): s3_path = getattr(custom_logger, "s3_path") + from litellm.integrations.s3_v3 import S3Logger as S3V2Logger + if isinstance(custom_logger, S3V2Logger): partition_granularity = custom_logger.resolve_partition_granularity() except Exception: diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index fe5340cb6fe..4f22647fee5 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -21,6 +21,16 @@ _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {} _callback_lists_lock: Final = threading.RLock() +def _v2_companion(callback: object) -> CustomLoggerV2 | None: + companion: Final[object] = getattr(callback, "v2_companion", None) + return companion if isinstance(companion, CustomLoggerV2) else None + + +def _with_v2_companions(callbacks: Iterable[object]) -> tuple[object, ...]: + listed: Final = tuple(callbacks) + return (*listed, *(companion for callback in listed if (companion := _v2_companion(callback)) is not None)) + + def _instance_attributes(target: object) -> Mapping[str, object]: return getattr(target, "__dict__", {}) @@ -292,16 +302,48 @@ class LoggingCallbackManager: parent_list=parent_list, ) elif isinstance(callback, CustomLoggerV2): - v2_key: Final = self._get_custom_logger_key(callback) - if not any( - isinstance(existing, CustomLoggerV2) and self._get_custom_logger_key(existing) == v2_key - for existing in parent_list - ): - parent_list.append(callback) # pyright: ignore[reportArgumentType] # the event lists also hold V2 callbacks + listed: Final = self._listed_v2(callback, parent_list) + if listed is None: + self._append_v2(callback, parent_list) + self._place_v2_companion(callback if listed is None else listed, parent_list) elif callable(callback): self._add_callback_function_to_list(callback=callback, parent_list=parent_list) + def _append_v2( + self, callback: CustomLoggerV2, parent_list: list[CustomLogger | Callable[..., object] | str] + ) -> None: + parent_list.append(callback) # pyright: ignore[reportArgumentType] # the event lists also hold V2 callbacks + + def _listed_v2(self, callback: CustomLoggerV2, parent_list: Iterable[object]) -> CustomLoggerV2 | None: + v2_key: Final = self._get_custom_logger_key(callback) + return next( + ( + existing + for existing in parent_list + if isinstance(existing, CustomLoggerV2) and self._get_custom_logger_key(existing) == v2_key + ), + None, + ) + + def _place_v2_companion( + self, callback: CustomLoggerV2, parent_list: list[CustomLogger | Callable[..., object] | str] + ) -> None: + companion: Final = _v2_companion(callback) + if companion is None or self._listed_v2(companion, parent_list) is not None: + return + if not self._check_callback_list_size(parent_list): + return + self._append_v2(companion, parent_list) + position: Final = next(index for index, existing in enumerate(parent_list) if existing is callback) + parent_list.insert(position, parent_list.pop()) + + def place_v2_companions(self, parent_list: list[CustomLogger | Callable[..., object] | str]) -> None: + with _callback_lists_lock: + for callback in tuple(parent_list): + if isinstance(callback, CustomLoggerV2): + self._place_v2_companion(callback, parent_list) + def _add_callback_function_to_list( self, callback: Callable[..., object], @@ -364,7 +406,7 @@ class LoggingCallbackManager: callback_lists_lock: Final = _callback_lists_lock def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]: - dropped_ids: Final = frozenset(map(id, callbacks)) + dropped_ids: Final = frozenset(map(id, _with_v2_companions(callbacks))) owner_ids: Final = frozenset(map(id, method_owners)) def keep(callback: object) -> bool: @@ -444,6 +486,7 @@ class LoggingCallbackManager: Remove a callback object from every callback list it may have been promoted into, so a re-initialized callback leaves no stale instance behind. """ + companion: Final = None if require_self else _v2_companion(obj) with _callback_lists_lock: for callback_list in ( litellm.callbacks, @@ -454,6 +497,8 @@ class LoggingCallbackManager: litellm._async_failure_callback, ): self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self) + if companion is not None: + callback_list[:] = [callback for callback in callback_list if callback is not companion] def get_active_additional_logging_utils_from_custom_logger( self, @@ -536,7 +581,7 @@ class LoggingCallbackManager: if not isinstance(callback, str) ) - def _get_callback_string(self, callback: CustomLogger | Callable[..., object] | str) -> str: + def _get_callback_string(self, callback: CustomLogger | CustomLoggerV2 | Callable[..., object] | str) -> str: from litellm.integrations.opentelemetry import OpenTelemetry from litellm.litellm_core_utils.custom_logger_registry import ( CustomLoggerRegistry, @@ -547,7 +592,7 @@ class LoggingCallbackManager: return callback elif isinstance(callback, OpenTelemetry) and callback.callback_name is not None: return callback.callback_name - elif isinstance(callback, CustomLogger): + elif isinstance(callback, CustomLogger | CustomLoggerV2): # Try to get the string representation from the registry callback_str: Final = CustomLoggerRegistry.get_callback_str_from_class_type(type(callback)) return callback_str if callback_str is not None else type(callback).__name__ diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index aa31f44dd13..65b64a297c9 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -4,9 +4,9 @@ Functions to create audit logs for LiteLLM Proxy import asyncio import json -from collections.abc import Awaitable, Sequence +from collections.abc import Awaitable, Mapping, Sequence from datetime import datetime, timezone -from typing import Final, TypeVar +from typing import TYPE_CHECKING, Final, TypeVar import litellm from litellm._logging import verbose_proxy_logger @@ -21,7 +21,10 @@ from litellm.proxy._types import ( from litellm.repositories.table_repositories import AuditLogRepository from litellm.types.utils import StandardAuditLogPayload -_audit_log_callback_cache: Final[dict[str, CustomLogger]] = {} +if TYPE_CHECKING: + from litellm.integrations.s3_v3 import S3Logger + +_audit_log_callback_cache: Final[dict[str, "CustomLogger | S3Logger"]] = {} ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY: Final = "allow_litellm_changed_by_header" @@ -62,7 +65,7 @@ def get_audit_log_changed_by( return user_api_key_dict.user_id or litellm_proxy_admin_name -def _resolve_audit_log_callback(name: str) -> CustomLogger | None: +def _resolve_audit_log_callback(name: str) -> "CustomLogger | S3Logger | None": """Resolve a string callback name to a CustomLogger instance, with caching. For "s3_v2" with `litellm.s3_audit_callback_params` set, constructs a @@ -72,11 +75,12 @@ def _resolve_audit_log_callback(name: str) -> CustomLogger | None: if name in _audit_log_callback_cache: return _audit_log_callback_cache[name] - instance: CustomLogger | None - if name == "s3_v2" and getattr(litellm, "s3_audit_callback_params", None) is not None: - from litellm.integrations.s3_v2 import S3Logger as S3V2Logger + instance: CustomLogger | S3Logger | None + audit_params: Final[Mapping[str, object] | None] = getattr(litellm, "s3_audit_callback_params", None) + if name == "s3_v2" and audit_params is not None: + from litellm.integrations import s3_v3 - instance = S3V2Logger(s3_callback_params_override=litellm.s3_audit_callback_params) + instance = s3_v3.S3Logger(s3_callback_params_override=audit_params) else: from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, @@ -196,14 +200,14 @@ async def _dispatch_audit_log_to_callbacks( for callback in litellm.audit_log_callbacks: try: - resolved: CustomLogger | None = callback if isinstance(callback, CustomLogger) else None + resolved: CustomLogger | S3Logger | None = callback if isinstance(callback, CustomLogger) else None if isinstance(callback, str): resolved = _resolve_audit_log_callback(callback) if resolved is None: verbose_proxy_logger.warning("Could not resolve audit log callback: %s", callback) continue - if isinstance(resolved, CustomLogger): + if resolved is not None: track_audit_task( asyncio.create_task( resolved.async_log_audit_log_event(payload), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b41f240f89b..b161fef588a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7446,7 +7446,7 @@ class ProxyConfig: if key == "request_timeout": litellm.request_timeout_explicitly_set = True if key in {"s3_audit_callback_params", "s3_callback_params"}: - from litellm.integrations.s3_v2 import S3Logger as S3V2Logger + from litellm.integrations.s3_v3 import S3Logger as S3V2Logger from litellm.litellm_core_utils.litellm_logging import ( _in_memory_loggers, ) @@ -16441,12 +16441,13 @@ async def _routed_count_deployment( if llm_router is None: return None try: - return await llm_router.select(model, dict(_responses_routing_kwargs(responses_request))) + routed: Final = await llm_router.select(model, dict(_responses_routing_kwargs(responses_request))) except Exception: verbose_proxy_logger.exception( "litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment" ) return None + return routed if isinstance(routed, Mapping) else None def _with_chat_timeout(litellm_params: Mapping[str, object]) -> Mapping[str, object]: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ce7a041e147..53863c881ae 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2514,6 +2514,7 @@ class ProxyLogging: # Replace string entries in litellm.callbacks with initialized instances for idx, initialized_callback in string_callbacks_to_replace.items(): litellm.callbacks[idx] = initialized_callback + litellm.logging_callback_manager.place_v2_companions(litellm.callbacks) # Fan ``litellm.callbacks`` (the "all events" registry) out into the # success/failure event lists eagerly, at startup. ``completion()`` does diff --git a/litellm/rust/Cargo.lock b/litellm/rust/Cargo.lock index 3a66beb7c42..811bda4f798 100644 --- a/litellm/rust/Cargo.lock +++ b/litellm/rust/Cargo.lock @@ -3739,6 +3739,34 @@ dependencies = [ "tokio", ] +[[package]] +name = "litellm-callbacks-s3-log" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-util", + "litellm-auth-aws", + "litellm-callbacks", + "litellm-http", + "litellm-python-compat", + "litellm-s3", + "litellm-secret-value", + "litellm-tracing", + "md-5", + "rand 0.8.7", + "reqwest", + "rstest", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", + "tokio", + "tracing", + "uuid", + "wiremock", +] + [[package]] name = "litellm-circuit-breaker" version = "0.1.0" @@ -4451,7 +4479,10 @@ dependencies = [ "litellm-auth-azure", "litellm-cache", "litellm-cache-response", + "litellm-callbacks", "litellm-callbacks-legacy-python", + "litellm-callbacks-python", + "litellm-callbacks-s3-log", "litellm-circuit-breaker", "litellm-core-utils", "litellm-error", @@ -4691,11 +4722,15 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", + "bytes", "http 1.4.2", + "hyper 1.12.0", + "litellm-auth-aws", "litellm-http", "reqwest", "rstest", "thiserror 2.0.19", + "tokio", ] [[package]] diff --git a/litellm/rust/Cargo.toml b/litellm/rust/Cargo.toml index b0dc4b2c279..3fc881102c1 100644 --- a/litellm/rust/Cargo.toml +++ b/litellm/rust/Cargo.toml @@ -38,6 +38,7 @@ litellm-host-native = { path = "crates/host-native" } litellm-callbacks = { path = "crates/callbacks" } litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" } litellm-callbacks-python = { path = "crates/callbacks-python" } +litellm-callbacks-s3-log = { path = "crates/callbacks-s3-log" } litellm-framer = { path = "crates/framer" } litellm-auth = { path = "crates/auth" } litellm-auth-types = { path = "crates/auth-types" } diff --git a/litellm/rust/crates/auth-aws/Cargo.toml b/litellm/rust/crates/auth-aws/Cargo.toml index c8ed7bb0f14..1a7392afc52 100644 --- a/litellm/rust/crates/auth-aws/Cargo.toml +++ b/litellm/rust/crates/auth-aws/Cargo.toml @@ -17,6 +17,7 @@ serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } sha2.workspace = true thiserror.workspace = true +tokio.workspace = true url.workspace = true veil.workspace = true diff --git a/litellm/rust/crates/auth-aws/src/aws.rs b/litellm/rust/crates/auth-aws/src/aws.rs index 8508ac0c47c..a94a4e8d66b 100644 --- a/litellm/rust/crates/auth-aws/src/aws.rs +++ b/litellm/rust/crates/auth-aws/src/aws.rs @@ -9,8 +9,8 @@ use std::{future::Future, pin::Pin, sync::Arc}; use aws_credential_types::Credentials; use aws_credential_types::provider::ProvideCredentials; use aws_sigv4::http_request::{ - SignableBody, SignableRequest, SignatureLocation, SigningInstructions, SigningParams, - SigningSettings, sign, + PercentEncodingMode, SignableBody, SignableRequest, SignatureLocation, SigningInstructions, + SigningParams, SigningSettings, UriPathNormalizationMode, sign, }; use aws_sigv4::sign::v4; use aws_smithy_runtime_api::client::identity::Identity; @@ -38,10 +38,15 @@ pub struct AwsAuthService { impl Default for AwsAuthService { fn default() -> Self { - Self::new(Arc::new(NativeProviderLoader)) + Self::new(Arc::new(NativeProviderLoader { + token: None, + policy: None, + })) } } +pub type WebIdentityTokenResolver = Arc Result + Send + Sync>; + impl AwsAuthService { fn new(loader: Arc) -> Self { Self { @@ -53,6 +58,16 @@ impl AwsAuthService { loader, } } + + pub fn for_web_identity( + resolver: WebIdentityTokenResolver, + session_policy: Option, + ) -> Self { + Self::new(Arc::new(NativeProviderLoader { + token: Some(resolver), + policy: session_policy, + })) + } } struct CredentialExpiry; @@ -81,11 +96,19 @@ trait ProviderLoader: Send + Sync { fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_>; } -struct NativeProviderLoader; +struct NativeProviderLoader { + token: Option, + policy: Option, +} impl ProviderLoader for NativeProviderLoader { fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> { - Box::pin(build_provider(config, flow)) + Box::pin(build_provider( + config, + flow, + self.token.clone(), + self.policy.clone(), + )) } } @@ -271,6 +294,16 @@ impl AwsAuthService { self.fresh_credentials(&resolved, &flow).await } + pub async fn invalidate_credentials( + &self, + config: AwsAuthConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) { + self.credentials + .invalidate(&config.with_environment(env_lookup)) + .await; + } + /// The credentials botocore's default chain finds in `region`, without LiteLLM's Bedrock settings /// (`AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, `AWS_SESSION_NAME`) that `resolve_credentials` also reads. pub async fn default_chain_credentials(&self, region: &str) -> Result { @@ -347,6 +380,8 @@ impl AwsAuthFlow { async fn build_provider( resolved: AwsAuthConfig, flow: AwsAuthFlow, + token_resolver: Option, + policy: Option, ) -> Result { match flow { AwsAuthFlow::SessionToken { @@ -417,8 +452,10 @@ async fn build_provider( role, session_name, } => Ok(SharedCredentialsProvider::new(WebIdentityProvider { + policy, client: aws_sdk_sts::Client::new(&sdk_config(resolved).await), token, + token_resolver, role, session_name, })), @@ -448,14 +485,27 @@ async fn sdk_config(config: AwsAuthConfig) -> aws_types::SdkConfig { loader.load().await } -#[derive(Debug)] struct WebIdentityProvider { client: aws_sdk_sts::Client, token: SecretValue, + token_resolver: Option, + policy: Option, role: String, session_name: String, } +impl std::fmt::Debug for WebIdentityProvider { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("WebIdentityProvider") + .field("token", &self.token) + .field("policy", &self.policy) + .field("role", &self.role) + .field("session_name", &self.session_name) + .finish_non_exhaustive() + } +} + impl ProvideCredentials for WebIdentityProvider { fn provide_credentials<'a>( &'a self, @@ -464,12 +514,31 @@ impl ProvideCredentials for WebIdentityProvider { Self: 'a, { aws_credential_types::provider::future::ProvideCredentials::new(async { - let response = self + let token = match &self.token_resolver { + Some(resolve) => { + let resolve = Arc::clone(resolve); + let reference = self.token.expose().to_owned(); + tokio::task::spawn_blocking(move || resolve(&reference)) + .await + .map_err(|error| error.to_string()) + .and_then(|token| token) + .map_err(|message| { + CredentialsError::provider_error(std::io::Error::other(message)) + })? + } + None => self.token.expose().to_owned(), + }; + let request = self .client .assume_role_with_web_identity() .role_arn(&self.role) .role_session_name(&self.session_name) - .web_identity_token(self.token.expose()) + .web_identity_token(token); + let request = match &self.policy { + Some(policy) => request.policy(policy).duration_seconds(3600), + None => request, + }; + let response = request .send() .await .map_err(CredentialsError::provider_error)?; @@ -605,6 +674,32 @@ pub fn sign_headers( .collect() } +pub fn sign_s3_headers( + method: &str, + url: &str, + body: &[u8], + headers: &[(String, String)], + scope: SigningScope<'_>, +) -> Result, Error> { + let mut settings = SigningSettings::default(); + settings.percent_encoding_mode = PercentEncodingMode::Single; + settings.uri_path_normalization_mode = UriPathNormalizationMode::Disabled; + let instructions = sign_request( + method, + url, + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_str())), + body, + settings, + scope, + )?; + Ok(instructions + .headers() + .map(|(name, value)| (name.to_owned(), value.to_owned())) + .collect()) +} + pub fn presign_url( url: &str, expires_in: Duration, diff --git a/litellm/rust/crates/callbacks-s3-log/Cargo.toml b/litellm/rust/crates/callbacks-s3-log/Cargo.toml new file mode 100644 index 00000000000..ae4513a2a48 --- /dev/null +++ b/litellm/rust/crates/callbacks-s3-log/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "litellm-callbacks-s3-log" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-auth-aws.workspace = true +litellm-callbacks.workspace = true +litellm-http.workspace = true +litellm-python-compat.workspace = true +litellm-s3.workspace = true +litellm-secret-value.workspace = true +litellm-tracing.workspace = true + +base64.workspace = true +bytes.workspace = true +futures-util.workspace = true +md-5 = "0.11.0" +rand.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } +sha2.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["sync", "time", "rt"] } +tracing.workspace = true +uuid.workspace = true + +[dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } +rstest.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } +wiremock.workspace = true diff --git a/litellm/rust/crates/callbacks-s3-log/src/key.rs b/litellm/rust/crates/callbacks-s3-log/src/key.rs new file mode 100644 index 00000000000..a0f3fc12b10 --- /dev/null +++ b/litellm/rust/crates/callbacks-s3-log/src/key.rs @@ -0,0 +1,367 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +use litellm_callbacks::Moment; +use sha2::{Digest, Sha256}; + +use crate::settings::{Granularity, Settings}; + +const MAX_KEY_BYTES: usize = 1024; +const MAX_FILE_NAME_BYTES: usize = 255; +const MAX_DOWNLOAD_NAME_BYTES: usize = 1024; +const BOUNDED_HEAD_BYTES: i64 = 64; +const MIN_BOUNDED_FILE_NAME_BYTES: usize = 64; +const PREFIX_DIGEST_CHARS: usize = 16; +const EXTENSION: &str = ".json"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Stamp { + pub year: i64, + pub month: u32, + pub day: u32, + pub hour: u32, + pub minute: u32, + pub second: u32, + pub micros: u32, +} + +impl From<[i64; 7]> for Stamp { + fn from(parts: [i64; 7]) -> Self { + let [year, month, day, hour, minute, second, micros] = parts; + let part = |value: i64| u32::try_from(value).unwrap_or_default(); + Self { + year, + month: part(month), + day: part(day), + hour: part(hour), + minute: part(minute), + second: part(second), + micros: part(micros), + } + } +} + +impl From<&Moment> for Stamp { + fn from(moment: &Moment) -> Self { + Self { + year: i64::from(moment.year), + month: u32::from(moment.month), + day: u32::from(moment.day), + hour: u32::from(moment.hour), + minute: u32::from(moment.minute), + second: u32::from(moment.second), + micros: moment.microsecond, + } + } +} + +impl Stamp { + pub fn utc(time: SystemTime) -> Self { + let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default(); + let seconds = elapsed.as_secs(); + let (year, month, day) = civil_from_days((seconds / 86_400) as i64); + let of_day = seconds % 86_400; + Self { + year, + month, + day, + hour: (of_day / 3600) as u32, + minute: (of_day / 60 % 60) as u32, + second: (of_day % 60) as u32, + micros: elapsed.subsec_micros(), + } + } + + pub fn date(&self) -> String { + format!("{:04}-{:02}-{:02}", self.year, self.month, self.day) + } + + pub fn clock(&self) -> String { + format!("{:02}-{:02}-{:02}", self.hour, self.minute, self.second) + } + + fn clock_micros(&self) -> String { + format!("{}-{:06}", self.clock(), self.micros) + } +} + +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let shifted = days + 719_468; + let era = shifted.div_euclid(146_097); + let day_of_era = shifted.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + } as u32; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +pub fn logging_id(start: &Stamp, id: &str) -> String { + format!("time-{}_{id}", start.clock_micros()) +} + +pub fn object_key( + path: &str, + prefix: &str, + start: &Stamp, + file_name: &str, + granularity: Granularity, +) -> String { + let sanitized = file_name.replace(['/', ':'], "_"); + let configured = if path.is_empty() { + prefix.to_owned() + } else { + format!("{}/{prefix}", path.trim_end_matches('/')) + }; + let date = match granularity { + Granularity::Day => format!("{}/", start.date()), + Granularity::Hour => format!("{}/{:02}/", start.date(), start.hour), + }; + let file_name_budget = MAX_FILE_NAME_BYTES - EXTENSION.len(); + let key = format!("{configured}{date}{sanitized}{EXTENSION}"); + if key.len() <= MAX_KEY_BYTES && sanitized.len() <= file_name_budget { + return key; + } + let budget = MAX_KEY_BYTES - date.len() - EXTENSION.len(); + if configured.len() + MIN_BOUNDED_FILE_NAME_BYTES <= budget { + let bounded = bounded_name( + file_name, + &sanitized, + file_name_budget.min(budget - configured.len()) as i64, + ); + return format!("{configured}{date}{bounded}{EXTENSION}"); + } + let shortest = bounded_name(file_name, &sanitized, MIN_BOUNDED_FILE_NAME_BYTES as i64); + let prefix = bounded_prefix(&configured, budget as i64 - shortest.len() as i64); + format!("{prefix}{date}{shortest}{EXTENSION}") +} + +pub fn download_name(start: &Stamp, id: &str) -> String { + let stamp = format!("time-{}T{}", start.date(), start.clock_micros()); + let name = format!("{stamp}_{id}"); + let sanitized = format!("{stamp}_{}", id.replace(['/', '"'], "_")); + let budget = MAX_DOWNLOAD_NAME_BYTES - EXTENSION.len(); + if sanitized.len() <= budget { + return format!("{sanitized}{EXTENSION}"); + } + format!( + "{}{EXTENSION}", + bounded_name(&name, &sanitized, budget as i64) + ) +} + +fn bounded_name(name: &str, sanitized: &str, max_bytes: i64) -> String { + let digest = hex_sha256(name); + let head = truncate( + sanitized, + BOUNDED_HEAD_BYTES.min(max_bytes - digest.len() as i64 - 1), + ); + if head.is_empty() { + digest + } else { + format!("{head}_{digest}") + } +} + +fn bounded_prefix(prefix: &str, max_bytes: i64) -> String { + let digest = format!("{}/", &hex_sha256(prefix)[..PREFIX_DIGEST_CHARS]); + if max_bytes < digest.len() as i64 { + return String::new(); + } + let head = truncate(prefix, max_bytes - digest.len() as i64 - 1).trim_end_matches('/'); + if head.is_empty() { + digest + } else { + format!("{head}/{digest}") + } +} + +fn truncate(value: &str, max_bytes: i64) -> &str { + let Ok(max_bytes) = usize::try_from(max_bytes) else { + return ""; + }; + if max_bytes == 0 { + return ""; + } + if value.len() <= max_bytes { + return value; + } + let end = (0..=max_bytes) + .rev() + .find(|end| value.is_char_boundary(*end)) + .unwrap_or(0); + &value[..end] +} + +pub fn hex_sha256(value: impl AsRef<[u8]>) -> String { + hex(&Sha256::digest(value.as_ref())) +} + +pub fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +pub fn parent(key: &str) -> &str { + key.rsplit_once('/').map_or("", |(parent, _)| parent) +} + +pub fn quote(key: &str) -> String { + let mut encoded = String::with_capacity(key.len()); + for byte in key.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') { + encoded.push(byte as char); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +pub fn dns_suffix(region: Option<&str>) -> &'static str { + let Some(region) = region.filter(|region| !region.is_empty()) else { + return "amazonaws.com"; + }; + [ + ("cn-", "amazonaws.com.cn"), + ("us-gov-", "amazonaws.com"), + ("us-isob-", "sc2s.sgov.gov"), + ("us-isof-", "csp.hci.ic.gov"), + ("us-iso-", "c2s.ic.gov"), + ("eu-isoe-", "cloud.adc-e.uk"), + ] + .into_iter() + .find(|(prefix, _)| region.starts_with(prefix)) + .map_or("amazonaws.com", |(_, suffix)| suffix) +} + +pub fn object_url(settings: &Settings, key: &str) -> String { + let encoded = quote(key); + match (&settings.endpoint_url, &settings.bucket_name) { + (Some(endpoint), Some(bucket)) if settings.use_virtual_hosted_style => { + let host = endpoint.replace("https://", "").replace("http://", ""); + let scheme = if endpoint.starts_with("https://") { + "https://" + } else { + "http://" + }; + format!("{scheme}{bucket}.{host}/{encoded}") + } + (Some(endpoint), Some(bucket)) => format!("{endpoint}/{bucket}/{encoded}"), + _ => { + let region = settings.url_region(); + let suffix = dns_suffix( + settings + .region_name + .as_ref() + .and_then(serde_json::Value::as_str), + ); + format!( + "https://{}.s3.{region}.{suffix}/{encoded}", + settings.bucket_name.as_deref().unwrap_or("None") + ) + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn stamp() -> Stamp { + Stamp::from([2026, 11, 13, 9, 5, 7, 42]) + } + + #[rstest] + #[case("", "", Granularity::Day, "2026-11-13/time-09-05-07-000042_abc.json")] + #[case( + "logs/", + "team/", + Granularity::Hour, + "logs/team/2026-11-13/09/time-09-05-07-000042_abc.json" + )] + #[case( + "a/b", + "", + Granularity::Day, + "a/b/2026-11-13/time-09-05-07-000042_abc.json" + )] + fn keys_are_laid_out_as_python_lays_them_out( + #[case] path: &str, + #[case] prefix: &str, + #[case] granularity: Granularity, + #[case] expected: &str, + ) { + let name = logging_id(&stamp(), "abc"); + assert_eq!( + object_key(path, prefix, &stamp(), &name, granularity), + expected + ); + } + + #[rstest] + fn a_long_id_is_bounded_with_its_digest() { + let name = logging_id(&stamp(), &"x".repeat(400)); + let key = object_key("p", "", &stamp(), &name, Granularity::Day); + let file = key.rsplit('/').next().unwrap(); + assert!(file.len() <= MAX_FILE_NAME_BYTES); + assert!(file.ends_with(&format!("_{}.json", hex_sha256(&name)))); + } + + #[rstest] + fn utc_stamps_split_the_clock() { + let time = UNIX_EPOCH + std::time::Duration::from_micros(1_763_024_707_000_042); + assert_eq!(Stamp::utc(time), Stamp::from([2025, 11, 13, 9, 5, 7, 42])); + } + + #[rstest] + fn keys_are_quoted_like_urllib() { + assert_eq!(quote("a b/c=d+e~f.json"), "a%20b/c%3Dd%2Be~f.json"); + assert_eq!(quote("é"), "%C3%A9"); + } + + #[rstest] + #[case( + "cn-north-1", + "https://audit-bucket.s3.cn-north-1.amazonaws.com.cn/2025-01-01/key.json" + )] + #[case( + "cn-northwest-1", + "https://audit-bucket.s3.cn-northwest-1.amazonaws.com.cn/2025-01-01/key.json" + )] + #[case( + "us-east-1", + "https://audit-bucket.s3.us-east-1.amazonaws.com/2025-01-01/key.json" + )] + #[case( + "us-gov-west-1", + "https://audit-bucket.s3.us-gov-west-1.amazonaws.com/2025-01-01/key.json" + )] + fn object_urls_follow_the_partition_of_the_region( + #[case] region: &str, + #[case] expected: &str, + ) { + let params = + serde_json::json!({"s3_bucket_name": "audit-bucket", "s3_region_name": region}); + let settings = Settings::from_params( + params.as_object().unwrap(), + &crate::settings::Constants::default(), + ); + assert_eq!(object_url(&settings, "2025-01-01/key.json"), expected); + } + + #[rstest] + fn download_names_replace_slashes_and_quotes() { + assert_eq!( + download_name(&stamp(), "a/b\"c"), + "time-2026-11-13T09-05-07-000042_a_b_c.json" + ); + } +} diff --git a/litellm/rust/crates/callbacks-s3-log/src/lib.rs b/litellm/rust/crates/callbacks-s3-log/src/lib.rs new file mode 100644 index 00000000000..c8328c6ace6 --- /dev/null +++ b/litellm/rust/crates/callbacks-s3-log/src/lib.rs @@ -0,0 +1,11 @@ +mod key; +mod limiter; +mod pyjson; +mod settings; +mod sink; +mod upload; + +pub use key::{Stamp, download_name, logging_id, object_key, object_url}; +pub use settings::{Constants, Environment, Granularity, Settings}; +pub use sink::{BatchUploadFailure, Host, NAME, S3Log, S3LogParts, system_clock}; +pub use upload::{Backoff, Clock, Outcome, panic_message, python_backoff}; diff --git a/litellm/rust/crates/callbacks-s3-log/src/limiter.rs b/litellm/rust/crates/callbacks-s3-log/src/limiter.rs new file mode 100644 index 00000000000..c3b0db22647 --- /dev/null +++ b/litellm/rust/crates/callbacks-s3-log/src/limiter.rs @@ -0,0 +1,304 @@ +use std::{ + collections::VecDeque, + sync::{Arc, Mutex, PoisonError}, +}; + +use tokio::sync::{OwnedSemaphorePermit, Semaphore, oneshot}; + +pub enum Limiter { + Fixed(Arc), + Adaptive(Arc), +} + +pub enum Permit { + Fixed(#[allow(dead_code)] OwnedSemaphorePermit), + Adaptive(Arc), +} + +impl Drop for Permit { + fn drop(&mut self) { + if let Self::Adaptive(limiter) = self { + limiter.release(); + } + } +} + +impl Limiter { + pub fn new(width: usize, adaptive: bool, ceiling: usize) -> Self { + if adaptive { + Self::Adaptive(Arc::new(Adaptive::new(width, width, width.max(ceiling)))) + } else { + Self::Fixed(Arc::new(Semaphore::new(width.min(Semaphore::MAX_PERMITS)))) + } + } + + pub async fn acquire(&self) -> Permit { + match self { + Self::Fixed(semaphore) => Permit::Fixed( + Arc::clone(semaphore) + .acquire_owned() + .await + .unwrap_or_else(|_| unreachable!("the upload semaphore is never closed")), + ), + Self::Adaptive(limiter) => { + limiter.acquire().await; + Permit::Adaptive(Arc::clone(limiter)) + } + } + } + + pub fn record(&self, throttled: bool) { + if let Self::Adaptive(limiter) = self { + limiter.record(throttled); + } + } + + #[cfg(test)] + pub fn limit(&self) -> Option { + match self { + Self::Fixed(_) => None, + Self::Adaptive(limiter) => Some(limiter.state().limit), + } + } +} + +struct State { + limit: usize, + clean_streak: usize, + in_flight: usize, + waiters: VecDeque>, +} + +pub struct Adaptive { + floor: usize, + ceiling: usize, + state: Mutex, +} + +impl Adaptive { + fn new(initial: usize, floor: usize, ceiling: usize) -> Self { + Self { + floor, + ceiling, + state: Mutex::new(State { + limit: initial.max(floor).min(ceiling), + clean_streak: 0, + in_flight: 0, + waiters: VecDeque::new(), + }), + } + } + + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + async fn acquire(&self) { + let waiter = { + let mut state = self.state(); + if state.in_flight < state.limit { + state.in_flight += 1; + return; + } + let (sender, receiver) = oneshot::channel(); + state.waiters.push_back(sender); + receiver + }; + let _ = waiter.await; + } + + fn grant(state: &mut State) { + while state.in_flight < state.limit { + let Some(waiter) = state.waiters.pop_front() else { + return; + }; + if waiter.send(()).is_ok() { + state.in_flight += 1; + } + } + } + + fn release(&self) { + let mut state = self.state(); + state.in_flight = state.in_flight.saturating_sub(1); + Self::grant(&mut state); + } + + fn record(&self, throttled: bool) { + let mut state = self.state(); + if throttled { + state.limit = self.floor.max(state.limit / 2); + state.clean_streak = 0; + return; + } + state.clean_streak += 1; + if state.clean_streak >= state.limit && state.limit < self.ceiling { + state.limit += 1; + state.clean_streak = 0; + Self::grant(&mut state); + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + fn the_limit_grows_after_a_clean_streak_and_halves_on_a_throttle() { + let limiter = Limiter::new(2, true, 8); + for _ in 0..2 { + limiter.record(false); + } + assert_eq!(limiter.limit(), Some(3)); + for _ in 0..3 { + limiter.record(false); + } + assert_eq!(limiter.limit(), Some(4)); + limiter.record(true); + assert_eq!(limiter.limit(), Some(2)); + limiter.record(true); + assert_eq!(limiter.limit(), Some(2)); + } + + #[rstest] + #[tokio::test] + async fn a_bound_past_the_semaphore_maximum_acts_as_no_bound() { + let limiter = Limiter::new(usize::MAX, false, 0); + let _first = limiter.acquire().await; + let _second = limiter.acquire().await; + } + + #[rstest] + #[tokio::test] + async fn a_full_limit_admits_the_next_waiter_when_a_permit_drops() { + let limiter = Arc::new(Limiter::new(1, true, 1)); + let first = limiter.acquire().await; + let waiting = { + let limiter = Arc::clone(&limiter); + tokio::spawn(async move { + let _permit = limiter.acquire().await; + }) + }; + tokio::task::yield_now().await; + assert!(!waiting.is_finished()); + drop(first); + waiting.await.unwrap(); + } + + fn adaptive(initial: usize, floor: usize, ceiling: usize) -> Arc { + Arc::new(Limiter::Adaptive(Arc::new(Adaptive::new( + initial, floor, ceiling, + )))) + } + + fn in_flight(limiter: &Limiter) -> usize { + match limiter { + Limiter::Fixed(_) => 0, + Limiter::Adaptive(adaptive) => adaptive.state().in_flight, + } + } + + fn record(limiter: &Limiter, throttled: bool, times: usize) { + for _ in 0..times { + limiter.record(throttled); + } + } + + #[rstest] + #[case(4, 1, 16, &[(false, 4)], 5)] + #[case(4, 1, 16, &[(false, 3)], 4)] + #[case(16, 1, 16, &[(true, 1)], 8)] + #[case(4, 4, 16, &[(true, 1)], 4)] + #[case(15, 1, 16, &[(false, 1000)], 16)] + #[case(4, 1, 32, &[(false, 3), (true, 1), (false, 1)], 2)] + fn the_limit_follows_the_samples( + #[case] initial: usize, + #[case] floor: usize, + #[case] ceiling: usize, + #[case] samples: &[(bool, usize)], + #[case] expected: usize, + ) { + let limiter = adaptive(initial, floor, ceiling); + for &(throttled, times) in samples { + record(&limiter, throttled, times); + } + assert_eq!(limiter.limit(), Some(expected)); + } + + #[rstest] + #[tokio::test] + async fn growing_the_limit_wakes_a_waiting_acquirer() { + let limiter = adaptive(1, 1, 4); + let holder = limiter.acquire().await; + let waiting = { + let limiter = Arc::clone(&limiter); + tokio::spawn(async move { + let _permit = limiter.acquire().await; + }) + }; + tokio::task::yield_now().await; + assert!(!waiting.is_finished()); + limiter.record(false); + waiting.await.unwrap(); + drop(holder); + assert_eq!(in_flight(&limiter), 0); + } + + #[rstest] + #[tokio::test] + async fn releasing_a_slot_wakes_exactly_one_waiter() { + let limiter = adaptive(1, 1, 4); + let holder = limiter.acquire().await; + let acquired = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let (release, released) = tokio::sync::watch::channel(false); + let waiters: Vec<_> = (0..3) + .map(|_| { + let limiter = Arc::clone(&limiter); + let acquired = Arc::clone(&acquired); + let mut released = released.clone(); + tokio::spawn(async move { + let _permit = limiter.acquire().await; + acquired.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let _ = released.wait_for(|released| *released).await; + }) + }) + .collect(); + tokio::task::yield_now().await; + drop(holder); + tokio::task::yield_now().await; + assert_eq!(acquired.load(std::sync::atomic::Ordering::SeqCst), 1); + for _ in 0..3 { + limiter.record(false); + tokio::task::yield_now().await; + } + assert_eq!(acquired.load(std::sync::atomic::Ordering::SeqCst), 3); + release.send(true).unwrap(); + for waiter in waiters { + waiter.await.unwrap(); + } + assert_eq!(in_flight(&limiter), 0); + } + + #[rstest] + #[tokio::test] + async fn a_cancelled_waiter_is_skipped_and_leaves_in_flight_at_zero() { + let limiter = adaptive(1, 1, 1); + let holder = limiter.acquire().await; + let spawn_waiter = |limiter: Arc| { + tokio::spawn(async move { + let _permit = limiter.acquire().await; + }) + }; + let doomed = spawn_waiter(Arc::clone(&limiter)); + let next = spawn_waiter(Arc::clone(&limiter)); + tokio::task::yield_now().await; + doomed.abort(); + assert!(doomed.await.unwrap_err().is_cancelled()); + drop(holder); + next.await.unwrap(); + assert_eq!(in_flight(&limiter), 0); + } +} diff --git a/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs b/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs new file mode 100644 index 00000000000..c2f442c9af9 --- /dev/null +++ b/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs @@ -0,0 +1,214 @@ +use serde_json::Value; + +const HEX: &[u8; 16] = b"0123456789abcdef"; + +pub fn write_str(out: &mut Vec, text: &str) { + out.push(b'"'); + let mut units = [0u16; 2]; + for character in text.chars() { + match character { + '"' => out.extend_from_slice(b"\\\""), + '\\' => out.extend_from_slice(b"\\\\"), + '\n' => out.extend_from_slice(b"\\n"), + '\r' => out.extend_from_slice(b"\\r"), + '\t' => out.extend_from_slice(b"\\t"), + '\u{8}' => out.extend_from_slice(b"\\b"), + '\u{c}' => out.extend_from_slice(b"\\f"), + ' '..='~' => out.push(character as u8), + _ => { + for unit in character.encode_utf16(&mut units) { + write_unit(out, *unit); + } + } + } + } + out.push(b'"'); +} + +fn write_unit(out: &mut Vec, unit: u16) { + out.extend_from_slice(b"\\u"); + for shift in [12, 8, 4, 0] { + out.push(HEX[usize::from((unit >> shift) & 0xf)]); + } +} + +pub fn float_repr(value: f64) -> String { + if value.is_nan() { + return "NaN".to_owned(); + } + if value.is_infinite() { + return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned(); + } + let shortest = format!("{value:e}"); + let length = shortest + .split('e') + .next() + .unwrap_or_default() + .chars() + .filter(char::is_ascii_digit) + .count(); + let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1)); + let (negative, scientific) = match scientific.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, scientific.as_str()), + }; + let (mantissa, exponent) = scientific.split_once('e').unwrap_or((scientific, "0")); + let exponent: i32 = exponent.parse().unwrap_or(0); + let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + let mut text = String::with_capacity(digits.len() + 8); + if negative { + text.push('-'); + } + let point = exponent + 1; + if (-3..=16).contains(&point) { + let count = digits.len() as i32; + if point <= 0 { + text.push_str("0."); + text.extend(std::iter::repeat_n('0', (-point) as usize)); + text.push_str(&digits); + } else if point >= count { + text.push_str(&digits); + text.extend(std::iter::repeat_n('0', (point - count) as usize)); + text.push_str(".0"); + } else { + let (whole, fraction) = digits.split_at(point as usize); + text.push_str(whole); + text.push('.'); + text.push_str(fraction); + } + return text; + } + let (first, rest) = digits.split_at(1); + text.push_str(first); + if !rest.is_empty() { + text.push('.'); + text.push_str(rest); + } + text.push('e'); + text.push(if exponent < 0 { '-' } else { '+' }); + text.push_str(&format!("{:02}", exponent.unsigned_abs())); + text +} + +pub fn write_value(out: &mut Vec, value: &Value) { + match value { + Value::Null => out.extend_from_slice(b"null"), + Value::Bool(true) => out.extend_from_slice(b"true"), + Value::Bool(false) => out.extend_from_slice(b"false"), + Value::Number(number) => match (number.as_i64(), number.as_u64(), number.as_f64()) { + (Some(integer), _, _) => out.extend_from_slice(integer.to_string().as_bytes()), + (None, Some(integer), _) => out.extend_from_slice(integer.to_string().as_bytes()), + (None, None, Some(float)) => out.extend_from_slice(float_repr(float).as_bytes()), + (None, None, None) => out.extend_from_slice(number.to_string().as_bytes()), + }, + Value::String(text) => write_str(out, text), + Value::Array(items) => { + out.push(b'['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.extend_from_slice(b", "); + } + write_value(out, item); + } + out.push(b']'); + } + Value::Object(entries) => { + out.push(b'{'); + for (index, (key, item)) in entries.iter().enumerate() { + if index > 0 { + out.extend_from_slice(b", "); + } + write_str(out, key); + out.extend_from_slice(b": "); + write_value(out, item); + } + out.push(b'}'); + } + } +} + +pub fn dumps(value: &Value) -> String { + let mut out = Vec::new(); + write_value(&mut out, value); + String::from_utf8(out).unwrap_or_default() +} + +pub fn py_str(value: &Value) -> String { + match value { + Value::Null => "None".to_owned(), + Value::Bool(true) => "True".to_owned(), + Value::Bool(false) => "False".to_owned(), + Value::String(text) => text.clone(), + Value::Number(number) => match number.as_f64() { + Some(float) if number.is_f64() => float_repr(float), + _ => number.to_string(), + }, + Value::Array(_) | Value::Object(_) => dumps(value), + } +} + +pub fn truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64().is_some_and(|float| float != 0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(entries) => !entries.is_empty(), + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case(0.0, "0.0")] + #[case(-0.0, "-0.0")] + #[case(1.0, "1.0")] + #[case(0.1, "0.1")] + #[case(1.5e-5, "1.5e-05")] + #[case(1e-4, "0.0001")] + #[case(1e-5, "1e-05")] + #[case(1e15, "1000000000000000.0")] + #[case(1e16, "1e+16")] + #[case(1.2345678901234568e17, "1.2345678901234568e+17")] + #[case(123.456, "123.456")] + #[case(1e100, "1e+100")] + #[case(-2.5e-300, "-2.5e-300")] + #[case(f64::NAN, "NaN")] + #[case(f64::INFINITY, "Infinity")] + #[case(f64::NEG_INFINITY, "-Infinity")] + #[case(0.30000000000000004, "0.30000000000000004")] + #[case(0.07872390747070312, "0.07872390747070312")] + #[case(1794700894.4414062, "1794700894.4414062")] + fn floats_print_as_python_repr_prints_them(#[case] value: f64, #[case] expected: &str) { + assert_eq!(float_repr(value), expected); + } + + #[rstest] + #[case("plain", "\"plain\"")] + #[case("quote\" back\\", "\"quote\\\" back\\\\\"")] + #[case("\n\r\t\u{8}\u{c}\u{1}\u{7f}", "\"\\n\\r\\t\\b\\f\\u0001\\u007f\"")] + #[case("café", "\"caf\\u00e9\"")] + #[case("😀", "\"\\ud83d\\ude00\"")] + fn strings_are_escaped_as_ensure_ascii_escapes_them( + #[case] text: &str, + #[case] expected: &str, + ) { + let mut out = Vec::new(); + write_str(&mut out, text); + assert_eq!(String::from_utf8(out).unwrap(), expected); + } + + #[rstest] + fn containers_use_the_default_separators_and_keep_key_order() { + assert_eq!( + dumps(&json!({"b": [1, 2.5, null, true], "a": {"x": "y"}})), + "{\"b\": [1, 2.5, null, true], \"a\": {\"x\": \"y\"}}" + ); + } +} diff --git a/litellm/rust/crates/callbacks-s3-log/src/settings.rs b/litellm/rust/crates/callbacks-s3-log/src/settings.rs new file mode 100644 index 00000000000..c266f612c94 --- /dev/null +++ b/litellm/rust/crates/callbacks-s3-log/src/settings.rs @@ -0,0 +1,656 @@ +use std::time::Duration; + +use litellm_auth_aws::AwsAuthConfig; +use litellm_secret_value::SecretValue; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::pyjson::{py_str, truthy}; + +fn py_repr(value: &Value) -> String { + litellm_python_compat::repr::repr(&litellm_python_compat::json::from_json(value.clone())) +} + +fn py_optional(value: Option) -> String { + value.map_or_else(|| "None".to_owned(), |value| value.to_string()) +} + +fn py_type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) => "dict", + } +} + +pub const PARTITION_GRANULARITY_ENV_VAR: &str = "S3_PARTITION_GRANULARITY"; +pub const LOG_PROMPTS_ONLY_ENV_VAR: &str = "S3_LOG_PROMPTS_ONLY"; +const DEFAULT_MAX_QUEUE_SIZE: usize = 50_000; +const DEFAULT_MAX_RETRY_AGE_SECONDS: u64 = 3600; +const DEFAULT_REGION: &str = "us-west-2"; + +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub struct Constants { + pub batch_size: usize, + pub max_concurrent_uploads: usize, + pub max_adaptive_concurrency: usize, + pub upload_attempt_timeout_seconds: f64, + pub download_read_timeout_seconds: f64, + pub download_connect_timeout_seconds: f64, + pub web_identity_session_policy: Option, +} + +impl Default for Constants { + fn default() -> Self { + Self { + batch_size: 512, + max_concurrent_uploads: 16, + max_adaptive_concurrency: 200, + upload_attempt_timeout_seconds: 30.0, + download_read_timeout_seconds: 600.0, + download_connect_timeout_seconds: 5.0, + web_identity_session_policy: None, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Granularity { + #[default] + Day, + Hour, +} + +impl Granularity { + pub fn as_str(self) -> &'static str { + match self { + Self::Day => "day", + Self::Hour => "hour", + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Settings { + pub bucket_name: Option, + pub region_name: Option, + pub endpoint_url: Option, + pub path: Option, + pub verify: Option, + pub auth: AwsAuthConfig, + pub use_team_prefix: bool, + pub use_key_prefix: bool, + pub strip_base64_files: bool, + pub use_virtual_hosted_style: bool, + pub log_prompts_only: Option, + pub partition_granularity: Option, + pub server_side_encryption: Option, + pub sse_kms_key_id: Option, + pub max_concurrent_uploads: usize, + pub max_queue_size: usize, + pub max_retry_age: Option, + pub drop_on_terminal_error: bool, + pub adaptive_concurrency: bool, + pub max_adaptive_concurrency: usize, + pub batch_file_upload: bool, + pub batch_size: usize, + pub attempt_timeout: Duration, + pub download_read_timeout: Duration, + pub web_identity_session_policy: Option, +} + +pub type Environment = dyn Fn(&str) -> Option + Send + Sync; + +fn text(params: &Map, name: &str) -> Option { + params.get(name).filter(|value| truthy(value)).map(py_str) +} + +fn secret(params: &Map, name: &str) -> Option { + text(params, name).map(SecretValue::new) +} + +fn present(params: &Map, name: &str) -> Option { + params.get(name).filter(|value| !value.is_null()).cloned() +} + +fn is_blank(value: &Value) -> bool { + matches!(value, Value::String(text) if text.is_empty()) +} + +pub fn lax_bool(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(*flag), + Value::Number(number) => number + .as_f64() + .and_then(|float| { + [(0.0, false), (1.0, true)] + .into_iter() + .find(|(bound, _)| *bound == float) + }) + .map(|(_, flag)| flag), + Value::String(text) => match text.trim().to_ascii_lowercase().as_str() { + "0" | "off" | "f" | "false" | "n" | "no" => Some(false), + "1" | "on" | "t" | "true" | "y" | "yes" => Some(true), + _ => None, + }, + _ => None, + } +} + +pub fn lax_int(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(i128::from(*flag)), + Value::Number(number) => { + if let Some(integer) = number.as_i64() { + return Some(i128::from(integer)); + } + if let Some(integer) = number.as_u64() { + return Some(i128::from(integer)); + } + let float = number.as_f64()?; + (float.is_finite() && float.fract() == 0.0 && float.abs() <= i64::MAX as f64) + .then_some(float as i128) + } + Value::String(text) => int_text(text.trim()), + _ => None, + } +} + +fn int_text(text: &str) -> Option { + let (whole, fraction) = text.split_once('.').unwrap_or((text, "")); + if !fraction.chars().all(|character| character == '0') { + return None; + } + if text.contains('.') && fraction.is_empty() { + return None; + } + let (sign, digits) = match whole.strip_prefix('-') { + Some(rest) => (-1, rest), + None => (1, whole.strip_prefix('+').unwrap_or(whole)), + }; + let well_formed = !digits.is_empty() + && !digits.starts_with('_') + && !digits.ends_with('_') + && !digits.contains("__") + && digits + .chars() + .all(|character| character.is_ascii_digit() || character == '_'); + if !well_formed { + return None; + } + digits + .replace('_', "") + .parse::() + .ok() + .map(|number| sign * number) +} + +fn strip_text(value: &Value) -> Value { + match value { + Value::String(text) => Value::String(text.trim().to_owned()), + other => other.clone(), + } +} + +fn positive_int( + setting: &str, + configured: Option<&Value>, + fallback: usize, + reject_bool: bool, +) -> usize { + let Some(configured) = configured.filter(|value| !value.is_null() && !is_blank(value)) else { + return fallback; + }; + if reject_bool && configured.is_boolean() { + tracing::warn!( + "s3 logging: {setting}={} is a boolean, not an integer, using {fallback}", + py_repr(configured) + ); + return fallback; + } + match lax_int(&strip_text(configured)) { + None => { + tracing::warn!( + "s3 logging: {setting}={} is not an integer, using {fallback}", + py_repr(configured) + ); + fallback + } + Some(bound) if bound < 1 => { + tracing::warn!( + "s3 logging: {setting}={} must be at least 1, using {fallback}", + py_repr(configured) + ); + fallback + } + Some(bound) => usize::try_from(bound).unwrap_or(usize::MAX), + } +} + +fn retry_age(configured: &Value, fallback: Option) -> Option { + if configured.is_null() || is_blank(configured) { + return None; + } + if configured.is_boolean() { + tracing::warn!( + "s3 logging: s3_max_retry_age_seconds={} is a boolean, not an integer, falling back to {}", + py_repr(configured), + py_optional(fallback) + ); + return fallback; + } + match lax_int(&strip_text(configured)) { + None => { + tracing::warn!( + "s3 logging: s3_max_retry_age_seconds={} is not an integer, falling back to {}", + py_repr(configured), + py_optional(fallback) + ); + fallback + } + Some(bound) if bound < 0 => { + tracing::warn!( + "s3 logging: s3_max_retry_age_seconds={} must be at least 0, falling back to {}", + py_repr(configured), + py_optional(fallback) + ); + fallback + } + Some(0) => None, + Some(bound) => Some(u64::try_from(bound).unwrap_or(u64::MAX)), + } +} + +fn flag( + setting: &str, + configured: Option<&Value>, + unset: bool, + invalid: bool, + consequence: &str, +) -> bool { + let Some(configured) = configured.filter(|value| !value.is_null() && !is_blank(value)) else { + return unset; + }; + lax_bool(&strip_text(configured)).unwrap_or_else(|| { + tracing::warn!( + "s3 logging: {setting}={} is not a boolean, {consequence}", + py_repr(configured) + ); + invalid + }) +} + +fn sse_value(name: &str, value: Option<&Value>) -> Option { + match value { + None => None, + Some(Value::String(text)) => Some(text.clone()), + Some(other) => { + tracing::warn!( + "s3 logging: ignoring {name} because it has invalid type {}; expected a string", + py_type_name(other) + ); + None + } + } +} + +pub fn resolve_sse( + algorithm: Option<&Value>, + key_id: Option<&Value>, +) -> (Option, Option) { + let valid_sse = sse_value("s3_server_side_encryption", algorithm); + let valid_key_id = sse_value("s3_sse_kms_key_id", key_id).filter(|key| !key.is_empty()); + let Some(algorithm) = valid_sse + .filter(|algorithm| !algorithm.is_empty()) + .or_else(|| valid_key_id.as_ref().map(|_| "aws:kms".to_owned())) + else { + return (None, None); + }; + if valid_key_id.is_some() && !algorithm.starts_with("aws:kms") { + tracing::warn!( + "s3 logging: ignoring s3_sse_kms_key_id because s3_server_side_encryption is {algorithm}; set it to aws:kms to encrypt with the KMS key" + ); + return (Some(algorithm), None); + } + (Some(algorithm), valid_key_id) +} + +pub fn resolve_partition_granularity(raw: Option<&Value>) -> Granularity { + match raw { + Some(Value::String(text)) if text == "hour" => Granularity::Hour, + None => Granularity::Day, + Some(Value::String(text)) if text.is_empty() || text == "day" => Granularity::Day, + Some(other) => { + tracing::warn!( + "s3 logging: s3_partition_granularity={} is not one of day, hour, using day", + py_repr(other) + ); + Granularity::Day + } + } +} + +impl Settings { + pub fn from_params(params: &Map, constants: &Constants) -> Self { + let truthy_param = |name: &str| params.get(name).is_some_and(truthy); + let (server_side_encryption, sse_kms_key_id) = resolve_sse( + params + .get("s3_server_side_encryption") + .filter(|value| truthy(value)), + params + .get("s3_sse_kms_key_id") + .filter(|value| truthy(value)), + ); + let max_concurrent_uploads = positive_int( + "s3_max_concurrent_uploads", + Some( + params + .get("s3_max_concurrent_uploads") + .filter(|value| !value.is_null() && !is_blank(value)) + .cloned() + .unwrap_or_else(|| Value::from(constants.max_concurrent_uploads)), + ) + .as_ref(), + constants.max_concurrent_uploads, + false, + ); + let max_queue_size = positive_int( + "s3_max_queue_size", + params.get("s3_max_queue_size"), + DEFAULT_MAX_QUEUE_SIZE, + true, + ); + let max_retry_age = match params.get("s3_max_retry_age_seconds") { + Some(value) if !value.is_null() && !is_blank(value) => { + retry_age(value, Some(DEFAULT_MAX_RETRY_AGE_SECONDS)) + } + _ => Some(DEFAULT_MAX_RETRY_AGE_SECONDS), + }; + Self { + bucket_name: text(params, "s3_bucket_name"), + region_name: params + .get("s3_region_name") + .filter(|value| truthy(value)) + .cloned(), + endpoint_url: text(params, "s3_endpoint_url"), + path: text(params, "s3_path"), + verify: present(params, "s3_verify"), + auth: AwsAuthConfig { + access_key_id: secret(params, "s3_aws_access_key_id"), + secret_access_key: secret(params, "s3_aws_secret_access_key"), + session_token: secret(params, "s3_aws_session_token"), + region_name: text(params, "s3_region_name"), + session_name: text(params, "s3_aws_session_name"), + profile_name: text(params, "s3_aws_profile_name"), + role_name: text(params, "s3_aws_role_name"), + web_identity_token: secret(params, "s3_aws_web_identity_token"), + sts_endpoint: text(params, "s3_aws_sts_endpoint"), + external_id: None, + }, + use_team_prefix: truthy_param("s3_use_team_prefix"), + use_key_prefix: truthy_param("s3_use_key_prefix"), + strip_base64_files: truthy_param("s3_strip_base64_files"), + use_virtual_hosted_style: truthy_param("s3_use_virtual_hosted_style"), + log_prompts_only: present(params, "s3_log_prompts_only"), + partition_granularity: present(params, "s3_partition_granularity"), + server_side_encryption, + sse_kms_key_id, + max_concurrent_uploads, + max_queue_size, + max_retry_age: max_retry_age.map(Duration::from_secs), + drop_on_terminal_error: flag( + "s3_drop_on_terminal_error", + params.get("s3_drop_on_terminal_error"), + true, + true, + "dropping terminal-failed uploads", + ), + adaptive_concurrency: flag( + "s3_adaptive_concurrency", + params.get("s3_adaptive_concurrency"), + false, + false, + "keeping the fixed upload width", + ), + max_adaptive_concurrency: positive_int( + "s3_max_adaptive_concurrency", + params.get("s3_max_adaptive_concurrency"), + constants.max_adaptive_concurrency, + true, + ), + batch_file_upload: flag( + "s3_batch_file_upload", + params.get("s3_batch_file_upload"), + false, + false, + "keeping per-request objects", + ), + batch_size: constants.batch_size, + attempt_timeout: Duration::from_secs_f64( + constants.upload_attempt_timeout_seconds.max(0.0), + ), + download_read_timeout: Duration::from_secs_f64( + constants.download_read_timeout_seconds.max(0.0), + ), + web_identity_session_policy: constants.web_identity_session_policy.clone(), + } + } + + pub fn partition_granularity(&self, environment: &Environment) -> Granularity { + resolve_partition_granularity(self.partition_granularity_raw(environment).as_ref()) + } + + pub fn partition_granularity_raw(&self, environment: &Environment) -> Option { + match &self.partition_granularity { + Some(value) => Some(value.clone()), + None => environment(PARTITION_GRANULARITY_ENV_VAR).map(Value::String), + } + } + + pub fn log_prompts_only(&self, environment: &Environment) -> bool { + let raw = match &self.log_prompts_only { + Some(value) => Some(value.clone()), + None => environment(LOG_PROMPTS_ONLY_ENV_VAR).map(Value::String), + }; + let Some(raw) = raw.filter(|value| !is_blank(value)) else { + return false; + }; + lax_bool(&strip_text(&raw)).unwrap_or_else(|| { + tracing::warn!( + "s3 logging: s3_log_prompts_only={} is not a boolean, logging prompts only", + py_repr(&raw) + ); + true + }) + } + + pub fn signing_region( + &self, + from_environment: impl FnOnce() -> Result, String>, + ) -> Result { + let region = match &self.region_name { + Some(Value::String(region)) => region.clone(), + Some(other) => return Err(invalid_region(&py_str(other))), + None => from_environment()?.unwrap_or_else(|| DEFAULT_REGION.to_owned()), + }; + if valid_region(®ion) { + Ok(region) + } else { + Err(invalid_region(®ion)) + } + } + + pub fn url_region(&self) -> String { + self.region_name + .as_ref() + .map_or_else(|| "None".to_owned(), py_str) + } +} + +fn invalid_region(region: &str) -> String { + format!( + "Invalid AWS region format: {region}. Region names must contain only lowercase letters, digits, and hyphens." + ) +} + +fn valid_region(region: &str) -> bool { + !region.is_empty() + && region + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn settings(params: Value) -> Settings { + Settings::from_params(params.as_object().unwrap(), &Constants::default()) + } + + fn no_env(_: &str) -> Option { + None + } + + #[rstest] + #[case(json!(true), Some(1))] + #[case(json!(2.0), Some(2))] + #[case(json!(2.5), None)] + #[case(json!(" 4 "), Some(4))] + #[case(json!("4.0"), Some(4))] + #[case(json!("+4"), Some(4))] + #[case(json!("1_000"), Some(1000))] + #[case(json!("0x10"), None)] + #[case(json!("1e3"), None)] + #[case(json!("4.5"), None)] + #[case(json!("007"), Some(7))] + #[case(json!(1e20), None)] + fn integers_parse_as_pydantic_parses_them( + #[case] value: Value, + #[case] expected: Option, + ) { + assert_eq!(lax_int(&strip_text(&value)), expected); + } + + #[rstest] + #[case(json!(0), Some(false))] + #[case(json!(2), None)] + #[case(json!(1.0), Some(true))] + #[case(json!(0.5), None)] + #[case(json!(" yes "), Some(true))] + #[case(json!("TRUE"), Some(true))] + #[case(json!("off"), Some(false))] + #[case(json!("1.0"), None)] + #[case(json!("2"), None)] + fn booleans_parse_as_pydantic_parses_them( + #[case] value: Value, + #[case] expected: Option, + ) { + assert_eq!(lax_bool(&strip_text(&value)), expected); + } + + #[rstest] + fn defaults_match_the_python_constructor() { + let settings = settings(json!({})); + assert_eq!(settings.bucket_name, None); + assert_eq!(settings.path, None); + assert_eq!(settings.max_concurrent_uploads, 16); + assert_eq!(settings.max_queue_size, 50_000); + assert_eq!(settings.max_retry_age, Some(Duration::from_secs(3600))); + assert!(settings.drop_on_terminal_error); + assert!(!settings.adaptive_concurrency); + assert_eq!(settings.max_adaptive_concurrency, 200); + assert!(!settings.batch_file_upload); + assert_eq!(settings.partition_granularity(&no_env), Granularity::Day); + assert!(!settings.log_prompts_only(&no_env)); + } + + #[rstest] + fn prefix_flags_take_python_truthiness() { + let settings = settings(json!({"s3_use_team_prefix": "false", "s3_use_key_prefix": 0})); + assert!(settings.use_team_prefix); + assert!(!settings.use_key_prefix); + } + + #[rstest] + #[case(json!({"s3_max_retry_age_seconds": 0}), None)] + #[case(json!({"s3_max_retry_age_seconds": "60"}), Some(60))] + #[case(json!({"s3_max_retry_age_seconds": -1}), Some(3600))] + #[case(json!({"s3_max_retry_age_seconds": true}), Some(3600))] + #[case(json!({"s3_max_retry_age_seconds": ""}), Some(3600))] + fn retry_age_parses_as_python(#[case] params: Value, #[case] seconds: Option) { + assert_eq!( + settings(params).max_retry_age, + seconds.map(Duration::from_secs) + ); + } + + #[rstest] + #[case(json!({"s3_sse_kms_key_id": "k"}), Some("aws:kms"), Some("k"))] + #[case(json!({"s3_server_side_encryption": "AES256", "s3_sse_kms_key_id": "k"}), Some("AES256"), None)] + #[case(json!({"s3_server_side_encryption": 3}), None, None)] + fn encryption_combines_as_python( + #[case] params: Value, + #[case] algorithm: Option<&str>, + #[case] key: Option<&str>, + ) { + let settings = settings(params); + assert_eq!(settings.server_side_encryption.as_deref(), algorithm); + assert_eq!(settings.sse_kms_key_id.as_deref(), key); + } + + #[rstest] + #[case(json!({"s3_partition_granularity": "hour"}), None, Granularity::Hour)] + #[case(json!({}), Some("hour"), Granularity::Hour)] + #[case(json!({"s3_partition_granularity": "day"}), Some("hour"), Granularity::Day)] + #[case(json!({"s3_partition_granularity": "minute"}), None, Granularity::Day)] + fn granularity_reads_the_param_then_the_environment( + #[case] params: Value, + #[case] env: Option<&'static str>, + #[case] expected: Granularity, + ) { + let environment = move |name: &str| { + (name == PARTITION_GRANULARITY_ENV_VAR) + .then_some(env) + .flatten() + .map(str::to_owned) + }; + assert_eq!( + settings(params).partition_granularity(&environment), + expected + ); + } + + #[rstest] + #[case::default(json!({}), Ok(None), Ok("us-west-2"))] + #[case::param(json!({"s3_region_name": "eu-west-1"}), Ok(Some("ap-south-1")), Ok("eu-west-1"))] + #[case::bad_param(json!({"s3_region_name": "EU"}), Ok(None), Err(()))] + #[case::environment(json!({}), Ok(Some("ap-south-1")), Ok("ap-south-1"))] + #[case::bad_environment(json!({}), Ok(Some("Ap_South")), Err(()))] + #[case::empty_environment(json!({}), Ok(Some("")), Err(()))] + #[case::secret_failure(json!({}), Err(()), Err(()))] + fn the_signing_region_falls_back_like_python( + #[case] params: Value, + #[case] environment: Result, ()>, + #[case] expected: Result<&str, ()>, + ) { + let from_environment = || { + environment + .map(|region| region.map(str::to_owned)) + .map_err(|()| "secret manager down".to_owned()) + }; + assert_eq!( + settings(params) + .signing_region(from_environment) + .map_err(|_| ()), + expected.map(str::to_owned) + ); + } +} diff --git a/litellm/rust/crates/callbacks-s3-log/src/sink.rs b/litellm/rust/crates/callbacks-s3-log/src/sink.rs new file mode 100644 index 00000000000..1374390e01b --- /dev/null +++ b/litellm/rust/crates/callbacks-s3-log/src/sink.rs @@ -0,0 +1,670 @@ +use std::{ + collections::BTreeMap, + sync::{ + Arc, Mutex, MutexGuard, PoisonError, + atomic::{AtomicUsize, Ordering}, + }, + time::SystemTime, +}; + +use bytes::Bytes; +use futures_util::{ + FutureExt, + future::{BoxFuture, join_all}, +}; +use litellm_auth_aws::AwsAuthService; +use litellm_callbacks::{ + Activation, Callback, Field, FieldError, FieldSet, FieldValues, Hook, HookInput, HookOutput, + LoggedKeyIdentity, Manifest, MetadataKind, OptOutBy, PayloadRef, RenderOptions, Scalar, + Selection, Subscription, typed, +}; +use litellm_tracing::Logger; +use serde_json::Value; +use tokio::time::Instant; + +use crate::{ + key::{Stamp, download_name, logging_id, object_key, parent}, + limiter::Limiter, + settings::{Environment, Granularity, Settings, resolve_partition_granularity}, + upload::{Backoff, Body, Clock, Held, Object, Outcome, Uploader, panic_message, render_bodies}, +}; + +pub const NAME: &str = "s3_v2"; +const JSON: &str = "application/json"; +const NDJSON: &str = "application/x-ndjson"; +const PROMPTS_ONLY_NULL_KEYS: [&str; 1] = ["response"]; + +pub trait Host: Send + Sync { + fn callback_failed(&self); + fn web_identity_token(&self, reference: &str) -> Result; +} + +pub struct S3LogParts { + pub settings: Settings, + pub client: litellm_http::Client, + pub download_client: litellm_http::Client, + pub environment: Arc, + pub clock: Clock, + pub backoff: Backoff, + pub host: Arc, + pub cold_storage_is_s3_v2: bool, + pub region_from_environment: Option, + pub logger: Logger, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BatchUploadFailure { + pub queued: usize, + pub failed: usize, + pub total: usize, +} + +#[derive(Clone, Debug)] +struct Element { + object: Object, + retrying_since: Option, + is_file: bool, +} + +#[derive(Default)] +struct Queue { + items: Vec, + requeued: usize, +} + +pub struct S3Log { + manifest: Manifest, + uploader: Uploader, + limiter: Limiter, + host: Arc, + cold_storage_is_s3_v2: bool, + logger: Logger, + granularity: Mutex, Granularity)>>, + queue: Mutex, + flush_lock: tokio::sync::Mutex<()>, + held: Arc, +} + +#[derive(Debug, PartialEq)] +enum Alias { + Absent, + Text(String), + NotText, +} + +impl Alias { + fn of(identity: &LoggedKeyIdentity, name: &str, value: Option<&Scalar>) -> Self { + let rejected = identity + .rejected + .as_deref() + .is_some_and(|rejected| rejected.iter().any(|member| member == name)); + match value { + _ if rejected => Self::NotText, + Some(value) if !value.truthy() => Self::Absent, + None => Self::Absent, + Some(Scalar::Text(text)) => Self::Text(text.clone()), + Some(_) => Self::NotText, + } + } +} + +#[derive(Debug)] +struct Event { + start: Option, + id: Option, + metadata: MetadataKind, + team_alias: Alias, + key_alias: Alias, + cold_storage_object_key: Option, + cold_storage_s3_v2: bool, +} + +fn manifest() -> Manifest { + Manifest { + selection: Selection { + activation: Activation::DefaultOn, + opt_out_by: vec![OptOutBy::Admin], + honors_no_log: true, + ..Selection::named(NAME) + }, + subscriptions: vec![Subscription { + reads: FieldSet::of(&[ + Field::DeprecatedKwargs, + Field::StandardLoggingPayloadRef, + Field::LogId, + Field::ColdStorageObjectKey, + Field::LoggedKeyIdentity, + Field::AttemptTiming, + ]), + awaited: true, + ..Subscription::on(Hook::AttemptEnded) + }], + has_request_state: false, + } +} + +impl S3Log { + pub fn new(parts: S3LogParts) -> Arc { + let settings = Arc::new(parts.settings); + let limiter = Limiter::new( + settings.max_concurrent_uploads, + settings.adaptive_concurrency, + settings.max_adaptive_concurrency, + ); + let resolver_host = Arc::clone(&parts.host); + let session_policy = settings.web_identity_session_policy.clone(); + Arc::new(Self { + manifest: manifest(), + uploader: Uploader { + settings, + client: parts.client, + download_client: parts.download_client, + environment_region: parts.region_from_environment, + auth: AwsAuthService::for_web_identity( + Arc::new(move |reference: &str| resolver_host.web_identity_token(reference)), + session_policy, + ), + environment: parts.environment, + clock: parts.clock, + backoff: parts.backoff, + retries: Default::default(), + }, + limiter, + host: parts.host, + cold_storage_is_s3_v2: parts.cold_storage_is_s3_v2, + logger: parts.logger, + granularity: Mutex::default(), + queue: Mutex::default(), + flush_lock: tokio::sync::Mutex::new(()), + held: Arc::default(), + }) + } + + pub fn settings(&self) -> &Settings { + &self.uploader.settings + } + + pub fn partition_granularity(&self) -> Granularity { + let raw = self + .settings() + .partition_granularity_raw(self.uploader.environment.as_ref()); + let mut cached = self + .granularity + .lock() + .unwrap_or_else(PoisonError::into_inner); + if let Some((seen, resolved)) = cached.as_ref() + && *seen == raw + { + return *resolved; + } + let resolved = resolve_partition_granularity(raw.as_ref()); + *cached = Some((raw, resolved)); + resolved + } + + pub fn log_prompts_only(&self) -> bool { + self.settings() + .log_prompts_only(self.uploader.environment.as_ref()) + } + + fn queue(&self) -> MutexGuard<'_, Queue> { + self.queue.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub fn queued_keys(&self) -> Vec { + self.queue() + .items + .iter() + .map(|element| element.object.key.clone()) + .collect() + } + + fn push(&self, element: Element) -> usize { + let mut queue = self.queue(); + queue.items.push(element); + queue.items.len() + } + + pub fn held_payloads(&self) -> usize { + self.held.load(Ordering::Relaxed) + } + + fn hold(&self, payload: Arc, options: RenderOptions) -> Body { + Body::Payload(Arc::new(Held::new( + payload, + options, + Arc::clone(&self.held), + ))) + } + + fn element(&self, event: Event, body: Body) -> Result { + let settings = self.settings(); + let start = event + .start + .ok_or_else(|| "the payload carries no start time".to_owned())?; + let mut prefixes = Vec::new(); + for (enabled, alias) in [ + (settings.use_team_prefix, &event.team_alias), + (settings.use_key_prefix, &event.key_alias), + ] { + if !enabled { + continue; + } + if matches!(event.metadata, MetadataKind::Null | MetadataKind::Other) { + return Err("metadata has no get".to_owned()); + } + match alias { + Alias::Absent => {} + Alias::Text(alias) => prefixes.push(alias.clone()), + Alias::NotText => return Err("sequence item: expected str instance".to_owned()), + } + } + let mut prefix = prefixes.join("/"); + if !prefix.is_empty() { + prefix.push('/'); + } + let file_name = event + .id + .as_deref() + .map(|id| logging_id(&start, id)) + .unwrap_or_default(); + let path = settings.path.clone().unwrap_or_default(); + let key_for = |granularity| object_key(&path, &prefix, &start, &file_name, granularity); + let cold_storage_object_key = match (event.cold_storage_s3_v2, &event.metadata) { + (true, MetadataKind::Other) => return Err("metadata has no get".to_owned()), + (true, MetadataKind::Mapping) => event.cold_storage_object_key, + _ => None, + }; + let key = match cold_storage_object_key { + Some(cold) + if cold == key_for(Granularity::Day) || cold == key_for(Granularity::Hour) => + { + cold + } + _ => key_for(self.partition_granularity()), + }; + let id = event + .id + .ok_or_else(|| "the payload id is missing or not a string".to_owned())?; + Ok(Element { + object: Object { + key, + body, + content_type: JSON, + download_name: download_name(&start, &id), + }, + retrying_since: None, + is_file: false, + }) + } + + fn event(&self, fields: &FieldValues) -> Result { + let identity = fields + .get::()? + .unwrap_or_default(); + let metadata = identity.metadata_kind.unwrap_or(MetadataKind::Missing); + let cold_storage_object_key = fields.get::()?; + let reads_cold_storage = match metadata { + MetadataKind::Other => true, + MetadataKind::Mapping => cold_storage_object_key.is_some(), + MetadataKind::Missing | MetadataKind::Null => false, + }; + let cold_storage_s3_v2 = reads_cold_storage && self.cold_storage_is_s3_v2; + Ok(Event { + start: fields + .get::()? + .and_then(|timing| timing.start) + .as_ref() + .map(Stamp::from), + id: fields.get::()?, + metadata, + team_alias: Alias::of(&identity, "team_alias", identity.team_alias.as_ref()), + key_alias: Alias::of(&identity, "key_alias", identity.key_alias.as_ref()), + cold_storage_object_key, + cold_storage_s3_v2, + }) + } + + pub async fn submit(&self, fields: &FieldValues) -> Result<(), FieldError> { + let Some(payload) = fields.payload()? else { + return Ok(()); + }; + let event = self.event(fields)?; + let options = if self.log_prompts_only() { + RenderOptions { + null_keys: PROMPTS_ONLY_NULL_KEYS.map(str::to_owned).to_vec(), + } + } else { + RenderOptions::default() + }; + match self.element(event, self.hold(payload, options)) { + Ok(element) => { + self.push(element); + } + Err(error) => self.failed(&error), + } + Ok(()) + } + + fn failed(&self, error: &str) { + tracing::error!("s3 Layer Error - {error}"); + self.host.callback_failed(); + } + + pub async fn submit_audit(&self, payload: PayloadRef, id: &str) -> Option { + let body = self.hold(Arc::new(payload), RenderOptions::default()); + let now = Stamp::utc((self.uploader.clock)()); + let key = object_key( + self.settings().path.as_deref().unwrap_or_default(), + "audit_logs/", + &now, + &format!("{}_{id}", now.clock()), + self.partition_granularity(), + ); + let queued = self.push(Element { + object: Object { + key, + body, + content_type: JSON, + download_name: format!("audit-{id}.json"), + }, + retrying_since: None, + is_file: false, + }); + if queued >= self.settings().batch_size { + return self.flush().await; + } + None + } + + pub async fn upload_now(&self, key: String, body: String, download_name: String) -> bool { + let body = Bytes::from(body); + let object = Object { + key, + body: Body::Ready(body.clone()), + content_type: NDJSON, + download_name, + }; + let outcome = self + .uploader + .upload(&object, &body, &self.limiter, false) + .await; + if outcome != Outcome::Delivered { + self.host.callback_failed(); + } + outcome == Outcome::Delivered + } + + pub async fn download(&self, key: &str) -> Option { + self.uploader.download(key).await + } + + pub async fn flush(&self) -> Option { + let _flushing = self.flush_lock.lock().await; + let (batch, requeued) = { + let queue = self.queue(); + if queue.items.is_empty() { + return None; + } + (queue.items.clone(), queue.requeued) + }; + let length = batch.len(); + let failure = match std::panic::AssertUnwindSafe(self.send_batch(batch, requeued)) + .catch_unwind() + .await + { + Ok(Ok(())) => { + self.queue().items.drain(..length); + return None; + } + Ok(Err(failure)) => failure, + Err(panic) => { + tracing::error!( + "CustomLogger: async_send_batch raised; preserving {length} events in queue for retry: {}", + panic_message(panic.as_ref()) + ); + None + } + }; + let max_queue_size = self.settings().max_queue_size; + let mut queue = self.queue(); + let overflow = queue.items.len().saturating_sub(max_queue_size); + if overflow > 0 { + queue.items.drain(..overflow); + tracing::warn!( + "CustomLogger: log queue exceeded max_queue_size={max_queue_size}; dropped {overflow} oldest events." + ); + } + failure + } + + fn batch_file_mode(&self) -> bool { + if !self.settings().batch_file_upload { + return false; + } + if self.cold_storage_is_s3_v2 { + tracing::warn!( + "s3 logging: s3_batch_file_upload is ignored because s3_v2 is the cold storage logger; per-request objects are required for spend log lookups" + ); + return false; + } + true + } + + async fn batch_files(&self, batch: &[Element]) -> Result, String> { + let now = Stamp::utc((self.uploader.clock)()); + let mut groups: BTreeMap<&str, Vec<&Element>> = BTreeMap::new(); + for element in batch.iter().filter(|element| !element.is_file) { + groups + .entry(parent(&element.object.key)) + .or_default() + .push(element); + } + let bodies: Vec<&Body> = groups + .values() + .flatten() + .map(|element| &element.object.body) + .collect(); + let mut rendered = render_bodies(&bodies).await?.into_iter(); + let mut files = Vec::with_capacity(groups.len()); + for (folder, elements) in groups { + let name = format!("batch_{}_{}", now.clock(), uuid::Uuid::new_v4().simple()); + let mut body = Vec::new(); + for index in 0..elements.len() { + if index > 0 { + body.push(b'\n'); + } + body.extend_from_slice(&rendered.next().unwrap_or_default()); + } + files.push(Element { + object: Object { + key: if folder.is_empty() { + format!("{name}.jsonl") + } else { + format!("{folder}/{name}.jsonl") + }, + body: Body::Ready(Bytes::from(body)), + content_type: NDJSON, + download_name: format!("{name}.jsonl"), + }, + retrying_since: elements + .iter() + .filter_map(|element| element.retrying_since) + .min(), + is_file: true, + }); + } + Ok(batch + .iter() + .filter(|element| element.is_file) + .cloned() + .chain(files) + .collect()) + } + + async fn upload_bounded(&self, element: &Element) -> Outcome { + let _permit = self.limiter.acquire().await; + let outcome = match element.object.body.render().await { + Ok(body) => { + self.uploader + .upload(&element.object, &body, &self.limiter, true) + .await + } + Err(error) => { + tracing::error!("Error uploading to s3: {error}"); + Outcome::Retry + } + }; + if outcome != Outcome::Delivered { + self.host.callback_failed(); + } + outcome + } + + async fn send_batch( + &self, + batch: Vec, + requeued_count: usize, + ) -> Result<(), Option> { + let uploads = if self.batch_file_mode() { + match self.batch_files(&batch).await { + Ok(files) => files, + Err(error) => { + tracing::error!( + "CustomLogger: async_send_batch raised; preserving {} events in queue for retry: {error}", + batch.len() + ); + return Err(None); + } + } + } else { + batch.clone() + }; + self.uploader.retries.store(0, Ordering::Relaxed); + let stale = if uploads.len() == batch.len() { + requeued_count.min(uploads.len()) + } else { + 0 + }; + let order: Vec = (stale..uploads.len()).chain(0..stale).collect(); + let finished = join_all( + order + .iter() + .map(|index| self.upload_bounded(&uploads[*index])), + ) + .await; + let mut outcomes = vec![Outcome::Retry; uploads.len()]; + for (index, outcome) in order.into_iter().zip(finished) { + outcomes[index] = outcome; + } + let retries = self.uploader.retries.load(Ordering::Relaxed); + if retries > 0 { + tracing::warn!( + "s3 logging: {retries} in-call retries across {} uploads this flush", + uploads.len() + ); + } + let delivered = outcomes + .iter() + .filter(|outcome| **outcome == Outcome::Delivered) + .count(); + let failed: Vec<(&Element, Outcome)> = uploads + .iter() + .zip(outcomes) + .filter(|(_, outcome)| *outcome != Outcome::Delivered) + .collect(); + let now = Instant::now(); + let max_age = self.settings().max_retry_age; + let requeued: Vec = if delivered == 0 { + failed + .iter() + .map(|(element, _)| (*element).clone()) + .collect() + } else { + failed + .iter() + .filter(|(element, outcome)| { + *outcome != Outcome::Dropped + && !matches!( + (max_age, element.retrying_since), + (Some(max_age), Some(since)) if now.duration_since(since) > max_age + ) + }) + .map(|(element, _)| { + let mut element = (*element).clone(); + if element.retrying_since.is_none() && max_age.is_some() { + element.retrying_since = Some(now); + } + element + }) + .collect() + }; + let dropped = failed.len() - requeued.len(); + if dropped > 0 { + let terminal = failed + .iter() + .filter(|(_, outcome)| *outcome == Outcome::Dropped) + .count(); + tracing::warn!( + "s3 logging: {dropped} uploads dropped this flush: {terminal} rejected as terminal by S3, {} retrying longer than s3_max_retry_age_seconds={}", + dropped - terminal, + max_age.map_or_else(|| "None".to_owned(), |age| age.as_secs().to_string()) + ); + } + let mut queue = self.queue(); + if requeued.is_empty() { + queue.requeued = 0; + return Ok(()); + } + let max_queue_size = self.settings().max_queue_size; + let split = batch.len().min(queue.items.len()); + let arrivals = queue.items.split_off(split); + let overflow = (requeued.len() + arrivals.len()).saturating_sub(max_queue_size); + if overflow > 0 { + tracing::warn!( + "s3 logging: queue exceeded max_queue_size={max_queue_size} after a failed flush, dropped {overflow} oldest events" + ); + } + let requeued_length = requeued.len(); + queue.items = requeued + .into_iter() + .chain(arrivals) + .skip(overflow) + .collect(); + queue.requeued = requeued_length.saturating_sub(overflow); + Err(Some(BatchUploadFailure { + queued: batch.len(), + failed: failed.len(), + total: uploads.len(), + })) + } +} + +impl Callback for S3Log { + fn manifest(&self) -> &Manifest { + &self.manifest + } + + type State = (); + + fn handle<'a>( + &'a self, + input: &'a HookInput, + _state: Option<&'a ()>, + ) -> BoxFuture<'a, HookOutput> { + Box::pin(async move { + if input.hook == Hook::AttemptEnded { + self.logger.instrument(self.submit(&input.fields)).await?; + } + Ok(None) + }) + } + + fn flush(&self) -> BoxFuture<'_, ()> { + Box::pin(self.logger.instrument(S3Log::flush(self)).map(drop)) + } +} + +pub fn system_clock() -> SystemTime { + SystemTime::now() +} diff --git a/litellm/rust/crates/callbacks-s3-log/src/upload.rs b/litellm/rust/crates/callbacks-s3-log/src/upload.rs new file mode 100644 index 00000000000..b5e4f0aa39f --- /dev/null +++ b/litellm/rust/crates/callbacks-s3-log/src/upload.rs @@ -0,0 +1,470 @@ +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, SystemTime}, +}; + +use base64::Engine; +use bytes::Bytes; +use futures_util::FutureExt; +use litellm_auth_aws::{AwsAuthService, Credentials}; +use litellm_callbacks::{PayloadRef, RenderOptions, render_all}; +use litellm_s3::signed::{Reply, SendError, SignedRequest, send}; +use reqwest::Method; + +use crate::{ + key::{hex_sha256, object_url}, + limiter::Limiter, + settings::{Environment, Settings}, +}; + +const ATTEMPTS: u32 = 3; +const RETRYABLE: [u16; 4] = [403, 429, 500, 503]; +const TERMINAL_CODES: [&str; 7] = [ + "EntityTooLarge", + "InvalidArgument", + "MalformedXML", + "InvalidDigest", + "KeyTooLongError", + "BadDigest", + "InvalidRequest", +]; +const CACHE_CONTROL: &str = "private, immutable, max-age=31536000, s-maxage=0"; +const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Outcome { + Delivered, + Retry, + Dropped, +} + +pub struct Held { + payload: Arc, + options: RenderOptions, + live: Arc, +} + +impl Held { + pub fn new(payload: Arc, options: RenderOptions, live: Arc) -> Self { + live.fetch_add(1, Ordering::Relaxed); + Self { + payload, + options, + live, + } + } +} + +impl std::fmt::Debug for Held { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("Held") + .field("payload", &self.payload) + .field("options", &self.options) + .finish() + } +} + +impl Drop for Held { + fn drop(&mut self) { + self.live.fetch_sub(1, Ordering::Relaxed); + } +} + +#[derive(Clone, Debug)] +pub enum Body { + Ready(Bytes), + Payload(Arc), +} + +#[derive(Clone, Debug)] +pub struct Object { + pub key: String, + pub body: Body, + pub content_type: &'static str, + pub download_name: String, +} + +pub fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { + panic + .downcast_ref::<&str>() + .map(|message| (*message).to_owned()) + .or_else(|| panic.downcast_ref::().cloned()) + .unwrap_or_else(|| "a panic without a message".to_owned()) +} + +pub async fn render_bodies(bodies: &[&Body]) -> Result, String> { + let held: Vec<&Held> = bodies + .iter() + .filter_map(|body| match body { + Body::Payload(held) => Some(held.as_ref()), + Body::Ready(_) => None, + }) + .collect(); + let mut groups: Vec<(&RenderOptions, Vec)> = Vec::new(); + for (index, item) in held.iter().enumerate() { + match groups + .iter_mut() + .find(|(options, _)| **options == item.options) + { + Some((_, indexes)) => indexes.push(index), + None => groups.push((&item.options, vec![index])), + } + } + let mut rendered: Vec> = vec![None; held.len()]; + for (options, indexes) in groups { + let payloads: Vec<&PayloadRef> = indexes + .iter() + .map(|index| held[*index].payload.as_ref()) + .collect(); + let encoded = std::panic::AssertUnwindSafe(render_all(&payloads, options.clone())) + .catch_unwind() + .await + .map_err(|panic| panic_message(panic.as_ref()))??; + for (index, bytes) in indexes.into_iter().zip(encoded) { + rendered[index] = Some(Bytes::from(bytes)); + } + } + let mut rendered = rendered.into_iter(); + bodies + .iter() + .map(|body| match body { + Body::Ready(bytes) => Ok(bytes.clone()), + Body::Payload(_) => rendered + .next() + .flatten() + .ok_or_else(|| "the host rendered too few payloads".to_owned()), + }) + .collect() +} + +impl Body { + pub async fn render(&self) -> Result { + render_bodies(&[self]) + .await? + .pop() + .ok_or_else(|| "the host rendered no payload".to_owned()) + } +} + +pub type Clock = fn() -> SystemTime; +pub type Backoff = Arc Duration + Send + Sync>; + +pub struct Uploader { + pub settings: Arc, + pub client: litellm_http::Client, + pub download_client: litellm_http::Client, + pub environment_region: Option, + pub auth: AwsAuthService, + pub environment: Arc, + pub clock: Clock, + pub backoff: Backoff, + pub retries: AtomicUsize, +} + +enum Attempt { + Reply(Reply), + Failed(SendError), +} + +pub fn error_code(body: &str) -> Option<&str> { + let mut rest = body; + while let Some(start) = rest.find("") { + let after = &rest[start + "".len()..]; + let code_length = after.find('<').unwrap_or(after.len()); + if code_length > 0 && after[code_length..].starts_with("") { + return Some(&after[..code_length]); + } + rest = after; + } + None +} + +pub fn is_terminal(status: u16, body: &str) -> bool { + matches!(status, 400 | 403) + && error_code(body).is_some_and(|code| TERMINAL_CODES.contains(&code)) +} + +pub fn python_backoff(attempt: u32) -> Duration { + Duration::from_secs_f64(f64::from(1u32 << attempt) * (0.5 + rand::random::())) +} + +fn throttled(attempt: &Attempt) -> bool { + match attempt { + Attempt::Failed(_) => true, + Attempt::Reply(reply) => { + matches!(reply.status, 429 | 503) || error_code(&reply.text()) == Some("SlowDown") + } + } +} + +impl Uploader { + pub fn headers(&self, object: &Object, body: &Bytes) -> Vec<(String, String)> { + let md5 = base64::engine::general_purpose::STANDARD + .encode(::digest(body)); + let mut headers = vec![ + ("Content-Type".to_owned(), object.content_type.to_owned()), + ("Content-MD5".to_owned(), md5), + ("x-amz-content-sha256".to_owned(), hex_sha256(body)), + ("Content-Language".to_owned(), "en".to_owned()), + ( + "Content-Disposition".to_owned(), + format!("inline; filename=\"{}\"", object.download_name), + ), + ("Cache-Control".to_owned(), CACHE_CONTROL.to_owned()), + ]; + if let Some(algorithm) = &self.settings.server_side_encryption { + headers.push(("x-amz-server-side-encryption".to_owned(), algorithm.clone())); + } + if let Some(key_id) = &self.settings.sse_kms_key_id { + headers.push(( + "x-amz-server-side-encryption-aws-kms-key-id".to_owned(), + key_id.clone(), + )); + } + headers + } + + async fn credentials(&self) -> Result { + let environment = Arc::clone(&self.environment); + let lookup = move |name: &str| environment(name); + self.auth + .resolve_credentials(self.settings.auth.clone(), &lookup) + .await + .map_err(|error| error.to_string()) + } + + async fn forget_credentials(&self) { + let environment = Arc::clone(&self.environment); + let lookup = move |name: &str| environment(name); + self.auth + .invalidate_credentials(self.settings.auth.clone(), &lookup) + .await; + } + + async fn attempt( + &self, + url: &str, + headers: &[(String, String)], + body: &Bytes, + ) -> Result { + let credentials = self.credentials().await?; + let region = self + .settings + .signing_region(|| Ok(self.environment_region.clone()))?; + let put = || { + send( + &self.client, + SignedRequest { + method: Method::PUT, + url, + headers, + body: body.clone(), + region: ®ion, + credentials: &credentials, + time: (self.clock)(), + read_timeout: Some(self.settings.attempt_timeout), + }, + ) + }; + let sent = match put().await { + Err(error) if error.is_reconnectable() => put().await, + sent => sent, + }; + match sent { + Ok(reply) => Ok(Attempt::Reply(reply)), + Err(error) if error.is_transient() => Ok(Attempt::Failed(error)), + Err(error) => Err(error.to_string()), + } + } + + fn retry_reason(&self, attempt: &Attempt) -> Option { + match attempt { + Attempt::Failed(error) => Some(error.to_string()), + Attempt::Reply(reply) => { + if !RETRYABLE.contains(&reply.status) { + return None; + } + if self.settings.drop_on_terminal_error && is_terminal(reply.status, &reply.text()) + { + return None; + } + Some(reply.status.to_string()) + } + } + } + + pub async fn upload( + &self, + object: &Object, + body: &Bytes, + limiter: &Limiter, + in_flush: bool, + ) -> Outcome { + let url = object_url(&self.settings, &object.key); + let headers = self.headers(object, body); + let mut attempt_number = 0; + let last = loop { + let attempt = match self.attempt(&url, &headers, body).await { + Ok(attempt) => attempt, + Err(error) => { + tracing::error!("Error uploading to s3: {error}"); + return Outcome::Retry; + } + }; + limiter.record(throttled(&attempt)); + if attempt_number + 1 >= ATTEMPTS { + break attempt; + } + let Some(reason) = self.retry_reason(&attempt) else { + break attempt; + }; + if matches!(&attempt, Attempt::Reply(reply) if reply.status == 403) { + self.forget_credentials().await; + } + let wait = (self.backoff)(attempt_number); + if in_flush { + tracing::debug!( + "S3 upload attempt {}/{ATTEMPTS} failed with {reason}, retrying in {:.2}s key={}", + attempt_number + 1, + wait.as_secs_f64(), + object.key + ); + } else { + tracing::warn!( + "S3 upload attempt {}/{ATTEMPTS} failed with {reason}, retrying in {:.2}s key={}", + attempt_number + 1, + wait.as_secs_f64(), + object.key + ); + } + self.retries.fetch_add(1, Ordering::Relaxed); + tokio::time::sleep(wait).await; + attempt_number += 1; + }; + match last { + Attempt::Reply(reply) if (200..300).contains(&reply.status) => Outcome::Delivered, + Attempt::Reply(reply) => { + let body = reply.text(); + tracing::error!( + "Error uploading to s3: status {} for {}: {}", + reply.status, + object.key, + body.chars().take(512).collect::() + ); + if self.settings.drop_on_terminal_error && is_terminal(reply.status, &body) { + tracing::warn!( + "s3 logging: dropping object {} after terminal status {}", + object.key, + reply.status + ); + return Outcome::Dropped; + } + Outcome::Retry + } + Attempt::Failed(error) => { + tracing::error!("Error uploading to s3: {error}"); + Outcome::Retry + } + } + } + + pub async fn download(&self, key: &str) -> Option { + let credentials = match self.credentials().await { + Ok(credentials) => credentials, + Err(error) => { + tracing::error!("Error downloading from S3: {error}"); + return None; + } + }; + let Some(region) = self + .settings + .region_name + .as_ref() + .map(crate::pyjson::py_str) + else { + tracing::error!("Error downloading from S3: no s3_region_name to sign with"); + return None; + }; + let headers = [("x-amz-content-sha256".to_owned(), EMPTY_SHA256.to_owned())]; + let reply = send( + &self.download_client, + SignedRequest { + method: Method::GET, + url: &object_url(&self.settings, key), + headers: &headers, + body: Bytes::new(), + region: ®ion, + credentials: &credentials, + time: (self.clock)(), + read_timeout: Some(self.settings.download_read_timeout), + }, + ) + .await; + match reply { + Ok(reply) if reply.status == 200 => Some(reply.body), + Ok(reply) => { + tracing::error!("S3 object not found, saw response={}", reply.text()); + None + } + Err(error) => { + tracing::error!("Error downloading from S3: {error}"); + None + } + } + } +} + +#[cfg(test)] +pub fn md5_base64(body: &[u8]) -> String { + base64::engine::general_purpose::STANDARD.encode(::digest(body)) +} + +#[cfg(test)] +pub fn sha256_hex(body: &[u8]) -> String { + crate::key::hex(&::digest(body)) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case(400, "EntityTooLarge", true)] + #[case(403, "BadDigest", true)] + #[case(403, "AccessDenied", false)] + #[case(503, "InvalidRequest", false)] + #[case(400, "bad request", false)] + #[case(400, "InvalidDigest", true)] + fn only_object_rejections_are_terminal( + #[case] status: u16, + #[case] body: &str, + #[case] expected: bool, + ) { + assert_eq!(is_terminal(status, body), expected); + } + + #[rstest] + #[case(0, 0.5, 1.5)] + #[case(1, 1.0, 3.0)] + fn backoff_doubles_with_jitter(#[case] attempt: u32, #[case] low: f64, #[case] high: f64) { + for _ in 0..50 { + let wait = python_backoff(attempt).as_secs_f64(); + assert!((low..high).contains(&wait), "{wait}"); + } + } + + #[rstest] + fn digests_match_python() { + assert_eq!(md5_base64(b"{}"), "mZFLkyvTelC5g8XnyQrpOw=="); + assert_eq!( + sha256_hex(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + } +} diff --git a/litellm/rust/crates/callbacks-s3-log/tests/sink.rs b/litellm/rust/crates/callbacks-s3-log/tests/sink.rs new file mode 100644 index 00000000000..7a2e19d30b3 --- /dev/null +++ b/litellm/rust/crates/callbacks-s3-log/tests/sink.rs @@ -0,0 +1,693 @@ +use std::{ + collections::HashMap, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, AtomicUsize, Ordering}, + }, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use futures_util::future::BoxFuture; +use litellm_callbacks::{ + Callback, Field, FieldSet, FieldValues, Hook, HookInput, LoggedKeyIdentity, MetadataKind, + PayloadHost, PayloadRef, RenderOptions, Rendered, Scalar, +}; +use litellm_callbacks_s3_log::{BatchUploadFailure, Constants, Host, S3Log, S3LogParts, Settings}; +use rstest::rstest; +use serde_json::{Value, json}; +use wiremock::{ + Mock, MockServer, Request, ResponseTemplate, + matchers::{method, path_regex}, +}; + +#[derive(Default)] +struct Recorder { + failures: AtomicUsize, + next: AtomicU64, + payloads: Mutex>>, +} + +const PANICS: &str = "panics while rendering"; + +impl Recorder { + fn stash(self: &Arc, body: Option<&str>) -> PayloadRef { + let id = self.next.fetch_add(1, Ordering::Relaxed); + self.payloads + .lock() + .unwrap() + .insert(id, body.map(str::to_owned)); + PayloadRef::new(id, Arc::clone(self) as Arc) + } + + fn rewrite(&self, id: u64, body: &str) { + self.payloads + .lock() + .unwrap() + .insert(id, Some(body.to_owned())); + } + + fn held(&self) -> usize { + self.payloads.lock().unwrap().len() + } + + fn failures(&self) -> usize { + self.failures.load(Ordering::Relaxed) + } +} + +impl PayloadHost for Recorder { + fn render(&self, payloads: Vec, options: RenderOptions) -> BoxFuture<'static, Rendered> { + let bodies: Vec> = payloads + .iter() + .map(|payload| { + self.payloads + .lock() + .unwrap() + .get(payload) + .cloned() + .flatten() + }) + .collect(); + let rendered = bodies + .into_iter() + .map(|body| match body { + Some(body) if body == PANICS => panic!("dictionary changed size during iteration"), + Some(body) if options.null_keys.is_empty() => Ok(body.into_bytes()), + Some(body) => { + Ok(format!("{body} without {}", options.null_keys.join(",")).into_bytes()) + } + None => Err("the payload cannot be encoded".to_owned()), + }) + .collect(); + Box::pin(std::future::ready(rendered)) + } + + fn release(&self, payload: u64) { + self.payloads.lock().unwrap().remove(&payload); + } +} + +impl Host for Recorder { + fn callback_failed(&self) { + self.failures.fetch_add(1, Ordering::Relaxed); + } + + fn web_identity_token(&self, reference: &str) -> Result { + Ok(reference.to_owned()) + } +} + +fn fixed_clock() -> SystemTime { + UNIX_EPOCH + Duration::from_secs(1_763_024_707) +} + +fn no_env(_: &str) -> Option { + None +} + +fn sink(server: &MockServer, extra: Value) -> (Arc, Arc) { + sink_with(server, extra, Constants::default()) +} + +fn sink_with( + server: &MockServer, + extra: Value, + constants: Constants, +) -> (Arc, Arc) { + sink_from(server, extra, constants, false, None) +} + +fn sink_from( + server: &MockServer, + extra: Value, + constants: Constants, + cold_storage_is_s3_v2: bool, + region_from_environment: Option<&str>, +) -> (Arc, Arc) { + let mut params = json!({ + "s3_bucket_name": "logs", + "s3_region_name": "us-east-1", + "s3_endpoint_url": server.uri(), + "s3_aws_access_key_id": "AKIDEXAMPLE", + "s3_aws_secret_access_key": "secret", + "s3_path": "base", + }); + params + .as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + params + .as_object_mut() + .unwrap() + .retain(|_, value| !value.is_null()); + let host = Arc::new(Recorder::default()); + let sink = S3Log::new(S3LogParts { + settings: Settings::from_params(params.as_object().unwrap(), &constants), + client: litellm_http::Client::for_test(), + download_client: litellm_http::Client::for_test(), + environment: Arc::new(no_env), + clock: fixed_clock, + backoff: Arc::new(|_| Duration::ZERO), + host: Arc::clone(&host) as Arc, + cold_storage_is_s3_v2, + region_from_environment: region_from_environment.map(str::to_owned), + logger: Default::default(), + }); + (sink, host) +} + +const READS: [Field; 6] = [ + Field::DeprecatedKwargs, + Field::StandardLoggingPayloadRef, + Field::LogId, + Field::ColdStorageObjectKey, + Field::LoggedKeyIdentity, + Field::AttemptTiming, +]; + +struct Event { + payload: Option, + id: &'static str, + identity: Value, + cold_storage_object_key: Option<&'static str>, +} + +impl Event { + fn input(self) -> HookInput { + let mut fields = FieldValues::new(FieldSet::of(&READS)) + .with(Field::DeprecatedKwargs, Value::Null) + .with(Field::LogId, json!(self.id)) + .with(Field::LoggedKeyIdentity, self.identity) + .with( + Field::AttemptTiming, + json!({"start": {"year": 2026, "month": 11, "day": 13, "hour": 9, "minute": 5, "second": 7, "microsecond": 42}}), + ); + if let Some(key) = self.cold_storage_object_key { + fields = fields.with(Field::ColdStorageObjectKey, json!(key)); + } + if let Some(payload) = self.payload { + fields.set_payload(payload); + } + HookInput { + hook: Hook::AttemptEnded, + fields, + call: None, + } + } +} + +fn event(host: &Arc, id: &'static str, body: &str) -> Event { + event_with(host.stash(Some(body)), id) +} + +fn event_with(payload: PayloadRef, id: &'static str) -> Event { + Event { + payload: Some(payload), + id, + identity: json!({"metadata_kind": "mapping", "team_alias": "team-a"}), + cold_storage_object_key: None, + } +} + +async fn submit(sink: &S3Log, event: Event) { + sink.handle(&event.input(), None).await.unwrap(); +} + +fn header(request: &Request, name: &str) -> Option { + request + .headers + .get(name) + .map(|value| value.to_str().unwrap().to_owned()) +} + +#[rstest] +#[tokio::test] +async fn a_flush_puts_one_object_per_event_with_the_python_headers() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink( + &server, + json!({"s3_use_team_prefix": true, "s3_sse_kms_key_id": "kms-1"}), + ); + submit( + &sink, + event(&host, "chatcmpl-1", "{\"id\": \"chatcmpl-1\"}"), + ) + .await; + let failure = sink.flush().await; + + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + let put = &requests[0]; + assert_eq!( + put.url.path(), + "/logs/base/team-a/2026-11-13/time-09-05-07-000042_chatcmpl-1.json" + ); + assert_eq!(put.body, b"{\"id\": \"chatcmpl-1\"}"); + assert_eq!( + header(put, "content-type").as_deref(), + Some("application/json") + ); + assert_eq!( + header(put, "content-md5").as_deref(), + Some("Ma6yKrRUR7fU10HikmP0wA==") + ); + assert_eq!( + header(put, "content-disposition").as_deref(), + Some("inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"") + ); + assert_eq!( + header(put, "x-amz-server-side-encryption").as_deref(), + Some("aws:kms") + ); + assert_eq!( + header(put, "x-amz-date").as_deref(), + Some("20251113T090507Z") + ); + assert!( + header(put, "authorization").unwrap().starts_with( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20251113/us-east-1/s3/aws4_request" + ) + ); + assert!(sink.queued_keys().is_empty()); + assert_eq!(host.failures.load(Ordering::Relaxed), 0); + assert_eq!(failure, None); +} + +#[rstest] +#[tokio::test] +async fn a_bucket_that_fails_every_upload_keeps_the_queue() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let (sink, host) = sink(&server, json!({})); + submit(&sink, event(&host, "a", "{}")).await; + submit(&sink, event(&host, "b", "{}")).await; + let failure = sink.flush().await; + + assert_eq!(server.received_requests().await.unwrap().len(), 6); + assert_eq!(sink.queued_keys().len(), 2); + assert_eq!(host.failures.load(Ordering::Relaxed), 2); + assert_eq!( + failure, + Some(BatchUploadFailure { + queued: 2, + failed: 2, + total: 2 + }) + ); +} + +#[rstest] +#[tokio::test] +async fn a_terminal_rejection_is_dropped_when_another_upload_lands() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .and(path_regex("_bad\\.json$")) + .respond_with( + ResponseTemplate::new(400) + .set_body_string("EntityTooLarge"), + ) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(path_regex("_slow\\.json$")) + .respond_with(ResponseTemplate::new(500)) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink(&server, json!({})); + for id in ["ok", "bad", "slow"] { + submit(&sink, event(&host, id, "{}")).await; + } + sink.flush().await; + + assert_eq!( + sink.queued_keys(), + ["base/2026-11-13/time-09-05-07-000042_slow.json"] + ); + assert_eq!(host.held(), 1); +} + +#[rstest] +#[tokio::test] +async fn batch_files_group_one_flush_by_folder() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink(&server, json!({"s3_batch_file_upload": "yes"})); + submit(&sink, event(&host, "a", "{\"n\": 1}")).await; + submit(&sink, event(&host, "b", "{\"n\": 2}")).await; + sink.flush().await; + + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert!( + requests[0] + .url + .path() + .starts_with("/logs/base/2026-11-13/batch_09-05-07_") + ); + assert_eq!(requests[0].body, b"{\"n\": 1}\n{\"n\": 2}"); + assert_eq!( + header(&requests[0], "content-type").as_deref(), + Some("application/x-ndjson") + ); +} + +#[rstest] +#[tokio::test] +async fn an_event_python_would_reject_is_counted_and_skipped() { + let server = MockServer::start().await; + let (sink, host) = sink(&server, json!({"s3_use_key_prefix": true})); + let mut bad = event(&host, "a", "{}"); + bad.identity["key_alias"] = json!(true); + submit(&sink, bad).await; + submit( + &sink, + Event { + payload: None, + id: "b", + identity: json!({"metadata_kind": "mapping"}), + cold_storage_object_key: None, + }, + ) + .await; + + assert!(sink.queued_keys().is_empty()); + assert_eq!(host.failures(), 1); + assert_eq!(host.held(), 0); +} + +#[rstest] +#[tokio::test] +async fn the_body_is_encoded_at_flush_and_released_once_delivered() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink(&server, json!({})); + let payload = host.stash(Some("{\"n\": 1}")); + let handle = payload.handle(); + submit(&sink, event_with(payload, "a")).await; + host.rewrite(handle, "{\"n\": 2}"); + sink.flush().await; + + assert_eq!( + server.received_requests().await.unwrap()[0].body, + b"{\"n\": 2}" + ); + assert_eq!(host.held(), 0); +} + +#[rstest] +#[tokio::test] +async fn a_body_that_cannot_be_encoded_stays_queued_and_counts_a_failure() { + let server = MockServer::start().await; + let (sink, host) = sink(&server, json!({})); + submit(&sink, event_with(host.stash(None), "a")).await; + sink.flush().await; + + assert!(server.received_requests().await.unwrap().is_empty()); + assert_eq!(sink.queued_keys().len(), 1); + assert_eq!(host.failures(), 1); +} + +#[rstest] +#[case::environment(Some("ap-south-1"), Some("/ap-south-1/"))] +#[case::default(None, Some("/us-west-2/"))] +#[case::invalid(Some("Ap_South"), None)] +#[tokio::test] +async fn without_a_region_param_the_constructor_region_signs( + #[case] region: Option<&str>, + #[case] scope: Option<&str>, +) { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink_from( + &server, + json!({"s3_region_name": null, "s3_aws_session_token": "session"}), + Constants::default(), + false, + region, + ); + submit(&sink, event(&host, "a", "{}")).await; + sink.flush().await; + + let requests = server.received_requests().await.unwrap(); + let signed = requests + .first() + .and_then(|put| header(put, "authorization")) + .map(|authorization| scope.is_some_and(|scope| authorization.contains(scope))); + assert_eq!(signed, scope.map(|_| true)); + assert_eq!(sink.queued_keys().len(), usize::from(scope.is_none())); +} + +#[rstest] +#[tokio::test] +async fn a_reply_slower_than_the_attempt_timeout_is_retried_then_kept() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200).set_delay(Duration::from_millis(600))) + .mount(&server) + .await; + let (sink, host) = sink_with( + &server, + json!({}), + Constants { + upload_attempt_timeout_seconds: 0.1, + ..Constants::default() + }, + ); + submit(&sink, event(&host, "a", "{}")).await; + sink.flush().await; + + assert_eq!(server.received_requests().await.unwrap().len(), 3); + assert_eq!(sink.queued_keys().len(), 1); + assert_eq!(host.failures(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_panic_while_rendering_keeps_that_object_and_the_flush_goes_on() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink(&server, json!({})); + let racing = host.stash(Some(PANICS)); + let handle = racing.handle(); + submit(&sink, event_with(racing, "a")).await; + submit(&sink, event(&host, "b", "{}")).await; + sink.flush().await; + + let paths: Vec = server + .received_requests() + .await + .unwrap() + .iter() + .map(|put| put.url.path().to_owned()) + .collect(); + assert_eq!(paths, ["/logs/base/2026-11-13/time-09-05-07-000042_b.json"]); + assert_eq!( + sink.queued_keys(), + ["base/2026-11-13/time-09-05-07-000042_a.json"] + ); + assert_eq!(host.failures(), 1); + + host.rewrite(handle, "{}"); + sink.flush().await; + assert!(sink.queued_keys().is_empty()); + assert_eq!(server.received_requests().await.unwrap().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn a_panic_while_rendering_a_batch_file_keeps_the_whole_queue() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink(&server, json!({"s3_batch_file_upload": true})); + let racing = host.stash(Some(PANICS)); + let handle = racing.handle(); + submit(&sink, event_with(racing, "a")).await; + submit(&sink, event(&host, "b", "{}")).await; + sink.flush().await; + + assert!(server.received_requests().await.unwrap().is_empty()); + assert_eq!(sink.queued_keys().len(), 2); + + host.rewrite(handle, "{}"); + sink.flush().await; + assert!(sink.queued_keys().is_empty()); + assert_eq!(server.received_requests().await.unwrap().len(), 1); +} + +#[rstest] +#[case::bool_alias(json!({"metadata_kind": "mapping", "team_alias": true}))] +#[case::int_alias(json!({"metadata_kind": "mapping", "team_alias": 7}))] +#[case::nan_alias(serde_json::to_value(LoggedKeyIdentity { + metadata_kind: Some(MetadataKind::Mapping), + team_alias: Some(Scalar::Float(f64::NAN)), + ..LoggedKeyIdentity::default() +}).unwrap())] +#[case::list_alias(json!({"metadata_kind": "mapping", "team_alias": {"text": "['team-a']", "truthy": true, "hashable": false, "integer": false, "type_name": "list", "hash_error": "unhashable type: 'list'"}}))] +#[case::alias_whose_str_raises(json!({"metadata_kind": "mapping", "rejected": ["team_alias"]}))] +#[case::null_metadata(json!({"metadata_kind": "null"}))] +#[case::metadata_without_get(json!({"metadata_kind": "other"}))] +#[tokio::test] +async fn a_team_prefix_python_cannot_build_is_counted_and_released(#[case] identity: Value) { + let server = MockServer::start().await; + let (sink, host) = sink(&server, json!({"s3_use_team_prefix": true})); + let mut bad = event(&host, "a", "{}"); + bad.identity = identity; + submit(&sink, bad).await; + + assert!(sink.queued_keys().is_empty()); + assert_eq!(host.failures(), 1); + assert_eq!(host.held(), 0); +} + +#[rstest] +#[case::empty_text(json!(""))] +#[case::zero(json!(0))] +#[case::zero_float(json!(0.0))] +#[case::false_flag(json!(false))] +#[case::null(Value::Null)] +#[case::empty_list(json!({"text": "[]", "truthy": false, "hashable": false, "integer": false, "type_name": "list", "hash_error": "unhashable type: 'list'"}))] +#[tokio::test] +async fn a_falsy_team_alias_adds_no_prefix(#[case] alias: Value) { + let server = MockServer::start().await; + let (sink, host) = sink(&server, json!({"s3_use_team_prefix": true})); + let mut falsy = event(&host, "a", "{}"); + falsy.identity = json!({"metadata_kind": "mapping", "team_alias": alias}); + submit(&sink, falsy).await; + + assert_eq!( + sink.queued_keys(), + ["base/2026-11-13/time-09-05-07-000042_a.json"] + ); + assert_eq!(host.failures(), 0); +} + +#[rstest] +#[case::prompts_only(json!({"s3_log_prompts_only": true}), "{} without response")] +#[case::whole_payload(json!({}), "{}")] +#[tokio::test] +async fn prompts_only_renders_the_response_as_null(#[case] extra: Value, #[case] body: &str) { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink(&server, extra); + submit(&sink, event(&host, "a", "{}")).await; + sink.flush().await; + + assert_eq!( + server.received_requests().await.unwrap()[0].body, + body.as_bytes() + ); +} + +#[rstest] +#[tokio::test] +async fn a_region_param_wins_over_the_constructor_region() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink_from( + &server, + json!({}), + Constants::default(), + false, + Some("ap-south-1"), + ); + submit(&sink, event(&host, "a", "{}")).await; + sink.flush().await; + + let requests = server.received_requests().await.unwrap(); + assert!( + header(&requests[0], "authorization") + .unwrap() + .contains("/us-east-1/") + ); +} + +const HOUR_KEY: &str = "base/2026-11-13/09/time-09-05-07-000042_a.json"; +const DAY_KEY: &str = "base/2026-11-13/time-09-05-07-000042_a.json"; + +#[rstest] +#[case::s3_v2_reuses_its_hour_key(true, json!({"metadata_kind": "mapping"}), Some(HOUR_KEY))] +#[case::another_owner_keeps_the_day_key(false, json!({"metadata_kind": "mapping"}), Some(DAY_KEY))] +#[case::s3_v2_rejects_metadata_without_get(true, json!({"metadata_kind": "other"}), None)] +#[case::another_owner_ignores_metadata_without_get(false, json!({"metadata_kind": "other"}), Some(DAY_KEY))] +#[tokio::test] +async fn the_constructor_cold_storage_owner_decides_the_cold_key( + #[case] cold_storage_is_s3_v2: bool, + #[case] identity: Value, + #[case] key: Option<&str>, +) { + let server = MockServer::start().await; + let (sink, host) = sink_from( + &server, + json!({}), + Constants::default(), + cold_storage_is_s3_v2, + None, + ); + let mut cold = event(&host, "a", "{}"); + cold.identity = identity; + cold.cold_storage_object_key = Some(HOUR_KEY); + submit(&sink, cold).await; + + assert_eq!(sink.queued_keys(), key.into_iter().collect::>()); + assert_eq!(host.failures(), usize::from(key.is_none())); +} + +#[rstest] +#[case::s3_v2_owner(true, "/logs/base/2026-11-13/time-09-05-07-000042_a.json")] +#[case::another_owner(false, "/logs/base/2026-11-13/batch_09-05-07_")] +#[tokio::test] +async fn the_constructor_cold_storage_owner_turns_batch_files_off( + #[case] cold_storage_is_s3_v2: bool, + #[case] path: &str, +) { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (sink, host) = sink_from( + &server, + json!({"s3_batch_file_upload": true}), + Constants::default(), + cold_storage_is_s3_v2, + None, + ); + submit(&sink, event(&host, "a", "{}")).await; + sink.flush().await; + + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].url.path().starts_with(path)); +} diff --git a/litellm/rust/crates/python-bridge/Cargo.toml b/litellm/rust/crates/python-bridge/Cargo.toml index 618a27fc4ee..4a52308cef1 100644 --- a/litellm/rust/crates/python-bridge/Cargo.toml +++ b/litellm/rust/crates/python-bridge/Cargo.toml @@ -46,6 +46,9 @@ litellm-auth-aws.workspace = true litellm-auth-azure.workspace = true azure_core = "1.0.0" litellm-callbacks-legacy-python.workspace = true +litellm-callbacks.workspace = true +litellm-callbacks-python.workspace = true +litellm-callbacks-s3-log.workspace = true litellm-inference.workspace = true litellm-inference-transcription.workspace = true litellm-inference-responses.workspace = true diff --git a/litellm/rust/crates/python-bridge/src/lib.rs b/litellm/rust/crates/python-bridge/src/lib.rs index 304421bb5d6..c4b8a7f5e0c 100644 --- a/litellm/rust/crates/python-bridge/src/lib.rs +++ b/litellm/rust/crates/python-bridge/src/lib.rs @@ -16,6 +16,7 @@ mod python_settings; mod redis; mod router; mod routes; +mod s3_log; mod secrets; mod tokenizer; @@ -97,6 +98,8 @@ mod _native { use crate::routes::token_counter::TokenCounter; #[pymodule_export] use crate::routes::traces::trace_encode_error; + #[pymodule_export] + use crate::s3_log::NativeS3Log; #[cfg(feature = "huggingface")] #[pymodule_export] use crate::tokenizer::HuggingFaceEncoding; @@ -202,6 +205,7 @@ mod tests { "process_state_started", "reserve_process_for_forking", "unsupported_tls_settings", + "NativeS3Log", ]; #[cfg(feature = "huggingface")] expected.push("HuggingFaceEncoding"); diff --git a/litellm/rust/crates/python-bridge/src/s3_log.rs b/litellm/rust/crates/python-bridge/src/s3_log.rs new file mode 100644 index 00000000000..bf89974be32 --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/s3_log.rs @@ -0,0 +1,225 @@ +use std::{sync::Arc, time::Duration}; + +use litellm_callbacks::ErasedCallback; +use litellm_callbacks_python::{RustCallback, payload}; +use litellm_callbacks_s3_log::{ + BatchUploadFailure, Constants, Host, S3Log, S3LogParts, Settings, python_backoff, system_clock, +}; +use litellm_host_python::{from_py, json_from_py}; +use litellm_http::ClientVariant; +use pyo3::{ + prelude::*, + types::{PyBool, PyDict, PyInt, PyString}, +}; +use serde_json::{Map, Value}; + +struct PythonHost { + host: Arc>, +} + +impl PythonHost { + fn ask( + &self, + method: &str, + argument: Option<&str>, + read: impl for<'py> FnOnce(&Bound<'py, PyAny>) -> PyResult, + ) -> Result { + Python::attach(|py| { + let answer = match argument { + Some(argument) => self.host.call_method1(py, method, (argument,)), + None => self.host.call_method0(py, method), + }; + answer + .and_then(|answer| read(answer.bind(py))) + .map_err(|error| error.to_string()) + }) + } +} + +impl Host for PythonHost { + fn callback_failed(&self) { + let host = Arc::clone(&self.host); + let report = move || { + Python::attach(|py| { + if let Err(error) = host.call_method0(py, "callback_failed") { + error.print(py); + } + }); + }; + match tokio::runtime::Handle::try_current() { + Ok(runtime) => drop(runtime.spawn_blocking(report)), + Err(_) => report(), + } + } + + fn web_identity_token(&self, reference: &str) -> Result { + self.ask("web_identity_token", Some(reference), |answer| { + answer.extract::() + }) + } +} + +type Failure = Option<(usize, usize, usize)>; + +fn failure(failure: Option) -> Failure { + failure.map(|failure| (failure.queued, failure.failed, failure.total)) +} + +#[pyclass(frozen, name = "NativeS3Log")] +pub struct NativeS3Log { + sink: Arc, +} + +fn client( + py: Python<'_>, + params: &Bound<'_, PyDict>, + connect_timeout_seconds: f64, +) -> PyResult { + let kwargs = PyDict::new(py); + kwargs.set_item("ssl_verify", params.get_item("s3_verify")?)?; + let mut config = crate::http::call_config(py, &kwargs, true)?; + config.connect_timeout = Duration::from_secs_f64(connect_timeout_seconds.max(0.0)); + crate::http::pool() + .client(&config, ClientVariant::Provider) + .map_err(crate::http::client_error) +} + +fn settings_param(value: &Bound<'_, PyAny>) -> Value { + if let Ok(integer) = value.cast::() + && !value.is_instance_of::() + && integer.extract::().is_err() + && integer.extract::().is_err() + { + return value + .str() + .map_or(Value::Null, |text| Value::String(text.to_string())); + } + json_from_py(value).unwrap_or(Value::Null) +} + +fn settings_params(params: &Bound<'_, PyDict>) -> Map { + params + .iter() + .filter_map(|(key, value)| { + let key = key.cast_into::().ok()?; + Some((key.to_str().ok()?.to_owned(), settings_param(&value))) + }) + .collect() +} + +#[pymethods] +impl NativeS3Log { + #[new] + #[pyo3(signature = (params, constants, host, cold_storage_is_s3_v2, region_from_environment))] + fn new( + py: Python<'_>, + params: &Bound<'_, PyDict>, + constants: &Bound<'_, PyAny>, + host: Py, + cold_storage_is_s3_v2: bool, + region_from_environment: Option, + ) -> PyResult { + let resolved = settings_params(params); + let constants: Constants = from_py(constants)?; + let logger = crate::logger::capture(py); + let settings = logger.scope(|| Settings::from_params(&resolved, &constants)); + let host = Arc::new(PythonHost { + host: Arc::new(host), + }); + let sink = S3Log::new(S3LogParts { + settings, + client: client(py, params, constants.upload_attempt_timeout_seconds)?, + download_client: client(py, params, constants.download_connect_timeout_seconds)?, + environment: Arc::new(|name: &str| std::env::var(name).ok()), + clock: system_clock, + backoff: Arc::new(python_backoff), + host: host as Arc, + cold_storage_is_s3_v2, + region_from_environment, + logger, + }); + Ok(Self { sink }) + } + + fn callback(&self) -> PyResult { + Ok(RustCallback::new( + Arc::clone(&self.sink) as Arc + )?) + } + + fn stashed(&self) -> usize { + self.sink.held_payloads() + } + + fn partition_granularity(&self) -> &'static str { + self.sink.partition_granularity().as_str() + } + + #[getter] + fn s3_path(&self) -> Option { + self.sink.settings().path.clone() + } + + #[getter] + fn s3_bucket_name(&self) -> Option { + self.sink.settings().bucket_name.clone() + } + + #[getter] + fn strip_base64_files(&self) -> bool { + self.sink.settings().strip_base64_files + } + + fn queued_keys(&self) -> Vec { + self.sink.queued_keys() + } + + fn submit_audit<'py>( + &self, + py: Python<'py>, + audit_log: Py, + id: String, + ) -> PyResult> { + let sink = Arc::clone(&self.sink); + let stashed = payload::stash(audit_log); + let event_loop = payload::running_loop(py); + crate::execution::run_async_value( + py, + payload::on_render_loop(event_loop, async move { + Ok(failure(sink.submit_audit(stashed, &id).await)) + }), + ) + } + + fn upload_line_items<'py>( + &self, + py: Python<'py>, + key: String, + body: String, + download_name: String, + ) -> PyResult> { + let sink = Arc::clone(&self.sink); + crate::execution::run_async_value(py, async move { + Ok(sink.upload_now(key, body, download_name).await) + }) + } + + fn download<'py>(&self, py: Python<'py>, key: String) -> PyResult> { + let sink = Arc::clone(&self.sink); + crate::execution::run_async_value(py, async move { + Ok(sink + .download(&key) + .await + .map(|body| String::from_utf8_lossy(&body).into_owned())) + }) + } + + fn flush<'py>(&self, py: Python<'py>) -> PyResult> { + let sink = Arc::clone(&self.sink); + let event_loop = payload::running_loop(py); + crate::execution::run_async_value( + py, + payload::on_render_loop(event_loop, async move { Ok(failure(sink.flush().await)) }), + ) + } +} diff --git a/litellm/rust/crates/python-compat/generated/values.json b/litellm/rust/crates/python-compat/generated/values.json index 7f7e1e35a7d..024377dac76 100644 --- a/litellm/rust/crates/python-compat/generated/values.json +++ b/litellm/rust/crates/python-compat/generated/values.json @@ -1,5 +1,5 @@ { - "python": "3.14.7", + "python": "3.14.6", "rows": [ { "name": "None", @@ -495,6 +495,25 @@ }, "view": "1e+22" }, + { + "name": "0.07872390747070312", + "source": "0.07872390747070312", + "literal": true, + "plain": true, + "repr": "0.07872390747070312", + "str": "0.07872390747070312", + "truthy": true, + "json": "0.07872390747070312", + "pickle": { + "0": "46302e30373837323339303734373037303331320a2e", + "1": "473fb42740000000002e", + "2": "8002473fb42740000000002e", + "3": "8003473fb42740000000002e", + "4": "8004950a00000000000000473fb42740000000002e", + "5": "8005950a00000000000000473fb42740000000002e" + }, + "view": "0.07872390747070312" + }, { "name": "float('inf')", "source": "float('inf')", diff --git a/litellm/rust/crates/python-compat/scripts/generate_fixtures.py b/litellm/rust/crates/python-compat/scripts/generate_fixtures.py index 26456a638f6..e9ce26f5be3 100644 --- a/litellm/rust/crates/python-compat/scripts/generate_fixtures.py +++ b/litellm/rust/crates/python-compat/scripts/generate_fixtures.py @@ -47,6 +47,7 @@ CORPUS = [ "5e-324", "1.7976931348623157e308", "1e22", + "0.07872390747070312", "float('inf')", "float('-inf')", "float('nan')", diff --git a/litellm/rust/crates/python-compat/src/repr.rs b/litellm/rust/crates/python-compat/src/repr.rs index 1cf20d87716..407184cf2e1 100644 --- a/litellm/rust/crates/python-compat/src/repr.rs +++ b/litellm/rust/crates/python-compat/src/repr.rs @@ -113,8 +113,14 @@ fn format_float(value: f64, add_dot_0: bool) -> String { if value.is_infinite() { return if value > 0.0 { "inf" } else { "-inf" }.to_owned(); } - // Rust's `{:e}` prints the shortest round-trip digits, like CPython's 'r' mode. - let scientific = format!("{value:e}"); + let shortest = format!("{value:e}"); + let length = shortest + .split_once('e') + .map_or(shortest.as_str(), |(mantissa, _)| mantissa) + .chars() + .filter(char::is_ascii_digit) + .count(); + let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1)); let (mantissa, exponent) = scientific .split_once('e') .expect("`{:e}` always prints an exponent"); diff --git a/litellm/rust/crates/s3/Cargo.toml b/litellm/rust/crates/s3/Cargo.toml index 4f071d45dc6..b6f84d8ef64 100644 --- a/litellm/rust/crates/s3/Cargo.toml +++ b/litellm/rust/crates/s3/Cargo.toml @@ -6,7 +6,10 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-auth-aws.workspace = true litellm-http.workspace = true +hyper = "1.12.0" +bytes.workspace = true aws-sdk-s3.workspace = true aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] } aws-types = "1.4.0" @@ -16,6 +19,7 @@ aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] } http.workspace = true reqwest.workspace = true thiserror.workspace = true +tokio = { workspace = true, features = ["time"] } [dev-dependencies] rstest.workspace = true diff --git a/litellm/rust/crates/s3/src/lib.rs b/litellm/rust/crates/s3/src/lib.rs index be595d2bef4..8b45e7807f0 100644 --- a/litellm/rust/crates/s3/src/lib.rs +++ b/litellm/rust/crates/s3/src/lib.rs @@ -1,4 +1,5 @@ mod error; +pub mod signed; mod transport; mod uri; diff --git a/litellm/rust/crates/s3/src/signed.rs b/litellm/rust/crates/s3/src/signed.rs new file mode 100644 index 00000000000..f44a812a81c --- /dev/null +++ b/litellm/rust/crates/s3/src/signed.rs @@ -0,0 +1,137 @@ +use std::time::{Duration, SystemTime}; + +use bytes::Bytes; +use litellm_auth_aws::{Credentials, SigningScope, sign_s3_headers}; +use reqwest::Method; + +pub struct SignedRequest<'a> { + pub method: Method, + pub url: &'a str, + pub headers: &'a [(String, String)], + pub body: Bytes, + pub region: &'a str, + pub credentials: &'a Credentials, + pub time: SystemTime, + pub read_timeout: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Reply { + pub status: u16, + pub body: Bytes, +} + +impl Reply { + pub fn text(&self) -> String { + String::from_utf8_lossy(&self.body).into_owned() + } +} + +#[derive(Debug, thiserror::Error)] +pub enum SendError { + #[error("the request cannot be signed: {0}")] + Signing(#[from] litellm_auth_aws::Error), + #[error("the request was not sent: {0}")] + Request(String), + #[error("{kind}: {message}")] + Transport { kind: &'static str, message: String }, +} + +impl SendError { + pub fn is_transient(&self) -> bool { + matches!(self, Self::Transport { .. }) + } + + pub fn is_reconnectable(&self) -> bool { + matches!( + self, + Self::Transport { + kind: "ConnectError" | "RemoteProtocolError", + .. + } + ) + } +} + +pub fn signature_headers(request: &SignedRequest<'_>) -> Result, SendError> { + Ok(sign_s3_headers( + request.method.as_str(), + request.url, + &request.body, + request.headers, + SigningScope { + region: request.region, + service: "s3", + credentials: request.credentials, + time: request.time, + }, + )?) +} + +pub async fn send( + client: &litellm_http::Client, + request: SignedRequest<'_>, +) -> Result { + let signature = signature_headers(&request)?; + let mut builder = client.request(request.method.clone(), request.url); + for (name, value) in request.headers.iter().chain(&signature) { + builder = builder.header(name, value); + } + if !request.body.is_empty() || request.method == Method::PUT { + builder = builder.body(request.body.clone()); + } + let mut response = timed(request.read_timeout, builder.send()).await?; + let status = response.status().as_u16(); + let mut body = Vec::new(); + while let Some(chunk) = timed(request.read_timeout, response.chunk()).await? { + body.extend_from_slice(&chunk); + } + Ok(Reply { + status, + body: Bytes::from(body), + }) +} + +async fn timed( + limit: Option, + pending: impl Future>, +) -> Result { + let finished = match limit { + Some(limit) => tokio::time::timeout(limit, pending) + .await + .map_err(|elapsed| SendError::Transport { + kind: "ReadTimeout", + message: elapsed.to_string(), + })?, + None => pending.await, + }; + finished.map_err(transport) +} + +fn transport(error: reqwest::Error) -> SendError { + let message = error.to_string(); + if error.is_builder() || error.is_redirect() { + return SendError::Request(message); + } + let kind = if error.is_timeout() { + "Timeout" + } else if error.is_connect() { + "ConnectError" + } else if remote_protocol(&error) { + "RemoteProtocolError" + } else { + "TransportError" + }; + SendError::Transport { kind, message } +} + +fn remote_protocol(error: &reqwest::Error) -> bool { + let mut cause = std::error::Error::source(error); + while let Some(current) = cause { + if let Some(hyper) = current.downcast_ref::() { + return hyper.is_incomplete_message() || hyper.is_parse(); + } + cause = current.source(); + } + false +} diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 86fac69ba83..e03b88d7d4d 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -6,6 +6,7 @@ import httpx from pydantic import JsonValue from typing_extensions import disjoint_base +from litellm.integrations.custom_logger_v2.rust import RustHandle from litellm.redis.store_settings import RedisBreakerSettings, RedisStoreSettings from litellm.redis.stores.router_session_pins.keys import PinKey from litellm.rust_bridge.execution import Execution @@ -119,6 +120,35 @@ class NativeModelCostMap: @final class NativeModelCostMapState: ... +class _S3LogHost(Protocol): + def callback_failed(self) -> None: ... + def web_identity_token(self, reference: str) -> str: ... + +@final +class NativeS3Log: + def __new__( + cls, + params: dict[str, object], + constants: Mapping[str, object], + host: _S3LogHost, + cold_storage_is_s3_v2: bool, + region_from_environment: str | None, + ) -> NativeS3Log: ... + @property + def s3_path(self) -> str | None: ... + @property + def s3_bucket_name(self) -> str | None: ... + @property + def strip_base64_files(self) -> bool: ... + def callback(self) -> RustHandle: ... + def stashed(self) -> int: ... + def partition_granularity(self) -> Literal["day", "hour"]: ... + def queued_keys(self) -> list[str]: ... + def submit_audit(self, audit_log: dict[str, object], id: str) -> Future[tuple[int, int, int] | None]: ... + def upload_line_items(self, key: str, body: str, download_name: str) -> Future[bool]: ... + def download(self, key: str) -> Future[str | None]: ... + def flush(self) -> Future[tuple[int, int, int] | None]: ... + @final class NativeClickHouseSpendConfig: def __new__(cls, database: str, url: str, retention_days: int) -> NativeClickHouseSpendConfig: ... @@ -786,6 +816,7 @@ __all__ = [ "NativeRouterCooldowns", "NativeRouterSessionPins", "NativeRouterUsage", + "NativeS3Log", "NativeSend", "NativeSensitiveRoutePins", "NativeSpendCounters", diff --git a/litellm/types/integrations/s3_v2.py b/litellm/types/integrations/s3_v2.py index c0f719aacfb..61f63e6c869 100644 --- a/litellm/types/integrations/s3_v2.py +++ b/litellm/types/integrations/s3_v2.py @@ -1,18 +1,41 @@ from typing import Literal -from litellm.types.llms.base import LiteLLMBaseModel +from typing_extensions import ReadOnly, TypedDict S3PartitionGranularity = Literal["day", "hour"] -class s3BatchLoggingElement(LiteLLMBaseModel): - """ - Type of element stored in self.log_queue in S3Logger - """ - - payload: dict - s3_object_key: str - s3_object_download_filename: str - body: str | None = None - content_type: str = "application/json" - retrying_since: float | None = None +class S3LoggerSettings(TypedDict, total=False): + s3_bucket_name: ReadOnly[str | None] + s3_path: ReadOnly[str | None] + s3_region_name: ReadOnly[str | None] + s3_api_version: ReadOnly[str | None] + s3_use_ssl: ReadOnly[bool] + s3_verify: ReadOnly[bool | str | None] + s3_endpoint_url: ReadOnly[str | None] + s3_aws_access_key_id: ReadOnly[str | None] + s3_aws_secret_access_key: ReadOnly[str | None] + s3_aws_session_token: ReadOnly[str | None] + s3_aws_session_name: ReadOnly[str | None] + s3_aws_profile_name: ReadOnly[str | None] + s3_aws_role_name: ReadOnly[str | None] + s3_aws_web_identity_token: ReadOnly[str | None] + s3_aws_sts_endpoint: ReadOnly[str | None] + s3_flush_interval: ReadOnly[int | None] + s3_batch_size: ReadOnly[int | None] + s3_config: ReadOnly[object] + s3_use_team_prefix: ReadOnly[bool] + s3_strip_base64_files: ReadOnly[bool] + s3_use_key_prefix: ReadOnly[bool] + s3_use_virtual_hosted_style: ReadOnly[bool] + s3_server_side_encryption: ReadOnly[str | None] + s3_sse_kms_key_id: ReadOnly[str | None] + s3_log_prompts_only: ReadOnly[bool | None] + s3_partition_granularity: ReadOnly[str | None] + s3_max_concurrent_uploads: ReadOnly[int] + s3_max_queue_size: ReadOnly[int | None] + s3_max_retry_age_seconds: ReadOnly[int | None] + s3_drop_on_terminal_error: ReadOnly[bool] + s3_adaptive_concurrency: ReadOnly[bool] + s3_max_adaptive_concurrency: ReadOnly[int | None] + s3_batch_file_upload: ReadOnly[bool] diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index 69cad8eb22e..6eebb3c5173 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -1,7 +1,7 @@ # Logging integration delivery (behavior features). Grounded in litellm/integrations/. -- {id: logging.s3.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/s3_v2.py", rationale: "Primary audit trail; batch flush no-drop"} -- {id: logging.s3.failure.writes_object, module: logging, tier: P0, event: failure, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/s3_v2.py", rationale: "Failed calls persisted for compliance"} -- {id: logging.s3.success.partition_layout, module: logging, tier: P1, event: success, assertions: [object_key_layout], exercised_on: [chat_completions], source: "integrations/s3_v2.py / LIT-8985", rationale: "s3_partition_granularity picks the date or date/hour folder every downstream query and lifecycle rule reads"} +- {id: logging.s3.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/s3_v3.py", rationale: "Primary audit trail; batch flush no-drop"} +- {id: logging.s3.failure.writes_object, module: logging, tier: P0, event: failure, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/s3_v3.py", rationale: "Failed calls persisted for compliance"} +- {id: logging.s3.success.partition_layout, module: logging, tier: P1, event: success, assertions: [object_key_layout], exercised_on: [chat_completions], source: "integrations/s3_v3.py / LIT-8985", rationale: "s3_partition_granularity picks the date or date/hour folder every downstream query and lifecycle rule reads"} - {id: logging.gcs_bucket.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/gcs_bucket/gcs_bucket.py", rationale: "GCS parallel to S3"} - {id: logging.datadog.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/datadog/datadog.py", rationale: "Powers dashboards/alerts; cardinality regressions common"} - {id: logging.datadog.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/datadog/datadog.py", rationale: "Streaming aggregates usage after the last chunk; delivery and cost must survive that path"} diff --git a/tests/integration/observability/test_s3_v2_flush_surfaces.py b/tests/integration/observability/test_s3_v2_flush_surfaces_rust.py similarity index 95% rename from tests/integration/observability/test_s3_v2_flush_surfaces.py rename to tests/integration/observability/test_s3_v2_flush_surfaces_rust.py index 5a94eaa248f..2e1d61fe5b9 100644 --- a/tests/integration/observability/test_s3_v2_flush_surfaces.py +++ b/tests/integration/observability/test_s3_v2_flush_surfaces_rust.py @@ -21,6 +21,7 @@ from integration._support.process import owned_proxy from integration._support.wire import wire_server from integration.caching.response_cache_case import entry_keys, redis_store + pytestmark: Final = pytest.mark.shares_worker_state PER_REQUEST_KEY: Final = re.compile(rf"^/{BUCKET}/{PREFIX}/\d{{4}}-\d{{2}}-\d{{2}}/.+\.json$") @@ -29,7 +30,6 @@ BATCH_KEY: Final = re.compile( ) -@pytest.mark.covers("other.observability.s3_v2.mixed_surface_burst_bounds_puts_one_object_per_response_id") def test_s3_v2_mixed_surface_burst_bounds_puts_one_object_per_response_id(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3mix" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink() @@ -54,7 +54,6 @@ def test_s3_v2_mixed_surface_burst_bounds_puts_one_object_per_response_id(gatewa assert matched_ids(payloads, answered) -@pytest.mark.covers("other.observability.s3_v2.mixed_surface_batch_writes_ndjson_lines_per_response_id") def test_s3_v2_mixed_surface_batch_writes_ndjson_lines_per_response_id(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3mixb" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink() @@ -80,7 +79,6 @@ def test_s3_v2_mixed_surface_batch_writes_ndjson_lines_per_response_id(gateway: assert len(payloads) == 48 -@pytest.mark.covers("other.observability.s3_v2.sink_outage_mid_mixed_burst_recovers_every_response_id") def test_s3_v2_sink_outage_mid_mixed_burst_recovers_every_response_id(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3mixo" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink(fail_attempts=30, fail_status=503, delay_seconds=0.2) diff --git a/tests/integration/observability/test_s3_v2_partition_granularity.py b/tests/integration/observability/test_s3_v2_partition_granularity_rust.py similarity index 99% rename from tests/integration/observability/test_s3_v2_partition_granularity.py rename to tests/integration/observability/test_s3_v2_partition_granularity_rust.py index d2b05fd50a3..a6bbf315997 100644 --- a/tests/integration/observability/test_s3_v2_partition_granularity.py +++ b/tests/integration/observability/test_s3_v2_partition_granularity_rust.py @@ -33,6 +33,7 @@ from integration._support.database_relay import database_relay from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process from integration._support.wire import Reply, Request, wire_server + pytestmark: Final = pytest.mark.shares_worker_state FLUSH: Final = {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "1"} diff --git a/tests/integration/observability/test_s3_v2_streaming_failure_dedupe.py b/tests/integration/observability/test_s3_v2_streaming_failure_dedupe_rust.py similarity index 96% rename from tests/integration/observability/test_s3_v2_streaming_failure_dedupe.py rename to tests/integration/observability/test_s3_v2_streaming_failure_dedupe_rust.py index d35a644ea3d..1631ad5222d 100644 --- a/tests/integration/observability/test_s3_v2_streaming_failure_dedupe.py +++ b/tests/integration/observability/test_s3_v2_streaming_failure_dedupe_rust.py @@ -14,6 +14,7 @@ from integration._support.client import Gateway, JsonValue, Scenario, eventually from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, wire_server + Surface = Literal["chat", "messages", "responses"] SURFACES: Final[tuple[Surface, ...]] = ("chat", "messages", "responses") FAILURE_BODY: Final[bytes] = json.dumps( @@ -77,9 +78,7 @@ def _responses_stream_id(response: httpx.Response) -> str: if line.startswith("data: {") ) completed: Final = tuple( - object_value(event["response"]) - for event in events - if event.get("type") == "response.completed" + object_value(event["response"]) for event in events if event.get("type") == "response.completed" ) assert len(completed) == 1, events response_id: Final = completed[0]["id"] @@ -137,13 +136,11 @@ def _assert_one_payload( return_last_on_timeout=True, ) assert len(payloads) == 1, ( - f"expected one {status} payload after {upstream_posts} upstream POSTs, " - f"observed {len(payloads)} payloads" + f"expected one {status} payload after {upstream_posts} upstream POSTs, observed {len(payloads)} payloads" ) - assert ( - _request_key(payloads[0], response, marker) == response.headers.get("x-litellm-call-id") - or marker in json.dumps(payloads[0]) - ), f"payload did not match request id or marker: {payloads[0]!r}" + assert _request_key(payloads[0], response, marker) == response.headers.get( + "x-litellm-call-id" + ) or marker in json.dumps(payloads[0]), f"payload did not match request id or marker: {payloads[0]!r}" return payloads @@ -337,9 +334,7 @@ def test_failure_burst_through_sink_outage_lands_each_request_once( with ThreadPoolExecutor(max_workers=len(jobs)) as pool: responses: Final = tuple(pool.map(call, jobs)) - upstream_requests: Final = tuple( - request for request in upstream.drain() if request.method == "POST" - ) + upstream_requests: Final = tuple(request for request in upstream.drain() if request.method == "POST") route_counts: Final = Counter(request.target for request in upstream_requests) expected_route_counts: Final = { _surface_target(surface): 2 * requests_per_variant * 3 for surface in SURFACES @@ -358,8 +353,7 @@ def test_failure_burst_through_sink_outage_lands_each_request_once( rejected: Final = sink.attempts - len(sink.store) assert rejected >= 1, f"expected at least one rejected sink upload, rejected={rejected}" assert len(sink.store) == 0, ( - f"expected all sink uploads to fail before recovery, rejected={rejected}, " - f"attempts={sink.attempts}" + f"expected all sink uploads to fail before recovery, rejected={rejected}, attempts={sink.attempts}" ) sink.fail_until = 0.0 payloads: Final = eventually( diff --git a/tests/integration/observability/test_s3_v2_upload_fanout.py b/tests/integration/observability/test_s3_v2_upload_fanout_rust.py similarity index 97% rename from tests/integration/observability/test_s3_v2_upload_fanout.py rename to tests/integration/observability/test_s3_v2_upload_fanout_rust.py index 863aefeef0e..6834f459246 100644 --- a/tests/integration/observability/test_s3_v2_upload_fanout.py +++ b/tests/integration/observability/test_s3_v2_upload_fanout_rust.py @@ -20,6 +20,7 @@ from integration._support.process import owned_proxy, owned_proxy_process from integration._support.timing import assert_enforced_within from integration._support.wire import Reply, Request, Wire, wire_server + BUCKET: Final = "integration-bucket" PREFIX: Final = "integration-logs" REQUESTS: Final = 64 @@ -121,7 +122,6 @@ BATCH_KEY: Final = re.compile( ) -@pytest.mark.covers("other.observability.s3_v2.flush_bounds_concurrent_puts_to_default_ceiling_and_keeps_every_log") def test_s3_v2_flush_bounds_concurrent_puts_to_the_default_ceiling(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3fan" + uuid.uuid4().hex[:8] sink: Final = S3Sink() @@ -144,7 +144,6 @@ def test_s3_v2_flush_bounds_concurrent_puts_to_the_default_ceiling(gateway: Gate assert len({put.target for put in puts}) == REQUESTS -@pytest.mark.covers("other.observability.s3_v2.configured_bound_and_env_backed_false_keeps_per_request_objects") def test_s3_v2_honors_configured_bound_and_env_backed_false_batch_flag(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3cap" + uuid.uuid4().hex[:8] sink: Final = S3Sink() @@ -173,7 +172,6 @@ def test_s3_v2_honors_configured_bound_and_env_backed_false_batch_flag(gateway: assert frozenset(json.loads(put.body)["id"] for put in puts) == ids -@pytest.mark.covers("other.observability.s3_v2.batch_file_upload_writes_one_ndjson_object_per_flush") def test_s3_v2_batch_file_upload_writes_one_jsonl_object_per_flush(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3jsonl" + uuid.uuid4().hex[:8] sink: Final = S3Sink() @@ -196,7 +194,6 @@ def test_s3_v2_batch_file_upload_writes_one_jsonl_object_per_flush(gateway: Gate assert len(lines) == REQUESTS -@pytest.mark.covers("other.observability.s3_v2.batch_file_upload_keeps_team_prefix_in_object_key") def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3team" + uuid.uuid4().hex[:8] team_alias: Final = f"alpha-{uuid.uuid4().hex[:8]}" @@ -223,7 +220,6 @@ def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_p assert len(lines) == REQUESTS -@pytest.mark.covers("other.observability.s3_v2.upstream_failure_events_land_alongside_successes") def test_s3_v2_upstream_failure_events_land_alongside_successes(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3fail" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink() @@ -276,7 +272,6 @@ def test_s3_v2_upstream_failure_events_land_alongside_successes(gateway: Gateway assert all("synthetic upstream rejection" in json.dumps(payload["error_information"]) for payload in failures) -@pytest.mark.covers("other.observability.s3_v2.invalid_or_empty_bound_falls_back_to_default_ceiling") @pytest.mark.parametrize( ("bad", "warns"), [ @@ -313,7 +308,6 @@ def test_s3_v2_invalid_or_empty_bound_falls_back_to_default_ceiling( assert frozenset(payload["id"] for payload in payloads) == ids -@pytest.mark.covers("other.observability.s3_v2.sink_rejection_requeues_and_delivers_every_id_once") def test_s3_v2_sink_rejection_requeues_and_delivers_every_id_once(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3deny" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink(fail_status=503, delay_seconds=0.2) @@ -632,7 +626,6 @@ def test_s3_v2_default_config_retries_access_denied_and_every_event_lands(gatewa ) -@pytest.mark.covers("other.observability.s3_v2.batch_retry_resends_identical_key_and_body") def test_s3_v2_batch_retry_resends_identical_key_and_body(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3retry" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink(fail_status=500, delay_seconds=0.2) @@ -658,7 +651,6 @@ def test_s3_v2_batch_retry_resends_identical_key_and_body(gateway: Gateway, tmp_ assert len(payloads) == REQUESTS -@pytest.mark.covers("other.observability.s3_v2.unknown_model_rejection_keeps_other_requests_logging") def test_s3_v2_unknown_model_rejection_keeps_other_requests_logging(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3ghost" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink() @@ -690,7 +682,6 @@ def test_s3_v2_unknown_model_rejection_keeps_other_requests_logging(gateway: Gat assert all(payload["status"] == "failure" for payload in extras), extras -@pytest.mark.covers("other.observability.s3_v2.batch_flag_ignored_when_s3_v2_is_cold_storage_logger") def test_s3_v2_batch_flag_ignored_when_s3_v2_is_cold_storage_logger(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3cold" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink() @@ -734,7 +725,6 @@ def test_s3_v2_batch_flag_ignored_when_s3_v2_is_cold_storage_logger(gateway: Gat assert payloads[0]["id"] == request_id -@pytest.mark.covers("other.observability.s3_v2.identical_requests_land_distinct_objects") def test_s3_v2_identical_requests_land_distinct_objects(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3same" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink() @@ -766,7 +756,6 @@ def test_s3_v2_identical_requests_land_distinct_objects(gateway: Gateway, tmp_pa assert all(payload["id"] == marker for payload in payloads) -@pytest.mark.covers("other.observability.s3_v2.two_workers_bound_and_deliver_every_id") def test_s3_v2_two_workers_bound_and_deliver_every_id(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3work" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink() @@ -788,7 +777,6 @@ def test_s3_v2_two_workers_bound_and_deliver_every_id(gateway: Gateway, tmp_path assert frozenset(payload["id"] for payload in payloads) == ids -@pytest.mark.covers("other.observability.s3_v2.slow_sink_never_duplicates_or_stalls_readiness") def test_s3_v2_slow_sink_never_duplicates_or_stalls_readiness(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3slow" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink(delay_seconds=1.5) @@ -817,7 +805,6 @@ def test_s3_v2_slow_sink_never_duplicates_or_stalls_readiness(gateway: Gateway, assert len(payloads) == REQUESTS -@pytest.mark.covers("other.observability.s3_v2.worker_kill_mid_burst_keeps_surviving_deliveries") def test_s3_v2_worker_kill_mid_burst_keeps_surviving_deliveries(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3kill" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink() @@ -874,7 +861,6 @@ def test_s3_v2_worker_kill_mid_burst_keeps_surviving_deliveries(gateway: Gateway assert landed <= frozenset(burst) | frozenset(after), "an id that was never sent landed" -@pytest.mark.covers("other.observability.s3_v2.sigterm_mid_burst_loses_only_inflight_without_duplicates") def test_s3_v2_sigterm_mid_burst_loses_only_inflight_without_duplicates(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "s3term" + uuid.uuid4().hex[:8] sink: Final = RecordingS3Sink() diff --git a/tests/integration/providers/test_s3_wire.py b/tests/integration/providers/test_s3_wire.py index 5d9691ab811..02d67d355fd 100644 --- a/tests/integration/providers/test_s3_wire.py +++ b/tests/integration/providers/test_s3_wire.py @@ -37,14 +37,26 @@ def test_sigv4_verifier_matches_published_put_and_rejects_corruption() -> None: @pytest.mark.covers("other.provider_wire.s3.sync_async_reserved_keys_are_signed_and_accepted") -async def test_s3_sync_and_async_uploads_pass_independent_wire_verification(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_s3_sync_and_async_uploads_pass_independent_wire_verification() -> None: from litellm.integrations.s3_v2 import S3Logger - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement + from litellm.types.utils import StandardBatchLineItem, StandardBatchLineItemMetadata, StandardBatchLineItemsPayload - monkeypatch.setattr("botocore.auth.get_current_datetime", lambda: datetime(2026, 9, 14)) - payload: Final = {"id": "synthetic-event", "content": "synthetic snow 雪"} + metadata: Final = StandardBatchLineItemMetadata( + user_api_key_hash=None, user_api_key_alias=None, user_api_key_team_id=None, + user_api_key_team_alias=None, user_api_key_user_id=None, user_api_key_org_id=None, + ) + lines: Final = ( + StandardBatchLineItem( + batch_id="synthetic-batch", custom_id="synthetic-event", status="success", + request={"content": "synthetic snow 雪"}, response=None, usage=None, response_cost=0.0, + model="synthetic-model", custom_llm_provider=None, metadata=metadata, + ), + ) expected_path = "" + def scope_of(request: Request) -> str: + return f"{request.headers.get('x-amz-date', '')[:8]}/us-east-1/s3/aws4_request" + def verify(request: Request) -> Reply: if request.method != "PUT" or request.target != expected_path: return Reply(status=403) @@ -53,8 +65,8 @@ async def test_s3_sync_and_async_uploads_pass_independent_wire_verification(monk assert authorization.startswith("AWS4-HMAC-SHA256 ") fields: Final = dict(part.split("=", 1) for part in authorization.removeprefix("AWS4-HMAC-SHA256 ").split(", ")) access, scope = fields["Credential"].split("/", 1) - assert access == ACCESS and scope == "20260914/us-east-1/s3/aws4_request" - assert request.headers["x-amz-date"] == "20260914T000000Z" + assert access == ACCESS and scope == scope_of(request) + assert datetime.strptime(request.headers["x-amz-date"], "%Y%m%dT%H%M%SZ") signed: Final = fields["SignedHeaders"].split(";") assert signed == sorted(set(signed)) assert {"host", "content-md5", "x-amz-date"}.issubset(signed) @@ -69,40 +81,37 @@ async def test_s3_sync_and_async_uploads_pass_independent_wire_verification(monk with wire_server(verify) as wire: prior: Final = asyncio.all_tasks() logger: Final = S3Logger(s3_bucket_name="integration-bucket", s3_region_name="us-east-1", s3_endpoint_url=wire.url, - s3_aws_access_key_id=ACCESS, s3_aws_secret_access_key=SECRET, s3_callback_params_override={}) - owned: Final = {task for task in asyncio.all_tasks() - prior if task.get_coro().__qualname__ == "CustomBatchLogger.periodic_flush"} - assert len(owned) == 1 + s3_aws_access_key_id=ACCESS, s3_aws_secret_access_key=SECRET, s3_callback_params_override={}, + s3_path="a=b+c/d e/雪") + owned: Final = asyncio.all_tasks() - prior + assert owned == {logger.periodic_flush_task} try: - for mode in ("sync", "async"): - for key in ("plain.json", "a=b+c/d e/雪.json", "percent%2Fplus+.json"): - expected_path = encoded_path(f"/integration-bucket/{key}") - element: Final = s3BatchLoggingElement(payload=payload, s3_object_key=key, s3_object_download_filename="event.json") - if mode == "sync": - await asyncio.to_thread(logger.upload_data_to_s3, element) - else: - await logger.async_upload_data_to_s3(element) - requests: Final = wire.drain() - assert len(requests) == 1, "Upload must be accepted on its first actual PUT" - request: Final = requests[0] - assert request.target == expected_path - assert json.loads(request.body) == payload - assert verify(request).status == 200 - with httpx.Client(timeout=5, trust_env=False) as client: - corrupt: Final = {**request.headers, "authorization": request.headers["authorization"][:-1] + ("0" if request.headers["authorization"][-1] != "0" else "1")} - assert client.put(wire.url + expected_path, content=request.body, headers=corrupt).status_code == 403 - assert client.put(wire.url + expected_path + "-wrong", content=request.body, headers=request.headers).status_code == 403 - assert client.put(wire.url + expected_path, content=request.body + b" ", headers={name: value for name, value in request.headers.items() if name != "content-length"}).status_code == 403 - fields: Final = dict(part.split("=", 1) for part in request.headers["authorization"].removeprefix("AWS4-HMAC-SHA256 ").split(", ")) - for signed, scope, md5 in ( - (fields["SignedHeaders"].replace("host;", ""), "20260914/us-east-1/s3/aws4_request", request.headers["content-md5"]), - (fields["SignedHeaders"], "20260914/us-west-2/s3/aws4_request", request.headers["content-md5"]), - (fields["SignedHeaders"], "20260914/us-east-1/s3/aws4_request", "AAAAAAAAAAAAAAAAAAAAAA=="), - ): - candidate_headers: Final = {**request.headers, "content-md5": md5} - digest: Final = signature("PUT", request.target, candidate_headers, signed, request.body, SECRET, scope)[1] - candidate_headers["authorization"] = f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, SignedHeaders={signed}, Signature={digest}" - assert client.put(wire.url + expected_path, content=request.body, headers=candidate_headers).status_code == 403 - assert len(wire.drain()) == 6 + for batch_id in ("plain", "a=b+c d e", "percent%2Fplus+"): + expected_path = encoded_path(f"/integration-bucket/a=b+c/d e/雪/batch_line_items/2026-09-14/{batch_id}/part-00001.jsonl") + await logger.async_log_batch_line_items(StandardBatchLineItemsPayload(batch_id=batch_id, finished_at=1789344000, part=1, lines=lines)) + requests: Final = wire.drain() + assert len(requests) == 1, "Upload must be accepted on its first actual PUT" + request: Final = requests[0] + assert request.target == expected_path + assert tuple(json.loads(line) for line in request.body.decode().splitlines()) == lines + assert verify(request).status == 200 + day: Final = scope_of(request).split("/", 1)[0] + with httpx.Client(timeout=5, trust_env=False) as client: + corrupt: Final = {**request.headers, "authorization": request.headers["authorization"][:-1] + ("0" if request.headers["authorization"][-1] != "0" else "1")} + assert client.put(wire.url + expected_path, content=request.body, headers=corrupt).status_code == 403 + assert client.put(wire.url + expected_path + "-wrong", content=request.body, headers=request.headers).status_code == 403 + assert client.put(wire.url + expected_path, content=request.body + b" ", headers={name: value for name, value in request.headers.items() if name != "content-length"}).status_code == 403 + fields: Final = dict(part.split("=", 1) for part in request.headers["authorization"].removeprefix("AWS4-HMAC-SHA256 ").split(", ")) + for signed, scope, md5 in ( + (fields["SignedHeaders"].replace("host;", ""), f"{day}/us-east-1/s3/aws4_request", request.headers["content-md5"]), + (fields["SignedHeaders"], f"{day}/us-west-2/s3/aws4_request", request.headers["content-md5"]), + (fields["SignedHeaders"], f"{day}/us-east-1/s3/aws4_request", "AAAAAAAAAAAAAAAAAAAAAA=="), + ): + candidate_headers: Final = {**request.headers, "content-md5": md5} + digest: Final = signature("PUT", request.target, candidate_headers, signed, request.body, SECRET, scope)[1] + candidate_headers["authorization"] = f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, SignedHeaders={signed}, Signature={digest}" + assert client.put(wire.url + expected_path, content=request.body, headers=candidate_headers).status_code == 403 + assert len(wire.drain()) == 6 finally: for task in owned: diff --git a/tests/unit/integrations/s3_v2_golden/__init__.py b/tests/unit/integrations/s3_v2_golden/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/integrations/s3_v2_golden/python_s3_v2.json b/tests/unit/integrations/s3_v2_golden/python_s3_v2.json new file mode 100644 index 00000000000..e443fa4ca57 --- /dev/null +++ b/tests/unit/integrations/s3_v2_golden/python_s3_v2.json @@ -0,0 +1,5945 @@ +{ + "test_a_cold_storage_key_is_reused_when_it_matches[day]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"cold_storage_object_key\": \"cold/2026-11-13/time-09-05-07-000042_chatcmpl-1.json\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "qgQYsgYDpwWlOnazqKk0dA==", + "content-type": "application/json", + "x-amz-content-sha256": "b63f0b729194706ee189be3b44a2cff62396f8dc89c0b02df2f53462b021387c", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/cold/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"chatcmpl-2\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"cold_storage_object_key\": \"elsewhere/x.json\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-2.json\"", + "content-language": "en", + "content-md5": "qFaKqRtnuMe76kO19WvF1Q==", + "content-type": "application/json", + "x-amz-content-sha256": "6661e10cc6895fa064d8f7391c38bd5f82608d16b842189bee02cf8338cc9e8b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/cold/2026-11-13/time-09-05-07-000042_chatcmpl-2.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_cold_storage_key_is_reused_when_it_matches[hour]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"cold_storage_object_key\": \"cold/2026-11-13/09/time-09-05-07-000042_chatcmpl-1.json\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "tL3WbKpQG+1Fp6RxU/1p6g==", + "content-type": "application/json", + "x-amz-content-sha256": "5bca59a39fcfdaf3faffd5e7164d7133af2bff52a8eb1e544c9f0c16017a19d6", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/cold/2026-11-13/09/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"chatcmpl-2\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"cold_storage_object_key\": \"elsewhere/x.json\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-2.json\"", + "content-language": "en", + "content-md5": "qFaKqRtnuMe76kO19WvF1Q==", + "content-type": "application/json", + "x-amz-content-sha256": "6661e10cc6895fa064d8f7391c38bd5f82608d16b842189bee02cf8338cc9e8b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/cold/2026-11-13/time-09-05-07-000042_chatcmpl-2.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_failed_acompletion_reaches_both_loggers_alike::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": false, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 0, + "cost_breakdown": null, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "RateLimitError", + "error_code": "429", + "error_cooldown_deployments": null, + "error_message": "litellm.RateLimitError: this is a mock rate limit error", + "error_provider_request_id": null, + "error_rate_limit_category": "vendor_rate_limit", + "error_rate_limit_type": null, + "llm_provider": "openai", + "normalized_error": "429_RATE_LIMIT_EXCEEDED", + "traceback": " File \"/home/user/wt/rust-s3/litellm/utils.py\", line 2633, in wrapper_async\n result = await original_function(*args, **call_kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 1020, in acompletion\n raise exception_type(\n ...<5 lines>...\n )\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 956, in acompletion\n init_response: Final = await loop.run_in_executor(None, func_with_context)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.13/concurrent/futures/thread.py\", line 59, in run\n result = self.fn(*self.args, **self.kwargs)\n File \"/home/user/wt/rust-s3/litellm/utils.py\", line 2259, in wrapper\n result = original_function(*args, **kwargs)\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 6718, in completion\n raise exception_type(\n ...<5 lines>...\n )\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 6369, in completion\n return mock_completion(\n model,\n ...<13 lines>...\n ),\n )\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 1349, in mock_completion\n raise e\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 1235, in mock_completion\n _handle_mock_potential_exceptions(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n mock_response=mock_response,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n model=model,\n ^^^^^^^^^^^^\n custom_llm_provider=custom_llm_provider,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 1081, in _handle_mock_potential_exceptions\n raise litellm.RateLimitError(\n ...<3 lines>...\n )\n" + }, + "error_str": "litellm.RateLimitError: this is a mock rate limit error", + "guardrail_information": null, + "hidden_params": { + "additional_headers": null, + "api_base": null, + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": null, + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 0.0, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "messages": [ + { + "content": "hello", + "role": "user" + } + ], + "metadata": { + "applied_guardrails": [], + "cold_storage_object_key": null, + "kubernetes_pod_routing": null, + "mcp_tool_call_metadata": null, + "prompt_management_metadata": null, + "requester_custom_headers": null, + "requester_ip_address": null, + "requester_metadata": null, + "routing_decision": null, + "spend_logs_metadata": null, + "team_alias": null, + "team_id": null, + "usage_object": {}, + "used_client_oauth_token": null, + "user_agent": null, + "user_api_key_alias": "key-a", + "user_api_key_auth_metadata": null, + "user_api_key_budget_reset_at": null, + "user_api_key_customer_member_id": null, + "user_api_key_customer_org_id": null, + "user_api_key_customer_team_id": null, + "user_api_key_end_user_id": null, + "user_api_key_hash": null, + "user_api_key_max_budget": null, + "user_api_key_org_alias": null, + "user_api_key_org_id": null, + "user_api_key_project_alias": null, + "user_api_key_project_id": null, + "user_api_key_request_route": null, + "user_api_key_spend": null, + "user_api_key_team_alias": null, + "user_api_key_team_id": null, + "user_api_key_team_max_budget": null, + "user_api_key_team_spend": null, + "user_api_key_user_email": null, + "user_api_key_user_id": null, + "user_api_key_user_max_budget": null, + "user_api_key_user_spend": null, + "vector_store_request_metadata": null + }, + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "", + "model_map_value": null + }, + "model_parameters": {}, + "prompt_tokens": 0, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response": {}, + "response_cost": 0.0, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "failure", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "failure" + }, + "stream": null, + "total_tokens": 0, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "metadata", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "messages", + "response", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs/key-a//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_falsy_alias_of_any_type_adds_no_prefix_like_python[empty-dict]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": {}}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "7sHDt6uJOFtGd8tNM6x1eg==", + "content-type": "application/json", + "x-amz-content-sha256": "5e797ec948da67bace9bfd5630c7d082f85cf57368110a1b74e3a0701ffe5044", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_falsy_alias_of_any_type_adds_no_prefix_like_python[empty-list]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": []}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "aZfExCUfAFFqtLcUfA6nkg==", + "content-type": "application/json", + "x-amz-content-sha256": "23e6c48876d59d6bc1086374a18fa1782f2d04bfb4e56e2ba0c6588f38c4e62d", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_falsy_alias_of_any_type_adds_no_prefix_like_python[empty-set]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": []}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "aZfExCUfAFFqtLcUfA6nkg==", + "content-type": "application/json", + "x-amz-content-sha256": "23e6c48876d59d6bc1086374a18fa1782f2d04bfb4e56e2ba0c6588f38c4e62d", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_falsy_alias_of_any_type_adds_no_prefix_like_python[empty-text]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "TSkyWgUekXTOcZoufH41XQ==", + "content-type": "application/json", + "x-amz-content-sha256": "2d141a391497680f3476c1d7dbf8d794d256364a7d702144201062f49a03d5cc", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_falsy_alias_of_any_type_adds_no_prefix_like_python[falsy-object]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"falsy\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "j/LWey3LukJq3JLvzP86+A==", + "content-type": "application/json", + "x-amz-content-sha256": "c65b375b8789c7f6a42edd98267e994732021c32e17ac8c6ca6c731be7c47916", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_falsy_alias_of_any_type_adds_no_prefix_like_python[zero-float]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": 0.0}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "+s+X90s7WIXlhZbcQ/3ayQ==", + "content-type": "application/json", + "x-amz-content-sha256": "a42195396188cb7a180027135766cca052d3a3fe775437907eda23c42816949e", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_missing_payload_logs_nothing::puts": [], + "test_a_payload_changed_before_the_flush_is_stored_as_it_is_at_the_flush::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\", \"late\": \"written after the event\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "YqaIEKreGpn5GnOBO1hxwA==", + "content-type": "application/json", + "x-amz-content-sha256": "6efb02f17319999e36da39fc47a88884f732ee98c765ef907344edf891085c43", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_payload_that_changes_while_it_is_rendered_is_kept_for_retry_by_both_loggers::failures": 1, + "test_a_payload_that_changes_while_it_is_rendered_is_kept_for_retry_by_both_loggers::puts": [ + { + "body": "{\"id\": \"steady\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_steady.json\"", + "content-language": "en", + "content-md5": "/VoNMW/RPp1bpH/J3ga3fg==", + "content-type": "application/json", + "x-amz-content-sha256": "ed903fa041017af6c171eb545bf0d67799827ed3034b4ec239bf478b9899d0df", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_steady.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_payload_that_changes_while_it_is_rendered_is_kept_for_retry_by_both_loggers::queue": [ + "2026-11-13/time-09-05-07-000042_racing.json" + ], + "test_a_payload_whose_alias_lookup_fails_is_counted_and_released::failures": 1, + "test_a_payload_whose_alias_lookup_fails_is_counted_and_released::puts": [], + "test_a_streamed_acompletion_reaches_both_loggers_alike::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": false, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 4, + "cost_breakdown": { + "data_residency": null, + "discount_amount": 0.0, + "discount_percent": 0.0, + "input_cost": 2e-05, + "margin_fixed_amount": 0.0, + "margin_percent": 0.0, + "margin_total_amount": 0.0, + "original_cost": 6.000000000000001e-05, + "output_cost": 4e-05, + "service_tier": null, + "tool_usage_cost": 0.0, + "total_cost": 6.000000000000001e-05, + "vertex_location": null + }, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "", + "error_code": "", + "error_cooldown_deployments": null, + "error_message": "", + "error_provider_request_id": null, + "error_rate_limit_category": null, + "error_rate_limit_type": null, + "llm_provider": "", + "normalized_error": null, + "traceback": "" + }, + "error_str": null, + "guardrail_information": null, + "hidden_params": { + "additional_headers": {}, + "api_base": "", + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": null, + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 6.000000000000001e-05, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "messages": [ + { + "content": "hello", + "role": "user" + } + ], + "metadata": { + "applied_guardrails": [], + "cold_storage_object_key": null, + "kubernetes_pod_routing": null, + "mcp_tool_call_metadata": null, + "prompt_management_metadata": null, + "requester_custom_headers": null, + "requester_ip_address": null, + "requester_metadata": null, + "routing_decision": null, + "spend_logs_metadata": null, + "team_alias": null, + "team_id": null, + "usage_object": { + "completion_tokens": 4, + "completion_tokens_details": { + "accepted_prediction_tokens": null, + "audio_tokens": null, + "image_tokens": null, + "reasoning_tokens": 0, + "rejected_prediction_tokens": null, + "text_tokens": 4, + "video_tokens": null + }, + "cost": 6.000000000000001e-05, + "prompt_tokens": 8, + "prompt_tokens_details": null, + "total_tokens": 12 + }, + "used_client_oauth_token": null, + "user_agent": null, + "user_api_key_alias": null, + "user_api_key_auth_metadata": null, + "user_api_key_budget_reset_at": null, + "user_api_key_customer_member_id": null, + "user_api_key_customer_org_id": null, + "user_api_key_customer_team_id": null, + "user_api_key_end_user_id": null, + "user_api_key_hash": null, + "user_api_key_max_budget": null, + "user_api_key_org_alias": null, + "user_api_key_org_id": null, + "user_api_key_project_alias": null, + "user_api_key_project_id": null, + "user_api_key_request_route": null, + "user_api_key_spend": null, + "user_api_key_team_alias": null, + "user_api_key_team_id": null, + "user_api_key_team_max_budget": null, + "user_api_key_team_spend": null, + "user_api_key_user_email": null, + "user_api_key_user_id": null, + "user_api_key_user_max_budget": null, + "user_api_key_user_spend": null, + "vector_store_request_metadata": null + }, + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "gpt-4o", + "model_map_value": { + "annotation_cost_per_page": null, + "annotation_cost_per_page_batches": null, + "bedrock_converse_supports_strict_tools": null, + "bedrock_output_config_effort_ceiling": null, + "cache_creation_input_token_cost": null, + "cache_creation_input_token_cost_above_12hr": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_above_12hr_batches": null, + "cache_creation_input_token_cost_above_12hr_flex": null, + "cache_creation_input_token_cost_above_12hr_priority": null, + "cache_creation_input_token_cost_above_12hr_ultrafast": null, + "cache_creation_input_token_cost_above_1hr": null, + "cache_creation_input_token_cost_above_200k_tokens": null, + "cache_creation_input_token_cost_above_200k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens": null, + "cache_creation_input_token_cost_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_batches": null, + "cache_creation_input_token_cost_flex": null, + "cache_creation_input_token_cost_priority": null, + "cache_creation_input_token_cost_ultrafast": null, + "cache_read_input_audio_token_cost": null, + "cache_read_input_audio_token_cost_batches": null, + "cache_read_input_audio_token_cost_flex": null, + "cache_read_input_audio_token_cost_priority": null, + "cache_read_input_image_token_cost": null, + "cache_read_input_image_token_cost_batches": null, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_200k_tokens": null, + "cache_read_input_token_cost_above_200k_tokens_batches": null, + "cache_read_input_token_cost_above_200k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens": null, + "cache_read_input_token_cost_above_272k_tokens_batches": null, + "cache_read_input_token_cost_above_272k_tokens_flex": null, + "cache_read_input_token_cost_above_272k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_read_input_token_cost_above_512k_tokens": null, + "cache_read_input_token_cost_balanced": null, + "cache_read_input_token_cost_batches": null, + "cache_read_input_token_cost_flex": null, + "cache_read_input_token_cost_priority": 2.125e-06, + "cache_read_input_token_cost_ultrafast": null, + "citation_cost_per_token": null, + "cost_per_second": null, + "default_reasoning_effort": null, + "explicit_cache_read_input_token_cost": null, + "fedramp_uplift_multiplier": 1.1, + "google_maps_grounding_cost_per_query": null, + "input_cost_per_audio_per_second": null, + "input_cost_per_audio_token": null, + "input_cost_per_audio_token_above_200k_tokens": null, + "input_cost_per_audio_token_above_200k_tokens_batches": null, + "input_cost_per_audio_token_above_200k_tokens_flex": null, + "input_cost_per_audio_token_above_200k_tokens_priority": null, + "input_cost_per_audio_token_batches": null, + "input_cost_per_audio_token_flex": null, + "input_cost_per_audio_token_priority": null, + "input_cost_per_character": null, + "input_cost_per_image": null, + "input_cost_per_image_token": null, + "input_cost_per_image_token_batches": null, + "input_cost_per_query": null, + "input_cost_per_second": null, + "input_cost_per_text_message": null, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": null, + "input_cost_per_token_above_200k_tokens": null, + "input_cost_per_token_above_200k_tokens_batches": null, + "input_cost_per_token_above_200k_tokens_flex": null, + "input_cost_per_token_above_200k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens": null, + "input_cost_per_token_above_272k_tokens_batches": null, + "input_cost_per_token_above_272k_tokens_flex": null, + "input_cost_per_token_above_272k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens_ultrafast": null, + "input_cost_per_token_above_512k_tokens": null, + "input_cost_per_token_balanced": null, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": null, + "input_cost_per_token_priority": 4.25e-06, + "input_cost_per_token_ultrafast": null, + "input_cost_per_video_per_second": null, + "input_cost_per_video_token": null, + "input_cost_per_video_token_batches": null, + "key": "gpt-4o", + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "ocr_cost_per_credit": null, + "ocr_cost_per_page": null, + "ocr_cost_per_page_batches": null, + "off_peak_pricing": null, + "output_cost_per_audio_token": null, + "output_cost_per_billable_unit": null, + "output_cost_per_character": null, + "output_cost_per_character_above_128k_tokens": null, + "output_cost_per_image": null, + "output_cost_per_image_1K": null, + "output_cost_per_image_2K": null, + "output_cost_per_image_token": null, + "output_cost_per_image_token_batches": null, + "output_cost_per_image_token_priority": null, + "output_cost_per_pixel": null, + "output_cost_per_reasoning_token": null, + "output_cost_per_reasoning_token_flex": null, + "output_cost_per_reasoning_token_priority": null, + "output_cost_per_second": null, + "output_cost_per_second_1080p": null, + "output_cost_per_second_2k": null, + "output_cost_per_second_480p": null, + "output_cost_per_second_4k": null, + "output_cost_per_second_720p": null, + "output_cost_per_second_768p": null, + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_128k_tokens": null, + "output_cost_per_token_above_200k_tokens": null, + "output_cost_per_token_above_200k_tokens_batches": null, + "output_cost_per_token_above_200k_tokens_flex": null, + "output_cost_per_token_above_200k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens": null, + "output_cost_per_token_above_272k_tokens_batches": null, + "output_cost_per_token_above_272k_tokens_flex": null, + "output_cost_per_token_above_272k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens_ultrafast": null, + "output_cost_per_token_above_512k_tokens": null, + "output_cost_per_token_balanced": null, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": null, + "output_cost_per_token_priority": 1.7e-05, + "output_cost_per_token_ultrafast": null, + "output_cost_per_video": null, + "output_cost_per_video_clip": null, + "output_cost_per_video_per_second": null, + "output_cost_per_video_token": null, + "output_vector_size": null, + "prompt_cache_min_tokens": null, + "provider_specific_entry": null, + "reasoning_contexts": null, + "reasoning_effort_levels": null, + "reasoning_modes": null, + "regional_endpoint_uplift_multiplier": null, + "regional_processing_uplift_multiplier_eu": null, + "regional_processing_uplift_multiplier_us": null, + "rpm": null, + "search_context_cost_per_query": null, + "search_free_requests": null, + "search_queries_cost_per_query": null, + "supported_audio_formats": null, + "supported_endpoints": null, + "supported_openai_params": [ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "max_retries", + "extra_headers", + "parallel_tool_calls", + "audio", + "web_search_options", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "response_format", + "user" + ], + "supports_adaptive_thinking": null, + "supports_anthropic_compaction": null, + "supports_anthropic_thinking_payload": null, + "supports_assistant_prefill": null, + "supports_audio_input": null, + "supports_audio_output": null, + "supports_bedrock_native_structured_output": null, + "supports_bedrock_native_structured_output_by_default": null, + "supports_box_questions": null, + "supports_computer_use": null, + "supports_continuation": null, + "supports_embedding_image_input": null, + "supports_function_calling": true, + "supports_image_size": null, + "supports_legacy_thinking": null, + "supports_low_reasoning_effort": null, + "supports_max_reasoning_effort": null, + "supports_mid_conversation_system": null, + "supports_minimal_reasoning_effort": null, + "supports_native_streaming": null, + "supports_native_structured_output": null, + "supports_none_reasoning_effort": null, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_per_turn_control": null, + "supports_prompt_cache_breakpoint": null, + "supports_prompt_caching": true, + "supports_reasoning": null, + "supports_regex_lookaround": null, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_thinking_cache_preservation": null, + "supports_tool_choice": true, + "supports_tool_search": null, + "supports_url_context": null, + "supports_video_input": null, + "supports_vision": true, + "supports_web_search": null, + "supports_xhigh_reasoning_effort": null, + "thinking_always_on": null, + "thinking_on_by_default": null, + "tiered_pricing": null, + "token_count_framing": null, + "token_count_model": null, + "token_count_responses_framing": null, + "token_count_responses_tool_framing": null, + "token_count_tokenizer": null, + "token_count_tool_framing": null, + "tpm": null, + "uses_embed_content": null, + "vertex_ai_audio_api": null, + "web_search_billing_unit": null + } + }, + "model_parameters": { + "stream": true + }, + "prompt_tokens": 8, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hi there, streamed", + "function_call": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": "", + "id": "", + "model": "gpt-4o", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 4, + "completion_tokens_details": { + "accepted_prediction_tokens": null, + "audio_tokens": null, + "image_tokens": null, + "reasoning_tokens": 0, + "rejected_prediction_tokens": null, + "text_tokens": 4, + "video_tokens": null + }, + "cost": 6.000000000000001e-05, + "prompt_tokens": 8, + "prompt_tokens_details": null, + "total_tokens": 12 + } + }, + "response_cost": 6.000000000000001e-05, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "success", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "success" + }, + "stream": true, + "total_tokens": 12, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "metadata", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "messages", + "response", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_a_sync_completion_reaches_both_loggers_alike[False]::puts": [], + "test_a_sync_completion_reaches_both_loggers_alike[True]::puts": [], + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[7]::error": "Parameter validation failed:\nInvalid type for parameter WebIdentityToken, value: 7, type: , valid types: ", + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[7]::failures": 1, + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[7]::puts": [], + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[7]::queue": [ + "2026-11-13/time-09-05-07-000042_chatcmpl-1.json" + ], + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[True]::error": "Parameter validation failed:\nInvalid type for parameter WebIdentityToken, value: True, type: , valid types: ", + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[True]::failures": 1, + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[True]::puts": [], + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[True]::queue": [ + "2026-11-13/time-09-05-07-000042_chatcmpl-1.json" + ], + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[]::error": "Parameter validation failed:\nInvalid length for parameter WebIdentityToken, value: 0, valid min length: 4", + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[]::failures": 1, + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[]::puts": [], + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[]::queue": [ + "2026-11-13/time-09-05-07-000042_chatcmpl-1.json" + ], + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[abc]::error": "Parameter validation failed:\nInvalid length for parameter WebIdentityToken, value: 3, valid min length: 4", + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[abc]::failures": 1, + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[abc]::puts": [], + "test_a_web_identity_token_botocore_rejects_fails_both_before_sts[abc]::queue": [ + "2026-11-13/time-09-05-07-000042_chatcmpl-1.json" + ], + "test_an_acompletion_reaches_both_loggers_alike[False-False-False]::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": null, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 20, + "cost_breakdown": { + "data_residency": null, + "discount_amount": 0.0, + "discount_percent": 0.0, + "input_cost": 2.5e-05, + "margin_fixed_amount": 0.0, + "margin_percent": 0.0, + "margin_total_amount": 0.0, + "original_cost": 0.00022500000000000002, + "output_cost": 0.0002, + "service_tier": null, + "tool_usage_cost": 0.0, + "total_cost": 0.00022500000000000002, + "vertex_location": null + }, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "", + "error_code": "", + "error_cooldown_deployments": null, + "error_message": "", + "error_provider_request_id": null, + "error_rate_limit_category": null, + "error_rate_limit_type": null, + "llm_provider": "", + "normalized_error": null, + "traceback": "" + }, + "error_str": null, + "guardrail_information": null, + "hidden_params": { + "additional_headers": {}, + "api_base": "https://api.openai.com", + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": "gpt-4o", + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 0.00022500000000000002, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "messages": [ + { + "content": "hello «world»", + "role": "user" + } + ], + "metadata": { + "applied_guardrails": [], + "cold_storage_object_key": null, + "kubernetes_pod_routing": null, + "mcp_tool_call_metadata": null, + "prompt_management_metadata": null, + "requester_custom_headers": null, + "requester_ip_address": null, + "requester_metadata": null, + "routing_decision": null, + "spend_logs_metadata": null, + "team_alias": null, + "team_id": null, + "usage_object": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + }, + "used_client_oauth_token": null, + "user_agent": null, + "user_api_key_alias": null, + "user_api_key_auth_metadata": null, + "user_api_key_budget_reset_at": null, + "user_api_key_customer_member_id": null, + "user_api_key_customer_org_id": null, + "user_api_key_customer_team_id": null, + "user_api_key_end_user_id": null, + "user_api_key_hash": null, + "user_api_key_max_budget": null, + "user_api_key_org_alias": null, + "user_api_key_org_id": null, + "user_api_key_project_alias": null, + "user_api_key_project_id": null, + "user_api_key_request_route": null, + "user_api_key_spend": null, + "user_api_key_team_alias": "team-a", + "user_api_key_team_id": null, + "user_api_key_team_max_budget": null, + "user_api_key_team_spend": null, + "user_api_key_user_email": null, + "user_api_key_user_id": null, + "user_api_key_user_max_budget": null, + "user_api_key_user_spend": null, + "vector_store_request_metadata": null + }, + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "gpt-4o", + "model_map_value": { + "annotation_cost_per_page": null, + "annotation_cost_per_page_batches": null, + "bedrock_converse_supports_strict_tools": null, + "bedrock_output_config_effort_ceiling": null, + "cache_creation_input_token_cost": null, + "cache_creation_input_token_cost_above_12hr": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_above_12hr_batches": null, + "cache_creation_input_token_cost_above_12hr_flex": null, + "cache_creation_input_token_cost_above_12hr_priority": null, + "cache_creation_input_token_cost_above_12hr_ultrafast": null, + "cache_creation_input_token_cost_above_1hr": null, + "cache_creation_input_token_cost_above_200k_tokens": null, + "cache_creation_input_token_cost_above_200k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens": null, + "cache_creation_input_token_cost_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_batches": null, + "cache_creation_input_token_cost_flex": null, + "cache_creation_input_token_cost_priority": null, + "cache_creation_input_token_cost_ultrafast": null, + "cache_read_input_audio_token_cost": null, + "cache_read_input_audio_token_cost_batches": null, + "cache_read_input_audio_token_cost_flex": null, + "cache_read_input_audio_token_cost_priority": null, + "cache_read_input_image_token_cost": null, + "cache_read_input_image_token_cost_batches": null, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_200k_tokens": null, + "cache_read_input_token_cost_above_200k_tokens_batches": null, + "cache_read_input_token_cost_above_200k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens": null, + "cache_read_input_token_cost_above_272k_tokens_batches": null, + "cache_read_input_token_cost_above_272k_tokens_flex": null, + "cache_read_input_token_cost_above_272k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_read_input_token_cost_above_512k_tokens": null, + "cache_read_input_token_cost_balanced": null, + "cache_read_input_token_cost_batches": null, + "cache_read_input_token_cost_flex": null, + "cache_read_input_token_cost_priority": 2.125e-06, + "cache_read_input_token_cost_ultrafast": null, + "citation_cost_per_token": null, + "cost_per_second": null, + "default_reasoning_effort": null, + "explicit_cache_read_input_token_cost": null, + "fedramp_uplift_multiplier": 1.1, + "google_maps_grounding_cost_per_query": null, + "input_cost_per_audio_per_second": null, + "input_cost_per_audio_token": null, + "input_cost_per_audio_token_above_200k_tokens": null, + "input_cost_per_audio_token_above_200k_tokens_batches": null, + "input_cost_per_audio_token_above_200k_tokens_flex": null, + "input_cost_per_audio_token_above_200k_tokens_priority": null, + "input_cost_per_audio_token_batches": null, + "input_cost_per_audio_token_flex": null, + "input_cost_per_audio_token_priority": null, + "input_cost_per_character": null, + "input_cost_per_image": null, + "input_cost_per_image_token": null, + "input_cost_per_image_token_batches": null, + "input_cost_per_query": null, + "input_cost_per_second": null, + "input_cost_per_text_message": null, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": null, + "input_cost_per_token_above_200k_tokens": null, + "input_cost_per_token_above_200k_tokens_batches": null, + "input_cost_per_token_above_200k_tokens_flex": null, + "input_cost_per_token_above_200k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens": null, + "input_cost_per_token_above_272k_tokens_batches": null, + "input_cost_per_token_above_272k_tokens_flex": null, + "input_cost_per_token_above_272k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens_ultrafast": null, + "input_cost_per_token_above_512k_tokens": null, + "input_cost_per_token_balanced": null, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": null, + "input_cost_per_token_priority": 4.25e-06, + "input_cost_per_token_ultrafast": null, + "input_cost_per_video_per_second": null, + "input_cost_per_video_token": null, + "input_cost_per_video_token_batches": null, + "key": "gpt-4o", + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "ocr_cost_per_credit": null, + "ocr_cost_per_page": null, + "ocr_cost_per_page_batches": null, + "off_peak_pricing": null, + "output_cost_per_audio_token": null, + "output_cost_per_billable_unit": null, + "output_cost_per_character": null, + "output_cost_per_character_above_128k_tokens": null, + "output_cost_per_image": null, + "output_cost_per_image_1K": null, + "output_cost_per_image_2K": null, + "output_cost_per_image_token": null, + "output_cost_per_image_token_batches": null, + "output_cost_per_image_token_priority": null, + "output_cost_per_pixel": null, + "output_cost_per_reasoning_token": null, + "output_cost_per_reasoning_token_flex": null, + "output_cost_per_reasoning_token_priority": null, + "output_cost_per_second": null, + "output_cost_per_second_1080p": null, + "output_cost_per_second_2k": null, + "output_cost_per_second_480p": null, + "output_cost_per_second_4k": null, + "output_cost_per_second_720p": null, + "output_cost_per_second_768p": null, + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_128k_tokens": null, + "output_cost_per_token_above_200k_tokens": null, + "output_cost_per_token_above_200k_tokens_batches": null, + "output_cost_per_token_above_200k_tokens_flex": null, + "output_cost_per_token_above_200k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens": null, + "output_cost_per_token_above_272k_tokens_batches": null, + "output_cost_per_token_above_272k_tokens_flex": null, + "output_cost_per_token_above_272k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens_ultrafast": null, + "output_cost_per_token_above_512k_tokens": null, + "output_cost_per_token_balanced": null, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": null, + "output_cost_per_token_priority": 1.7e-05, + "output_cost_per_token_ultrafast": null, + "output_cost_per_video": null, + "output_cost_per_video_clip": null, + "output_cost_per_video_per_second": null, + "output_cost_per_video_token": null, + "output_vector_size": null, + "prompt_cache_min_tokens": null, + "provider_specific_entry": null, + "reasoning_contexts": null, + "reasoning_effort_levels": null, + "reasoning_modes": null, + "regional_endpoint_uplift_multiplier": null, + "regional_processing_uplift_multiplier_eu": null, + "regional_processing_uplift_multiplier_us": null, + "rpm": null, + "search_context_cost_per_query": null, + "search_free_requests": null, + "search_queries_cost_per_query": null, + "supported_audio_formats": null, + "supported_endpoints": null, + "supported_openai_params": [ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "max_retries", + "extra_headers", + "parallel_tool_calls", + "audio", + "web_search_options", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "response_format", + "user" + ], + "supports_adaptive_thinking": null, + "supports_anthropic_compaction": null, + "supports_anthropic_thinking_payload": null, + "supports_assistant_prefill": null, + "supports_audio_input": null, + "supports_audio_output": null, + "supports_bedrock_native_structured_output": null, + "supports_bedrock_native_structured_output_by_default": null, + "supports_box_questions": null, + "supports_computer_use": null, + "supports_continuation": null, + "supports_embedding_image_input": null, + "supports_function_calling": true, + "supports_image_size": null, + "supports_legacy_thinking": null, + "supports_low_reasoning_effort": null, + "supports_max_reasoning_effort": null, + "supports_mid_conversation_system": null, + "supports_minimal_reasoning_effort": null, + "supports_native_streaming": null, + "supports_native_structured_output": null, + "supports_none_reasoning_effort": null, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_per_turn_control": null, + "supports_prompt_cache_breakpoint": null, + "supports_prompt_caching": true, + "supports_reasoning": null, + "supports_regex_lookaround": null, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_thinking_cache_preservation": null, + "supports_tool_choice": true, + "supports_tool_search": null, + "supports_url_context": null, + "supports_video_input": null, + "supports_vision": true, + "supports_web_search": null, + "supports_xhigh_reasoning_effort": null, + "thinking_always_on": null, + "thinking_on_by_default": null, + "tiered_pricing": null, + "token_count_framing": null, + "token_count_model": null, + "token_count_responses_framing": null, + "token_count_responses_tool_framing": null, + "token_count_tokenizer": null, + "token_count_tool_framing": null, + "tpm": null, + "uses_embed_content": null, + "vertex_ai_audio_api": null, + "web_search_billing_unit": null + } + }, + "model_parameters": {}, + "prompt_tokens": 10, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hi there", + "function_call": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": "", + "id": "", + "model": "gpt-4o", + "object": "chat.completion", + "usage": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + } + }, + "response_cost": 0.00022500000000000002, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "success", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "success" + }, + "stream": null, + "total_tokens": 30, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "metadata", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "messages", + "response", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs/team-a//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_an_acompletion_reaches_both_loggers_alike[False-False-True]::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": null, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 20, + "cost_breakdown": { + "data_residency": null, + "discount_amount": 0.0, + "discount_percent": 0.0, + "input_cost": 2.5e-05, + "margin_fixed_amount": 0.0, + "margin_percent": 0.0, + "margin_total_amount": 0.0, + "original_cost": 0.00022500000000000002, + "output_cost": 0.0002, + "service_tier": null, + "tool_usage_cost": 0.0, + "total_cost": 0.00022500000000000002, + "vertex_location": null + }, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "", + "error_code": "", + "error_cooldown_deployments": null, + "error_message": "", + "error_provider_request_id": null, + "error_rate_limit_category": null, + "error_rate_limit_type": null, + "llm_provider": "", + "normalized_error": null, + "traceback": "" + }, + "error_str": null, + "guardrail_information": null, + "hidden_params": { + "additional_headers": {}, + "api_base": "https://api.openai.com", + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": "gpt-4o", + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 0.00022500000000000002, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "messages": [ + { + "content": "hello «world»", + "role": "user" + } + ], + "metadata": { + "applied_guardrails": [], + "cold_storage_object_key": null, + "kubernetes_pod_routing": null, + "mcp_tool_call_metadata": null, + "prompt_management_metadata": null, + "requester_custom_headers": null, + "requester_ip_address": null, + "requester_metadata": null, + "routing_decision": null, + "spend_logs_metadata": null, + "team_alias": null, + "team_id": null, + "usage_object": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + }, + "used_client_oauth_token": null, + "user_agent": null, + "user_api_key_alias": null, + "user_api_key_auth_metadata": null, + "user_api_key_budget_reset_at": null, + "user_api_key_customer_member_id": null, + "user_api_key_customer_org_id": null, + "user_api_key_customer_team_id": null, + "user_api_key_end_user_id": null, + "user_api_key_hash": null, + "user_api_key_max_budget": null, + "user_api_key_org_alias": null, + "user_api_key_org_id": null, + "user_api_key_project_alias": null, + "user_api_key_project_id": null, + "user_api_key_request_route": null, + "user_api_key_spend": null, + "user_api_key_team_alias": "team-a", + "user_api_key_team_id": null, + "user_api_key_team_max_budget": null, + "user_api_key_team_spend": null, + "user_api_key_user_email": null, + "user_api_key_user_id": null, + "user_api_key_user_max_budget": null, + "user_api_key_user_spend": null, + "vector_store_request_metadata": null + }, + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "gpt-4o", + "model_map_value": { + "annotation_cost_per_page": null, + "annotation_cost_per_page_batches": null, + "bedrock_converse_supports_strict_tools": null, + "bedrock_output_config_effort_ceiling": null, + "cache_creation_input_token_cost": null, + "cache_creation_input_token_cost_above_12hr": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_above_12hr_batches": null, + "cache_creation_input_token_cost_above_12hr_flex": null, + "cache_creation_input_token_cost_above_12hr_priority": null, + "cache_creation_input_token_cost_above_12hr_ultrafast": null, + "cache_creation_input_token_cost_above_1hr": null, + "cache_creation_input_token_cost_above_200k_tokens": null, + "cache_creation_input_token_cost_above_200k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens": null, + "cache_creation_input_token_cost_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_batches": null, + "cache_creation_input_token_cost_flex": null, + "cache_creation_input_token_cost_priority": null, + "cache_creation_input_token_cost_ultrafast": null, + "cache_read_input_audio_token_cost": null, + "cache_read_input_audio_token_cost_batches": null, + "cache_read_input_audio_token_cost_flex": null, + "cache_read_input_audio_token_cost_priority": null, + "cache_read_input_image_token_cost": null, + "cache_read_input_image_token_cost_batches": null, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_200k_tokens": null, + "cache_read_input_token_cost_above_200k_tokens_batches": null, + "cache_read_input_token_cost_above_200k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens": null, + "cache_read_input_token_cost_above_272k_tokens_batches": null, + "cache_read_input_token_cost_above_272k_tokens_flex": null, + "cache_read_input_token_cost_above_272k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_read_input_token_cost_above_512k_tokens": null, + "cache_read_input_token_cost_balanced": null, + "cache_read_input_token_cost_batches": null, + "cache_read_input_token_cost_flex": null, + "cache_read_input_token_cost_priority": 2.125e-06, + "cache_read_input_token_cost_ultrafast": null, + "citation_cost_per_token": null, + "cost_per_second": null, + "default_reasoning_effort": null, + "explicit_cache_read_input_token_cost": null, + "fedramp_uplift_multiplier": 1.1, + "google_maps_grounding_cost_per_query": null, + "input_cost_per_audio_per_second": null, + "input_cost_per_audio_token": null, + "input_cost_per_audio_token_above_200k_tokens": null, + "input_cost_per_audio_token_above_200k_tokens_batches": null, + "input_cost_per_audio_token_above_200k_tokens_flex": null, + "input_cost_per_audio_token_above_200k_tokens_priority": null, + "input_cost_per_audio_token_batches": null, + "input_cost_per_audio_token_flex": null, + "input_cost_per_audio_token_priority": null, + "input_cost_per_character": null, + "input_cost_per_image": null, + "input_cost_per_image_token": null, + "input_cost_per_image_token_batches": null, + "input_cost_per_query": null, + "input_cost_per_second": null, + "input_cost_per_text_message": null, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": null, + "input_cost_per_token_above_200k_tokens": null, + "input_cost_per_token_above_200k_tokens_batches": null, + "input_cost_per_token_above_200k_tokens_flex": null, + "input_cost_per_token_above_200k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens": null, + "input_cost_per_token_above_272k_tokens_batches": null, + "input_cost_per_token_above_272k_tokens_flex": null, + "input_cost_per_token_above_272k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens_ultrafast": null, + "input_cost_per_token_above_512k_tokens": null, + "input_cost_per_token_balanced": null, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": null, + "input_cost_per_token_priority": 4.25e-06, + "input_cost_per_token_ultrafast": null, + "input_cost_per_video_per_second": null, + "input_cost_per_video_token": null, + "input_cost_per_video_token_batches": null, + "key": "gpt-4o", + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "ocr_cost_per_credit": null, + "ocr_cost_per_page": null, + "ocr_cost_per_page_batches": null, + "off_peak_pricing": null, + "output_cost_per_audio_token": null, + "output_cost_per_billable_unit": null, + "output_cost_per_character": null, + "output_cost_per_character_above_128k_tokens": null, + "output_cost_per_image": null, + "output_cost_per_image_1K": null, + "output_cost_per_image_2K": null, + "output_cost_per_image_token": null, + "output_cost_per_image_token_batches": null, + "output_cost_per_image_token_priority": null, + "output_cost_per_pixel": null, + "output_cost_per_reasoning_token": null, + "output_cost_per_reasoning_token_flex": null, + "output_cost_per_reasoning_token_priority": null, + "output_cost_per_second": null, + "output_cost_per_second_1080p": null, + "output_cost_per_second_2k": null, + "output_cost_per_second_480p": null, + "output_cost_per_second_4k": null, + "output_cost_per_second_720p": null, + "output_cost_per_second_768p": null, + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_128k_tokens": null, + "output_cost_per_token_above_200k_tokens": null, + "output_cost_per_token_above_200k_tokens_batches": null, + "output_cost_per_token_above_200k_tokens_flex": null, + "output_cost_per_token_above_200k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens": null, + "output_cost_per_token_above_272k_tokens_batches": null, + "output_cost_per_token_above_272k_tokens_flex": null, + "output_cost_per_token_above_272k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens_ultrafast": null, + "output_cost_per_token_above_512k_tokens": null, + "output_cost_per_token_balanced": null, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": null, + "output_cost_per_token_priority": 1.7e-05, + "output_cost_per_token_ultrafast": null, + "output_cost_per_video": null, + "output_cost_per_video_clip": null, + "output_cost_per_video_per_second": null, + "output_cost_per_video_token": null, + "output_vector_size": null, + "prompt_cache_min_tokens": null, + "provider_specific_entry": null, + "reasoning_contexts": null, + "reasoning_effort_levels": null, + "reasoning_modes": null, + "regional_endpoint_uplift_multiplier": null, + "regional_processing_uplift_multiplier_eu": null, + "regional_processing_uplift_multiplier_us": null, + "rpm": null, + "search_context_cost_per_query": null, + "search_free_requests": null, + "search_queries_cost_per_query": null, + "supported_audio_formats": null, + "supported_endpoints": null, + "supported_openai_params": [ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "max_retries", + "extra_headers", + "parallel_tool_calls", + "audio", + "web_search_options", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "response_format", + "user" + ], + "supports_adaptive_thinking": null, + "supports_anthropic_compaction": null, + "supports_anthropic_thinking_payload": null, + "supports_assistant_prefill": null, + "supports_audio_input": null, + "supports_audio_output": null, + "supports_bedrock_native_structured_output": null, + "supports_bedrock_native_structured_output_by_default": null, + "supports_box_questions": null, + "supports_computer_use": null, + "supports_continuation": null, + "supports_embedding_image_input": null, + "supports_function_calling": true, + "supports_image_size": null, + "supports_legacy_thinking": null, + "supports_low_reasoning_effort": null, + "supports_max_reasoning_effort": null, + "supports_mid_conversation_system": null, + "supports_minimal_reasoning_effort": null, + "supports_native_streaming": null, + "supports_native_structured_output": null, + "supports_none_reasoning_effort": null, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_per_turn_control": null, + "supports_prompt_cache_breakpoint": null, + "supports_prompt_caching": true, + "supports_reasoning": null, + "supports_regex_lookaround": null, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_thinking_cache_preservation": null, + "supports_tool_choice": true, + "supports_tool_search": null, + "supports_url_context": null, + "supports_video_input": null, + "supports_vision": true, + "supports_web_search": null, + "supports_xhigh_reasoning_effort": null, + "thinking_always_on": null, + "thinking_on_by_default": null, + "tiered_pricing": null, + "token_count_framing": null, + "token_count_model": null, + "token_count_responses_framing": null, + "token_count_responses_tool_framing": null, + "token_count_tokenizer": null, + "token_count_tool_framing": null, + "tpm": null, + "uses_embed_content": null, + "vertex_ai_audio_api": null, + "web_search_billing_unit": null + } + }, + "model_parameters": {}, + "prompt_tokens": 10, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hi there", + "function_call": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": "", + "id": "", + "model": "gpt-4o", + "object": "chat.completion", + "usage": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + } + }, + "response_cost": 0.00022500000000000002, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "success", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "success" + }, + "stream": null, + "total_tokens": 30, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "metadata", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "messages", + "response", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs/team-a//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_an_acompletion_reaches_both_loggers_alike[False-True-False]::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": null, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 20, + "cost_breakdown": { + "data_residency": null, + "discount_amount": 0.0, + "discount_percent": 0.0, + "input_cost": 2.5e-05, + "margin_fixed_amount": 0.0, + "margin_percent": 0.0, + "margin_total_amount": 0.0, + "original_cost": 0.00022500000000000002, + "output_cost": 0.0002, + "service_tier": null, + "tool_usage_cost": 0.0, + "total_cost": 0.00022500000000000002, + "vertex_location": null + }, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "", + "error_code": "", + "error_cooldown_deployments": null, + "error_message": "", + "error_provider_request_id": null, + "error_rate_limit_category": null, + "error_rate_limit_type": null, + "llm_provider": "", + "normalized_error": null, + "traceback": "" + }, + "error_str": null, + "guardrail_information": null, + "hidden_params": { + "additional_headers": {}, + "api_base": "https://api.openai.com", + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": "gpt-4o", + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 0.00022500000000000002, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "messages": [ + { + "content": "hello «world»", + "role": "user" + } + ], + "metadata": { + "applied_guardrails": [], + "cold_storage_object_key": null, + "kubernetes_pod_routing": null, + "mcp_tool_call_metadata": null, + "prompt_management_metadata": null, + "requester_custom_headers": null, + "requester_ip_address": null, + "requester_metadata": null, + "routing_decision": null, + "spend_logs_metadata": null, + "team_alias": null, + "team_id": null, + "usage_object": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + }, + "used_client_oauth_token": null, + "user_agent": null, + "user_api_key_alias": null, + "user_api_key_auth_metadata": null, + "user_api_key_budget_reset_at": null, + "user_api_key_customer_member_id": null, + "user_api_key_customer_org_id": null, + "user_api_key_customer_team_id": null, + "user_api_key_end_user_id": null, + "user_api_key_hash": null, + "user_api_key_max_budget": null, + "user_api_key_org_alias": null, + "user_api_key_org_id": null, + "user_api_key_project_alias": null, + "user_api_key_project_id": null, + "user_api_key_request_route": null, + "user_api_key_spend": null, + "user_api_key_team_alias": "team-a", + "user_api_key_team_id": null, + "user_api_key_team_max_budget": null, + "user_api_key_team_spend": null, + "user_api_key_user_email": null, + "user_api_key_user_id": null, + "user_api_key_user_max_budget": null, + "user_api_key_user_spend": null, + "vector_store_request_metadata": null + }, + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "gpt-4o", + "model_map_value": { + "annotation_cost_per_page": null, + "annotation_cost_per_page_batches": null, + "bedrock_converse_supports_strict_tools": null, + "bedrock_output_config_effort_ceiling": null, + "cache_creation_input_token_cost": null, + "cache_creation_input_token_cost_above_12hr": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_above_12hr_batches": null, + "cache_creation_input_token_cost_above_12hr_flex": null, + "cache_creation_input_token_cost_above_12hr_priority": null, + "cache_creation_input_token_cost_above_12hr_ultrafast": null, + "cache_creation_input_token_cost_above_1hr": null, + "cache_creation_input_token_cost_above_200k_tokens": null, + "cache_creation_input_token_cost_above_200k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens": null, + "cache_creation_input_token_cost_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_batches": null, + "cache_creation_input_token_cost_flex": null, + "cache_creation_input_token_cost_priority": null, + "cache_creation_input_token_cost_ultrafast": null, + "cache_read_input_audio_token_cost": null, + "cache_read_input_audio_token_cost_batches": null, + "cache_read_input_audio_token_cost_flex": null, + "cache_read_input_audio_token_cost_priority": null, + "cache_read_input_image_token_cost": null, + "cache_read_input_image_token_cost_batches": null, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_200k_tokens": null, + "cache_read_input_token_cost_above_200k_tokens_batches": null, + "cache_read_input_token_cost_above_200k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens": null, + "cache_read_input_token_cost_above_272k_tokens_batches": null, + "cache_read_input_token_cost_above_272k_tokens_flex": null, + "cache_read_input_token_cost_above_272k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_read_input_token_cost_above_512k_tokens": null, + "cache_read_input_token_cost_balanced": null, + "cache_read_input_token_cost_batches": null, + "cache_read_input_token_cost_flex": null, + "cache_read_input_token_cost_priority": 2.125e-06, + "cache_read_input_token_cost_ultrafast": null, + "citation_cost_per_token": null, + "cost_per_second": null, + "default_reasoning_effort": null, + "explicit_cache_read_input_token_cost": null, + "fedramp_uplift_multiplier": 1.1, + "google_maps_grounding_cost_per_query": null, + "input_cost_per_audio_per_second": null, + "input_cost_per_audio_token": null, + "input_cost_per_audio_token_above_200k_tokens": null, + "input_cost_per_audio_token_above_200k_tokens_batches": null, + "input_cost_per_audio_token_above_200k_tokens_flex": null, + "input_cost_per_audio_token_above_200k_tokens_priority": null, + "input_cost_per_audio_token_batches": null, + "input_cost_per_audio_token_flex": null, + "input_cost_per_audio_token_priority": null, + "input_cost_per_character": null, + "input_cost_per_image": null, + "input_cost_per_image_token": null, + "input_cost_per_image_token_batches": null, + "input_cost_per_query": null, + "input_cost_per_second": null, + "input_cost_per_text_message": null, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": null, + "input_cost_per_token_above_200k_tokens": null, + "input_cost_per_token_above_200k_tokens_batches": null, + "input_cost_per_token_above_200k_tokens_flex": null, + "input_cost_per_token_above_200k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens": null, + "input_cost_per_token_above_272k_tokens_batches": null, + "input_cost_per_token_above_272k_tokens_flex": null, + "input_cost_per_token_above_272k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens_ultrafast": null, + "input_cost_per_token_above_512k_tokens": null, + "input_cost_per_token_balanced": null, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": null, + "input_cost_per_token_priority": 4.25e-06, + "input_cost_per_token_ultrafast": null, + "input_cost_per_video_per_second": null, + "input_cost_per_video_token": null, + "input_cost_per_video_token_batches": null, + "key": "gpt-4o", + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "ocr_cost_per_credit": null, + "ocr_cost_per_page": null, + "ocr_cost_per_page_batches": null, + "off_peak_pricing": null, + "output_cost_per_audio_token": null, + "output_cost_per_billable_unit": null, + "output_cost_per_character": null, + "output_cost_per_character_above_128k_tokens": null, + "output_cost_per_image": null, + "output_cost_per_image_1K": null, + "output_cost_per_image_2K": null, + "output_cost_per_image_token": null, + "output_cost_per_image_token_batches": null, + "output_cost_per_image_token_priority": null, + "output_cost_per_pixel": null, + "output_cost_per_reasoning_token": null, + "output_cost_per_reasoning_token_flex": null, + "output_cost_per_reasoning_token_priority": null, + "output_cost_per_second": null, + "output_cost_per_second_1080p": null, + "output_cost_per_second_2k": null, + "output_cost_per_second_480p": null, + "output_cost_per_second_4k": null, + "output_cost_per_second_720p": null, + "output_cost_per_second_768p": null, + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_128k_tokens": null, + "output_cost_per_token_above_200k_tokens": null, + "output_cost_per_token_above_200k_tokens_batches": null, + "output_cost_per_token_above_200k_tokens_flex": null, + "output_cost_per_token_above_200k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens": null, + "output_cost_per_token_above_272k_tokens_batches": null, + "output_cost_per_token_above_272k_tokens_flex": null, + "output_cost_per_token_above_272k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens_ultrafast": null, + "output_cost_per_token_above_512k_tokens": null, + "output_cost_per_token_balanced": null, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": null, + "output_cost_per_token_priority": 1.7e-05, + "output_cost_per_token_ultrafast": null, + "output_cost_per_video": null, + "output_cost_per_video_clip": null, + "output_cost_per_video_per_second": null, + "output_cost_per_video_token": null, + "output_vector_size": null, + "prompt_cache_min_tokens": null, + "provider_specific_entry": null, + "reasoning_contexts": null, + "reasoning_effort_levels": null, + "reasoning_modes": null, + "regional_endpoint_uplift_multiplier": null, + "regional_processing_uplift_multiplier_eu": null, + "regional_processing_uplift_multiplier_us": null, + "rpm": null, + "search_context_cost_per_query": null, + "search_free_requests": null, + "search_queries_cost_per_query": null, + "supported_audio_formats": null, + "supported_endpoints": null, + "supported_openai_params": [ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "max_retries", + "extra_headers", + "parallel_tool_calls", + "audio", + "web_search_options", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "response_format", + "user" + ], + "supports_adaptive_thinking": null, + "supports_anthropic_compaction": null, + "supports_anthropic_thinking_payload": null, + "supports_assistant_prefill": null, + "supports_audio_input": null, + "supports_audio_output": null, + "supports_bedrock_native_structured_output": null, + "supports_bedrock_native_structured_output_by_default": null, + "supports_box_questions": null, + "supports_computer_use": null, + "supports_continuation": null, + "supports_embedding_image_input": null, + "supports_function_calling": true, + "supports_image_size": null, + "supports_legacy_thinking": null, + "supports_low_reasoning_effort": null, + "supports_max_reasoning_effort": null, + "supports_mid_conversation_system": null, + "supports_minimal_reasoning_effort": null, + "supports_native_streaming": null, + "supports_native_structured_output": null, + "supports_none_reasoning_effort": null, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_per_turn_control": null, + "supports_prompt_cache_breakpoint": null, + "supports_prompt_caching": true, + "supports_reasoning": null, + "supports_regex_lookaround": null, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_thinking_cache_preservation": null, + "supports_tool_choice": true, + "supports_tool_search": null, + "supports_url_context": null, + "supports_video_input": null, + "supports_vision": true, + "supports_web_search": null, + "supports_xhigh_reasoning_effort": null, + "thinking_always_on": null, + "thinking_on_by_default": null, + "tiered_pricing": null, + "token_count_framing": null, + "token_count_model": null, + "token_count_responses_framing": null, + "token_count_responses_tool_framing": null, + "token_count_tokenizer": null, + "token_count_tool_framing": null, + "tpm": null, + "uses_embed_content": null, + "vertex_ai_audio_api": null, + "web_search_billing_unit": null + } + }, + "model_parameters": {}, + "prompt_tokens": 10, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hi there", + "function_call": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": "", + "id": "", + "model": "gpt-4o", + "object": "chat.completion", + "usage": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + } + }, + "response_cost": 0.00022500000000000002, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "success", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "success" + }, + "stream": null, + "total_tokens": 30, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "metadata", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "messages", + "response", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs/team-a//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_an_acompletion_reaches_both_loggers_alike[False-True-True]::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": null, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 20, + "cost_breakdown": { + "data_residency": null, + "discount_amount": 0.0, + "discount_percent": 0.0, + "input_cost": 2.5e-05, + "margin_fixed_amount": 0.0, + "margin_percent": 0.0, + "margin_total_amount": 0.0, + "original_cost": 0.00022500000000000002, + "output_cost": 0.0002, + "service_tier": null, + "tool_usage_cost": 0.0, + "total_cost": 0.00022500000000000002, + "vertex_location": null + }, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "", + "error_code": "", + "error_cooldown_deployments": null, + "error_message": "", + "error_provider_request_id": null, + "error_rate_limit_category": null, + "error_rate_limit_type": null, + "llm_provider": "", + "normalized_error": null, + "traceback": "" + }, + "error_str": null, + "guardrail_information": null, + "hidden_params": { + "additional_headers": {}, + "api_base": "https://api.openai.com", + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": "gpt-4o", + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 0.00022500000000000002, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "messages": [ + { + "content": "hello «world»", + "role": "user" + } + ], + "metadata": { + "applied_guardrails": [], + "cold_storage_object_key": null, + "kubernetes_pod_routing": null, + "mcp_tool_call_metadata": null, + "prompt_management_metadata": null, + "requester_custom_headers": null, + "requester_ip_address": null, + "requester_metadata": null, + "routing_decision": null, + "spend_logs_metadata": null, + "team_alias": null, + "team_id": null, + "usage_object": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + }, + "used_client_oauth_token": null, + "user_agent": null, + "user_api_key_alias": null, + "user_api_key_auth_metadata": null, + "user_api_key_budget_reset_at": null, + "user_api_key_customer_member_id": null, + "user_api_key_customer_org_id": null, + "user_api_key_customer_team_id": null, + "user_api_key_end_user_id": null, + "user_api_key_hash": null, + "user_api_key_max_budget": null, + "user_api_key_org_alias": null, + "user_api_key_org_id": null, + "user_api_key_project_alias": null, + "user_api_key_project_id": null, + "user_api_key_request_route": null, + "user_api_key_spend": null, + "user_api_key_team_alias": "team-a", + "user_api_key_team_id": null, + "user_api_key_team_max_budget": null, + "user_api_key_team_spend": null, + "user_api_key_user_email": null, + "user_api_key_user_id": null, + "user_api_key_user_max_budget": null, + "user_api_key_user_spend": null, + "vector_store_request_metadata": null + }, + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "gpt-4o", + "model_map_value": { + "annotation_cost_per_page": null, + "annotation_cost_per_page_batches": null, + "bedrock_converse_supports_strict_tools": null, + "bedrock_output_config_effort_ceiling": null, + "cache_creation_input_token_cost": null, + "cache_creation_input_token_cost_above_12hr": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_above_12hr_batches": null, + "cache_creation_input_token_cost_above_12hr_flex": null, + "cache_creation_input_token_cost_above_12hr_priority": null, + "cache_creation_input_token_cost_above_12hr_ultrafast": null, + "cache_creation_input_token_cost_above_1hr": null, + "cache_creation_input_token_cost_above_200k_tokens": null, + "cache_creation_input_token_cost_above_200k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens": null, + "cache_creation_input_token_cost_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_batches": null, + "cache_creation_input_token_cost_flex": null, + "cache_creation_input_token_cost_priority": null, + "cache_creation_input_token_cost_ultrafast": null, + "cache_read_input_audio_token_cost": null, + "cache_read_input_audio_token_cost_batches": null, + "cache_read_input_audio_token_cost_flex": null, + "cache_read_input_audio_token_cost_priority": null, + "cache_read_input_image_token_cost": null, + "cache_read_input_image_token_cost_batches": null, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_200k_tokens": null, + "cache_read_input_token_cost_above_200k_tokens_batches": null, + "cache_read_input_token_cost_above_200k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens": null, + "cache_read_input_token_cost_above_272k_tokens_batches": null, + "cache_read_input_token_cost_above_272k_tokens_flex": null, + "cache_read_input_token_cost_above_272k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_read_input_token_cost_above_512k_tokens": null, + "cache_read_input_token_cost_balanced": null, + "cache_read_input_token_cost_batches": null, + "cache_read_input_token_cost_flex": null, + "cache_read_input_token_cost_priority": 2.125e-06, + "cache_read_input_token_cost_ultrafast": null, + "citation_cost_per_token": null, + "cost_per_second": null, + "default_reasoning_effort": null, + "explicit_cache_read_input_token_cost": null, + "fedramp_uplift_multiplier": 1.1, + "google_maps_grounding_cost_per_query": null, + "input_cost_per_audio_per_second": null, + "input_cost_per_audio_token": null, + "input_cost_per_audio_token_above_200k_tokens": null, + "input_cost_per_audio_token_above_200k_tokens_batches": null, + "input_cost_per_audio_token_above_200k_tokens_flex": null, + "input_cost_per_audio_token_above_200k_tokens_priority": null, + "input_cost_per_audio_token_batches": null, + "input_cost_per_audio_token_flex": null, + "input_cost_per_audio_token_priority": null, + "input_cost_per_character": null, + "input_cost_per_image": null, + "input_cost_per_image_token": null, + "input_cost_per_image_token_batches": null, + "input_cost_per_query": null, + "input_cost_per_second": null, + "input_cost_per_text_message": null, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": null, + "input_cost_per_token_above_200k_tokens": null, + "input_cost_per_token_above_200k_tokens_batches": null, + "input_cost_per_token_above_200k_tokens_flex": null, + "input_cost_per_token_above_200k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens": null, + "input_cost_per_token_above_272k_tokens_batches": null, + "input_cost_per_token_above_272k_tokens_flex": null, + "input_cost_per_token_above_272k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens_ultrafast": null, + "input_cost_per_token_above_512k_tokens": null, + "input_cost_per_token_balanced": null, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": null, + "input_cost_per_token_priority": 4.25e-06, + "input_cost_per_token_ultrafast": null, + "input_cost_per_video_per_second": null, + "input_cost_per_video_token": null, + "input_cost_per_video_token_batches": null, + "key": "gpt-4o", + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "ocr_cost_per_credit": null, + "ocr_cost_per_page": null, + "ocr_cost_per_page_batches": null, + "off_peak_pricing": null, + "output_cost_per_audio_token": null, + "output_cost_per_billable_unit": null, + "output_cost_per_character": null, + "output_cost_per_character_above_128k_tokens": null, + "output_cost_per_image": null, + "output_cost_per_image_1K": null, + "output_cost_per_image_2K": null, + "output_cost_per_image_token": null, + "output_cost_per_image_token_batches": null, + "output_cost_per_image_token_priority": null, + "output_cost_per_pixel": null, + "output_cost_per_reasoning_token": null, + "output_cost_per_reasoning_token_flex": null, + "output_cost_per_reasoning_token_priority": null, + "output_cost_per_second": null, + "output_cost_per_second_1080p": null, + "output_cost_per_second_2k": null, + "output_cost_per_second_480p": null, + "output_cost_per_second_4k": null, + "output_cost_per_second_720p": null, + "output_cost_per_second_768p": null, + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_128k_tokens": null, + "output_cost_per_token_above_200k_tokens": null, + "output_cost_per_token_above_200k_tokens_batches": null, + "output_cost_per_token_above_200k_tokens_flex": null, + "output_cost_per_token_above_200k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens": null, + "output_cost_per_token_above_272k_tokens_batches": null, + "output_cost_per_token_above_272k_tokens_flex": null, + "output_cost_per_token_above_272k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens_ultrafast": null, + "output_cost_per_token_above_512k_tokens": null, + "output_cost_per_token_balanced": null, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": null, + "output_cost_per_token_priority": 1.7e-05, + "output_cost_per_token_ultrafast": null, + "output_cost_per_video": null, + "output_cost_per_video_clip": null, + "output_cost_per_video_per_second": null, + "output_cost_per_video_token": null, + "output_vector_size": null, + "prompt_cache_min_tokens": null, + "provider_specific_entry": null, + "reasoning_contexts": null, + "reasoning_effort_levels": null, + "reasoning_modes": null, + "regional_endpoint_uplift_multiplier": null, + "regional_processing_uplift_multiplier_eu": null, + "regional_processing_uplift_multiplier_us": null, + "rpm": null, + "search_context_cost_per_query": null, + "search_free_requests": null, + "search_queries_cost_per_query": null, + "supported_audio_formats": null, + "supported_endpoints": null, + "supported_openai_params": [ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "max_retries", + "extra_headers", + "parallel_tool_calls", + "audio", + "web_search_options", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "response_format", + "user" + ], + "supports_adaptive_thinking": null, + "supports_anthropic_compaction": null, + "supports_anthropic_thinking_payload": null, + "supports_assistant_prefill": null, + "supports_audio_input": null, + "supports_audio_output": null, + "supports_bedrock_native_structured_output": null, + "supports_bedrock_native_structured_output_by_default": null, + "supports_box_questions": null, + "supports_computer_use": null, + "supports_continuation": null, + "supports_embedding_image_input": null, + "supports_function_calling": true, + "supports_image_size": null, + "supports_legacy_thinking": null, + "supports_low_reasoning_effort": null, + "supports_max_reasoning_effort": null, + "supports_mid_conversation_system": null, + "supports_minimal_reasoning_effort": null, + "supports_native_streaming": null, + "supports_native_structured_output": null, + "supports_none_reasoning_effort": null, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_per_turn_control": null, + "supports_prompt_cache_breakpoint": null, + "supports_prompt_caching": true, + "supports_reasoning": null, + "supports_regex_lookaround": null, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_thinking_cache_preservation": null, + "supports_tool_choice": true, + "supports_tool_search": null, + "supports_url_context": null, + "supports_video_input": null, + "supports_vision": true, + "supports_web_search": null, + "supports_xhigh_reasoning_effort": null, + "thinking_always_on": null, + "thinking_on_by_default": null, + "tiered_pricing": null, + "token_count_framing": null, + "token_count_model": null, + "token_count_responses_framing": null, + "token_count_responses_tool_framing": null, + "token_count_tokenizer": null, + "token_count_tool_framing": null, + "tpm": null, + "uses_embed_content": null, + "vertex_ai_audio_api": null, + "web_search_billing_unit": null + } + }, + "model_parameters": {}, + "prompt_tokens": 10, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hi there", + "function_call": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": "", + "id": "", + "model": "gpt-4o", + "object": "chat.completion", + "usage": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + } + }, + "response_cost": 0.00022500000000000002, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "success", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "success" + }, + "stream": null, + "total_tokens": 30, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "metadata", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "messages", + "response", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs/team-a//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_an_acompletion_reaches_both_loggers_alike[True-False-False]::live-puts": [], + "test_an_acompletion_reaches_both_loggers_alike[True-False-True]::live-puts": [], + "test_an_acompletion_reaches_both_loggers_alike[True-True-False]::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": null, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 20, + "cost_breakdown": { + "data_residency": null, + "discount_amount": 0.0, + "discount_percent": 0.0, + "input_cost": 2.5e-05, + "margin_fixed_amount": 0.0, + "margin_percent": 0.0, + "margin_total_amount": 0.0, + "original_cost": 0.00022500000000000002, + "output_cost": 0.0002, + "service_tier": null, + "tool_usage_cost": 0.0, + "total_cost": 0.00022500000000000002, + "vertex_location": null + }, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "", + "error_code": "", + "error_cooldown_deployments": null, + "error_message": "", + "error_provider_request_id": null, + "error_rate_limit_category": null, + "error_rate_limit_type": null, + "llm_provider": "", + "normalized_error": null, + "traceback": "" + }, + "error_str": null, + "guardrail_information": null, + "hidden_params": { + "additional_headers": {}, + "api_base": "https://api.openai.com", + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": "gpt-4o", + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 0.00022500000000000002, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "messages": [ + { + "content": "hello «world»", + "role": "user" + } + ], + "metadata": { + "applied_guardrails": [], + "cold_storage_object_key": null, + "kubernetes_pod_routing": null, + "mcp_tool_call_metadata": null, + "prompt_management_metadata": null, + "requester_custom_headers": null, + "requester_ip_address": null, + "requester_metadata": null, + "routing_decision": null, + "spend_logs_metadata": null, + "team_alias": null, + "team_id": null, + "usage_object": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + }, + "used_client_oauth_token": null, + "user_agent": null, + "user_api_key_alias": null, + "user_api_key_auth_metadata": null, + "user_api_key_budget_reset_at": null, + "user_api_key_customer_member_id": null, + "user_api_key_customer_org_id": null, + "user_api_key_customer_team_id": null, + "user_api_key_end_user_id": null, + "user_api_key_hash": null, + "user_api_key_max_budget": null, + "user_api_key_org_alias": null, + "user_api_key_org_id": null, + "user_api_key_project_alias": null, + "user_api_key_project_id": null, + "user_api_key_request_route": null, + "user_api_key_spend": null, + "user_api_key_team_alias": "team-a", + "user_api_key_team_id": null, + "user_api_key_team_max_budget": null, + "user_api_key_team_spend": null, + "user_api_key_user_email": null, + "user_api_key_user_id": null, + "user_api_key_user_max_budget": null, + "user_api_key_user_spend": null, + "vector_store_request_metadata": null + }, + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "gpt-4o", + "model_map_value": { + "annotation_cost_per_page": null, + "annotation_cost_per_page_batches": null, + "bedrock_converse_supports_strict_tools": null, + "bedrock_output_config_effort_ceiling": null, + "cache_creation_input_token_cost": null, + "cache_creation_input_token_cost_above_12hr": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_above_12hr_batches": null, + "cache_creation_input_token_cost_above_12hr_flex": null, + "cache_creation_input_token_cost_above_12hr_priority": null, + "cache_creation_input_token_cost_above_12hr_ultrafast": null, + "cache_creation_input_token_cost_above_1hr": null, + "cache_creation_input_token_cost_above_200k_tokens": null, + "cache_creation_input_token_cost_above_200k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens": null, + "cache_creation_input_token_cost_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_batches": null, + "cache_creation_input_token_cost_flex": null, + "cache_creation_input_token_cost_priority": null, + "cache_creation_input_token_cost_ultrafast": null, + "cache_read_input_audio_token_cost": null, + "cache_read_input_audio_token_cost_batches": null, + "cache_read_input_audio_token_cost_flex": null, + "cache_read_input_audio_token_cost_priority": null, + "cache_read_input_image_token_cost": null, + "cache_read_input_image_token_cost_batches": null, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_200k_tokens": null, + "cache_read_input_token_cost_above_200k_tokens_batches": null, + "cache_read_input_token_cost_above_200k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens": null, + "cache_read_input_token_cost_above_272k_tokens_batches": null, + "cache_read_input_token_cost_above_272k_tokens_flex": null, + "cache_read_input_token_cost_above_272k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_read_input_token_cost_above_512k_tokens": null, + "cache_read_input_token_cost_balanced": null, + "cache_read_input_token_cost_batches": null, + "cache_read_input_token_cost_flex": null, + "cache_read_input_token_cost_priority": 2.125e-06, + "cache_read_input_token_cost_ultrafast": null, + "citation_cost_per_token": null, + "cost_per_second": null, + "default_reasoning_effort": null, + "explicit_cache_read_input_token_cost": null, + "fedramp_uplift_multiplier": 1.1, + "google_maps_grounding_cost_per_query": null, + "input_cost_per_audio_per_second": null, + "input_cost_per_audio_token": null, + "input_cost_per_audio_token_above_200k_tokens": null, + "input_cost_per_audio_token_above_200k_tokens_batches": null, + "input_cost_per_audio_token_above_200k_tokens_flex": null, + "input_cost_per_audio_token_above_200k_tokens_priority": null, + "input_cost_per_audio_token_batches": null, + "input_cost_per_audio_token_flex": null, + "input_cost_per_audio_token_priority": null, + "input_cost_per_character": null, + "input_cost_per_image": null, + "input_cost_per_image_token": null, + "input_cost_per_image_token_batches": null, + "input_cost_per_query": null, + "input_cost_per_second": null, + "input_cost_per_text_message": null, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": null, + "input_cost_per_token_above_200k_tokens": null, + "input_cost_per_token_above_200k_tokens_batches": null, + "input_cost_per_token_above_200k_tokens_flex": null, + "input_cost_per_token_above_200k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens": null, + "input_cost_per_token_above_272k_tokens_batches": null, + "input_cost_per_token_above_272k_tokens_flex": null, + "input_cost_per_token_above_272k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens_ultrafast": null, + "input_cost_per_token_above_512k_tokens": null, + "input_cost_per_token_balanced": null, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": null, + "input_cost_per_token_priority": 4.25e-06, + "input_cost_per_token_ultrafast": null, + "input_cost_per_video_per_second": null, + "input_cost_per_video_token": null, + "input_cost_per_video_token_batches": null, + "key": "gpt-4o", + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "ocr_cost_per_credit": null, + "ocr_cost_per_page": null, + "ocr_cost_per_page_batches": null, + "off_peak_pricing": null, + "output_cost_per_audio_token": null, + "output_cost_per_billable_unit": null, + "output_cost_per_character": null, + "output_cost_per_character_above_128k_tokens": null, + "output_cost_per_image": null, + "output_cost_per_image_1K": null, + "output_cost_per_image_2K": null, + "output_cost_per_image_token": null, + "output_cost_per_image_token_batches": null, + "output_cost_per_image_token_priority": null, + "output_cost_per_pixel": null, + "output_cost_per_reasoning_token": null, + "output_cost_per_reasoning_token_flex": null, + "output_cost_per_reasoning_token_priority": null, + "output_cost_per_second": null, + "output_cost_per_second_1080p": null, + "output_cost_per_second_2k": null, + "output_cost_per_second_480p": null, + "output_cost_per_second_4k": null, + "output_cost_per_second_720p": null, + "output_cost_per_second_768p": null, + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_128k_tokens": null, + "output_cost_per_token_above_200k_tokens": null, + "output_cost_per_token_above_200k_tokens_batches": null, + "output_cost_per_token_above_200k_tokens_flex": null, + "output_cost_per_token_above_200k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens": null, + "output_cost_per_token_above_272k_tokens_batches": null, + "output_cost_per_token_above_272k_tokens_flex": null, + "output_cost_per_token_above_272k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens_ultrafast": null, + "output_cost_per_token_above_512k_tokens": null, + "output_cost_per_token_balanced": null, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": null, + "output_cost_per_token_priority": 1.7e-05, + "output_cost_per_token_ultrafast": null, + "output_cost_per_video": null, + "output_cost_per_video_clip": null, + "output_cost_per_video_per_second": null, + "output_cost_per_video_token": null, + "output_vector_size": null, + "prompt_cache_min_tokens": null, + "provider_specific_entry": null, + "reasoning_contexts": null, + "reasoning_effort_levels": null, + "reasoning_modes": null, + "regional_endpoint_uplift_multiplier": null, + "regional_processing_uplift_multiplier_eu": null, + "regional_processing_uplift_multiplier_us": null, + "rpm": null, + "search_context_cost_per_query": null, + "search_free_requests": null, + "search_queries_cost_per_query": null, + "supported_audio_formats": null, + "supported_endpoints": null, + "supported_openai_params": [ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "max_retries", + "extra_headers", + "parallel_tool_calls", + "audio", + "web_search_options", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "response_format", + "user" + ], + "supports_adaptive_thinking": null, + "supports_anthropic_compaction": null, + "supports_anthropic_thinking_payload": null, + "supports_assistant_prefill": null, + "supports_audio_input": null, + "supports_audio_output": null, + "supports_bedrock_native_structured_output": null, + "supports_bedrock_native_structured_output_by_default": null, + "supports_box_questions": null, + "supports_computer_use": null, + "supports_continuation": null, + "supports_embedding_image_input": null, + "supports_function_calling": true, + "supports_image_size": null, + "supports_legacy_thinking": null, + "supports_low_reasoning_effort": null, + "supports_max_reasoning_effort": null, + "supports_mid_conversation_system": null, + "supports_minimal_reasoning_effort": null, + "supports_native_streaming": null, + "supports_native_structured_output": null, + "supports_none_reasoning_effort": null, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_per_turn_control": null, + "supports_prompt_cache_breakpoint": null, + "supports_prompt_caching": true, + "supports_reasoning": null, + "supports_regex_lookaround": null, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_thinking_cache_preservation": null, + "supports_tool_choice": true, + "supports_tool_search": null, + "supports_url_context": null, + "supports_video_input": null, + "supports_vision": true, + "supports_web_search": null, + "supports_xhigh_reasoning_effort": null, + "thinking_always_on": null, + "thinking_on_by_default": null, + "tiered_pricing": null, + "token_count_framing": null, + "token_count_model": null, + "token_count_responses_framing": null, + "token_count_responses_tool_framing": null, + "token_count_tokenizer": null, + "token_count_tool_framing": null, + "tpm": null, + "uses_embed_content": null, + "vertex_ai_audio_api": null, + "web_search_billing_unit": null + } + }, + "model_parameters": {}, + "prompt_tokens": 10, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hi there", + "function_call": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": "", + "id": "", + "model": "gpt-4o", + "object": "chat.completion", + "usage": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + } + }, + "response_cost": 0.00022500000000000002, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "success", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "success" + }, + "stream": null, + "total_tokens": 30, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "metadata", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "messages", + "response", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs/team-a//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_an_acompletion_reaches_both_loggers_alike[True-True-True]::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": null, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 20, + "cost_breakdown": { + "data_residency": null, + "discount_amount": 0.0, + "discount_percent": 0.0, + "input_cost": 2.5e-05, + "margin_fixed_amount": 0.0, + "margin_percent": 0.0, + "margin_total_amount": 0.0, + "original_cost": 0.00022500000000000002, + "output_cost": 0.0002, + "service_tier": null, + "tool_usage_cost": 0.0, + "total_cost": 0.00022500000000000002, + "vertex_location": null + }, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "", + "error_code": "", + "error_cooldown_deployments": null, + "error_message": "", + "error_provider_request_id": null, + "error_rate_limit_category": null, + "error_rate_limit_type": null, + "llm_provider": "", + "normalized_error": null, + "traceback": "" + }, + "error_str": null, + "guardrail_information": null, + "hidden_params": { + "additional_headers": {}, + "api_base": "https://api.openai.com", + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": "gpt-4o", + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 0.00022500000000000002, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "messages": [ + { + "content": "hello «world»", + "role": "user" + } + ], + "metadata": { + "applied_guardrails": [], + "cold_storage_object_key": null, + "kubernetes_pod_routing": null, + "mcp_tool_call_metadata": null, + "prompt_management_metadata": null, + "requester_custom_headers": null, + "requester_ip_address": null, + "requester_metadata": null, + "routing_decision": null, + "spend_logs_metadata": null, + "team_alias": null, + "team_id": null, + "usage_object": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + }, + "used_client_oauth_token": null, + "user_agent": null, + "user_api_key_alias": null, + "user_api_key_auth_metadata": null, + "user_api_key_budget_reset_at": null, + "user_api_key_customer_member_id": null, + "user_api_key_customer_org_id": null, + "user_api_key_customer_team_id": null, + "user_api_key_end_user_id": null, + "user_api_key_hash": null, + "user_api_key_max_budget": null, + "user_api_key_org_alias": null, + "user_api_key_org_id": null, + "user_api_key_project_alias": null, + "user_api_key_project_id": null, + "user_api_key_request_route": null, + "user_api_key_spend": null, + "user_api_key_team_alias": "team-a", + "user_api_key_team_id": null, + "user_api_key_team_max_budget": null, + "user_api_key_team_spend": null, + "user_api_key_user_email": null, + "user_api_key_user_id": null, + "user_api_key_user_max_budget": null, + "user_api_key_user_spend": null, + "vector_store_request_metadata": null + }, + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "gpt-4o", + "model_map_value": { + "annotation_cost_per_page": null, + "annotation_cost_per_page_batches": null, + "bedrock_converse_supports_strict_tools": null, + "bedrock_output_config_effort_ceiling": null, + "cache_creation_input_token_cost": null, + "cache_creation_input_token_cost_above_12hr": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_above_12hr_batches": null, + "cache_creation_input_token_cost_above_12hr_flex": null, + "cache_creation_input_token_cost_above_12hr_priority": null, + "cache_creation_input_token_cost_above_12hr_ultrafast": null, + "cache_creation_input_token_cost_above_1hr": null, + "cache_creation_input_token_cost_above_200k_tokens": null, + "cache_creation_input_token_cost_above_200k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens": null, + "cache_creation_input_token_cost_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_batches": null, + "cache_creation_input_token_cost_flex": null, + "cache_creation_input_token_cost_priority": null, + "cache_creation_input_token_cost_ultrafast": null, + "cache_read_input_audio_token_cost": null, + "cache_read_input_audio_token_cost_batches": null, + "cache_read_input_audio_token_cost_flex": null, + "cache_read_input_audio_token_cost_priority": null, + "cache_read_input_image_token_cost": null, + "cache_read_input_image_token_cost_batches": null, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_200k_tokens": null, + "cache_read_input_token_cost_above_200k_tokens_batches": null, + "cache_read_input_token_cost_above_200k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens": null, + "cache_read_input_token_cost_above_272k_tokens_batches": null, + "cache_read_input_token_cost_above_272k_tokens_flex": null, + "cache_read_input_token_cost_above_272k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_read_input_token_cost_above_512k_tokens": null, + "cache_read_input_token_cost_balanced": null, + "cache_read_input_token_cost_batches": null, + "cache_read_input_token_cost_flex": null, + "cache_read_input_token_cost_priority": 2.125e-06, + "cache_read_input_token_cost_ultrafast": null, + "citation_cost_per_token": null, + "cost_per_second": null, + "default_reasoning_effort": null, + "explicit_cache_read_input_token_cost": null, + "fedramp_uplift_multiplier": 1.1, + "google_maps_grounding_cost_per_query": null, + "input_cost_per_audio_per_second": null, + "input_cost_per_audio_token": null, + "input_cost_per_audio_token_above_200k_tokens": null, + "input_cost_per_audio_token_above_200k_tokens_batches": null, + "input_cost_per_audio_token_above_200k_tokens_flex": null, + "input_cost_per_audio_token_above_200k_tokens_priority": null, + "input_cost_per_audio_token_batches": null, + "input_cost_per_audio_token_flex": null, + "input_cost_per_audio_token_priority": null, + "input_cost_per_character": null, + "input_cost_per_image": null, + "input_cost_per_image_token": null, + "input_cost_per_image_token_batches": null, + "input_cost_per_query": null, + "input_cost_per_second": null, + "input_cost_per_text_message": null, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": null, + "input_cost_per_token_above_200k_tokens": null, + "input_cost_per_token_above_200k_tokens_batches": null, + "input_cost_per_token_above_200k_tokens_flex": null, + "input_cost_per_token_above_200k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens": null, + "input_cost_per_token_above_272k_tokens_batches": null, + "input_cost_per_token_above_272k_tokens_flex": null, + "input_cost_per_token_above_272k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens_ultrafast": null, + "input_cost_per_token_above_512k_tokens": null, + "input_cost_per_token_balanced": null, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": null, + "input_cost_per_token_priority": 4.25e-06, + "input_cost_per_token_ultrafast": null, + "input_cost_per_video_per_second": null, + "input_cost_per_video_token": null, + "input_cost_per_video_token_batches": null, + "key": "gpt-4o", + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "ocr_cost_per_credit": null, + "ocr_cost_per_page": null, + "ocr_cost_per_page_batches": null, + "off_peak_pricing": null, + "output_cost_per_audio_token": null, + "output_cost_per_billable_unit": null, + "output_cost_per_character": null, + "output_cost_per_character_above_128k_tokens": null, + "output_cost_per_image": null, + "output_cost_per_image_1K": null, + "output_cost_per_image_2K": null, + "output_cost_per_image_token": null, + "output_cost_per_image_token_batches": null, + "output_cost_per_image_token_priority": null, + "output_cost_per_pixel": null, + "output_cost_per_reasoning_token": null, + "output_cost_per_reasoning_token_flex": null, + "output_cost_per_reasoning_token_priority": null, + "output_cost_per_second": null, + "output_cost_per_second_1080p": null, + "output_cost_per_second_2k": null, + "output_cost_per_second_480p": null, + "output_cost_per_second_4k": null, + "output_cost_per_second_720p": null, + "output_cost_per_second_768p": null, + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_128k_tokens": null, + "output_cost_per_token_above_200k_tokens": null, + "output_cost_per_token_above_200k_tokens_batches": null, + "output_cost_per_token_above_200k_tokens_flex": null, + "output_cost_per_token_above_200k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens": null, + "output_cost_per_token_above_272k_tokens_batches": null, + "output_cost_per_token_above_272k_tokens_flex": null, + "output_cost_per_token_above_272k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens_ultrafast": null, + "output_cost_per_token_above_512k_tokens": null, + "output_cost_per_token_balanced": null, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": null, + "output_cost_per_token_priority": 1.7e-05, + "output_cost_per_token_ultrafast": null, + "output_cost_per_video": null, + "output_cost_per_video_clip": null, + "output_cost_per_video_per_second": null, + "output_cost_per_video_token": null, + "output_vector_size": null, + "prompt_cache_min_tokens": null, + "provider_specific_entry": null, + "reasoning_contexts": null, + "reasoning_effort_levels": null, + "reasoning_modes": null, + "regional_endpoint_uplift_multiplier": null, + "regional_processing_uplift_multiplier_eu": null, + "regional_processing_uplift_multiplier_us": null, + "rpm": null, + "search_context_cost_per_query": null, + "search_free_requests": null, + "search_queries_cost_per_query": null, + "supported_audio_formats": null, + "supported_endpoints": null, + "supported_openai_params": [ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "max_retries", + "extra_headers", + "parallel_tool_calls", + "audio", + "web_search_options", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "response_format", + "user" + ], + "supports_adaptive_thinking": null, + "supports_anthropic_compaction": null, + "supports_anthropic_thinking_payload": null, + "supports_assistant_prefill": null, + "supports_audio_input": null, + "supports_audio_output": null, + "supports_bedrock_native_structured_output": null, + "supports_bedrock_native_structured_output_by_default": null, + "supports_box_questions": null, + "supports_computer_use": null, + "supports_continuation": null, + "supports_embedding_image_input": null, + "supports_function_calling": true, + "supports_image_size": null, + "supports_legacy_thinking": null, + "supports_low_reasoning_effort": null, + "supports_max_reasoning_effort": null, + "supports_mid_conversation_system": null, + "supports_minimal_reasoning_effort": null, + "supports_native_streaming": null, + "supports_native_structured_output": null, + "supports_none_reasoning_effort": null, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_per_turn_control": null, + "supports_prompt_cache_breakpoint": null, + "supports_prompt_caching": true, + "supports_reasoning": null, + "supports_regex_lookaround": null, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_thinking_cache_preservation": null, + "supports_tool_choice": true, + "supports_tool_search": null, + "supports_url_context": null, + "supports_video_input": null, + "supports_vision": true, + "supports_web_search": null, + "supports_xhigh_reasoning_effort": null, + "thinking_always_on": null, + "thinking_on_by_default": null, + "tiered_pricing": null, + "token_count_framing": null, + "token_count_model": null, + "token_count_responses_framing": null, + "token_count_responses_tool_framing": null, + "token_count_tokenizer": null, + "token_count_tool_framing": null, + "tpm": null, + "uses_embed_content": null, + "vertex_ai_audio_api": null, + "web_search_billing_unit": null + } + }, + "model_parameters": {}, + "prompt_tokens": 10, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hi there", + "function_call": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": "", + "id": "", + "model": "gpt-4o", + "object": "chat.completion", + "usage": { + "completion_tokens": 20, + "completion_tokens_details": null, + "prompt_tokens": 10, + "prompt_tokens_details": null, + "total_tokens": 30 + } + }, + "response_cost": 0.00022500000000000002, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "success", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "success" + }, + "stream": null, + "total_tokens": 30, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "metadata", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "messages", + "response", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs/team-a//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_audit_events_and_line_items_land_at_the_same_keys::puts": [ + { + "body": "{\"id\": \"audit-1\", \"action\": \"created\", \"before_value\": null, \"updated_at\": \"2026-11-13\"}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"audit-audit-1.json\"", + "content-language": "en", + "content-md5": "52hsEeQeSeWaPPgpyhPfoA==", + "content-type": "application/json", + "x-amz-content-sha256": "f2c84f8b1dd2c3c11d4555009e1ea2dd813685f7559624eaaf6707b968afa898", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/audit-base/audit_logs/audit-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"custom_id\": \"x\", \"status\": \"ok\", \"text\": \"ünïcode\"}\n", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"batch_1-part-00003.jsonl\"", + "content-language": "en", + "content-md5": "/KMGeOimblIMqTzvMo6Gdw==", + "content-type": "application/x-ndjson", + "x-amz-content-sha256": "cc8ed4a93b09d595d5218ebb22721707b9223c29604184cd7d111e7ada7ba77a", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/audit-base/batch_line_items/batch_1/part-00003.jsonl", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_audit_events_flush_at_the_same_batch_size[1-2]::bodies": [ + "{\"id\": \"audit-1\", \"action\": \"created\", \"updated_at\": \"2026-11-13\"}", + "{\"id\": \"audit-2\", \"action\": \"created\", \"updated_at\": \"2026-11-13\"}" + ], + "test_audit_events_flush_at_the_same_batch_size[2-2]::bodies": [ + "{\"id\": \"audit-1\", \"action\": \"created\", \"updated_at\": \"2026-11-13\"}", + "{\"id\": \"audit-2\", \"action\": \"created\", \"updated_at\": \"2026-11-13\"}" + ], + "test_audit_events_flush_at_the_same_batch_size[3-0]::bodies": [], + "test_batch_files_hold_the_same_lines::puts": [ + { + "body": "{\"id\": \"a\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"t1\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}\n{\"id\": \"c\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"t1\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "content-type": "application/x-ndjson" + }, + "method": "PUT", + "path": "/logs/t1/2026-11-13", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"b\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"t2\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "content-type": "application/x-ndjson" + }, + "method": "PUT", + "path": "/logs/t2/2026-11-13", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_drop_an_upload_retrying_past_the_max_retry_age[expired]::queue": [], + "test_both_loggers_drop_an_upload_retrying_past_the_max_retry_age[young]::queue": [ + "2026-11-13/time-09-05-07-000042_a.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[403-retried]::put-count": 5, + "test_both_loggers_keep_and_drop_the_same_failed_uploads[403-retried]::put-paths": [ + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[403-retried]::queue": [ + "2026-11-13/time-09-05-07-000042_a.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[403-retried]::queue-after-recovery": [], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[bucket-down-keeps-everything]::put-count": 9, + "test_both_loggers_keep_and_drop_the_same_failed_uploads[bucket-down-keeps-everything]::put-paths": [ + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[bucket-down-keeps-everything]::queue": [ + "2026-11-13/time-09-05-07-000042_a.json", + "2026-11-13/time-09-05-07-000042_b.json", + "2026-11-13/time-09-05-07-000042_c.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[bucket-down-keeps-everything]::queue-after-recovery": [], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[partial-503-keeps-the-failed-one]::put-count": 5, + "test_both_loggers_keep_and_drop_the_same_failed_uploads[partial-503-keeps-the-failed-one]::put-paths": [ + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[partial-503-keeps-the-failed-one]::queue": [ + "2026-11-13/time-09-05-07-000042_a.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[partial-503-keeps-the-failed-one]::queue-after-recovery": [], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[queue-cap-trims-oldest]::put-count": 9, + "test_both_loggers_keep_and_drop_the_same_failed_uploads[queue-cap-trims-oldest]::put-paths": [ + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[queue-cap-trims-oldest]::queue": [ + "2026-11-13/time-09-05-07-000042_b.json", + "2026-11-13/time-09-05-07-000042_c.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[queue-cap-trims-oldest]::queue-after-recovery": [], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[terminal-is-dropped]::put-count": 3, + "test_both_loggers_keep_and_drop_the_same_failed_uploads[terminal-is-dropped]::put-paths": [ + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[terminal-is-dropped]::queue": [], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[terminal-is-dropped]::queue-after-recovery": [], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[terminal-kept-when-dropping-is-off]::put-count": 3, + "test_both_loggers_keep_and_drop_the_same_failed_uploads[terminal-kept-when-dropping-is-off]::put-paths": [ + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[terminal-kept-when-dropping-is-off]::queue": [ + "2026-11-13/time-09-05-07-000042_a.json" + ], + "test_both_loggers_keep_and_drop_the_same_failed_uploads[terminal-kept-when-dropping-is-off]::queue-after-recovery": [], + "test_both_loggers_keep_uploads_that_fail_in_transport[dropped-connection]::failures": 1, + "test_both_loggers_keep_uploads_that_fail_in_transport[dropped-connection]::put-paths": [ + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json" + ], + "test_both_loggers_keep_uploads_that_fail_in_transport[dropped-connection]::queue": [ + "2026-11-13/time-09-05-07-000042_a.json" + ], + "test_both_loggers_keep_uploads_that_fail_in_transport[timeout]::failures": 1, + "test_both_loggers_keep_uploads_that_fail_in_transport[timeout]::put-paths": [ + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json" + ], + "test_both_loggers_keep_uploads_that_fail_in_transport[timeout]::queue": [ + "2026-11-13/time-09-05-07-000042_a.json" + ], + "test_both_loggers_name_the_object_from_the_same_start_time[aware]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794594600.000005, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T23-30-00-000005_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "Zu510WWIDxxf/ZRRas57Cg==", + "content-type": "application/json", + "x-amz-content-sha256": "6df9d6976625d613469d311a010b6fc398b45c585bfc99f51b206d69c5f2ddbf", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/23/time-23-30-00-000005_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_name_the_object_from_the_same_start_time[handler-start-differs]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "ri1XVSU2gNH4PxINvCEZsw==", + "content-type": "application/json", + "x-amz-content-sha256": "abcb4525830aa70ef9b1971bfd840fde25504d1127b1ed1062620ea127351c9b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/09/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_name_the_object_from_the_same_start_time[payload-without-start]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "REwfB63Se7sjNt7NLMaOVQ==", + "content-type": "application/json", + "x-amz-content-sha256": "33fffb5c360e9d03858b8c572128a076b04319339caf2e25581555f760dc0840", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/09/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_need_a_running_event_loop::error": "no running event loop", + "test_both_loggers_put_the_same_objects[defaults]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "ri1XVSU2gNH4PxINvCEZsw==", + "content-type": "application/json", + "x-amz-content-sha256": "abcb4525830aa70ef9b1971bfd840fde25504d1127b1ed1062620ea127351c9b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"cmpl:with/odd chars=+&#?%\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_cmpl:with_odd chars=+&#?%.json\"", + "content-language": "en", + "content-md5": "DEYxjYTtQxtiwjv1ooIefw==", + "content-type": "application/json", + "x-amz-content-sha256": "07cb6826d4016cb6bf9ce5d2b352fe73120af4e675c49b3a1fad68d52f79f13a", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_cmpl_with_odd%20chars%3D%2B%26%23%3F%25.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"odd\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}, \"repeated\": {\"first\": \"2026-01-02 03:04:05+00:00\", \"second\": \"CircularReference Detected\", \"amounts\": [\"1.50\", \"CircularReference Detected\"]}, \"numbers\": [1, 2.5, 1e+16, 1e-05, -0.0, NaN, Infinity, 1000000000000000000000000000000, true], \"members\": [1, 2, 3], \"model\": {\"name\": \"m\", \"when\": \"2026-11-13 09:05:07.000042\"}, \"opaque\": \"opaqueobject\", \"nul\": \"ab\", \"looped\": {\"name\": \"loop\", \"self\": \"CircularReference Detected\"}, \"deep\": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\"MaxDepthExceeded\"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]], \"keys\": {\"a\": 2}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_odd.json\"", + "content-language": "en", + "content-md5": "mh8BgOGxS8eoa56j9IvGHg==", + "content-type": "application/json", + "x-amz-content-sha256": "8573ec2072a7d5272c7c4af8e4b6648247001a89e873ee608891c43b078b5a2b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_odd.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"with-file\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"see data:image/png;base64,AAAA== inline\"}, {\"type\": \"file\", \"file\": {\"file_data\": \"data:application/pdf;base64,JVBER\"}}]}, \"not a dict\"], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_with-file.json\"", + "content-language": "en", + "content-md5": "j2y88nfIUMzrAGDTwd6qfw==", + "content-type": "application/json", + "x-amz-content-sha256": "a072eaa7053ca2021fd306700528c261eba82eeefe3dd89c88a78ae84e86dd87", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_with-file.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.json\"", + "content-language": "en", + "content-md5": "jePhZ3mguc7r6ubAaWFJMQ==", + "content-type": "application/json", + "x-amz-content-sha256": "2e1c7b69480494fbe108f4b1819d1605276b3afc5225d174bbf4930653b74e40", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_62a36d25347b01d5e6bad0cf8063897e26ea40858198d2c75824e78737752f31.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_put_the_same_objects[hourly-path]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "ri1XVSU2gNH4PxINvCEZsw==", + "content-type": "application/json", + "x-amz-content-sha256": "abcb4525830aa70ef9b1971bfd840fde25504d1127b1ed1062620ea127351c9b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/base/logs/2026-11-13/09/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"cmpl:with/odd chars=+&#?%\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_cmpl:with_odd chars=+&#?%.json\"", + "content-language": "en", + "content-md5": "DEYxjYTtQxtiwjv1ooIefw==", + "content-type": "application/json", + "x-amz-content-sha256": "07cb6826d4016cb6bf9ce5d2b352fe73120af4e675c49b3a1fad68d52f79f13a", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/base/logs/2026-11-13/09/time-09-05-07-000042_cmpl_with_odd%20chars%3D%2B%26%23%3F%25.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"odd\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}, \"repeated\": {\"first\": \"2026-01-02 03:04:05+00:00\", \"second\": \"CircularReference Detected\", \"amounts\": [\"1.50\", \"CircularReference Detected\"]}, \"numbers\": [1, 2.5, 1e+16, 1e-05, -0.0, NaN, Infinity, 1000000000000000000000000000000, true], \"members\": [1, 2, 3], \"model\": {\"name\": \"m\", \"when\": \"2026-11-13 09:05:07.000042\"}, \"opaque\": \"opaqueobject\", \"nul\": \"ab\", \"looped\": {\"name\": \"loop\", \"self\": \"CircularReference Detected\"}, \"deep\": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\"MaxDepthExceeded\"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]], \"keys\": {\"a\": 2}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_odd.json\"", + "content-language": "en", + "content-md5": "mh8BgOGxS8eoa56j9IvGHg==", + "content-type": "application/json", + "x-amz-content-sha256": "8573ec2072a7d5272c7c4af8e4b6648247001a89e873ee608891c43b078b5a2b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/base/logs/2026-11-13/09/time-09-05-07-000042_odd.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"with-file\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"see data:image/png;base64,AAAA== inline\"}, {\"type\": \"file\", \"file\": {\"file_data\": \"data:application/pdf;base64,JVBER\"}}]}, \"not a dict\"], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_with-file.json\"", + "content-language": "en", + "content-md5": "j2y88nfIUMzrAGDTwd6qfw==", + "content-type": "application/json", + "x-amz-content-sha256": "a072eaa7053ca2021fd306700528c261eba82eeefe3dd89c88a78ae84e86dd87", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/base/logs/2026-11-13/09/time-09-05-07-000042_with-file.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.json\"", + "content-language": "en", + "content-md5": "jePhZ3mguc7r6ubAaWFJMQ==", + "content-type": "application/json", + "x-amz-content-sha256": "2e1c7b69480494fbe108f4b1819d1605276b3afc5225d174bbf4930653b74e40", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/base/logs/2026-11-13/09/time-09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_62a36d25347b01d5e6bad0cf8063897e26ea40858198d2c75824e78737752f31.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_put_the_same_objects[prompts-only]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": null}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "eXJGHO/sno/E12MifrFhxg==", + "content-type": "application/json", + "x-amz-content-sha256": "f23201ad9f8c59d8ef66db4824ba919a3b1538ccb44d5fa4665d541ee8857dd8", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"cmpl:with/odd chars=+&#?%\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": null}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_cmpl:with_odd chars=+&#?%.json\"", + "content-language": "en", + "content-md5": "XW17wCr2xFSnml+upZDNYA==", + "content-type": "application/json", + "x-amz-content-sha256": "49dd91a784737f2074ea8c9eeea0669cfecba2930c97cb81dc7555471ded2f9d", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_cmpl_with_odd%20chars%3D%2B%26%23%3F%25.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"odd\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": null, \"repeated\": {\"first\": \"2026-01-02 03:04:05+00:00\", \"second\": \"CircularReference Detected\", \"amounts\": [\"1.50\", \"CircularReference Detected\"]}, \"numbers\": [1, 2.5, 1e+16, 1e-05, -0.0, NaN, Infinity, 1000000000000000000000000000000, true], \"members\": [1, 2, 3], \"model\": {\"name\": \"m\", \"when\": \"2026-11-13 09:05:07.000042\"}, \"opaque\": \"opaqueobject\", \"nul\": \"ab\", \"looped\": {\"name\": \"loop\", \"self\": \"CircularReference Detected\"}, \"deep\": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\"MaxDepthExceeded\"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]], \"keys\": {\"a\": 2}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_odd.json\"", + "content-language": "en", + "content-md5": "TQ4586JWEFZh+Zj022/KeA==", + "content-type": "application/json", + "x-amz-content-sha256": "2db2c054efb0f948e64baf4ece9feb00f0dad003017076b700baba1448987232", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_odd.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"with-file\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"see data:image/png;base64,AAAA== inline\"}, {\"type\": \"file\", \"file\": {\"file_data\": \"data:application/pdf;base64,JVBER\"}}]}, \"not a dict\"], \"response\": null}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_with-file.json\"", + "content-language": "en", + "content-md5": "U8ZlWb4pxsUts1f2w4KsWA==", + "content-type": "application/json", + "x-amz-content-sha256": "5c15da963acf4097312ae1907ea5b5ee45bff2a486f65ab595b6011255f24fea", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_with-file.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": null}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.json\"", + "content-language": "en", + "content-md5": "auWEJTPwfiVPNWK/hqJ1UA==", + "content-type": "application/json", + "x-amz-content-sha256": "bc4e3d5619209a972a5ae192a7042dc8a23a861a9569b64b54a8009185dd37f8", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_62a36d25347b01d5e6bad0cf8063897e26ea40858198d2c75824e78737752f31.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_put_the_same_objects[sse-aes-drops-key]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "ri1XVSU2gNH4PxINvCEZsw==", + "content-type": "application/json", + "x-amz-content-sha256": "abcb4525830aa70ef9b1971bfd840fde25504d1127b1ed1062620ea127351c9b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "AES256", + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption" + }, + { + "body": "{\"id\": \"cmpl:with/odd chars=+&#?%\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_cmpl:with_odd chars=+&#?%.json\"", + "content-language": "en", + "content-md5": "DEYxjYTtQxtiwjv1ooIefw==", + "content-type": "application/json", + "x-amz-content-sha256": "07cb6826d4016cb6bf9ce5d2b352fe73120af4e675c49b3a1fad68d52f79f13a", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "AES256", + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_cmpl_with_odd%20chars%3D%2B%26%23%3F%25.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption" + }, + { + "body": "{\"id\": \"odd\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}, \"repeated\": {\"first\": \"2026-01-02 03:04:05+00:00\", \"second\": \"CircularReference Detected\", \"amounts\": [\"1.50\", \"CircularReference Detected\"]}, \"numbers\": [1, 2.5, 1e+16, 1e-05, -0.0, NaN, Infinity, 1000000000000000000000000000000, true], \"members\": [1, 2, 3], \"model\": {\"name\": \"m\", \"when\": \"2026-11-13 09:05:07.000042\"}, \"opaque\": \"opaqueobject\", \"nul\": \"ab\", \"looped\": {\"name\": \"loop\", \"self\": \"CircularReference Detected\"}, \"deep\": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\"MaxDepthExceeded\"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]], \"keys\": {\"a\": 2}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_odd.json\"", + "content-language": "en", + "content-md5": "mh8BgOGxS8eoa56j9IvGHg==", + "content-type": "application/json", + "x-amz-content-sha256": "8573ec2072a7d5272c7c4af8e4b6648247001a89e873ee608891c43b078b5a2b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "AES256", + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_odd.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption" + }, + { + "body": "{\"id\": \"with-file\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"see data:image/png;base64,AAAA== inline\"}, {\"type\": \"file\", \"file\": {\"file_data\": \"data:application/pdf;base64,JVBER\"}}]}, \"not a dict\"], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_with-file.json\"", + "content-language": "en", + "content-md5": "j2y88nfIUMzrAGDTwd6qfw==", + "content-type": "application/json", + "x-amz-content-sha256": "a072eaa7053ca2021fd306700528c261eba82eeefe3dd89c88a78ae84e86dd87", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "AES256", + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_with-file.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption" + }, + { + "body": "{\"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.json\"", + "content-language": "en", + "content-md5": "jePhZ3mguc7r6ubAaWFJMQ==", + "content-type": "application/json", + "x-amz-content-sha256": "2e1c7b69480494fbe108f4b1819d1605276b3afc5225d174bbf4930653b74e40", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "AES256", + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_62a36d25347b01d5e6bad0cf8063897e26ea40858198d2c75824e78737752f31.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption" + } + ], + "test_both_loggers_put_the_same_objects[sse-kms]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "ri1XVSU2gNH4PxINvCEZsw==", + "content-type": "application/json", + "x-amz-content-sha256": "abcb4525830aa70ef9b1971bfd840fde25504d1127b1ed1062620ea127351c9b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "aws:kms", + "x-amz-server-side-encryption-aws-kms-key-id": "kms-1" + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id" + }, + { + "body": "{\"id\": \"cmpl:with/odd chars=+&#?%\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_cmpl:with_odd chars=+&#?%.json\"", + "content-language": "en", + "content-md5": "DEYxjYTtQxtiwjv1ooIefw==", + "content-type": "application/json", + "x-amz-content-sha256": "07cb6826d4016cb6bf9ce5d2b352fe73120af4e675c49b3a1fad68d52f79f13a", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "aws:kms", + "x-amz-server-side-encryption-aws-kms-key-id": "kms-1" + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_cmpl_with_odd%20chars%3D%2B%26%23%3F%25.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id" + }, + { + "body": "{\"id\": \"odd\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}, \"repeated\": {\"first\": \"2026-01-02 03:04:05+00:00\", \"second\": \"CircularReference Detected\", \"amounts\": [\"1.50\", \"CircularReference Detected\"]}, \"numbers\": [1, 2.5, 1e+16, 1e-05, -0.0, NaN, Infinity, 1000000000000000000000000000000, true], \"members\": [1, 2, 3], \"model\": {\"name\": \"m\", \"when\": \"2026-11-13 09:05:07.000042\"}, \"opaque\": \"opaqueobject\", \"nul\": \"ab\", \"looped\": {\"name\": \"loop\", \"self\": \"CircularReference Detected\"}, \"deep\": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\"MaxDepthExceeded\"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]], \"keys\": {\"a\": 2}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_odd.json\"", + "content-language": "en", + "content-md5": "mh8BgOGxS8eoa56j9IvGHg==", + "content-type": "application/json", + "x-amz-content-sha256": "8573ec2072a7d5272c7c4af8e4b6648247001a89e873ee608891c43b078b5a2b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "aws:kms", + "x-amz-server-side-encryption-aws-kms-key-id": "kms-1" + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_odd.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id" + }, + { + "body": "{\"id\": \"with-file\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"see data:image/png;base64,AAAA== inline\"}, {\"type\": \"file\", \"file\": {\"file_data\": \"data:application/pdf;base64,JVBER\"}}]}, \"not a dict\"], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_with-file.json\"", + "content-language": "en", + "content-md5": "j2y88nfIUMzrAGDTwd6qfw==", + "content-type": "application/json", + "x-amz-content-sha256": "a072eaa7053ca2021fd306700528c261eba82eeefe3dd89c88a78ae84e86dd87", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "aws:kms", + "x-amz-server-side-encryption-aws-kms-key-id": "kms-1" + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_with-file.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id" + }, + { + "body": "{\"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.json\"", + "content-language": "en", + "content-md5": "jePhZ3mguc7r6ubAaWFJMQ==", + "content-type": "application/json", + "x-amz-content-sha256": "2e1c7b69480494fbe108f4b1819d1605276b3afc5225d174bbf4930653b74e40", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": "aws:kms", + "x-amz-server-side-encryption-aws-kms-key-id": "kms-1" + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_62a36d25347b01d5e6bad0cf8063897e26ea40858198d2c75824e78737752f31.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id" + } + ], + "test_both_loggers_put_the_same_objects[strip-base64]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "ri1XVSU2gNH4PxINvCEZsw==", + "content-type": "application/json", + "x-amz-content-sha256": "abcb4525830aa70ef9b1971bfd840fde25504d1127b1ed1062620ea127351c9b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"cmpl:with/odd chars=+&#?%\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_cmpl:with_odd chars=+&#?%.json\"", + "content-language": "en", + "content-md5": "DEYxjYTtQxtiwjv1ooIefw==", + "content-type": "application/json", + "x-amz-content-sha256": "07cb6826d4016cb6bf9ce5d2b352fe73120af4e675c49b3a1fad68d52f79f13a", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_cmpl_with_odd%20chars%3D%2B%26%23%3F%25.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"odd\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}, \"repeated\": {\"first\": \"2026-01-02 03:04:05+00:00\", \"second\": \"CircularReference Detected\", \"amounts\": [\"1.50\", \"CircularReference Detected\"]}, \"numbers\": [1, 2.5, 1e+16, 1e-05, -0.0, NaN, Infinity, 1000000000000000000000000000000, true], \"members\": [1, 2, 3], \"model\": {\"name\": \"m\", \"when\": \"2026-11-13 09:05:07.000042\"}, \"opaque\": \"opaqueobject\", \"nul\": \"ab\", \"looped\": {\"name\": \"loop\", \"self\": \"CircularReference Detected\"}, \"deep\": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\"MaxDepthExceeded\"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]], \"keys\": {\"a\": 2}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_odd.json\"", + "content-language": "en", + "content-md5": "mh8BgOGxS8eoa56j9IvGHg==", + "content-type": "application/json", + "x-amz-content-sha256": "8573ec2072a7d5272c7c4af8e4b6648247001a89e873ee608891c43b078b5a2b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_odd.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"with-file\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"see [BASE64_REDACTED]\"}]}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_with-file.json\"", + "content-language": "en", + "content-md5": "TS+2rEiXbVNd/O1SWjnuhg==", + "content-type": "application/json", + "x-amz-content-sha256": "8097863d5bac5fc65b950ffc04de8e33d906ed190d1fe60ba36bdbb2b3683860", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_with-file.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.json\"", + "content-language": "en", + "content-md5": "jePhZ3mguc7r6ubAaWFJMQ==", + "content-type": "application/json", + "x-amz-content-sha256": "2e1c7b69480494fbe108f4b1819d1605276b3afc5225d174bbf4930653b74e40", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_62a36d25347b01d5e6bad0cf8063897e26ea40858198d2c75824e78737752f31.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_put_the_same_objects[team-and-key-prefix]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "ri1XVSU2gNH4PxINvCEZsw==", + "content-type": "application/json", + "x-amz-content-sha256": "abcb4525830aa70ef9b1971bfd840fde25504d1127b1ed1062620ea127351c9b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/team-%CE%B1/key/one/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"cmpl:with/odd chars=+&#?%\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_cmpl:with_odd chars=+&#?%.json\"", + "content-language": "en", + "content-md5": "DEYxjYTtQxtiwjv1ooIefw==", + "content-type": "application/json", + "x-amz-content-sha256": "07cb6826d4016cb6bf9ce5d2b352fe73120af4e675c49b3a1fad68d52f79f13a", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/team-%CE%B1/key/one/2026-11-13/time-09-05-07-000042_cmpl_with_odd%20chars%3D%2B%26%23%3F%25.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"odd\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}, \"repeated\": {\"first\": \"2026-01-02 03:04:05+00:00\", \"second\": \"CircularReference Detected\", \"amounts\": [\"1.50\", \"CircularReference Detected\"]}, \"numbers\": [1, 2.5, 1e+16, 1e-05, -0.0, NaN, Infinity, 1000000000000000000000000000000, true], \"members\": [1, 2, 3], \"model\": {\"name\": \"m\", \"when\": \"2026-11-13 09:05:07.000042\"}, \"opaque\": \"opaqueobject\", \"nul\": \"ab\", \"looped\": {\"name\": \"loop\", \"self\": \"CircularReference Detected\"}, \"deep\": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[\"MaxDepthExceeded\"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]], \"keys\": {\"a\": 2}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_odd.json\"", + "content-language": "en", + "content-md5": "mh8BgOGxS8eoa56j9IvGHg==", + "content-type": "application/json", + "x-amz-content-sha256": "8573ec2072a7d5272c7c4af8e4b6648247001a89e873ee608891c43b078b5a2b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/team-%CE%B1/key/one/2026-11-13/time-09-05-07-000042_odd.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"with-file\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"see data:image/png;base64,AAAA== inline\"}, {\"type\": \"file\", \"file\": {\"file_data\": \"data:application/pdf;base64,JVBER\"}}]}, \"not a dict\"], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_with-file.json\"", + "content-language": "en", + "content-md5": "j2y88nfIUMzrAGDTwd6qfw==", + "content-type": "application/json", + "x-amz-content-sha256": "a072eaa7053ca2021fd306700528c261eba82eeefe3dd89c88a78ae84e86dd87", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/team-%CE%B1/key/one/2026-11-13/time-09-05-07-000042_with-file.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.json\"", + "content-language": "en", + "content-md5": "jePhZ3mguc7r6ubAaWFJMQ==", + "content-type": "application/json", + "x-amz-content-sha256": "2e1c7b69480494fbe108f4b1819d1605276b3afc5225d174bbf4930653b74e40", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/team-%CE%B1/key/one/2026-11-13/time-09-05-07-000042_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_62a36d25347b01d5e6bad0cf8063897e26ea40858198d2c75824e78737752f31.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-a-dict]::failures": 1, + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-a-dict]::puts": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-a-dict]::queue": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-a-list]::failures": 1, + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-a-list]::puts": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-a-list]::queue": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-not-text]::failures": 1, + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-not-text]::puts": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-not-text]::queue": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-past-i128]::failures": 1, + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-past-i128]::puts": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-past-i128]::queue": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-whose-str-raises]::failures": 1, + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-whose-str-raises]::puts": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[alias-whose-str-raises]::queue": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[id-not-text]::failures": 1, + "test_both_loggers_skip_and_count_the_same_bad_payloads[id-not-text]::puts": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[id-not-text]::queue": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[metadata-none]::failures": 1, + "test_both_loggers_skip_and_count_the_same_bad_payloads[metadata-none]::puts": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[metadata-none]::queue": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[no-id]::failures": 1, + "test_both_loggers_skip_and_count_the_same_bad_payloads[no-id]::puts": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[no-id]::queue": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[payload-not-a-mapping]::failures": 1, + "test_both_loggers_skip_and_count_the_same_bad_payloads[payload-not-a-mapping]::puts": [], + "test_both_loggers_skip_and_count_the_same_bad_payloads[payload-not-a-mapping]::queue": [], + "test_both_loggers_upload_in_the_same_order_with_requeued_items_last::put-paths": [ + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json", + "/logs/2026-11-13/time-09-05-07-000042_c.json", + "/logs/2026-11-13/time-09-05-07-000042_d.json", + "/logs/2026-11-13/time-09-05-07-000042_e.json", + "/logs/2026-11-13/time-09-05-07-000042_a.json", + "/logs/2026-11-13/time-09-05-07-000042_b.json" + ], + "test_both_loggers_upload_in_the_same_order_with_requeued_items_last::queue": [ + "2026-11-13/time-09-05-07-000042_a.json", + "2026-11-13/time-09-05-07-000042_b.json" + ], + "test_both_loggers_widen_the_adaptive_upload_limit_alike[adaptive]::peak": 3, + "test_both_loggers_widen_the_adaptive_upload_limit_alike[adaptive]::puts": [ + { + "body": "{\"id\": \"r0\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r0.json\"", + "content-language": "en", + "content-md5": "1mU6wLsbIpOp3RNE4tE0QQ==", + "content-type": "application/json", + "x-amz-content-sha256": "bd4f0f90994caa9e106b4de68c036aa7542ef4ac2c2c0b78a5ad471709a38ad7", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r0.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r1.json\"", + "content-language": "en", + "content-md5": "jB8F5GmJy7xwvdI5PC/VPg==", + "content-type": "application/json", + "x-amz-content-sha256": "dc400ba7c13aba156cea5c5b32315dfa847bf9ce5420dc2b4dfcee5325392163", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r2\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r2.json\"", + "content-language": "en", + "content-md5": "cdUEoXxqg7NRqY/+Ehfbkg==", + "content-type": "application/json", + "x-amz-content-sha256": "57c8eecd242e32efd3c5c8a9b9f9d73b918a6211ede874e58e2b9e54ba7930db", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r2.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r3\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r3.json\"", + "content-language": "en", + "content-md5": "TdRj1WiBGgX0mpAEdm70Jg==", + "content-type": "application/json", + "x-amz-content-sha256": "16e5f2c1942d7a3508d5a4cef53efad88618fc0410f4f69cc7c25e90f15cae67", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r3.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r4\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r4.json\"", + "content-language": "en", + "content-md5": "stCbciIvzT/F6F+Dw83osQ==", + "content-type": "application/json", + "x-amz-content-sha256": "897d6ef147a8c98158ec4142678ee58ee990f0b5069c1544b405c2ae4dbc02f3", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r4.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r5\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r5.json\"", + "content-language": "en", + "content-md5": "GJLFGEl8tFA72uHogs9F9w==", + "content-type": "application/json", + "x-amz-content-sha256": "80c752466c14419f28b9cac426bb6ccfbc9d6aa3c1cb11759821dbc81ef5a36b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r5.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r6\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r6.json\"", + "content-language": "en", + "content-md5": "AFqMjAmDdHq0v7GFKEvnJw==", + "content-type": "application/json", + "x-amz-content-sha256": "4a7ddd0ef9a5bd38c0b56db429c8b6a832470cec23856c37fed0f1782469f19f", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r6.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r7\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r7.json\"", + "content-language": "en", + "content-md5": "oTQQNL1iXguyL3KuBi2vaA==", + "content-type": "application/json", + "x-amz-content-sha256": "29205bd5a2eeca154202d3fc86f736ff490299791b02193b6fdb3bceb7f8796b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r7.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r8\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r8.json\"", + "content-language": "en", + "content-md5": "01W6Glk5i9yASyf45ge5pQ==", + "content-type": "application/json", + "x-amz-content-sha256": "4070adc5a05d349ce64dd90a3a2f625b6473f0aa599592cf4849becda2693467", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r8.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r9\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r9.json\"", + "content-language": "en", + "content-md5": "SA5vw2XM6iCyGrloRD4fiA==", + "content-type": "application/json", + "x-amz-content-sha256": "abf8418b589a874d6b1493c0a0606139dee5d8233357daf2a970bf97270fa006", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r9.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_both_loggers_widen_the_adaptive_upload_limit_alike[fixed]::peak": 1, + "test_both_loggers_widen_the_adaptive_upload_limit_alike[fixed]::puts": [ + { + "body": "{\"id\": \"r0\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r0.json\"", + "content-language": "en", + "content-md5": "1mU6wLsbIpOp3RNE4tE0QQ==", + "content-type": "application/json", + "x-amz-content-sha256": "bd4f0f90994caa9e106b4de68c036aa7542ef4ac2c2c0b78a5ad471709a38ad7", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r0.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r1.json\"", + "content-language": "en", + "content-md5": "jB8F5GmJy7xwvdI5PC/VPg==", + "content-type": "application/json", + "x-amz-content-sha256": "dc400ba7c13aba156cea5c5b32315dfa847bf9ce5420dc2b4dfcee5325392163", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r2\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r2.json\"", + "content-language": "en", + "content-md5": "cdUEoXxqg7NRqY/+Ehfbkg==", + "content-type": "application/json", + "x-amz-content-sha256": "57c8eecd242e32efd3c5c8a9b9f9d73b918a6211ede874e58e2b9e54ba7930db", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r2.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r3\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r3.json\"", + "content-language": "en", + "content-md5": "TdRj1WiBGgX0mpAEdm70Jg==", + "content-type": "application/json", + "x-amz-content-sha256": "16e5f2c1942d7a3508d5a4cef53efad88618fc0410f4f69cc7c25e90f15cae67", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r3.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r4\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r4.json\"", + "content-language": "en", + "content-md5": "stCbciIvzT/F6F+Dw83osQ==", + "content-type": "application/json", + "x-amz-content-sha256": "897d6ef147a8c98158ec4142678ee58ee990f0b5069c1544b405c2ae4dbc02f3", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r4.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r5\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r5.json\"", + "content-language": "en", + "content-md5": "GJLFGEl8tFA72uHogs9F9w==", + "content-type": "application/json", + "x-amz-content-sha256": "80c752466c14419f28b9cac426bb6ccfbc9d6aa3c1cb11759821dbc81ef5a36b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r5.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r6\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r6.json\"", + "content-language": "en", + "content-md5": "AFqMjAmDdHq0v7GFKEvnJw==", + "content-type": "application/json", + "x-amz-content-sha256": "4a7ddd0ef9a5bd38c0b56db429c8b6a832470cec23856c37fed0f1782469f19f", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r6.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r7\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r7.json\"", + "content-language": "en", + "content-md5": "oTQQNL1iXguyL3KuBi2vaA==", + "content-type": "application/json", + "x-amz-content-sha256": "29205bd5a2eeca154202d3fc86f736ff490299791b02193b6fdb3bceb7f8796b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r7.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r8\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r8.json\"", + "content-language": "en", + "content-md5": "01W6Glk5i9yASyf45ge5pQ==", + "content-type": "application/json", + "x-amz-content-sha256": "4070adc5a05d349ce64dd90a3a2f625b6473f0aa599592cf4849becda2693467", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r8.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + { + "body": "{\"id\": \"r9\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_r9.json\"", + "content-language": "en", + "content-md5": "SA5vw2XM6iCyGrloRD4fiA==", + "content-type": "application/json", + "x-amz-content-sha256": "abf8418b589a874d6b1493c0a0606139dee5d8233357daf2a970bf97270fa006", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_r9.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_cold_storage_handles_odd_metadata_like_python[list]::failures": 1, + "test_cold_storage_handles_odd_metadata_like_python[list]::puts": [], + "test_cold_storage_handles_odd_metadata_like_python[missing]::failures": 0, + "test_cold_storage_handles_odd_metadata_like_python[missing]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "iuckydGRmc/cwCAjL7zFnA==", + "content-type": "application/json", + "x-amz-content-sha256": "a2bacd885a45c105a1bfced7363e066f1a20f6b501af377d935e1bafccad684e", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_cold_storage_handles_odd_metadata_like_python[none]::failures": 0, + "test_cold_storage_handles_odd_metadata_like_python[none]::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": null, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "/z1zyNb/uAhGsfKuS3Q+ag==", + "content-type": "application/json", + "x-amz-content-sha256": "40f755f4ba504746c2d4593421a8ed7bee97db7ed01ba4a0d2efb42440fe0ae0", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_cold_storage_handles_odd_metadata_like_python[text]::failures": 1, + "test_cold_storage_handles_odd_metadata_like_python[text]::puts": [], + "test_cold_storage_reads_return_the_same_object::get": { + "body": "", + "headers": { + "cache-control": null, + "content-disposition": null, + "content-language": null, + "content-md5": null, + "content-type": null, + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "GET", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + "test_cold_storage_reads_return_the_same_object::object": { + "call_type": "acompletion", + "endTime": 1794560708.250042, + "id": "chatcmpl-1", + "messages": [ + { + "content": "hello «world» 😀", + "role": "user" + } + ], + "metadata": { + "user_api_key_alias": "key/one", + "user_api_key_hash": "88dc28", + "user_api_key_team_alias": "team-α" + }, + "response": { + "choices": [ + { + "message": { + "content": "hi" + } + } + ] + }, + "response_cost": 0.000123, + "startTime": 1794560707.000042 + }, + "test_cold_storage_reads_without_a_region_send_nothing_like_python::object": null, + "test_cold_storage_reads_without_a_region_send_nothing_like_python::received": [], + "test_excluded_payload_fields_are_left_out_like_python[False]::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": null, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 20, + "cost_breakdown": { + "data_residency": null, + "discount_amount": 0.0, + "discount_percent": 0.0, + "input_cost": 2.5e-05, + "margin_fixed_amount": 0.0, + "margin_percent": 0.0, + "margin_total_amount": 0.0, + "original_cost": 0.00022500000000000002, + "output_cost": 0.0002, + "service_tier": null, + "tool_usage_cost": 0.0, + "total_cost": 0.00022500000000000002, + "vertex_location": null + }, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "", + "error_code": "", + "error_cooldown_deployments": null, + "error_message": "", + "error_provider_request_id": null, + "error_rate_limit_category": null, + "error_rate_limit_type": null, + "llm_provider": "", + "normalized_error": null, + "traceback": "" + }, + "error_str": null, + "guardrail_information": null, + "hidden_params": { + "additional_headers": {}, + "api_base": "https://api.openai.com", + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": "gpt-4o", + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 0.00022500000000000002, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "gpt-4o", + "model_map_value": { + "annotation_cost_per_page": null, + "annotation_cost_per_page_batches": null, + "bedrock_converse_supports_strict_tools": null, + "bedrock_output_config_effort_ceiling": null, + "cache_creation_input_token_cost": null, + "cache_creation_input_token_cost_above_12hr": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_above_12hr_batches": null, + "cache_creation_input_token_cost_above_12hr_flex": null, + "cache_creation_input_token_cost_above_12hr_priority": null, + "cache_creation_input_token_cost_above_12hr_ultrafast": null, + "cache_creation_input_token_cost_above_1hr": null, + "cache_creation_input_token_cost_above_200k_tokens": null, + "cache_creation_input_token_cost_above_200k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens": null, + "cache_creation_input_token_cost_above_272k_tokens_batches": null, + "cache_creation_input_token_cost_above_272k_tokens_flex": null, + "cache_creation_input_token_cost_above_272k_tokens_priority": null, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_creation_input_token_cost_batches": null, + "cache_creation_input_token_cost_flex": null, + "cache_creation_input_token_cost_priority": null, + "cache_creation_input_token_cost_ultrafast": null, + "cache_read_input_audio_token_cost": null, + "cache_read_input_audio_token_cost_batches": null, + "cache_read_input_audio_token_cost_flex": null, + "cache_read_input_audio_token_cost_priority": null, + "cache_read_input_image_token_cost": null, + "cache_read_input_image_token_cost_batches": null, + "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_above_200k_tokens": null, + "cache_read_input_token_cost_above_200k_tokens_batches": null, + "cache_read_input_token_cost_above_200k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens": null, + "cache_read_input_token_cost_above_272k_tokens_batches": null, + "cache_read_input_token_cost_above_272k_tokens_flex": null, + "cache_read_input_token_cost_above_272k_tokens_priority": null, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": null, + "cache_read_input_token_cost_above_512k_tokens": null, + "cache_read_input_token_cost_balanced": null, + "cache_read_input_token_cost_batches": null, + "cache_read_input_token_cost_flex": null, + "cache_read_input_token_cost_priority": 2.125e-06, + "cache_read_input_token_cost_ultrafast": null, + "citation_cost_per_token": null, + "cost_per_second": null, + "default_reasoning_effort": null, + "explicit_cache_read_input_token_cost": null, + "fedramp_uplift_multiplier": 1.1, + "google_maps_grounding_cost_per_query": null, + "input_cost_per_audio_per_second": null, + "input_cost_per_audio_token": null, + "input_cost_per_audio_token_above_200k_tokens": null, + "input_cost_per_audio_token_above_200k_tokens_batches": null, + "input_cost_per_audio_token_above_200k_tokens_flex": null, + "input_cost_per_audio_token_above_200k_tokens_priority": null, + "input_cost_per_audio_token_batches": null, + "input_cost_per_audio_token_flex": null, + "input_cost_per_audio_token_priority": null, + "input_cost_per_character": null, + "input_cost_per_image": null, + "input_cost_per_image_token": null, + "input_cost_per_image_token_batches": null, + "input_cost_per_query": null, + "input_cost_per_second": null, + "input_cost_per_text_message": null, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": null, + "input_cost_per_token_above_200k_tokens": null, + "input_cost_per_token_above_200k_tokens_batches": null, + "input_cost_per_token_above_200k_tokens_flex": null, + "input_cost_per_token_above_200k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens": null, + "input_cost_per_token_above_272k_tokens_batches": null, + "input_cost_per_token_above_272k_tokens_flex": null, + "input_cost_per_token_above_272k_tokens_priority": null, + "input_cost_per_token_above_272k_tokens_ultrafast": null, + "input_cost_per_token_above_512k_tokens": null, + "input_cost_per_token_balanced": null, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": null, + "input_cost_per_token_priority": 4.25e-06, + "input_cost_per_token_ultrafast": null, + "input_cost_per_video_per_second": null, + "input_cost_per_video_token": null, + "input_cost_per_video_token_batches": null, + "key": "gpt-4o", + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "ocr_cost_per_credit": null, + "ocr_cost_per_page": null, + "ocr_cost_per_page_batches": null, + "off_peak_pricing": null, + "output_cost_per_audio_token": null, + "output_cost_per_billable_unit": null, + "output_cost_per_character": null, + "output_cost_per_character_above_128k_tokens": null, + "output_cost_per_image": null, + "output_cost_per_image_1K": null, + "output_cost_per_image_2K": null, + "output_cost_per_image_token": null, + "output_cost_per_image_token_batches": null, + "output_cost_per_image_token_priority": null, + "output_cost_per_pixel": null, + "output_cost_per_reasoning_token": null, + "output_cost_per_reasoning_token_flex": null, + "output_cost_per_reasoning_token_priority": null, + "output_cost_per_second": null, + "output_cost_per_second_1080p": null, + "output_cost_per_second_2k": null, + "output_cost_per_second_480p": null, + "output_cost_per_second_4k": null, + "output_cost_per_second_720p": null, + "output_cost_per_second_768p": null, + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_128k_tokens": null, + "output_cost_per_token_above_200k_tokens": null, + "output_cost_per_token_above_200k_tokens_batches": null, + "output_cost_per_token_above_200k_tokens_flex": null, + "output_cost_per_token_above_200k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens": null, + "output_cost_per_token_above_272k_tokens_batches": null, + "output_cost_per_token_above_272k_tokens_flex": null, + "output_cost_per_token_above_272k_tokens_priority": null, + "output_cost_per_token_above_272k_tokens_ultrafast": null, + "output_cost_per_token_above_512k_tokens": null, + "output_cost_per_token_balanced": null, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": null, + "output_cost_per_token_priority": 1.7e-05, + "output_cost_per_token_ultrafast": null, + "output_cost_per_video": null, + "output_cost_per_video_clip": null, + "output_cost_per_video_per_second": null, + "output_cost_per_video_token": null, + "output_vector_size": null, + "prompt_cache_min_tokens": null, + "provider_specific_entry": null, + "reasoning_contexts": null, + "reasoning_effort_levels": null, + "reasoning_modes": null, + "regional_endpoint_uplift_multiplier": null, + "regional_processing_uplift_multiplier_eu": null, + "regional_processing_uplift_multiplier_us": null, + "rpm": null, + "search_context_cost_per_query": null, + "search_free_requests": null, + "search_queries_cost_per_query": null, + "supported_audio_formats": null, + "supported_endpoints": null, + "supported_openai_params": [ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "max_retries", + "extra_headers", + "parallel_tool_calls", + "audio", + "web_search_options", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "response_format", + "user" + ], + "supports_adaptive_thinking": null, + "supports_anthropic_compaction": null, + "supports_anthropic_thinking_payload": null, + "supports_assistant_prefill": null, + "supports_audio_input": null, + "supports_audio_output": null, + "supports_bedrock_native_structured_output": null, + "supports_bedrock_native_structured_output_by_default": null, + "supports_box_questions": null, + "supports_computer_use": null, + "supports_continuation": null, + "supports_embedding_image_input": null, + "supports_function_calling": true, + "supports_image_size": null, + "supports_legacy_thinking": null, + "supports_low_reasoning_effort": null, + "supports_max_reasoning_effort": null, + "supports_mid_conversation_system": null, + "supports_minimal_reasoning_effort": null, + "supports_native_streaming": null, + "supports_native_structured_output": null, + "supports_none_reasoning_effort": null, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_per_turn_control": null, + "supports_prompt_cache_breakpoint": null, + "supports_prompt_caching": true, + "supports_reasoning": null, + "supports_regex_lookaround": null, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_thinking_cache_preservation": null, + "supports_tool_choice": true, + "supports_tool_search": null, + "supports_url_context": null, + "supports_video_input": null, + "supports_vision": true, + "supports_web_search": null, + "supports_xhigh_reasoning_effort": null, + "thinking_always_on": null, + "thinking_on_by_default": null, + "tiered_pricing": null, + "token_count_framing": null, + "token_count_model": null, + "token_count_responses_framing": null, + "token_count_responses_tool_framing": null, + "token_count_tokenizer": null, + "token_count_tool_framing": null, + "tpm": null, + "uses_embed_content": null, + "vertex_ai_audio_api": null, + "web_search_billing_unit": null + } + }, + "model_parameters": {}, + "prompt_tokens": 10, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response_cost": 0.00022500000000000002, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "success", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "success" + }, + "stream": null, + "total_tokens": 30, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_excluded_payload_fields_are_left_out_like_python[True]::live-puts": [ + { + "body_json": { + "api_base": "", + "autorouter_baseline_observation": null, + "autorouter_savings": null, + "autorouter_savings_estimate": null, + "cache_hit": false, + "cache_key": null, + "call_type": "acompletion", + "completionStartTime": "", + "completion_tokens": 0, + "cost_breakdown": null, + "cost_finalized": false, + "custom_llm_provider": "openai", + "endTime": "", + "end_user": null, + "error_information": { + "error_budget_entity_id": null, + "error_budget_entity_type": null, + "error_budget_limit": null, + "error_budget_spend": null, + "error_class": "RateLimitError", + "error_code": "429", + "error_cooldown_deployments": null, + "error_message": "litellm.RateLimitError: this is a mock rate limit error", + "error_provider_request_id": null, + "error_rate_limit_category": "vendor_rate_limit", + "error_rate_limit_type": null, + "llm_provider": "openai", + "normalized_error": "429_RATE_LIMIT_EXCEEDED", + "traceback": " File \"/home/user/wt/rust-s3/litellm/utils.py\", line 2633, in wrapper_async\n result = await original_function(*args, **call_kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 1020, in acompletion\n raise exception_type(\n ...<5 lines>...\n )\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 956, in acompletion\n init_response: Final = await loop.run_in_executor(None, func_with_context)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.13/concurrent/futures/thread.py\", line 59, in run\n result = self.fn(*self.args, **self.kwargs)\n File \"/home/user/wt/rust-s3/litellm/utils.py\", line 2259, in wrapper\n result = original_function(*args, **kwargs)\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 6718, in completion\n raise exception_type(\n ...<5 lines>...\n )\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 6369, in completion\n return mock_completion(\n model,\n ...<13 lines>...\n ),\n )\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 1349, in mock_completion\n raise e\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 1235, in mock_completion\n _handle_mock_potential_exceptions(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n mock_response=mock_response,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n model=model,\n ^^^^^^^^^^^^\n custom_llm_provider=custom_llm_provider,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"/home/user/wt/rust-s3/litellm/main.py\", line 1081, in _handle_mock_potential_exceptions\n raise litellm.RateLimitError(\n ...<3 lines>...\n )\n" + }, + "error_str": "litellm.RateLimitError: this is a mock rate limit error", + "guardrail_information": null, + "hidden_params": { + "additional_headers": null, + "api_base": null, + "batch_failed_requests": null, + "batch_models": null, + "batch_successful_requests": null, + "cache_key": null, + "litellm_model_name": null, + "litellm_overhead_time_ms": null, + "model_id": null, + "response_cost": 0.0, + "usage_object": null + }, + "id": "", + "litellm_call_id": "", + "messages": [ + { + "content": "hello", + "role": "user" + } + ], + "metadata": { + "applied_guardrails": [], + "cold_storage_object_key": null, + "kubernetes_pod_routing": null, + "mcp_tool_call_metadata": null, + "prompt_management_metadata": null, + "requester_custom_headers": null, + "requester_ip_address": null, + "requester_metadata": null, + "routing_decision": null, + "spend_logs_metadata": null, + "team_alias": null, + "team_id": null, + "usage_object": {}, + "used_client_oauth_token": null, + "user_agent": null, + "user_api_key_alias": null, + "user_api_key_auth_metadata": null, + "user_api_key_budget_reset_at": null, + "user_api_key_customer_member_id": null, + "user_api_key_customer_org_id": null, + "user_api_key_customer_team_id": null, + "user_api_key_end_user_id": null, + "user_api_key_hash": null, + "user_api_key_max_budget": null, + "user_api_key_org_alias": null, + "user_api_key_org_id": null, + "user_api_key_project_alias": null, + "user_api_key_project_id": null, + "user_api_key_request_route": null, + "user_api_key_spend": null, + "user_api_key_team_alias": "team-a", + "user_api_key_team_id": null, + "user_api_key_team_max_budget": null, + "user_api_key_team_spend": null, + "user_api_key_user_email": null, + "user_api_key_user_id": null, + "user_api_key_user_max_budget": null, + "user_api_key_user_spend": null, + "vector_store_request_metadata": null + }, + "model": "gpt-4o", + "model_group": "", + "model_id": "", + "model_map_information": { + "model_map_key": "", + "model_map_value": null + }, + "model_parameters": {}, + "prompt_tokens": 0, + "request_model_access_groups": [], + "request_tags": [], + "requester_ip_address": null, + "response": {}, + "response_cost": 0.0, + "response_cost_failure_debug_info": null, + "response_time": "", + "saved_cache_cost": 0.0, + "session_id": "", + "standard_built_in_tools_params": { + "file_search": null, + "web_search_options": null + }, + "startTime": "", + "status": "failure", + "status_fields": { + "guardrail_status": "not_run", + "llm_api_status": "failure" + }, + "stream": null, + "total_tokens": 0, + "trace_id": "", + "user_agent": null, + "zero_cost_diagnostic": null + }, + "body_keys": [ + "id", + "litellm_call_id", + "trace_id", + "session_id", + "call_type", + "cache_hit", + "stream", + "status", + "status_fields", + "custom_llm_provider", + "saved_cache_cost", + "startTime", + "endTime", + "completionStartTime", + "response_time", + "model", + "metadata", + "cache_key", + "cost_finalized", + "response_cost", + "cost_breakdown", + "autorouter_savings", + "autorouter_savings_estimate", + "autorouter_baseline_observation", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "request_tags", + "request_model_access_groups", + "end_user", + "api_base", + "model_group", + "model_id", + "requester_ip_address", + "user_agent", + "messages", + "response", + "model_parameters", + "hidden_params", + "model_map_information", + "error_str", + "error_information", + "response_cost_failure_debug_info", + "zero_cost_diagnostic", + "guardrail_information", + "standard_built_in_tools_params" + ], + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "", + "content-language": "en", + "content-md5": "", + "content-type": "application/json", + "x-amz-content-sha256": "", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "path": "/logs/team-a//time-_.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_params_json_cannot_hold_still_build_the_native_logger::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "ri1XVSU2gNH4PxINvCEZsw==", + "content-type": "application/json", + "x-amz-content-sha256": "abcb4525830aa70ef9b1971bfd840fde25504d1127b1ed1062620ea127351c9b", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_prompts_only_adds_a_null_response_to_a_payload_without_one_like_python::puts": [ + { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": null}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "eXJGHO/sno/E12MifrFhxg==", + "content-type": "application/json", + "x-amz-content-sha256": "f23201ad9f8c59d8ef66db4824ba919a3b1538ccb44d5fa4665d541ee8857dd8", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_readiness_and_the_callbacks_view_see_the_native_logger_as_s3_v2::callbacks-view": { + "hidden": true, + "str-names-s3-logger": true + }, + "test_stripping_base64_rewrites_the_shared_payload_the_same_way::payload": { + "call_type": "acompletion", + "endTime": 1794560708.250042, + "id": "with-file", + "messages": [ + { + "content": [ + { + "text": "see [BASE64_REDACTED]", + "type": "text" + } + ], + "role": "user" + } + ], + "metadata": { + "user_api_key_alias": "key/one", + "user_api_key_hash": "88dc28", + "user_api_key_team_alias": "team-α" + }, + "response": { + "choices": [ + { + "message": { + "content": "hi" + } + } + ] + }, + "response_cost": 0.000123, + "startTime": 1794560707.000042 + }, + "test_stripping_base64_rewrites_the_shared_payload_the_same_way::puts": [ + { + "body": "{\"id\": \"with-file\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"see [BASE64_REDACTED]\"}]}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_with-file.json\"", + "content-language": "en", + "content-md5": "TS+2rEiXbVNd/O1SWjnuhg==", + "content-type": "application/json", + "x-amz-content-sha256": "8097863d5bac5fc65b950ffc04de8e33d906ed190d1fe60ba36bdbb2b3683860", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_with-file.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_the_attempt_ended_hook_only_enqueues_while_s3_is_slow::puts": [], + "test_the_attempt_ended_hook_only_enqueues_while_s3_is_slow::queue-length": 1, + "test_the_dispatched_native_strips_the_messages_before_the_payload_is_copied::puts": [ + { + "body": "{\"id\": \"stripped\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"see [BASE64_REDACTED]\"}]}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_stripped.json\"", + "content-language": "en", + "content-md5": "CT5il3y5UcYN1hkEqbk+ZA==", + "content-type": "application/json", + "x-amz-content-sha256": "1a45744feefe8bd41ac2613652cd2408d9194e2073d84225df00601c5a59f782", + "x-amz-security-token": "session-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_stripped.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + } + ], + "test_the_native_logger_answers_the_python_helpers::granularity": "hour", + "test_the_native_logger_answers_the_python_helpers::s3_path": "p", + "test_the_native_opt_out_matches_should_run_callback[None-None-False]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[None-None-True]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[None-True-False]::should_run_callback": false, + "test_the_native_opt_out_matches_should_run_callback[None-True-True]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[None-true-False]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[None-true-True]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[langfuse, S3_V2-None-False]::should_run_callback": false, + "test_the_native_opt_out_matches_should_run_callback[langfuse, S3_V2-None-True]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[langfuse, S3_V2-True-False]::should_run_callback": false, + "test_the_native_opt_out_matches_should_run_callback[langfuse, S3_V2-True-True]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[langfuse, S3_V2-true-False]::should_run_callback": false, + "test_the_native_opt_out_matches_should_run_callback[langfuse, S3_V2-true-True]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[s3_v2-None-False]::should_run_callback": false, + "test_the_native_opt_out_matches_should_run_callback[s3_v2-None-True]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[s3_v2-True-False]::should_run_callback": false, + "test_the_native_opt_out_matches_should_run_callback[s3_v2-True-True]::should_run_callback": true, + "test_the_native_opt_out_matches_should_run_callback[s3_v2-true-False]::should_run_callback": false, + "test_the_native_opt_out_matches_should_run_callback[s3_v2-true-True]::should_run_callback": true, + "test_web_identity_assumes_the_role_with_the_same_session_policy::policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"S3ObjectsLiteLLM\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\"],\"Resource\":\"*\",\"Condition\":{\"Bool\":{\"aws:SecureTransport\":\"true\"}}},{\"Sid\":\"KmsLiteLLM\",\"Effect\":\"Allow\",\"Action\":[\"kms:GenerateDataKey\",\"kms:Decrypt\"],\"Resource\":\"*\",\"Condition\":{\"Bool\":{\"aws:SecureTransport\":\"true\"},\"StringLike\":{\"kms:ViaService\":\"s3.*\"}}}]}", + "test_web_identity_assumes_the_role_with_the_same_session_policy::put": { + "body": "{\"id\": \"chatcmpl-1\", \"call_type\": \"acompletion\", \"startTime\": 1794560707.000042, \"endTime\": 1794560708.250042, \"response_cost\": 0.000123, \"metadata\": {\"user_api_key_team_alias\": \"team-\\u03b1\", \"user_api_key_alias\": \"key/one\", \"user_api_key_hash\": \"88dc28\"}, \"messages\": [{\"role\": \"user\", \"content\": \"hello \\u00abworld\\u00bb \\ud83d\\ude00\"}], \"response\": {\"choices\": [{\"message\": {\"content\": \"hi\"}}]}}", + "headers": { + "cache-control": "private, immutable, max-age=31536000, s-maxage=0", + "content-disposition": "inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"", + "content-language": "en", + "content-md5": "ri1XVSU2gNH4PxINvCEZsw==", + "content-type": "application/json", + "x-amz-content-sha256": "abcb4525830aa70ef9b1971bfd840fde25504d1127b1ed1062620ea127351c9b", + "x-amz-security-token": "sts-token", + "x-amz-server-side-encryption": null, + "x-amz-server-side-encryption-aws-kms-key-id": null + }, + "method": "PUT", + "path": "/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json", + "signed_headers": "cache-control;content-disposition;content-language;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token" + }, + "test_web_identity_assumes_the_role_with_the_same_session_policy::sts-call": { + "Action": [ + "AssumeRoleWithWebIdentity" + ], + "DurationSeconds": [ + "3600" + ], + "Policy": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"S3ObjectsLiteLLM\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\"],\"Resource\":\"*\",\"Condition\":{\"Bool\":{\"aws:SecureTransport\":\"true\"}}},{\"Sid\":\"KmsLiteLLM\",\"Effect\":\"Allow\",\"Action\":[\"kms:GenerateDataKey\",\"kms:Decrypt\"],\"Resource\":\"*\",\"Condition\":{\"Bool\":{\"aws:SecureTransport\":\"true\"},\"StringLike\":{\"kms:ViaService\":\"s3.*\"}}}]}" + ], + "RoleArn": [ + "arn:aws:iam::123456789012:role/logs" + ], + "RoleSessionName": [ + "s3-" + ], + "WebIdentityToken": [ + "oidc-token-value" + ] + }, + "test_without_a_region_param_both_sign_with_the_same_environment_region[default]::failures": 0, + "test_without_a_region_param_both_sign_with_the_same_environment_region[default]::put-count": 1, + "test_without_a_region_param_both_sign_with_the_same_environment_region[invalid]::failures": 1, + "test_without_a_region_param_both_sign_with_the_same_environment_region[invalid]::put-count": 0, + "test_without_a_region_param_both_sign_with_the_same_environment_region[region-name]::failures": 0, + "test_without_a_region_param_both_sign_with_the_same_environment_region[region-name]::put-count": 1, + "test_without_a_region_param_both_sign_with_the_same_environment_region[region-wins]::failures": 0, + "test_without_a_region_param_both_sign_with_the_same_environment_region[region-wins]::put-count": 1 +} diff --git a/tests/unit/integrations/s3_v3_rust_support.py b/tests/unit/integrations/s3_v3_rust_support.py new file mode 100644 index 00000000000..fcd566c98fb --- /dev/null +++ b/tests/unit/integrations/s3_v3_rust_support.py @@ -0,0 +1,145 @@ +import threading +import time +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final + +import pytest + +from litellm.integrations.custom_logger_v2.call import CallContext +from litellm.integrations.custom_logger_v2.derivations import DERIVED_ATTEMPT_FIELDS +from litellm.integrations.custom_logger_v2.enums import Hook +from litellm.integrations.custom_logger_v2.enforcement import FieldView +from litellm.integrations.custom_logger_v2.field_sources import HookArgs, with_sources +from litellm.integrations.custom_logger_v2.legacy import KwargsFields +from litellm.integrations.custom_logger_v2.rust import WireManifest +from litellm.integrations.s3_v3 import S3Logger +from litellm.litellm_core_utils.litellm_logging import LOGGING_KEYS, V2_ASYNC_LOGGING_LINE +from litellm.types.integrations.custom_logger_v2 import Outcome + +DROP: Final = -1 + + +@dataclass(frozen=True) +class Received: + method: str + path: str + headers: Mapping[str, str] + body: bytes + at: float + + +@dataclass +class ScriptedBucket: + port: int = 0 + received: list[Received] = field(default_factory=list) # mutable-ok: appended by the server thread + objects: dict[str, bytes] = field(default_factory=dict) # mutable-ok: GET serves what PUT stored + script: list[tuple[str, list[tuple[int, str]]]] = field(default_factory=list) # mutable-ok: replies are consumed + delay: float = 0.0 + in_flight: int = 0 + peak: int = 0 + lock: threading.Lock = field(default_factory=threading.Lock) + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def reply(self, fragment: str, *replies: tuple[int, str]) -> None: + self.script.append((fragment, list(replies))) + + def puts(self) -> list[Received]: + return [request for request in self.received if request.method == "PUT"] + + def next_reply(self, path: str) -> tuple[int, str]: + with self.lock: + for fragment, replies in self.script: + if fragment in path and replies: + return replies.pop(0) if len(replies) > 1 else replies[0] + return 200, "" + + +def _handler(bucket: ScriptedBucket) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_: object) -> None: + return + + def _send(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_PUT(self) -> None: + body: Final = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + headers: Final = {name.lower(): value for name, value in self.headers.items()} + with bucket.lock: + bucket.received.append(Received("PUT", self.path, headers, body, time.monotonic())) + bucket.in_flight += 1 + bucket.peak = max(bucket.peak, bucket.in_flight) + if bucket.delay: + time.sleep(bucket.delay) + with bucket.lock: + bucket.in_flight -= 1 + status, text = bucket.next_reply(self.path) + if status == DROP: + self.close_connection = True + return + if status == 200: + with bucket.lock: + bucket.objects[self.path] = body + self._send(status, text.encode()) + + def do_GET(self) -> None: + headers: Final = {name.lower(): value for name, value in self.headers.items()} + with bucket.lock: + bucket.received.append(Received("GET", self.path, headers, b"", time.monotonic())) + stored: Final = bucket.objects.get(self.path) + if stored is None: + self._send(404, b"NoSuchKey") + return + self._send(200, stored) + + return Handler + + +class _BucketServer(ThreadingHTTPServer): + request_queue_size = 256 + + +@contextmanager +def scripted_bucket() -> Iterator[ScriptedBucket]: + bucket: Final = ScriptedBucket() + server: Final = _BucketServer(("127.0.0.1", 0), _handler(bucket)) + bucket.port = server.server_address[1] + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield bucket + finally: + server.shutdown() + + +def rust_s3_logger(monkeypatch: pytest.MonkeyPatch, **params: object) -> S3Logger: # kwargs-ok: the S3Logger keywords + return S3Logger(params) if params else S3Logger() + + +async def log_attempt( + logger: S3Logger, kwargs: Mapping[str, object], start_time: datetime, outcome: Outcome = Outcome.success +) -> None: + args: Final = HookArgs(request=kwargs, start_time=start_time, end_time=start_time, outcome=outcome) + view: Final = FieldView(with_sources(KwargsFields(kwargs, LOGGING_KEYS), DERIVED_ATTEMPT_FIELDS, args)) + manifest: Final = WireManifest.model_validate_json(logger.rust.manifest()) + reads: Final = next(s.reads for s in manifest.subscriptions if s.hook is Hook.attempt_ended) + line: Final = V2_ASYNC_LOGGING_LINE.set(True) + try: + _ = await logger.run_rust(Hook.attempt_ended, reads, view, CallContext(Hook.attempt_ended, None, None)) + finally: + V2_ASYNC_LOGGING_LINE.reset(line) + + +async def log_payload(logger: S3Logger, payload: Mapping[str, object], start_time: datetime) -> None: + await log_attempt(logger, {"standard_logging_object": payload}, start_time) diff --git a/tests/unit/integrations/test_adaptive_concurrency.py b/tests/unit/integrations/test_adaptive_concurrency.py deleted file mode 100644 index 15b9d52a42e..00000000000 --- a/tests/unit/integrations/test_adaptive_concurrency.py +++ /dev/null @@ -1,179 +0,0 @@ -import asyncio -from typing import Final - -import pytest - -from litellm.integrations.adaptive_concurrency import AdaptiveConcurrencyLimiter, PutSample - -_real_sleep: Final = asyncio.sleep - - -def _limiter(initial: int = 4, floor: int = 1, ceiling: int = 16) -> AdaptiveConcurrencyLimiter: - return AdaptiveConcurrencyLimiter(initial=initial, floor=floor, ceiling=ceiling) - - -@pytest.mark.asyncio -async def test_limit_grows_after_limit_clean_samples() -> None: - limiter: Final = _limiter(initial=4) - for _ in range(4): - limiter.record(PutSample(throttled=False)) - assert limiter.limit == 5 - - -@pytest.mark.asyncio -async def test_limit_does_not_grow_before_the_streak_completes() -> None: - limiter: Final = _limiter(initial=4) - for _ in range(3): - limiter.record(PutSample(throttled=False)) - assert limiter.limit == 4 - - -@pytest.mark.asyncio -async def test_throttled_sample_halves_the_limit() -> None: - limiter: Final = _limiter(initial=16) - limiter.record(PutSample(throttled=True)) - assert limiter.limit == 8 - - -@pytest.mark.asyncio -async def test_limit_clamps_at_the_floor() -> None: - limiter: Final = _limiter(initial=4, floor=4) - limiter.record(PutSample(throttled=True)) - assert limiter.limit == 4 - - -@pytest.mark.asyncio -async def test_limit_clamps_at_the_ceiling() -> None: - limiter: Final = _limiter(initial=15, ceiling=16) - for _ in range(1000): - limiter.record(PutSample(throttled=False)) - assert limiter.limit == 16 - - -@pytest.mark.asyncio -async def test_throttled_sample_resets_the_clean_streak() -> None: - limiter: Final = _limiter(initial=4, ceiling=32) - for _ in range(3): - limiter.record(PutSample(throttled=False)) - limiter.record(PutSample(throttled=True)) - limiter.record(PutSample(throttled=False)) - assert limiter.limit == 2 - - -@pytest.mark.asyncio -async def test_growing_the_limit_wakes_a_waiting_acquirer() -> None: - limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=4) - acquired: Final[list[str]] = [] # mutable-ok: the waiter task appends to it across the await boundary - released: Final = asyncio.Event() - - async def hold() -> None: - async with limiter: - await released.wait() - - holder: Final = asyncio.create_task(hold()) - - async def waiter() -> None: - async with limiter: - acquired.append("waiter") - - pending: Final = asyncio.create_task(waiter()) - await _real_sleep(0) - assert not acquired - - limiter.record(PutSample(throttled=False)) - await asyncio.wait_for(asyncio.shield(pending), timeout=5) - released.set() - await asyncio.wait_for(holder, timeout=5) - assert tuple(acquired) == ("waiter",) - - -@pytest.mark.asyncio -async def test_releasing_a_slot_wakes_exactly_one_waiter() -> None: - limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=4) - acquired: Final[list[str]] = [] # mutable-ok: the waiter tasks append to it across the await boundary - release: Final = asyncio.Event() - - async def hold() -> None: - async with limiter: - await _real_sleep(0) - - async def waiter(name: str) -> None: - async with limiter: - acquired.append(name) - await release.wait() - - holder: Final = asyncio.create_task(hold()) - waiters: Final = tuple(asyncio.create_task(waiter(f"w{i}")) for i in range(3)) - await _real_sleep(0) - await asyncio.wait_for(holder, timeout=5) - await _real_sleep(0) - assert len(acquired) == 1 - - for _ in range(3): - limiter.record(PutSample(throttled=False)) - await _real_sleep(0) - assert len(acquired) == 3 - release.set() - await asyncio.gather(*waiters) - - -@pytest.mark.asyncio -async def test_double_cancel_during_release_leaves_in_flight_at_zero() -> None: - limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=1) - entered: Final = asyncio.Event() - release: Final = asyncio.Event() - - async def hold() -> None: - async with limiter: - entered.set() - await release.wait() - - holder: Final = asyncio.create_task(hold()) - await entered.wait() - - waiter: Final = asyncio.create_task(hold()) - await _real_sleep(0) - waiter.cancel() - with pytest.raises(asyncio.CancelledError): - await waiter - - release.set() - await asyncio.wait_for(holder, timeout=5) - holder.cancel() - try: - await holder - except asyncio.CancelledError: - pass - - assert limiter._in_flight == 0 - - -@pytest.mark.asyncio -async def test_a_cancelled_waiter_is_skipped_when_a_slot_frees() -> None: - limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=1) - acquired: Final[list[str]] = [] # mutable-ok: waiter tasks append across the await boundary - first_entered: Final = asyncio.Event() - release: Final = asyncio.Event() - - async def hold(name: str, entered: asyncio.Event | None = None) -> None: - async with limiter: - acquired.append(name) - if entered is not None: - entered.set() - await release.wait() - - holder: Final = asyncio.create_task(hold("holder", first_entered)) - await first_entered.wait() - doomed: Final = asyncio.create_task(hold("doomed")) - next_waiter: Final = asyncio.create_task(hold("next")) - await _real_sleep(0) - doomed.cancel() - with pytest.raises(asyncio.CancelledError): - await doomed - release.set() - await asyncio.wait_for(holder, timeout=5) - await asyncio.wait_for(next_waiter, timeout=5) - - assert "doomed" not in acquired - assert "next" in acquired - assert limiter._in_flight == 0 diff --git a/tests/unit/integrations/test_s3_v2.py b/tests/unit/integrations/test_s3_v2.py deleted file mode 100644 index e1db2743bc6..00000000000 --- a/tests/unit/integrations/test_s3_v2.py +++ /dev/null @@ -1,6044 +0,0 @@ -import asyncio, boto3, importlib, logging, os, pytest_asyncio -import copy -import json -import re -import sys -import textwrap -import threading -import time -import uuid -from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine -from contextlib import asynccontextmanager -from contextvars import Context -from datetime import datetime -from pathlib import Path -from typing import Final -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import pytest -import respx - -import litellm -from litellm.integrations.custom_logger import CustomLogger -from litellm.integrations.s3_v2 import S3BatchUploadError, S3Logger -from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.types.integrations.s3_v2 import s3BatchLoggingElement -from litellm.types.utils import StandardLoggingPayload -from collections import defaultdict -from litellm._logging import verbose_logger -from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE, S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome - -_real_sleep: Final = asyncio.sleep -_NOW: Final = 1_000_000.0 - - -def _assert_jittered_backoff(sleep_mock: MagicMock, retries: int) -> None: - """Each retry waits a jittered exponential backoff: attempt n sleeps somewhere in [2**n / 2, 2**n * 1.5).""" - waits: Final = [sleep_call.args[0] for sleep_call in sleep_mock.call_args_list] - assert len(waits) == retries, waits - for attempt, wait in enumerate(waits): - assert (1 << attempt) * 0.5 <= wait < (1 << attempt) * 1.5, (attempt, wait) - - -class TestS3V2UnitTests: - """Test that S3 v2 integration only uses safe_dumps and not json.dumps""" - - def test_s3_v2_source_code_analysis(self): - """Test that S3 v2 source code only imports and uses safe_dumps""" - import inspect - - from litellm.integrations import s3_v2 - - # Get the source code of the s3_v2 module - source_code = inspect.getsource(s3_v2) - - # Verify that json.dumps is not used directly in the code - assert "json.dumps(" not in source_code, "S3 v2 should not use json.dumps directly" - - @patch("asyncio.create_task") - @patch("litellm.integrations.s3_v2.CustomBatchLogger.periodic_flush") - def test_s3_v2_endpoint_url(self, mock_periodic_flush, mock_create_task): - """testing s3 endpoint url""" - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - # Mock periodic_flush and create_task to prevent async task creation during init - mock_periodic_flush.return_value = None - mock_create_task.return_value = None - - # Mock response for all tests - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - - # Create a test batch logging element - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-key.json", - payload={"test": "data"}, - s3_object_download_filename="test-file.json", - ) - - # Test 1: Custom endpoint URL with bucket name - s3_logger = S3Logger( - s3_bucket_name="test-bucket", - s3_endpoint_url="https://s3.amazonaws.com", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - s3_logger.async_httpx_client = AsyncMock() - s3_logger.async_httpx_client.put.return_value = mock_response - - asyncio.run(s3_logger.async_upload_data_to_s3(test_element)) - - call_args = s3_logger.async_httpx_client.put.call_args - assert call_args is not None - url = call_args[0][0] - expected_url = "https://s3.amazonaws.com/test-bucket/2025-09-14/test-key.json" - assert url == expected_url, f"Expected URL {expected_url}, got {url}" - - # Test 2: MinIO-compatible endpoint - s3_logger_minio = S3Logger( - s3_bucket_name="litellm-logs", - s3_endpoint_url="https://minio.example.com:9000", - s3_aws_access_key_id="minio-key", - s3_aws_secret_access_key="minio-secret", - s3_region_name="us-east-1", - ) - - s3_logger_minio.async_httpx_client = AsyncMock() - s3_logger_minio.async_httpx_client.put.return_value = mock_response - - asyncio.run(s3_logger_minio.async_upload_data_to_s3(test_element)) - - call_args_minio = s3_logger_minio.async_httpx_client.put.call_args - assert call_args_minio is not None - url_minio = call_args_minio[0][0] - expected_minio_url = "https://minio.example.com:9000/litellm-logs/2025-09-14/test-key.json" - assert url_minio == expected_minio_url, f"Expected MinIO URL {expected_minio_url}, got {url_minio}" - - # Test 3: Custom endpoint without bucket name (should fall back to default) - s3_logger_no_bucket = S3Logger( - s3_endpoint_url="https://s3.amazonaws.com", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - s3_logger_no_bucket.async_httpx_client = AsyncMock() - s3_logger_no_bucket.async_httpx_client.put.return_value = mock_response - - asyncio.run(s3_logger_no_bucket.async_upload_data_to_s3(test_element)) - - call_args_no_bucket = s3_logger_no_bucket.async_httpx_client.put.call_args - assert call_args_no_bucket is not None - url_no_bucket = call_args_no_bucket[0][0] - # Should use default S3 URL format when bucket is missing (bucket becomes None in URL) - assert "s3.us-east-1.amazonaws.com" in url_no_bucket - assert "https://" in url_no_bucket - # Should not include the custom endpoint since bucket is missing - assert "https://s3.amazonaws.com/" not in url_no_bucket - - # Test 4: Sync upload method with custom endpoint - s3_logger_sync = S3Logger( - s3_bucket_name="sync-bucket", - s3_endpoint_url="https://custom.s3.endpoint.com", - s3_aws_access_key_id="sync-key", - s3_aws_secret_access_key="sync-secret", - s3_region_name="us-east-1", - ) - - mock_sync_client = MagicMock() - mock_sync_client.put.return_value = mock_response - - with patch( - "litellm.integrations.s3_v2.get_httpx_client", - return_value=mock_sync_client, - ): - s3_logger_sync.upload_data_to_s3(test_element) - - call_args_sync = mock_sync_client.put.call_args - assert call_args_sync is not None - url_sync = call_args_sync[0][0] - expected_sync_url = "https://custom.s3.endpoint.com/sync-bucket/2025-09-14/test-key.json" - assert url_sync == expected_sync_url, f"Expected sync URL {expected_sync_url}, got {url_sync}" - - # Test 5: Download method with custom endpoint - s3_logger_download = S3Logger( - s3_bucket_name="download-bucket", - s3_endpoint_url="https://download.s3.endpoint.com", - s3_aws_access_key_id="download-key", - s3_aws_secret_access_key="download-secret", - s3_region_name="us-east-1", - ) - - mock_download_response = MagicMock() - mock_download_response.status_code = 200 - mock_download_response.json = MagicMock(return_value={"downloaded": "data"}) - s3_logger_download.async_httpx_client = AsyncMock() - s3_logger_download.async_httpx_client.get.return_value = mock_download_response - - result = asyncio.run(s3_logger_download._download_object_from_s3("2025-09-14/download-test-key.json")) - - call_args_download = s3_logger_download.async_httpx_client.get.call_args - assert call_args_download is not None - url_download = call_args_download[0][0] - expected_download_url = "https://download.s3.endpoint.com/download-bucket/2025-09-14/download-test-key.json" - assert url_download == expected_download_url, ( - f"Expected download URL {expected_download_url}, got {url_download}" - ) - - assert result == {"downloaded": "data"} - - @patch("asyncio.create_task") - @patch("litellm.integrations.s3_v2.CustomBatchLogger.periodic_flush") - def test_s3_v2_virtual_hosted_style(self, mock_periodic_flush, mock_create_task): - """Test s3_use_virtual_hosted_style parameter for virtual-hosted-style URLs""" - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - # Mock periodic_flush and create_task to prevent async task creation during init - mock_periodic_flush.return_value = None - mock_create_task.return_value = None - - # Mock response for all tests - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - - # Create a test batch logging element - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-key.json", - payload={"test": "data"}, - s3_object_download_filename="test-file.json", - ) - - # Test 1: Virtual-hosted-style with custom endpoint - s3_logger_virtual = S3Logger( - s3_bucket_name="test-bucket", - s3_endpoint_url="https://s3.custom-endpoint.com", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_use_virtual_hosted_style=True, - ) - - s3_logger_virtual.async_httpx_client = AsyncMock() - s3_logger_virtual.async_httpx_client.put.return_value = mock_response - - asyncio.run(s3_logger_virtual.async_upload_data_to_s3(test_element)) - - call_args = s3_logger_virtual.async_httpx_client.put.call_args - assert call_args is not None - url = call_args[0][0] - expected_url = "https://test-bucket.s3.custom-endpoint.com/2025-09-14/test-key.json" - assert url == expected_url, f"Expected virtual-hosted-style URL {expected_url}, got {url}" - - # Test 2: Path-style (default behavior with s3_use_virtual_hosted_style=False) - s3_logger_path = S3Logger( - s3_bucket_name="test-bucket", - s3_endpoint_url="https://s3.custom-endpoint.com", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_use_virtual_hosted_style=False, - ) - - s3_logger_path.async_httpx_client = AsyncMock() - s3_logger_path.async_httpx_client.put.return_value = mock_response - - asyncio.run(s3_logger_path.async_upload_data_to_s3(test_element)) - - call_args_path = s3_logger_path.async_httpx_client.put.call_args - assert call_args_path is not None - url_path = call_args_path[0][0] - expected_path_url = "https://s3.custom-endpoint.com/test-bucket/2025-09-14/test-key.json" - assert url_path == expected_path_url, f"Expected path-style URL {expected_path_url}, got {url_path}" - - # Test 3: Virtual-hosted-style with http protocol - s3_logger_http = S3Logger( - s3_bucket_name="http-bucket", - s3_endpoint_url="http://minio.local:9000", - s3_aws_access_key_id="minio-key", - s3_aws_secret_access_key="minio-secret", - s3_region_name="us-east-1", - s3_use_virtual_hosted_style=True, - ) - - s3_logger_http.async_httpx_client = AsyncMock() - s3_logger_http.async_httpx_client.put.return_value = mock_response - - asyncio.run(s3_logger_http.async_upload_data_to_s3(test_element)) - - call_args_http = s3_logger_http.async_httpx_client.put.call_args - assert call_args_http is not None - url_http = call_args_http[0][0] - expected_http_url = "http://http-bucket.minio.local:9000/2025-09-14/test-key.json" - assert url_http == expected_http_url, ( - f"Expected virtual-hosted-style URL with http {expected_http_url}, got {url_http}" - ) - - # Test 4: Sync upload method with virtual-hosted-style - s3_logger_sync_virtual = S3Logger( - s3_bucket_name="sync-bucket", - s3_endpoint_url="https://storage.example.com", - s3_aws_access_key_id="sync-key", - s3_aws_secret_access_key="sync-secret", - s3_region_name="us-east-1", - s3_use_virtual_hosted_style=True, - ) - - mock_sync_client = MagicMock() - mock_sync_client.put.return_value = mock_response - - with patch( - "litellm.integrations.s3_v2.get_httpx_client", - return_value=mock_sync_client, - ): - s3_logger_sync_virtual.upload_data_to_s3(test_element) - - call_args_sync = mock_sync_client.put.call_args - assert call_args_sync is not None - url_sync = call_args_sync[0][0] - expected_sync_url = "https://sync-bucket.storage.example.com/2025-09-14/test-key.json" - assert url_sync == expected_sync_url, ( - f"Expected virtual-hosted-style sync URL {expected_sync_url}, got {url_sync}" - ) - - # Test 5: Download method with virtual-hosted-style - s3_logger_download_virtual = S3Logger( - s3_bucket_name="download-bucket", - s3_endpoint_url="https://download.endpoint.com", - s3_aws_access_key_id="download-key", - s3_aws_secret_access_key="download-secret", - s3_region_name="us-east-1", - s3_use_virtual_hosted_style=True, - ) - - mock_download_response = MagicMock() - mock_download_response.status_code = 200 - mock_download_response.json = MagicMock(return_value={"downloaded": "data"}) - s3_logger_download_virtual.async_httpx_client = AsyncMock() - s3_logger_download_virtual.async_httpx_client.get.return_value = mock_download_response - - result = asyncio.run(s3_logger_download_virtual._download_object_from_s3("2025-09-14/download-test-key.json")) - - call_args_download = s3_logger_download_virtual.async_httpx_client.get.call_args - assert call_args_download is not None - url_download = call_args_download[0][0] - expected_download_url = "https://download-bucket.download.endpoint.com/2025-09-14/download-test-key.json" - assert url_download == expected_download_url, ( - f"Expected virtual-hosted-style download URL {expected_download_url}, got {url_download}" - ) - - assert result == {"downloaded": "data"} - - @patch("asyncio.create_task") - @patch("litellm.integrations.s3_v2.CustomBatchLogger.periodic_flush") - def test_s3_v2_put_url_encodes_spaces_in_object_key(self, mock_periodic_flush, mock_create_task): - from unittest.mock import AsyncMock - - import requests - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - mock_periodic_flush.return_value = None - mock_create_task.return_value = None - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - - s3_object_key = "My Team/2025-09-14/test-key.json" - test_element = s3BatchLoggingElement( - s3_object_key=s3_object_key, - payload={"test": "data"}, - s3_object_download_filename="test-file.json", - ) - - s3_logger = S3Logger( - s3_bucket_name="test-bucket", - s3_endpoint_url="https://s3.amazonaws.com", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - s3_logger.async_httpx_client = AsyncMock() - s3_logger.async_httpx_client.put.return_value = mock_response - - asyncio.run(s3_logger.async_upload_data_to_s3(test_element)) - - call_args = s3_logger.async_httpx_client.put.call_args - assert call_args is not None - actual_url = call_args[0][0] - raw_url = f"https://s3.amazonaws.com/test-bucket/{s3_object_key}" - expected_url = requests.Request("PUT", raw_url).prepare().url - assert actual_url == expected_url - assert " " not in actual_url - - -@pytest.mark.asyncio -async def test_async_upload_retries_on_s3_503(): - """ - Test that async_upload_data_to_s3 retries on transient S3 503 Slow Down - and succeeds on the second attempt. - """ - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-retry.json", - payload={"test": "retry"}, - s3_object_download_filename="test-retry.json", - ) - - # First call returns 503, second call returns 200 - response_503 = MagicMock() - response_503.status_code = 503 - response_503.text = "" - response_200 = MagicMock() - response_200.status_code = 200 - response_200.text = "" - response_200.raise_for_status = MagicMock() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = AsyncMock(side_effect=[response_503, response_200]) - - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - await logger.async_upload_data_to_s3(test_element) - - # Verify PUT was called twice (retry after 503) - assert logger.async_httpx_client.put.call_count == 2 - _assert_jittered_backoff(mock_sleep, 1) - - -@pytest.mark.asyncio -async def test_async_upload_retries_on_s3_500(): - """ - Test that async_upload_data_to_s3 retries on transient S3 500 errors. - """ - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-retry-500.json", - payload={"test": "retry-500"}, - s3_object_download_filename="test-retry-500.json", - ) - - response_500 = MagicMock() - response_500.status_code = 500 - response_500.text = "" - response_200 = MagicMock() - response_200.status_code = 200 - response_200.text = "" - response_200.raise_for_status = MagicMock() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = AsyncMock(side_effect=[response_500, response_200]) - - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - await logger.async_upload_data_to_s3(test_element) - - assert logger.async_httpx_client.put.call_count == 2 - _assert_jittered_backoff(mock_sleep, 1) - - -@pytest.mark.asyncio -async def test_async_upload_exhausts_retries_on_persistent_503(): - """ - Test that async_upload_data_to_s3 raises after exhausting all retries - on persistent S3 503. - """ - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-exhaust.json", - payload={"test": "exhaust"}, - s3_object_download_filename="test-exhaust.json", - ) - - # All 3 attempts return 503 - response_503 = MagicMock() - response_503.status_code = 503 - response_503.text = "" - response_503.raise_for_status = MagicMock(side_effect=Exception("503 Service Unavailable")) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = AsyncMock(return_value=response_503) - - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - with patch.object(logger, "handle_callback_failure") as mock_failure: - await logger.async_upload_data_to_s3(test_element) - - # 3 PUT attempts total - assert logger.async_httpx_client.put.call_count == 3 - # 2 sleeps (between attempts 1-2 and 2-3) - assert mock_sleep.call_count == 2 - # Callback failure handler called after exhausting retries - mock_failure.assert_called_once_with(callback_name="S3Logger") - - -@pytest.mark.asyncio -async def test_async_upload_retries_400_with_an_unknown_error_code(): - """ - A 400 is outside the retry set, so an unknown gets a single PUT and the "retry" outcome - for the flush-level requeue, never an in-call backoff. - """ - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-retry-400.json", - payload={"test": "retry-400"}, - s3_object_download_filename="test-retry-400.json", - ) - - response_400 = MagicMock() - response_400.status_code = 400 - response_400.text = "SomethingElse" - response_400.raise_for_status = MagicMock(side_effect=Exception("400 Bad Request")) - response_200 = MagicMock() - response_200.status_code = 200 - response_200.text = "" - response_200.raise_for_status = MagicMock() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = AsyncMock(side_effect=[response_400, response_200]) - - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - outcome = await logger.async_upload_data_to_s3(test_element) - - assert logger.async_httpx_client.put.call_count == 1 - mock_sleep.assert_not_awaited() - assert outcome is False - - -_SIGV4_ACCESS_KEY = re.compile(r"Credential=(AKIA\d+)/") - - -@pytest.fixture -def rotating_profile(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str: - """ - A real botocore profile whose credential_process hands out a new key generation on every call and - expires inside the advisory refresh window, so RefreshableCredentials re-runs it on every property read. - """ - counter = tmp_path / "generation" - script = tmp_path / "rotate_credentials.py" - script.write_text( - textwrap.dedent( - f""" - import json, sys - from datetime import datetime, timedelta, timezone - from pathlib import Path - - counter = Path({str(counter)!r}) - generation = int(counter.read_text()) if counter.exists() else 0 - counter.write_text(str(generation + 1)) - expiry = (datetime.now(timezone.utc) + timedelta(minutes=12)).strftime("%Y-%m-%dT%H:%M:%SZ") - json.dump( - {{ - "Version": 1, - "AccessKeyId": f"AKIA{{generation}}", - "SecretAccessKey": f"secret-{{generation}}", - "SessionToken": f"token-{{generation}}", - "Expiration": expiry, - }}, - sys.stdout, - ) - """ - ) - ) - profile = f"rotating-{uuid.uuid4().hex}" - (tmp_path / "config").write_text(f"[profile {profile}]\ncredential_process = {sys.executable} {script}\n") - monkeypatch.setenv("AWS_CONFIG_FILE", str(tmp_path / "config")) - return profile - - -def _generation(request: httpx.Request) -> tuple[str, str]: - """(access key generation, session token generation) SigV4 baked into one request.""" - access_key = _SIGV4_ACCESS_KEY.search(request.headers["Authorization"]) - assert access_key is not None - return access_key.group(1).removeprefix("AKIA"), request.headers["X-Amz-Security-Token"].removeprefix("token-") - - -@asynccontextmanager -async def _s3_logger_on_production_handler(profile: str, statuses: list[int | type[httpx.TransportError]]): - """ - S3Logger wired to the real AsyncHTTPHandler over an httpx MockTransport that answers with the given - statuses (or raises the given transport errors) in order, so the handler's own error mapping is exercised. - """ - requests: list[httpx.Request] = [] - replies = iter(statuses) - - def respond(request: httpx.Request) -> httpx.Response: - requests.append(request) - reply = next(replies) - if not isinstance(reply, int): - raise reply("S3 attempt failed", request=request) - return httpx.Response(reply, request=request, text="SignatureDoesNotMatch") - - handler = AsyncHTTPHandler() - handler.client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_region_name="us-east-1", - s3_aws_profile_name=profile, - s3_flush_interval=3600, - ) - logger.async_httpx_client = handler - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - yield logger, requests, mock_sleep - await handler.client.aclose() - - -@pytest.mark.asyncio -async def test_async_upload_signs_with_one_frozen_credential_snapshot(rotating_profile: str, caplog): - """ - RefreshableCredentials refreshes on every property read once inside the advisory window, so signing - off the live object would mix the access key of one generation with the token of the next. - """ - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-frozen.json", - payload={"test": "frozen"}, - s3_object_download_filename="test-frozen.json", - ) - async with _s3_logger_on_production_handler(rotating_profile, [200]) as (logger, requests, _): - await logger.async_upload_data_to_s3(test_element) - - assert len(requests) == 1 - access_key_generation, token_generation = _generation(requests[0]) - assert access_key_generation == token_generation - assert "Error uploading to s3" not in caplog.text - - -@pytest.mark.asyncio -async def test_async_upload_retries_403_with_fresh_credentials_and_signature(rotating_profile: str, caplog): - """ - A 403 (SignatureDoesNotMatch after an IMDS rotation) must be retried, and the retry must fetch - credentials again and carry a signature computed from that newer generation. - """ - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-403.json", - payload={"test": "403"}, - s3_object_download_filename="test-403.json", - ) - async with _s3_logger_on_production_handler(rotating_profile, [403, 200]) as (logger, requests, mock_sleep): - await logger.async_upload_data_to_s3(test_element) - - assert len(requests) == 2 - first_key, first_token = _generation(requests[0]) - second_key, second_token = _generation(requests[1]) - assert first_key == first_token - assert second_key == second_token - assert int(second_key) > int(first_key) - assert requests[1].headers["Authorization"] != requests[0].headers["Authorization"] - _assert_jittered_backoff(mock_sleep, 1) - assert "Error uploading to s3" not in caplog.text - - -@pytest.mark.asyncio -async def test_async_upload_exhausts_403_retries_through_production_http_handler(rotating_profile: str, caplog): - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-403-exhausted.json", - payload={"test": "403-exhausted"}, - s3_object_download_filename="test-403-exhausted.json", - ) - async with _s3_logger_on_production_handler(rotating_profile, [403, 403, 403]) as (logger, requests, mock_sleep): - await logger.async_upload_data_to_s3(test_element) - - assert len(requests) == 3 - _assert_jittered_backoff(mock_sleep, 2) - assert "Error uploading to s3" in caplog.text - - -@pytest.mark.asyncio -@pytest.mark.parametrize("transient_status", [500, 503]) -async def test_async_upload_recovers_from_transient_5xx_through_production_http_handler( - transient_status: int, rotating_profile: str, caplog: pytest.LogCaptureFixture -): - """ - AsyncHTTPHandler.put raises MaskedHTTPStatusError on 5xx instead of returning the response, so a retry - loop that only inspects returned status codes never runs (#42868). - """ - test_element = s3BatchLoggingElement( - s3_object_key=f"2025-09-14/test-{transient_status}.json", - payload={"test": str(transient_status)}, - s3_object_download_filename=f"test-{transient_status}.json", - ) - async with _s3_logger_on_production_handler(rotating_profile, [transient_status, 200]) as ( - logger, - requests, - mock_sleep, - ): - uploaded = await logger.async_upload_data_to_s3(test_element) - - assert uploaded is True - assert len(requests) == 2 - assert all(request.method == "PUT" for request in requests) - assert requests[0].url == requests[1].url - assert requests[0].content == requests[1].content - _assert_jittered_backoff(mock_sleep, 1) - assert "Error uploading to s3" not in caplog.text - - -@pytest.mark.asyncio -async def test_async_upload_is_single_attempted_on_404_through_production_http_handler(rotating_profile: str, caplog): - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-404.json", - payload={"test": "404"}, - s3_object_download_filename="test-404.json", - ) - async with _s3_logger_on_production_handler(rotating_profile, [404]) as (logger, requests, mock_sleep): - outcome = await logger.async_upload_data_to_s3(test_element) - - assert len(requests) == 1 - assert outcome is False - mock_sleep.assert_not_awaited() - assert "Error uploading to s3" in caplog.text - - -@pytest.mark.asyncio -async def test_async_upload_access_denied_403_is_retried_and_then_requeued(rotating_profile: str, caplog): - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-403-denied.json", - payload={"test": "403-denied"}, - s3_object_download_filename="test-403-denied.json", - ) - requests: list[httpx.Request] = [] - - def respond(request: httpx.Request) -> httpx.Response: - requests.append(request) - return httpx.Response(403, request=request, text="AccessDenied") - - handler = AsyncHTTPHandler() - handler.client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_region_name="us-east-1", - s3_aws_profile_name=rotating_profile, - s3_flush_interval=3600, - ) - logger.async_httpx_client = handler - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - outcome = await logger.async_upload_data_to_s3(test_element) - await handler.client.aclose() - - assert outcome is False - assert len(requests) == 3 - _assert_jittered_backoff(mock_sleep, 2) - assert "Error uploading to s3" in caplog.text - - -@pytest.mark.asyncio -async def test_async_upload_retries_429_through_production_http_handler(rotating_profile: str, caplog): - """S3 and S3-compatible stores answer throttling with 429 too, so it backs off and retries like a 503.""" - async with _s3_logger_on_production_handler(rotating_profile, [429, 200]) as (logger, requests, mock_sleep): - uploaded = await logger.async_upload_data_to_s3(_element({"test": "429"}, "429")) - - assert uploaded is True - assert len(requests) == 2 - assert requests[0].content == requests[1].content - _assert_jittered_backoff(mock_sleep, 1) - assert "Error uploading to s3" not in caplog.text - - -@pytest.mark.asyncio -async def test_async_upload_retries_a_timed_out_attempt_through_production_http_handler(rotating_profile: str, caplog): - """AsyncHTTPHandler.put turns an httpx timeout into litellm.Timeout, which must count as a retryable attempt.""" - async with _s3_logger_on_production_handler(rotating_profile, [httpx.ReadTimeout, 200]) as ( - logger, - requests, - mock_sleep, - ): - uploaded = await logger.async_upload_data_to_s3(_element({"test": "timeout"}, "timeout")) - - assert uploaded is True - assert len(requests) == 2 - _assert_jittered_backoff(mock_sleep, 1) - assert "Error uploading to s3" not in caplog.text - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "error", - [ - pytest.param(httpx.ConnectError("connection refused"), id="connect-error"), - pytest.param(httpx.RemoteProtocolError("server disconnected"), id="remote-protocol-error"), - pytest.param(httpx.ReadTimeout("read timed out"), id="read-timeout"), - ], -) -async def test_async_upload_retries_transport_errors_then_succeeds(error: Exception) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = AsyncMock(side_effect=[error, _ok_response()]) - - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - uploaded = await logger.async_upload_data_to_s3(_element({"test": "transport"}, "transport")) - - assert uploaded is True - assert logger.async_httpx_client.put.call_count == 2 - _assert_jittered_backoff(mock_sleep, 1) - - -@pytest.mark.asyncio -async def test_async_upload_exhausts_transport_error_retries_and_keeps_the_event_queued() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = AsyncMock(side_effect=httpx.ConnectError("connection refused")) - logger.log_queue = [_element({"test": "down"}, "down")] - - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - await logger.flush_queue() - - assert logger.async_httpx_client.put.call_count == 3 - _assert_jittered_backoff(mock_sleep, 2) - assert [element.s3_object_key for element in logger.log_queue] == ["2025-09-14/test-down.json"] - - -@pytest.mark.asyncio -async def test_async_upload_sends_every_attempt_with_a_finite_deadline(rotating_profile: str) -> None: - """ - The logger's handler is built with explicit params and no timeout, and the handler forwards that None to - httpx as "no timeout", so without a per-attempt deadline a stalled S3 endpoint pins the upload forever. - """ - async with _s3_logger_on_production_handler(rotating_profile, [503, 200]) as (logger, requests, _): - await logger.async_upload_data_to_s3(_element({"test": "deadline"}, "deadline")) - - assert len(requests) == 2 - for request in requests: - deadlines = request.extensions["timeout"] - assert all(isinstance(limit, (int, float)) and limit > 0 for limit in deadlines.values()), deadlines - - -@pytest.mark.asyncio -async def test_backoff_is_jittered_so_throttled_uploads_do_not_retry_in_lockstep() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = AsyncMock( - side_effect=[_transient_failure_response(), _ok_response(), _transient_failure_response(), _ok_response()] - ) - - with ( - patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep, - patch("random.random", side_effect=[0.0, 0.999]), - ): - await logger.async_upload_data_to_s3(_element({"i": 0}, "0")) - await logger.async_upload_data_to_s3(_element({"i": 1}, "1")) - - first, second = (sleep_call.args[0] for sleep_call in mock_sleep.call_args_list) - assert first != second - assert 0.5 <= first < 1.5 and 0.5 <= second < 1.5 - - -@pytest.mark.asyncio -async def test_dropped_uploads_are_logged_once_per_flush_naming_each_reason(caplog) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - terminal = (_element({"i": "terminal-0"}, "terminal-0"), _element({"i": "terminal-1"}, "terminal-1")) - expired = _element({"i": "expired"}, "expired").model_copy(update={"retrying_since": _NOW - 7200}) - fresh = _element({"i": "fresh"}, "fresh") - - async def put( - url: str, data: str | None = None, headers: dict[str, str] | None = None, timeout: httpx.Timeout | None = None - ) -> MagicMock: - if "test-terminal-" in url: - return _terminal_failure_response() - if url.endswith("test-expired.json"): - return _transient_failure_response() - return _ok_response() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [*terminal, expired, fresh] - - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=_NOW), - ): - await logger.flush_queue() - - assert logger.log_queue == [] - drop_lines = [record.getMessage() for record in caplog.records if "uploads dropped" in record.getMessage()] - assert len(drop_lines) == 1, drop_lines - assert "3 uploads dropped" in drop_lines[0] - assert "2 rejected as terminal" in drop_lines[0] - assert "1 retrying longer than s3_max_retry_age_seconds=3600" in drop_lines[0] - - -def test_sync_upload_retries_403_with_fresh_signature(rotating_profile: str, monkeypatch: pytest.MonkeyPatch): - monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) - monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) - monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False) - monkeypatch.setenv("AWS_PROFILE", rotating_profile) - logger = S3Logger(s3_bucket_name="test-bucket", s3_region_name="us-east-1", s3_flush_interval=3600) - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-sync-403.json", - payload={"test": "sync-403"}, - s3_object_download_filename="test-sync-403.json", - ) - requests: list[httpx.Request] = [] - replies = iter([403, 200]) - - def respond(request: httpx.Request) -> httpx.Response: - requests.append(request) - return httpx.Response(next(replies), request=request, text="SignatureDoesNotMatch") - - handler = HTTPHandler() - handler.client = httpx.Client(transport=httpx.MockTransport(respond)) - with ( - patch( # test-quality-ok: sync upload builds its HTTPHandler per call, there is no injection seam for it - "litellm.integrations.s3_v2.get_httpx_client", return_value=handler - ), - patch("time.sleep") as mock_sleep, - ): - logger.upload_data_to_s3(test_element) - - assert len(requests) == 2 - first_key, first_token = _generation(requests[0]) - second_key, second_token = _generation(requests[1]) - assert first_key == first_token - assert second_key == second_token - assert int(second_key) > int(first_key) - _assert_jittered_backoff(mock_sleep, 1) - - -def test_sync_upload_retries_on_s3_503(): - """ - Test that the sync upload_data_to_s3 retries on transient S3 503. - """ - from unittest.mock import MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-sync-retry.json", - payload={"test": "sync-retry"}, - s3_object_download_filename="test-sync-retry.json", - ) - - response_503 = MagicMock() - response_503.status_code = 503 - response_200 = MagicMock() - response_200.status_code = 200 - response_200.raise_for_status = MagicMock() - - mock_sync_client = MagicMock() - mock_sync_client.put = MagicMock(side_effect=[response_503, response_200]) - - with patch( - "litellm.integrations.s3_v2.get_httpx_client", - return_value=mock_sync_client, - ): - with patch("time.sleep") as mock_sleep: - logger.upload_data_to_s3(test_element) - - assert mock_sync_client.put.call_count == 2 - _assert_jittered_backoff(mock_sleep, 1) - - -@pytest.mark.asyncio -async def test_async_log_event_skips_when_standard_logging_object_missing(): - """ - Reproduces the bug where _async_log_event_base raises ValueError when - kwargs has no standard_logging_object (e.g. call_type=afile_delete). - - The S3 logger should skip gracefully, not raise. - """ - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_region_name="us-east-1", - s3_aws_access_key_id="fake", - s3_aws_secret_access_key="fake", - ) - - kwargs_without_slo = { - "call_type": "afile_delete", - "model": None, - "litellm_call_id": "test-call-id", - } - - start_time = datetime.utcnow() - end_time = datetime.utcnow() - - # Spy on handle_callback_failure — should NOT be called if we skip gracefully. - # Without the fix, the ValueError is caught by the except block which calls - # handle_callback_failure. With the fix, we return early and never hit except. - with patch.object(logger, "handle_callback_failure") as mock_failure: - await logger._async_log_event_base( - kwargs=kwargs_without_slo, - response_obj=None, - start_time=start_time, - end_time=end_time, - ) - - assert not mock_failure.called, ( - "handle_callback_failure should not be called — " - "missing standard_logging_object should be a graceful skip, not an error" - ) - - # Nothing should have been queued (catches the case where code falls - # through without returning and appends None to the queue) - assert len(logger.log_queue) == 0, "log_queue should be empty when standard_logging_object is missing" - - -@pytest.mark.asyncio -async def test_strip_base64_removes_file_and_nontext_entries(): - logger = S3Logger(s3_strip_base64_files=True) - - payload = { - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Hello world"}, - { - "type": "image", - "file": {"file_data": "data:image/png;base64,AAAA"}, - }, - { - "type": "file", - "file": {"file_data": "data:application/pdf;base64,BBBB"}, - }, - ], - }, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Response"}, - { - "type": "audio", - "file": {"file_data": "data:audio/wav;base64,CCCC"}, - }, - ], - }, - ] - } - - stripped = await logger._strip_base64_from_messages(payload) - - # 1️⃣ File/image/audio entries are removed - assert len(stripped["messages"][0]["content"]) == 1 - assert stripped["messages"][0]["content"][0]["text"] == "Hello world" - - assert len(stripped["messages"][1]["content"]) == 1 - assert stripped["messages"][1]["content"][0]["text"] == "Response" - - # 2️⃣ No 'file' keys remain - for msg in stripped["messages"]: - for content in msg["content"]: - assert "file" not in content - assert content.get("type") == "text" - - -@pytest.mark.asyncio -async def test_strip_base64_keeps_non_file_content(): - logger = S3Logger(s3_strip_base64_files=True) - - payload = { - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Just text"}, - {"type": "text", "text": "Another message"}, - ], - } - ] - } - - stripped = await logger._strip_base64_from_messages(payload) - - # Should not modify pure text messages - assert stripped["messages"][0]["content"] == payload["messages"][0]["content"] - - -@pytest.mark.asyncio -async def test_strip_base64_handles_empty_or_missing_messages(): - logger = S3Logger(s3_strip_base64_files=True) - - # Missing messages key - payload_no_messages = {} - stripped1 = await logger._strip_base64_from_messages(payload_no_messages) - assert stripped1 == payload_no_messages - - # Empty messages list - payload_empty = {"messages": []} - stripped2 = await logger._strip_base64_from_messages(payload_empty) - assert stripped2 == payload_empty - - -@pytest.mark.asyncio -async def test_strip_base64_mixed_nested_objects(): - """ - Handles weird/nested content structures gracefully. - """ - logger = S3Logger(s3_strip_base64_files=True) - - payload = { - "messages": [ - { - "role": "system", - "content": [ - {"type": "text", "text": "Keep me"}, - {"type": "custom", "metadata": "ignore but non-text"}, - {"foo": "bar"}, - {"file": {"file_data": "data:application/pdf;base64,XXX"}}, - ], - "extra": {"trace_id": "123"}, - } - ] - } - - stripped = await logger._strip_base64_from_messages(payload) - - # Custom/non-text and file entries removed - content = stripped["messages"][0]["content"] - assert len(content) == 2 - assert {"type": "text", "text": "Keep me"} in content - assert {"foo": "bar"} in content - # Extra metadata preserved - assert stripped["messages"][0]["extra"]["trace_id"] == "123" - - -@pytest.mark.asyncio -async def test_s3_verify_false_handling(monkeypatch: pytest.MonkeyPatch): - """ - Test that s3_verify=False is properly handled and not treated as None. - - This is a regression test for the bug where s3_verify=False was being - ignored because 'False or s3_verify' would evaluate to s3_verify (None). - """ - from unittest.mock import AsyncMock, patch - - import litellm - - # Set up s3_callback_params with s3_verify=False - monkeypatch.setattr( - litellm, - "s3_callback_params", - { - "s3_bucket_name": "test-bucket", - "s3_endpoint_url": "https://localhost:443", - "s3_aws_access_key_id": "minioadmin", - "s3_aws_secret_access_key": "minioadmin", - "s3_region_name": "us-east-1", - "s3_verify": False, # This should NOT be ignored - "s3_use_ssl": False, # This should also NOT be ignored - }, - ) - - with patch("asyncio.create_task"): - with patch("litellm.integrations.s3_v2.get_async_httpx_client") as mock_get_client: - mock_client = AsyncMock() - mock_get_client.return_value = mock_client - - # Create logger - logger = S3Logger() - - # Verify s3_verify is False, not None - assert logger.s3_verify is False, f"Expected s3_verify=False, got {logger.s3_verify}" - assert logger.s3_use_ssl is False, f"Expected s3_use_ssl=False, got {logger.s3_use_ssl}" - - # Verify that get_async_httpx_client was called with ssl_verify=False - mock_get_client.assert_called_once() - call_kwargs = mock_get_client.call_args.kwargs - assert "params" in call_kwargs, "params should be passed to get_async_httpx_client" - assert call_kwargs["params"] == {"ssl_verify": False}, ( - f"Expected ssl_verify=False in params, got {call_kwargs.get('params')}" - ) - - -@pytest.mark.asyncio -async def test_s3_verify_none_handling(monkeypatch: pytest.MonkeyPatch): - """ - Test that s3_verify=None uses default behavior. - """ - from unittest.mock import AsyncMock, patch - - import litellm - - # Set up s3_callback_params without s3_verify - monkeypatch.setattr( - litellm, - "s3_callback_params", - { - "s3_bucket_name": "test-bucket", - "s3_aws_access_key_id": "test-key", - "s3_aws_secret_access_key": "test-secret", - "s3_region_name": "us-east-1", - }, - ) - - with patch("asyncio.create_task"): - with patch("litellm.integrations.s3_v2.get_async_httpx_client") as mock_get_client: - mock_client = AsyncMock() - mock_get_client.return_value = mock_client - - # Create logger without explicit s3_verify - logger = S3Logger() - - # Verify s3_verify is None (default) - assert logger.s3_verify is None, f"Expected s3_verify=None, got {logger.s3_verify}" - - # Verify that get_async_httpx_client was called - mock_get_client.assert_called_once() - call_kwargs = mock_get_client.call_args.kwargs - # When s3_verify is None, params={'ssl_verify': None} which is fine - uses default behavior - # The important thing is it's not False - if "params" in call_kwargs and call_kwargs["params"] is not None: - assert call_kwargs["params"].get("ssl_verify") is None - # Either params is None or params={'ssl_verify': None} is acceptable - - -@pytest.mark.asyncio -async def test_s3_verify_false_creates_httpx_client_with_verify_false(monkeypatch: pytest.MonkeyPatch): - """ - Test that when s3_verify=False, the actual httpx client has verify=False. - - This validates that ssl_verify=False flows through to the httpx.AsyncClient. - """ - from unittest.mock import patch - - import litellm - - # Set up s3_callback_params with s3_verify=False - monkeypatch.setattr( - litellm, - "s3_callback_params", - { - "s3_bucket_name": "test-bucket", - "s3_endpoint_url": "https://localhost:443", - "s3_aws_access_key_id": "minioadmin", - "s3_aws_secret_access_key": "minioadmin", - "s3_region_name": "us-east-1", - "s3_verify": False, - }, - ) - - with patch("asyncio.create_task"): - # Create logger - this creates the httpx client - logger = S3Logger() - - # Verify the logger has s3_verify=False - assert logger.s3_verify is False - - # Check the actual httpx client has verify=False - # The async_httpx_client.client is the actual httpx.AsyncClient - if hasattr(logger.async_httpx_client, "client"): - httpx_client = logger.async_httpx_client.client - # Check the _verify attribute (httpx internal) - if hasattr(httpx_client, "_verify"): - assert httpx_client._verify is False, f"Expected httpx client _verify=False, got {httpx_client._verify}" - - -@pytest.mark.asyncio -async def test_s3_verify_false_async_client(monkeypatch: pytest.MonkeyPatch): - """ - Test that the async httpx client respects s3_verify=False. - """ - from unittest.mock import AsyncMock, MagicMock, patch - - import litellm - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - # Set up s3_callback_params with s3_verify=False - monkeypatch.setattr( - litellm, - "s3_callback_params", - { - "s3_bucket_name": "test-bucket", - "s3_endpoint_url": "https://localhost:443", - "s3_aws_access_key_id": "minioadmin", - "s3_aws_secret_access_key": "minioadmin", - "s3_region_name": "us-east-1", - "s3_verify": False, - }, - ) - - with patch("asyncio.create_task"): - logger = S3Logger() - - # Verify s3_verify is False - assert logger.s3_verify is False - - # Create test element - test_element = s3BatchLoggingElement( - s3_object_key="2025-11-03/test-key.json", - payload={"test": "data"}, - s3_object_download_filename="test-file.json", - ) - - # Mock the async httpx client's put method - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - logger.async_httpx_client.put = AsyncMock(return_value=mock_response) - - # Call async upload - await logger.async_upload_data_to_s3(test_element) - - # Verify put was called - assert logger.async_httpx_client.put.called - - # Check that the async httpx client was created with verify=False - if hasattr(logger.async_httpx_client, "client"): - httpx_client = logger.async_httpx_client.client - if hasattr(httpx_client, "_verify"): - assert httpx_client._verify is False, ( - f"Expected async httpx client _verify=False, got {httpx_client._verify}" - ) - - -@pytest.mark.asyncio -async def test_strip_base64_recursive_redaction(): - logger = S3Logger(s3_strip_base64_files=True) - payload = { - "messages": [ - { - "content": [ - {"type": "text", "text": "normal text"}, - { - "type": "text", - "text": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg", - }, - { - "type": "text", - "text": "Nested: {'data': 'data:application/pdf;base64,AAA...'}", - }, - {"file": {"file_data": "data:application/pdf;base64,AAAA"}}, - {"metadata": {"preview": "data:audio/mp3;base64,AAAAA=="}}, - ] - } - ] - } - - result = await logger._strip_base64_from_messages(payload) - content = result["messages"][0]["content"] - - # Dropped file-type entries - assert not any("file" in c for c in content) - - # Base64 redacted globally - import json - - for c in content: - if isinstance(c, dict): - s = json.dumps(c).lower() - # "[base64_redacted]" is fine, but raw base64 is not - assert "base64," not in s, f"Found real base64 blob in: {s}" - - -# -------------------------------------------------------------- -# Shared fixture that silences asyncio.create_task during tests -# -------------------------------------------------------------- -@pytest_asyncio.fixture(loop_scope="function") -async def cancel_s3_periodic_flush_tasks(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[None]: - periodic_flush_tasks: list[asyncio.Task[object]] = [] - original_create_task = asyncio.create_task - - def track_create_task( - coro: Coroutine[object, object, object], - *, - name: str | None = None, - context: Context | None = None, - ) -> asyncio.Task[object]: - if context is None: - task = original_create_task(coro, name=name) - else: - task = original_create_task(coro, name=name, context=context) - if coro.__qualname__.endswith("periodic_flush"): - periodic_flush_tasks.append(task) - return task - - monkeypatch.setattr(asyncio, "create_task", track_create_task) - yield - for task in periodic_flush_tasks: - if not task.done(): - task.cancel() - await asyncio.gather(*periodic_flush_tasks, return_exceptions=True) - - -@pytest.fixture(autouse=True) -def patch_asyncio_create_task(request): - """Prevent 'no running event loop' errors when S3Logger calls asyncio.create_task().""" - if request.node.originalname in { - "test_basic_s3_logging", - "test_basic_s3_v2_logging", - "test_basic_s3_v2_logging_failure", - }: - yield - return - with patch("asyncio.create_task"): - yield - - -# -------------------------------------------------------------- -# Parametrized prefix combination test -# -------------------------------------------------------------- -@pytest.mark.parametrize( - "use_team_prefix,use_key_prefix,team_alias,key_alias,expected_prefix", - [ - (False, False, "teamA", "keyA", ""), - (True, False, "teamA", "keyA", "teamA/"), - (False, True, "teamA", "keyA", "keyA/"), - (True, True, "teamA", "keyA", "teamA/keyA/"), - (True, True, None, "keyA", "keyA/"), - (True, True, "teamA", None, "teamA/"), - (True, True, None, None, ""), - ], -) -def test_s3_object_key_prefix_combinations(use_team_prefix, use_key_prefix, team_alias, key_alias, expected_prefix): - """ - Validate correct S3 prefix composition for team alias + key alias combinations. - """ - with patch("litellm.integrations.s3_v2.get_s3_object_key") as mock_get_key: - mock_get_key.return_value = "mocked/s3/object/key.json" - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_region_name="us-east-1", - s3_use_team_prefix=use_team_prefix, - s3_use_key_prefix=use_key_prefix, - ) - - payload = StandardLoggingPayload( - id="abc123", - metadata={ - "user_api_key_team_alias": team_alias, - "user_api_key_alias": key_alias, - }, - messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], - ) - - result = logger.create_s3_batch_logging_element(datetime.utcnow(), payload) - assert result is not None - mock_get_key.assert_called_once() - - prefix_arg = mock_get_key.call_args.kwargs.get("prefix") - assert prefix_arg == expected_prefix, ( - f"Expected prefix '{expected_prefix}', got '{prefix_arg}' " - f"for team={team_alias}, key={key_alias}, " - f"use_team_prefix={use_team_prefix}, use_key_prefix={use_key_prefix}" - ) - - -# -------------------------------------------------------------- -# Test prefix priority and concatenation -# -------------------------------------------------------------- -def test_prefix_priority_and_path_construction(): - """ - Validate that prefix components are ordered and joined with '/' only once. - """ - with patch("litellm.integrations.s3_v2.get_s3_object_key") as mock_get_key: - mock_get_key.return_value = "mocked/key" - - logger = S3Logger(s3_use_team_prefix=True, s3_use_key_prefix=True) - payload = StandardLoggingPayload( - id="xyz999", - metadata={ - "user_api_key_team_alias": "Team-Alpha", - "user_api_key_alias": "API-12345", - }, - messages=[], - ) - - logger.create_s3_batch_logging_element(datetime.utcnow(), payload) - prefix_arg = mock_get_key.call_args.kwargs.get("prefix", "") - - assert prefix_arg == "Team-Alpha/API-12345/" - assert "//" not in prefix_arg - - -# -------------------------------------------------------------- -# Test when prefixes are disabled -# -------------------------------------------------------------- -def test_prefix_absent_when_flags_disabled(): - """ - Verify prefix is omitted entirely when prefix flags are False. - """ - with patch("litellm.integrations.s3_v2.get_s3_object_key") as mock_get_key: - mock_get_key.return_value = "mocked/key" - - logger = S3Logger(s3_use_team_prefix=False, s3_use_key_prefix=False) - payload = StandardLoggingPayload( - id="no-prefix", - metadata={ - "user_api_key_team_alias": "team-x", - "user_api_key_alias": "key-x", - }, - messages=[], - ) - - logger.create_s3_batch_logging_element(datetime.utcnow(), payload) - prefix_arg = mock_get_key.call_args.kwargs.get("prefix", None) - assert prefix_arg == "", f"Expected empty prefix, got {prefix_arg}" - - -# -------------------------------------------------------------- -# Integration-style test (asyncio fixture will patch create_task) -# -------------------------------------------------------------- -@pytest.mark.asyncio -async def test_combined_prefix_reflects_in_s3_object_key(): - """ - Integration-style test ensuring final s3_object_key includes both prefixes correctly. - """ - logger = S3Logger(s3_use_team_prefix=True, s3_use_key_prefix=True) - payload = StandardLoggingPayload( - id="int-test", - metadata={ - "user_api_key_team_alias": "myteam", - "user_api_key_alias": "apikey", - }, - messages=[], - ) - - result = logger.create_s3_batch_logging_element(datetime.utcnow(), payload) - key = result.s3_object_key - assert "myteam/apikey/" in key, f"Expected both prefixes in key: {key}" - - -def test_s3_object_key_sanitizes_slashes_and_colons_in_file_name(): - from litellm.integrations.s3 import get_s3_object_key - - start_time = datetime(2026, 2, 11, 0, 35, 18, 391582) - file_name = "time-00-35-18-391582_arn:aws:bedrock:us-east-1:123456789012:model-invocation-job/gl18r6skk9yy" - - key = get_s3_object_key( - s3_path="LiteLLMAPPLogs", - prefix="myteam/", - start_time=start_time, - s3_file_name=file_name, - ) - - assert key == ( - "LiteLLMAPPLogs/myteam/2026-02-11/" - "time-00-35-18-391582_arn_aws_bedrock_us-east-1_123456789012_model-invocation-job_gl18r6skk9yy.json" - ) - - -@pytest.mark.parametrize( - "response_id", - [ - "s3://example-batch-bucket/litellm-bedrock-files/input.jsonl", - "gs://example-batch-bucket/litellm-vertex-files/input.jsonl", - ], -) -def test_s3_object_key_has_no_colon_for_cloud_uri_file_ids(response_id: str): - from litellm.integrations.s3 import get_s3_object_key - - key = get_s3_object_key( - s3_path="", - prefix="", - start_time=datetime(2026, 9, 7, 4, 51, 6, 685889), - s3_file_name=f"time-04-51-06-685889_{response_id}", - ) - - filename = key.rsplit("/", 1)[-1] - assert ":" not in filename - assert filename.endswith("_input.jsonl.json") - - -def test_create_s3_batch_logging_element_flat_key_for_arn_response_id(): - """End-to-end through the s3_v2 element builder: an ARN response id must - yield a flat file directly under the date segment.""" - logger = S3Logger(s3_use_team_prefix=False, s3_use_key_prefix=False) - payload = StandardLoggingPayload( - id="arn:aws:bedrock:us-east-1:123456789012:model-invocation-job/gl18r6skk9yy", - metadata={}, - messages=[], - ) - - start_time = datetime(2026, 2, 11, 0, 35, 18, 391582) - result = logger.create_s3_batch_logging_element(start_time, payload) - - assert result is not None - date_segment = "2026-02-11/" - file_segment = result.s3_object_key.split(date_segment, 1)[1] - assert "/" not in file_segment, f"Expected flat file under date segment, got: {result.s3_object_key}" - assert file_segment.endswith("model-invocation-job_gl18r6skk9yy.json") - - -# -------------------------------------------------------------- -# object keys bounded to S3 and filesystem-compatible limits -# -------------------------------------------------------------- -def _oversized_response_id() -> str: - return "resp_" + "A" * 1100 - - -def test_s3_object_key_at_the_segment_byte_limit_is_left_alone(): - """A final key segment that still fits is left byte-identical.""" - from litellm.integrations.s3 import get_s3_object_key - - start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) - file_name = "x" * 250 - - key = get_s3_object_key(s3_path="input", prefix="", start_time=start_time, s3_file_name=file_name) - - assert key == f"input/2026-08-24/{file_name}.json" - assert len(key.rsplit("/", 1)[1].encode("utf-8")) == 255 - - -def test_s3_object_key_just_over_the_file_name_limit_is_bounded(): - """A 256-byte final key segment is shortened below the local filesystem limit.""" - from litellm.integrations.s3 import get_s3_object_key - - key = get_s3_object_key( - s3_path="input", - prefix="", - start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), - s3_file_name="x" * 251, - ) - - assert len(key.rsplit("/", 1)[1].encode("utf-8")) <= 255 - - -def test_s3_object_key_bounds_the_file_segment_when_the_full_key_fits(): - """A locally stored object name is bounded even when its full S3 key is below 1024 bytes.""" - import hashlib - - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - from litellm.integrations.s3 import get_s3_object_key - - file_name = "time-01-14-19-015434_resp_" + "A" * 300 - - key = get_s3_object_key( - s3_path="requests", - prefix="", - start_time=datetime(2026, 9, 12, 1, 14, 19, 15434), - s3_file_name=file_name, - ) - - file_segment = key.rsplit("/", 1)[1] - assert len(key.encode("utf-8")) < MAX_S3_OBJECT_KEY_BYTES - assert len(file_segment.encode("utf-8")) <= 255 - assert file_segment.startswith("time-01-14-19-015434_resp_") - assert file_segment.endswith(f"_{hashlib.sha256(file_name.encode('utf-8')).hexdigest()}.json") - - -def test_s3_object_key_file_segment_limit_counts_utf8_bytes(): - """The final key segment is bounded by encoded bytes without splitting a character.""" - from litellm.integrations.s3 import get_s3_object_key - - key = get_s3_object_key( - s3_path="requests", - prefix="", - start_time=datetime(2026, 9, 12, 1, 14, 19, 15434), - s3_file_name="time-01-14-19-015434_resp_" + "日" * 100, - ) - - file_segment = key.rsplit("/", 1)[1] - assert len(file_segment.encode("utf-8")) <= 255 - assert "\ufffd" not in file_segment - - -def test_s3_object_key_at_the_full_key_byte_limit_is_left_alone(): - """A key of exactly 1024 bytes whose segments all fit is left byte-identical.""" - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - from litellm.integrations.s3 import get_s3_object_key - - file_name = "x" * 200 - fixed_len = len(f"/2026-08-24/{file_name}.json") - s3_path = "/".join(["p" * 200] * 4) + "/" + "q" * (MAX_S3_OBJECT_KEY_BYTES - fixed_len - 804) - - key = get_s3_object_key( - s3_path=s3_path, - prefix="", - start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), - s3_file_name=file_name, - ) - - assert key == f"{s3_path}/2026-08-24/{file_name}.json" - assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES - - -def test_s3_object_key_is_bounded_for_oversized_response_id(): - """An oversized Responses API id is shortened to a readable head plus a digest.""" - import hashlib - - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - from litellm.integrations.s3 import get_s3_object_key - - start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) - file_name = f"time-06-18-41-948021_{_oversized_response_id()}" - - key = get_s3_object_key(s3_path="input", prefix="DefaultTeamProd/", start_time=start_time, s3_file_name=file_name) - - assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES - assert key.startswith("input/DefaultTeamProd/2026-08-24/time-06-18-41-948021_resp_") - assert key.endswith(f"_{hashlib.sha256(file_name.encode('utf-8')).hexdigest()}.json") - - -@pytest.mark.parametrize( - "s3_path,prefix", - [ - ("input", ""), - ("a" * 900, ""), - ("input", "team-" + "b" * 900 + "/"), - ("c" * 600, "team-" + "d" * 600 + "/key-" + "e" * 600 + "/"), - # many short segments, so the trim lands exactly on the budget edge - ("", "ssss/" * 200), - ], -) -def test_s3_object_key_is_bounded_for_long_paths_and_aliases(s3_path: str, prefix: str): - """Long paths, team aliases and key aliases stay within the cap.""" - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - from litellm.integrations.s3 import get_s3_object_key - - key = get_s3_object_key( - s3_path=s3_path, - prefix=prefix, - start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), - s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}", - ) - - assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES - assert key.endswith(".json") - assert "/2026-08-24/" in key or key.startswith("2026-08-24/") - assert "/" not in key.rsplit("2026-08-24/", 1)[1] - - -def test_s3_object_key_trimmed_prefixes_stay_distinct_per_operator(): - """Prefixes that differ only past the trim point keep separate folders.""" - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - from litellm.integrations.s3 import get_s3_object_key - - start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) - keys = [ - get_s3_object_key( - s3_path="input", - prefix="team-" + "b" * 1000 + suffix + "/", - start_time=start_time, - s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}", - ) - for suffix in ("-one", "-two") - ] - - assert keys[0] != keys[1] - assert all(key.startswith("input/team-" + "b" * 900) for key in keys) - assert all(len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES for key in keys) - - -def test_s3_object_key_bounded_prefix_never_splits_a_multibyte_character(): - """A multibyte prefix is trimmed on a character boundary.""" - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - from litellm.integrations.s3 import get_s3_object_key - - s3_path = "\u65e5\u672c\u8a9e" * 200 - - key = get_s3_object_key( - s3_path=s3_path, - prefix="\u30c1\u30fc\u30e0" * 200 + "/", - start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), - s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}", - ) - - assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES - assert key.startswith(s3_path[:100]) - assert "\ufffd" not in key - - -def test_s3_object_key_stays_unique_for_ids_sharing_a_head(): - """Ids sharing a visible head still get distinct keys.""" - from litellm.integrations.s3 import get_s3_object_key - - start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) - keys = { - get_s3_object_key( - s3_path="input", - prefix="", - start_time=start_time, - s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}{suffix}", - ) - for suffix in ("first", "second", "third") - } - - assert len(keys) == 3 - - -def test_s3_object_key_bounding_matches_the_documented_layout(): - """The bounded key is `//_.json`.""" - import hashlib - - from litellm.integrations.s3 import get_s3_object_key - - file_name = f"time-06-18-41-948021_{_oversized_response_id()}" - - key = get_s3_object_key( - s3_path="input", - prefix="team/", - start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), - s3_file_name=file_name, - ) - - digest = hashlib.sha256(file_name.encode("utf-8")).hexdigest() - assert key == f"input/team/2026-08-24/{file_name[:64]}_{digest}.json" - - -def test_s3_object_key_keeps_the_configured_prefix_when_only_the_id_overflows(): - """A 940 byte configured prefix survives whole when only the id overflows.""" - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - from litellm.integrations.s3 import get_s3_object_key - - prefix = "team-" + "b" * 934 + "/" - - key = get_s3_object_key( - s3_path="", - prefix=prefix, - start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), - s3_file_name=f"time-06-18-41-948021_{_oversized_response_id()}", - ) - - assert key.startswith(prefix + "2026-08-24/") - assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES - - -def test_s3_object_key_spends_the_whole_budget_when_the_prefix_must_be_trimmed(): - """A trimmed prefix keeps every byte the budget allows, not whole segments.""" - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - from litellm.integrations.s3 import get_s3_object_key - - s3_path = "p" * 400 + "/" + "q" * 600 - - key = get_s3_object_key( - s3_path=s3_path, - prefix="", - start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), - s3_file_name="time-06-18-41-948021_abc", - ) - - assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES - assert key.startswith("p" * 400 + "/" + "q" * 500) - - -def test_s3_object_key_keeps_a_single_segment_path_as_far_as_it_fits(): - """A path with no separator is kept as far as it fits, never dropped to the bucket root.""" - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - from litellm.integrations.s3 import get_s3_object_key - - key = get_s3_object_key( - s3_path="a" * 1050, - prefix="", - start_time=datetime(2026, 8, 24, 6, 18, 41, 948021), - s3_file_name="time-06-18-41-948021_chatcmpl-xyz", - ) - - assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES - assert key.startswith("a" * 900) - - -def test_create_s3_batch_logging_element_bounds_key_and_keeps_full_response_id(): - """The batch element bounds the key and keeps the full response id in the payload.""" - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - - logger = S3Logger(s3_use_team_prefix=True, s3_use_key_prefix=True) - response_id = _oversized_response_id() - payload = StandardLoggingPayload( - id=response_id, - metadata={"user_api_key_team_alias": "DefaultTeamProd", "user_api_key_alias": "prod-key"}, - messages=[], - ) - - result = logger.create_s3_batch_logging_element(datetime(2026, 8, 24, 6, 18, 41, 948021), payload) - - assert result is not None - assert len(result.s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES - assert result.s3_object_key.startswith("DefaultTeamProd/prod-key/2026-08-24/") - assert result.payload["id"] == response_id - - -def test_s3_object_download_filename_is_bounded_for_oversized_response_id(): - """The Content-Disposition filename is bounded too, or the PUT fails with MetadataTooLarge.""" - from litellm.constants import MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES - from litellm.integrations.s3 import get_s3_object_download_filename - - file_name = get_s3_object_download_filename(datetime(2026, 8, 24, 6, 18, 41, 948021), _oversized_response_id()) - - assert len(file_name.encode("utf-8")) <= MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES - assert file_name.startswith("time-2026-08-24T06-18-41-948021_resp_") - assert file_name.endswith(".json") - - -def test_s3_object_download_filenames_stay_distinct_when_shortened(): - """Shortened filenames stay distinct.""" - from litellm.integrations.s3 import get_s3_object_download_filename - - start_time = datetime(2026, 8, 24, 6, 18, 41, 948021) - file_names = { - get_s3_object_download_filename(start_time, _oversized_response_id() + suffix) - for suffix in ("first", "second", "third") - } - - assert len(file_names) == 3 - - -def test_s3_object_download_filename_short_id_is_unchanged(): - """An ordinary response id keeps the filename it had before.""" - from litellm.integrations.s3 import get_s3_object_download_filename - - file_name = get_s3_object_download_filename(datetime(2026, 8, 24, 6, 18, 41, 948021), "resp_abc123") - - assert file_name == "time-2026-08-24T06-18-41-948021_resp_abc123.json" - - -def test_create_s3_batch_logging_element_bounds_the_download_filename(): - """The batch element carries a bounded Content-Disposition filename.""" - from litellm.constants import MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES - - logger = S3Logger() - payload = StandardLoggingPayload(id=_oversized_response_id(), metadata={}, messages=[]) - - result = logger.create_s3_batch_logging_element(datetime(2026, 8, 24, 6, 18, 41, 948021), payload) - - assert result is not None - assert len(result.s3_object_download_filename.encode("utf-8")) <= MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES - - -@pytest.mark.asyncio -async def test_audit_log_object_key_is_bounded_for_a_long_configured_path(): - """Audit log keys are bounded by the same builder.""" - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - - logger = S3Logger() - logger.s3_path = "audit-archive/" + "z" * 1100 - - await logger.async_log_audit_log_event({"id": "1a4f7bd0-6f1e-4d0a-9b3c-9f2e1d5a7c88"}) - - assert len(logger.log_queue) == 1 - assert len(logger.log_queue[0].s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES - assert logger.log_queue[0].s3_object_key.startswith("audit-archive/" + "z" * 900) - - -def test_s3_object_download_filename_drops_characters_that_break_the_header(): - """A quote or separator in the response id cannot escape the quoted header value.""" - from litellm.integrations.s3 import get_s3_object_download_filename - - file_name = get_s3_object_download_filename(datetime(2026, 8, 24, 6, 18, 41, 948021), 'resp_a"b/c') - - assert file_name == "time-2026-08-24T06-18-41-948021_resp_a_b_c.json" - - -# -------------------------------------------------------------- -# params_source / s3_callback_params_override (audit-log decoupling) -# -------------------------------------------------------------- -def test_s3_callback_params_override_uses_alternate_dict(monkeypatch): - """`s3_callback_params_override` makes the logger read its config from - the override dict instead of `litellm.s3_callback_params`.""" - import litellm - - monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "normal-bucket"}) - logger = S3Logger( - s3_callback_params_override={ - "s3_bucket_name": "audit-bucket", - "s3_path": "audit-prefix", - "s3_region_name": "us-west-2", - } - ) - assert logger.s3_bucket_name == "audit-bucket" - assert logger.s3_path == "audit-prefix" - assert logger.s3_region_name == "us-west-2" - - -def test_s3_callback_params_override_does_not_mutate_inputs(monkeypatch): - """Resolving `os.environ/X` markers must not mutate the override dict - or `litellm.s3_callback_params`.""" - import litellm - - monkeypatch.setenv("MY_AUDIT_BUCKET", "resolved-bucket") - override = {"s3_bucket_name": "os.environ/MY_AUDIT_BUCKET"} - monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "os.environ/MY_AUDIT_BUCKET"}) - logger = S3Logger(s3_callback_params_override=override) - assert logger.s3_bucket_name == "resolved-bucket" - assert override["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" - assert litellm.s3_callback_params["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" - - -def test_s3_callback_params_override_none_falls_back_to_global(monkeypatch): - """No override → behaves exactly as today (reads `litellm.s3_callback_params`).""" - import litellm - - monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "from-global"}) - logger = S3Logger() - assert logger.s3_bucket_name == "from-global" - - -def test_s3_callback_params_override_empty_dict_is_opt_in(monkeypatch): - """An empty override dict skips the global entirely (env/IAM-only config).""" - import litellm - - monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "from-global"}) - logger = S3Logger(s3_callback_params_override={}) - assert logger.s3_bucket_name is None - - -def _expected_content_md5(payload: dict) -> str: - import base64 - import hashlib - - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - - json_string = safe_dumps(payload) - return base64.b64encode(hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()).decode() - - -def _require_non_security_md5(monkeypatch): - import hashlib - - original_md5 = hashlib.md5 - - def fips_md5(data=b"", *, usedforsecurity=True): - if usedforsecurity: - raise ValueError("MD5 blocked for security use") - return original_md5(data, usedforsecurity=usedforsecurity) - - monkeypatch.setattr(hashlib, "md5", fips_md5) - - -@pytest.mark.asyncio -async def test_async_upload_sets_content_md5_header(monkeypatch): - """ - Object Lock buckets reject PUTs without a Content-MD5 header (AWS spec). - The async upload must send a base64 md5 of the exact signed body. - """ - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - payload = {"test": "content-md5"} - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-md5.json", - payload=payload, - s3_object_download_filename="test-md5.json", - ) - _require_non_security_md5(monkeypatch) - - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put.return_value = response - - await logger.async_upload_data_to_s3(test_element) - - headers = logger.async_httpx_client.put.call_args.kwargs["headers"] - assert headers["Content-MD5"] == _expected_content_md5(payload) - assert "x-amz-server-side-encryption" not in headers - - -def test_sync_upload_sets_content_md5_header(monkeypatch): - """The sync upload path must also send Content-MD5 for Object Lock buckets.""" - from unittest.mock import MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - payload = {"test": "sync-content-md5"} - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-sync-md5.json", - payload=payload, - s3_object_download_filename="test-sync-md5.json", - ) - _require_non_security_md5(monkeypatch) - - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - mock_sync_client = MagicMock() - mock_sync_client.put.return_value = response - - with patch( - "litellm.integrations.s3_v2.get_httpx_client", - return_value=mock_sync_client, - ): - logger.upload_data_to_s3(test_element) - - headers = mock_sync_client.put.call_args.kwargs["headers"] - assert headers["Content-MD5"] == _expected_content_md5(payload) - assert "x-amz-server-side-encryption" not in headers - - -@pytest.mark.asyncio -async def test_async_upload_sets_server_side_encryption_header_when_configured(): - """ - When s3_server_side_encryption is set (e.g. buckets with a KMS default - encryption policy), the PUT must carry x-amz-server-side-encryption. - """ - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_server_side_encryption="aws:kms", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-sse.json", - payload={"test": "sse"}, - s3_object_download_filename="test-sse.json", - ) - - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put.return_value = response - - await logger.async_upload_data_to_s3(test_element) - - headers = logger.async_httpx_client.put.call_args.kwargs["headers"] - assert headers["x-amz-server-side-encryption"] == "aws:kms" - - -def test_s3_server_side_encryption_read_from_callback_params(monkeypatch): - """s3_server_side_encryption can be configured via s3_callback_params.""" - import litellm - - monkeypatch.setattr( - litellm, - "s3_callback_params", - { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "aws:kms", - }, - ) - logger = S3Logger() - assert logger.s3_server_side_encryption == "aws:kms" - - -@pytest.mark.asyncio -async def test_async_upload_sets_sse_kms_key_id_header_when_configured(): - """ - When s3_sse_kms_key_id is set alongside aws:kms, the PUT must carry - x-amz-server-side-encryption-aws-kms-key-id so objects are encrypted - with the customer-managed KMS key instead of the bucket default. - """ - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_server_side_encryption="aws:kms", - s3_sse_kms_key_id="arn:aws:kms:us-east-1:111122223333:key/test-key-id", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-sse-kms.json", - payload={"test": "sse-kms"}, - s3_object_download_filename="test-sse-kms.json", - ) - - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put.return_value = response - - await logger.async_upload_data_to_s3(test_element) - - headers = logger.async_httpx_client.put.call_args.kwargs["headers"] - assert headers["x-amz-server-side-encryption"] == "aws:kms" - assert headers["x-amz-server-side-encryption-aws-kms-key-id"] == ( - "arn:aws:kms:us-east-1:111122223333:key/test-key-id" - ) - - -def test_sync_upload_sets_sse_kms_key_id_header_when_configured(): - """The sync upload path must carry the same SSE-KMS headers.""" - from unittest.mock import MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_server_side_encryption="aws:kms", - s3_sse_kms_key_id="arn:aws:kms:us-east-1:111122223333:key/test-key-id", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-sync-sse-kms.json", - payload={"test": "sync-sse-kms"}, - s3_object_download_filename="test-sync-sse-kms.json", - ) - - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - mock_sync_client = MagicMock() - mock_sync_client.put.return_value = response - - with patch( - "litellm.integrations.s3_v2.get_httpx_client", - return_value=mock_sync_client, - ): - logger.upload_data_to_s3(test_element) - - headers = mock_sync_client.put.call_args.kwargs["headers"] - assert headers["x-amz-server-side-encryption"] == "aws:kms" - assert headers["x-amz-server-side-encryption-aws-kms-key-id"] == ( - "arn:aws:kms:us-east-1:111122223333:key/test-key-id" - ) - - -@pytest.mark.asyncio -async def test_async_upload_omits_kms_key_id_header_when_not_configured(): - """SSE without a key id must not emit the KMS key id header.""" - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_server_side_encryption="AES256", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-aes256.json", - payload={"test": "aes256"}, - s3_object_download_filename="test-aes256.json", - ) - - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put.return_value = response - - await logger.async_upload_data_to_s3(test_element) - - headers = logger.async_httpx_client.put.call_args.kwargs["headers"] - assert headers["x-amz-server-side-encryption"] == "AES256" - assert "x-amz-server-side-encryption-aws-kms-key-id" not in headers - - -def test_s3_sse_kms_key_id_read_from_callback_params(monkeypatch): - """s3_sse_kms_key_id can be configured via s3_callback_params.""" - import litellm - - monkeypatch.setattr( - litellm, - "s3_callback_params", - { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "aws:kms", - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", - }, - ) - logger = S3Logger() - assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/test-key-id") - - -@pytest.mark.asyncio -async def test_async_upload_infers_aws_kms_when_only_key_id_set(): - """ - Setting only s3_sse_kms_key_id must not produce an invalid request - (S3 rejects a key id without an algorithm); aws:kms is inferred. - """ - from unittest.mock import AsyncMock, MagicMock - - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_sse_kms_key_id="arn:aws:kms:us-east-1:111122223333:key/test-key-id", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-kms-only.json", - payload={"test": "kms-only"}, - s3_object_download_filename="test-kms-only.json", - ) - - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put.return_value = response - - await logger.async_upload_data_to_s3(test_element) - - headers = logger.async_httpx_client.put.call_args.kwargs["headers"] - assert headers["x-amz-server-side-encryption"] == "aws:kms" - assert headers["x-amz-server-side-encryption-aws-kms-key-id"] == ( - "arn:aws:kms:us-east-1:111122223333:key/test-key-id" - ) - - -def test_s3_sse_kms_key_id_read_from_audit_override_params(monkeypatch): - """The audit-log override path must honor s3_sse_kms_key_id too.""" - import litellm - - monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "normal-logs-bucket"}) - logger = S3Logger( - s3_callback_params_override={ - "s3_bucket_name": "audit-logs-bucket", - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/audit-key-id", - } - ) - assert logger.s3_bucket_name == "audit-logs-bucket" - assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/audit-key-id") - - -def test_kms_key_id_dropped_when_algorithm_is_not_kms(monkeypatch): - """ - AES256 plus a KMS key id is an invalid S3 combination; the key id must be - dropped at init so uploads keep working instead of silently 400ing. - """ - import litellm - - monkeypatch.setattr( - litellm, - "s3_callback_params", - { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "AES256", - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", - }, - ) - logger = S3Logger() - assert logger.s3_server_side_encryption == "AES256" - assert logger.s3_sse_kms_key_id is None - - -def test_non_string_algorithm_is_dropped_and_valid_key_id_is_rescued(monkeypatch): - """ - A YAML boolean in s3_server_side_encryption must not crash logger init and - must not discard the valid key id; aws:kms is inferred from the key id. - """ - import litellm - - monkeypatch.setattr( - litellm, - "s3_callback_params", - { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": True, - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", - }, - ) - logger = S3Logger() - assert logger.s3_server_side_encryption == "aws:kms" - assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/test-key-id") - - -def test_non_string_key_id_is_dropped_and_valid_algorithm_is_kept(monkeypatch): - """A mistyped key id (unquoted YAML number) must not disable the valid algorithm.""" - import litellm - - monkeypatch.setattr( - litellm, - "s3_callback_params", - { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "aws:kms", - "s3_sse_kms_key_id": 12345, - }, - ) - logger = S3Logger() - assert logger.s3_server_side_encryption == "aws:kms" - assert logger.s3_sse_kms_key_id is None - - -_ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE" -_SECRET_KEY = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" -_KEY_WITH_SPACE = "LOGS/LLM AI Projects/2026-08-04/time-13-01-00-abc.json" - - -def _signature_for(signer_cls, url: str, method: str, body: bytes | None, headers: dict[str, str]) -> str: - from botocore.awsrequest import AWSRequest - from botocore.credentials import Credentials - - sent = {name.lower(): value for name, value in headers.items()} - signed_header_names = sent["authorization"].split("SignedHeaders=")[1].split(", ")[0].split(";") - request = AWSRequest( - method=method, - url=url, - data=body, - headers={name: sent[name] for name in signed_header_names if name in sent}, - ) - request.context["timestamp"] = sent["x-amz-date"] - signer = signer_cls(Credentials(_ACCESS_KEY, _SECRET_KEY), "s3", "us-east-1") - canonical_request = signer.canonical_request(request) - return signer.signature(signer.string_to_sign(request, canonical_request), request) - - -def _as_s3_canonicalizes(url: str) -> str: - """ - The path S3 rebuilds from the wire path: percent-encode everything outside the unreserved - set, without normalizing or double-encoding. `=` becomes `%3D`, `%20` stays `%20`. - """ - from urllib.parse import quote, unquote, urlsplit, urlunsplit - - split = urlsplit(url) - return urlunsplit(split._replace(path=quote(unquote(split.path), safe="/~"))) - - -def _assert_signed_for_s3_canonicalization(url: str, method: str, body: bytes | None, headers: dict[str, str]) -> None: - """ - S3 rebuilds the canonical request from the wire path with single percent-encoding, which - botocore models as S3SigV4Auth; plain SigV4Auth double-encodes it (%2520 for a space) and S3 - answers 403 SignatureDoesNotMatch. Assert we sent an already-encoded path and signed it the - way S3 reads it. - """ - from botocore.auth import S3SigV4Auth, SigV4Auth - - assert url == _as_s3_canonicalizes(url) - sent_signature = headers["Authorization"].split("Signature=")[1].strip() - assert sent_signature == _signature_for(S3SigV4Auth, url, method, body, headers) - assert sent_signature != _signature_for(SigV4Auth, url, method, body, headers) - - -def _logger_for_signing() -> S3Logger: - return S3Logger( - s3_bucket_name="logs-bucket", - s3_aws_access_key_id=_ACCESS_KEY, - s3_aws_secret_access_key=_SECRET_KEY, - s3_region_name="us-east-1", - ) - - -def _element_with_space(): - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - return s3BatchLoggingElement( - s3_object_key=_KEY_WITH_SPACE, - payload={"test": "sigv4"}, - s3_object_download_filename="log.json", - ) - - -@pytest.mark.asyncio -async def test_async_upload_signs_object_key_with_space_the_way_s3_does(): - from unittest.mock import AsyncMock, MagicMock - - logger = _logger_for_signing() - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put.return_value = response - - await logger.async_upload_data_to_s3(_element_with_space()) - - call = logger.async_httpx_client.put.call_args - _assert_signed_for_s3_canonicalization( - url=call[0][0], - method="PUT", - body=call.kwargs["data"].encode("utf-8"), - headers=call.kwargs["headers"], - ) - - -def test_sync_upload_signs_object_key_with_space_the_way_s3_does(): - from unittest.mock import MagicMock - - logger = _logger_for_signing() - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - mock_sync_client = MagicMock() - mock_sync_client.put.return_value = response - - with patch("litellm.integrations.s3_v2.get_httpx_client", return_value=mock_sync_client): - logger.upload_data_to_s3(_element_with_space()) - - call = mock_sync_client.put.call_args - _assert_signed_for_s3_canonicalization( - url=call[0][0], - method="PUT", - body=call.kwargs["data"].encode("utf-8"), - headers=call.kwargs["headers"], - ) - - -@pytest.mark.asyncio -async def test_download_signs_object_key_with_space_the_way_s3_does(): - from unittest.mock import AsyncMock, MagicMock - - logger = _logger_for_signing() - response = MagicMock() - response.status_code = 200 - response.json = MagicMock(return_value={"downloaded": "data"}) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.get.return_value = response - - assert await logger._download_object_from_s3(_KEY_WITH_SPACE) == {"downloaded": "data"} - - call = logger.async_httpx_client.get.call_args - _assert_signed_for_s3_canonicalization( - url=call[0][0], - method="GET", - body=None, - headers=call.kwargs["headers"], - ) - - -_RESERVED_CHAR_KEYS = ( - "2026-08-21/time-05-29-36_resp_bGl0ZWxsbTpjdXN0b20=.json", - "session=logs/2026-08-21/time-05-29-36_abc.json", - "a+b/2026-08-21/time-05-29-36_abc.json", - "a&b/2026-08-21/time-05-29-36_abc.json", - "a#b/2026-08-21/time-05-29-36_abc.json", - "a?b/2026-08-21/time-05-29-36_abc.json", - "a%b/2026-08-21/time-05-29-36_abc.json", - _KEY_WITH_SPACE, -) - - -def _element_for(s3_object_key: str): - from litellm.types.integrations.s3_v2 import s3BatchLoggingElement - - return s3BatchLoggingElement( - s3_object_key=s3_object_key, - payload={"test": "sigv4"}, - s3_object_download_filename="log.json", - ) - - -def _expected_wire_url(s3_object_key: str) -> str: - """The URL boto3 itself would put on the wire for this key.""" - from urllib.parse import quote - - return f"https://logs-bucket.s3.us-east-1.amazonaws.com/{quote(s3_object_key, safe='/')}" - - -@pytest.mark.parametrize("s3_object_key", _RESERVED_CHAR_KEYS) -@pytest.mark.asyncio -async def test_async_upload_percent_encodes_reserved_characters_in_object_key(s3_object_key): - from unittest.mock import AsyncMock, MagicMock - - logger = _logger_for_signing() - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put.return_value = response - - await logger.async_upload_data_to_s3(_element_for(s3_object_key)) - - call = logger.async_httpx_client.put.call_args - assert call[0][0] == _expected_wire_url(s3_object_key) - _assert_signed_for_s3_canonicalization( - url=call[0][0], - method="PUT", - body=call.kwargs["data"].encode("utf-8"), - headers=call.kwargs["headers"], - ) - - -@pytest.mark.parametrize("s3_object_key", _RESERVED_CHAR_KEYS) -def test_sync_upload_percent_encodes_reserved_characters_in_object_key(s3_object_key): - from unittest.mock import MagicMock - - logger = _logger_for_signing() - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - mock_sync_client = MagicMock() - mock_sync_client.put.return_value = response - - with patch("litellm.integrations.s3_v2.get_httpx_client", return_value=mock_sync_client): - logger.upload_data_to_s3(_element_for(s3_object_key)) - - call = mock_sync_client.put.call_args - assert call[0][0] == _expected_wire_url(s3_object_key) - _assert_signed_for_s3_canonicalization( - url=call[0][0], - method="PUT", - body=call.kwargs["data"].encode("utf-8"), - headers=call.kwargs["headers"], - ) - - -@pytest.mark.parametrize("s3_object_key", _RESERVED_CHAR_KEYS) -@pytest.mark.asyncio -async def test_download_percent_encodes_reserved_characters_in_object_key(s3_object_key): - from unittest.mock import AsyncMock, MagicMock - - logger = _logger_for_signing() - response = MagicMock() - response.status_code = 200 - response.json = MagicMock(return_value={"downloaded": "data"}) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.get.return_value = response - - assert await logger._download_object_from_s3(s3_object_key) == {"downloaded": "data"} - - call = logger.async_httpx_client.get.call_args - assert call[0][0] == _expected_wire_url(s3_object_key) - _assert_signed_for_s3_canonicalization( - url=call[0][0], - method="GET", - body=None, - headers=call.kwargs["headers"], - ) - - -def _s3_logger_for_region(region_name: str) -> S3Logger: - logger = S3Logger.__new__(S3Logger) - logger.s3_endpoint_url = None - logger.s3_bucket_name = "my-litellm-audit" - logger.s3_region_name = region_name - return logger - - -@pytest.mark.parametrize( - "region_name,expected_url", - [ - ( - "cn-northwest-1", - "https://my-litellm-audit.s3.cn-northwest-1.amazonaws.com.cn/2025-01-01/key.json", - ), - ( - "us-gov-west-1", - "https://my-litellm-audit.s3.us-gov-west-1.amazonaws.com/2025-01-01/key.json", - ), - ( - "us-east-1", - "https://my-litellm-audit.s3.us-east-1.amazonaws.com/2025-01-01/key.json", - ), - ], -) -def test_build_object_url_uses_partition_dns_suffix(region_name: str, expected_url: str) -> None: - assert _s3_logger_for_region(region_name)._build_object_url("2025-01-01/key.json") == expected_url - - -def _prompts_only_logger(s3_log_prompts_only: bool | None = None) -> S3Logger: - return S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_log_prompts_only=s3_log_prompts_only, - ) - - -def _chat_payload() -> StandardLoggingPayload: - return StandardLoggingPayload( - id="chatcmpl-prompts-only", - messages=[{"role": "user", "content": "Reply with exactly the word PINEAPPLE."}], - response={"choices": [{"message": {"role": "assistant", "content": "PINEAPPLE"}}]}, - metadata={"user_api_key_team_alias": None}, - ) - - -async def _queued_body_via_async_upload( - logger: S3Logger, log_event: Callable[..., Awaitable[None]] -) -> dict[str, object]: - payload = _chat_payload() - original = copy.deepcopy(payload) - await log_event( - kwargs={"standard_logging_object": payload}, - response_obj=None, - start_time=datetime(2026, 7, 30, 12, 0, 0), - end_time=datetime(2026, 7, 30, 12, 0, 1), - ) - assert payload == original, "the caller's standard_logging_object must not be mutated" - (element,) = logger.log_queue - - response = MagicMock() - response.status_code = 200 - response.raise_for_status = MagicMock() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put.return_value = response - await logger.async_upload_data_to_s3(element) - return json.loads(logger.async_httpx_client.put.call_args.kwargs["data"]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("event_name", ["async_log_success_event", "async_log_failure_event"]) -async def test_prompts_only_drops_response_but_keeps_messages_in_uploaded_object( - monkeypatch: pytest.MonkeyPatch, event_name: str -): - import litellm - - monkeypatch.setattr(litellm, "s3_callback_params", {"s3_log_prompts_only": True}) - logger = _prompts_only_logger() - - log_event: Callable[..., Awaitable[None]] = ( - logger.async_log_success_event if event_name == "async_log_success_event" else logger.async_log_failure_event - ) - body = await _queued_body_via_async_upload(logger, log_event) - - assert body["messages"] == _chat_payload()["messages"] - assert body["response"] is None - assert body["id"] == "chatcmpl-prompts-only" - - -@pytest.mark.asyncio -async def test_prompts_only_default_off_keeps_response_in_uploaded_object(monkeypatch: pytest.MonkeyPatch): - import litellm - - monkeypatch.setattr(litellm, "s3_callback_params", {}) - monkeypatch.delenv("S3_LOG_PROMPTS_ONLY", raising=False) - logger = _prompts_only_logger() - - body = await _queued_body_via_async_upload(logger, logger.async_log_success_event) - - assert body["response"] == _chat_payload()["response"] - assert body["messages"] == _chat_payload()["messages"] - - -@pytest.mark.asyncio -async def test_prompts_only_explicit_false_in_params_beats_env_var(monkeypatch: pytest.MonkeyPatch): - import litellm - - monkeypatch.setattr(litellm, "s3_callback_params", {"s3_log_prompts_only": False}) - monkeypatch.setenv("S3_LOG_PROMPTS_ONLY", "true") - logger = _prompts_only_logger() - - body = await _queued_body_via_async_upload(logger, logger.async_log_success_event) - - assert body["response"] == _chat_payload()["response"] - - -@pytest.mark.asyncio -async def test_prompts_only_env_var_applies_when_param_unset(monkeypatch: pytest.MonkeyPatch): - import litellm - - monkeypatch.setattr(litellm, "s3_callback_params", {}) - logger = _prompts_only_logger() - monkeypatch.setenv("S3_LOG_PROMPTS_ONLY", "true") - - body = await _queued_body_via_async_upload(logger, logger.async_log_success_event) - - assert body["response"] is None - assert body["messages"] == _chat_payload()["messages"] - - -@respx.mock -def test_prompts_only_constructor_kwarg_applies_to_sync_upload(monkeypatch: pytest.MonkeyPatch): - import litellm - - monkeypatch.setattr(litellm, "s3_callback_params", {}) - monkeypatch.delenv("S3_LOG_PROMPTS_ONLY", raising=False) - logger = _prompts_only_logger(s3_log_prompts_only=True) - payload = _chat_payload() - - element = logger.create_s3_batch_logging_element( - start_time=datetime(2026, 7, 30, 12, 0, 0), - standard_logging_payload=payload, - ) - assert element is not None - assert payload["response"] == _chat_payload()["response"] - - put_route = respx.put(url__regex=r"https://test-bucket\.s3\..*").mock(return_value=httpx.Response(200)) - logger.upload_data_to_s3(element) - - body = json.loads(put_route.calls.last.request.content) - assert body["response"] is None - assert body["messages"] == _chat_payload()["messages"] - - -@pytest.mark.parametrize("callback_name", ["s3", "s3_v2"]) -def test_prompts_only_toggle_is_exposed_to_admin_ui_for_both_s3_callbacks(callback_name: str): - from litellm.integrations.custom_logger import CustomLogger - - assert "S3_LOG_PROMPTS_ONLY" in CustomLogger.get_callback_env_vars(callback_name) - - -_PARTITION_START: Final = datetime(2026, 9, 29, 14, 5, 9, 123456) -_PARTITION_ID: Final = "chatcmpl-partition" - - -def _partition_payload(response_id: str = _PARTITION_ID) -> StandardLoggingPayload: - return StandardLoggingPayload( - id=response_id, - metadata={"user_api_key_team_alias": "team-a", "user_api_key_alias": "key-a"}, - messages=[], - ) - - -def _partition_logger( - monkeypatch: pytest.MonkeyPatch, callback_params: dict[str, object], **kwargs: object -) -> S3Logger: - import litellm - - monkeypatch.setattr( - litellm, - "s3_callback_params", - {"s3_bucket_name": "test-bucket", "s3_region_name": "us-east-1", "s3_path": "logs", **callback_params}, - ) - return S3Logger( - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_use_team_prefix=True, - s3_use_key_prefix=True, - **kwargs, - ) - - -_DAILY_KEY: Final = f"logs/team-a/key-a/2026-09-29/time-14-05-09-123456_{_PARTITION_ID}.json" -_HOURLY_KEY: Final = f"logs/team-a/key-a/2026-09-29/14/time-14-05-09-123456_{_PARTITION_ID}.json" - - -@pytest.mark.parametrize( - ("callback_params", "expected_key"), - [ - ({}, _DAILY_KEY), - ({"s3_partition_granularity": None}, _DAILY_KEY), - ({"s3_partition_granularity": "day"}, _DAILY_KEY), - ({"s3_partition_granularity": "hour"}, _HOURLY_KEY), - ], -) -def test_partition_granularity_sets_request_log_folder( - monkeypatch: pytest.MonkeyPatch, callback_params: dict[str, object], expected_key: str -) -> None: - monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) - logger = _partition_logger(monkeypatch, callback_params) - - element = logger.create_s3_batch_logging_element(_PARTITION_START, _partition_payload()) - - assert element is not None - assert element.s3_object_key == expected_key - - -@pytest.mark.parametrize("invalid", ["hourly", "HOUR", "1", 1, True]) -def test_invalid_partition_granularity_warns_and_keeps_daily_folder( - monkeypatch: pytest.MonkeyPatch, invalid: object -) -> None: - monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) - with patch("litellm.integrations.s3.verbose_logger") as mock_logger: - logger = _partition_logger(monkeypatch, {"s3_partition_granularity": invalid}) - element = logger.create_s3_batch_logging_element(_PARTITION_START, _partition_payload()) - second = logger.create_s3_batch_logging_element(_PARTITION_START, _partition_payload()) - - assert element is not None - assert second is not None - assert element.s3_object_key == second.s3_object_key == _DAILY_KEY - mock_logger.warning.assert_called_once() - assert mock_logger.warning.call_args.args[1:] == (invalid,) - - -def test_partition_granularity_reads_admin_ui_env_var_below_callback_params(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("S3_PARTITION_GRANULARITY", "hour") - - from_env = _partition_logger(monkeypatch, {}).create_s3_batch_logging_element( - _PARTITION_START, _partition_payload() - ) - from_params = _partition_logger(monkeypatch, {"s3_partition_granularity": "day"}).create_s3_batch_logging_element( - _PARTITION_START, _partition_payload() - ) - - assert from_env is not None and from_env.s3_object_key == _HOURLY_KEY - assert from_params is not None and from_params.s3_object_key == _DAILY_KEY - - -def test_partition_granularity_constructor_argument_and_os_environ_reference(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) - monkeypatch.setenv("MY_S3_PARTITION", "hour") - - from_ctor = _partition_logger(monkeypatch, {}, s3_partition_granularity="hour") - from_secret = _partition_logger(monkeypatch, {"s3_partition_granularity": "os.environ/MY_S3_PARTITION"}) - - for logger in (from_ctor, from_secret): - element = logger.create_s3_batch_logging_element(_PARTITION_START, _partition_payload()) - assert element is not None and element.s3_object_key == _HOURLY_KEY - - -def test_hourly_partition_long_key_keeps_hour_folder_within_s3_limit(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.constants import MAX_S3_OBJECT_KEY_BYTES - - monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) - logger = _partition_logger(monkeypatch, {"s3_partition_granularity": "hour", "s3_path": "p" * 1100}) - - element = logger.create_s3_batch_logging_element(_PARTITION_START, _partition_payload("r" * 600)) - - assert element is not None - assert len(element.s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES - assert re.search(r"/2026-09-29/14/[0-9a-f]{64}\.json$", element.s3_object_key) - - -@pytest.mark.asyncio -@pytest.mark.parametrize(("granularity", "hour_folder"), [("hour", True), ("day", False), (None, False)]) -async def test_audit_log_key_follows_audit_callback_params_partition_granularity( - monkeypatch: pytest.MonkeyPatch, granularity: str | None, hour_folder: bool -) -> None: - monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) - logger = S3Logger( - s3_callback_params_override={ - "s3_bucket_name": "audit-bucket", - "s3_path": "audit", - "s3_partition_granularity": granularity, - } - ) - - await logger.async_log_audit_log_event({"id": "audit-1"}) - - (element,) = logger.log_queue - match = re.fullmatch( - r"audit/audit_logs/\d{4}-\d{2}-\d{2}/(?:(\d{2})/)?(\d{2})-\d{2}-\d{2}_audit-1\.json", element.s3_object_key - ) - assert match is not None, element.s3_object_key - assert (match.group(1) is not None) is hour_folder - if hour_folder: - assert match.group(1) == match.group(2) - - -@pytest.mark.asyncio -async def test_hourly_batch_file_upload_writes_one_file_per_hour_folder(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) - logger = _partition_logger(monkeypatch, {"s3_partition_granularity": "hour"}, s3_batch_file_upload=True) - put = _RecordingPut() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - before = logger.create_s3_batch_logging_element(datetime(2026, 9, 29, 13, 59, 59), _partition_payload("before")) - after = logger.create_s3_batch_logging_element(datetime(2026, 9, 29, 14, 0, 1), _partition_payload("after")) - assert before is not None and after is not None - logger.log_queue = [before, after] - - await logger.async_send_batch() - - by_folder = { - re.sub(r"/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", "", url.split(".com/", 1)[-1]): data - for url, data, _headers in put.calls - } - assert sorted(by_folder) == ["logs/team-a/key-a/2026-09-29/13", "logs/team-a/key-a/2026-09-29/14"] - assert [json.loads(line)["id"] for line in (by_folder["logs/team-a/key-a/2026-09-29/13"] or "").splitlines()] == [ - "before" - ] - assert [json.loads(line)["id"] for line in (by_folder["logs/team-a/key-a/2026-09-29/14"] or "").splitlines()] == [ - "after" - ] - - -@pytest.mark.parametrize("granularity", [None, "day", "hour"]) -def test_cold_storage_object_key_matches_the_uploaded_request_log_key( - monkeypatch: pytest.MonkeyPatch, granularity: str | None -) -> None: - import litellm - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup - - monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) - monkeypatch.setattr( - litellm, - "s3_callback_params", - {"s3_bucket_name": "test-bucket", "s3_path": "coldlogs", "s3_partition_granularity": granularity}, - ) - monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2") - logger = S3Logger() - uploaded = logger.create_s3_batch_logging_element( - _PARTITION_START, StandardLoggingPayload(id=_PARTITION_ID, metadata={}, messages=[]) - ) - - monkeypatch.setattr(litellm, "callbacks", [logger]) - cold_key = StandardLoggingPayloadSetup._generate_cold_storage_object_key( - start_time=_PARTITION_START, response_id=_PARTITION_ID - ) - - assert uploaded is not None - assert cold_key == uploaded.s3_object_key - assert ("/2026-09-29/14/" in cold_key) is (granularity == "hour") - - -def test_cold_storage_key_matches_upload_when_env_var_changes_mid_request(monkeypatch: pytest.MonkeyPatch) -> None: - import litellm - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup - - monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) - monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "test-bucket", "s3_path": "coldlogs"}) - monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2") - logger = S3Logger() - monkeypatch.setattr(litellm, "callbacks", [logger]) - - cold_key = StandardLoggingPayloadSetup._generate_cold_storage_object_key( - start_time=_PARTITION_START, response_id=_PARTITION_ID - ) - monkeypatch.setenv("S3_PARTITION_GRANULARITY", "hour") - uploaded = logger.create_s3_batch_logging_element( - _PARTITION_START, - StandardLoggingPayload(id=_PARTITION_ID, metadata={"cold_storage_object_key": cold_key}, messages=[]), - ) - - assert uploaded is not None - assert cold_key == uploaded.s3_object_key == f"coldlogs/2026-09-29/time-14-05-09-123456_{_PARTITION_ID}.json" - - -def test_hour_upload_ignores_a_cold_storage_key_owned_by_another_logger(monkeypatch: pytest.MonkeyPatch) -> None: - import litellm - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup - - monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) - monkeypatch.setattr( - litellm, "s3_callback_params", {"s3_bucket_name": "test-bucket", "s3_partition_granularity": "hour"} - ) - monkeypatch.setattr(litellm, "cold_storage_custom_logger", "gcs_bucket") - logger = S3Logger() - cold_key = StandardLoggingPayloadSetup._generate_cold_storage_object_key( - start_time=_PARTITION_START, response_id=_PARTITION_ID - ) - uploaded = logger.create_s3_batch_logging_element( - _PARTITION_START, - StandardLoggingPayload(id=_PARTITION_ID, metadata={"cold_storage_object_key": cold_key}, messages=[]), - ) - - assert cold_key == f"2026-09-29/time-14-05-09-123456_{_PARTITION_ID}.json" - assert uploaded is not None - assert uploaded.s3_object_key == f"2026-09-29/14/time-14-05-09-123456_{_PARTITION_ID}.json" - - -@pytest.mark.parametrize("callback_name", ["s3", "s3_v2"]) -def test_partition_granularity_is_exposed_to_admin_ui(callback_name: str) -> None: - from litellm.integrations.custom_logger import CustomLogger - - assert "S3_PARTITION_GRANULARITY" in CustomLogger.get_callback_env_vars(callback_name) - - -def _element(payload: dict[str, object], key_suffix: str) -> s3BatchLoggingElement: - return s3BatchLoggingElement( - s3_object_key=f"2025-09-14/test-{key_suffix}.json", - payload=payload, - s3_object_download_filename=f"test-{key_suffix}.json", - ) - - -def _ok_response() -> MagicMock: - response = MagicMock() - response.status_code = 200 - response.text = "" - response.raise_for_status = MagicMock() - return response - - -class _CountingPut: - def __init__(self, width: int) -> None: - self.width = width - self.in_flight = 0 - self.peak = 0 - self.calls = 0 - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.in_flight += 1 - self.peak = max(self.peak, self.in_flight) - self.calls += 1 - for _ in range(50): - if self.in_flight >= self.width: - break - await _real_sleep(0) - self.in_flight -= 1 - return _ok_response() - - -class _RecordingPut: - def __init__(self) -> None: - self.calls: tuple[tuple[str, str | None, dict[str, str] | None], ...] = () - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls = (*self.calls, (url, data, headers)) - return _ok_response() - - -class _LateAppendingPut: - def __init__(self, logger: S3Logger, element: s3BatchLoggingElement, fail_first: bool = False) -> None: - self.logger = logger - self.element = element - self.fail_first = fail_first - self.appended = False - self.failed_key: str | None = None - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - if not self.appended: - self.appended = True - self.logger.log_queue.append(self.element) - if self.fail_first: - self.failed_key = url - if url == self.failed_key: - return _transient_failure_response() - return _ok_response() - - -class _AppendingFailingPut: - def __init__(self, logger: S3Logger, elements: tuple[s3BatchLoggingElement, ...]) -> None: - self.logger = logger - self.elements = elements - self.appended = False - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - if not self.appended: - self.appended = True - for element in self.elements: - self.logger.log_queue.append(element) - return _transient_failure_response() - - -class _FailOnSuffixPut: - def __init__(self, suffixes: tuple[str, ...]) -> None: - self.failing = True - self.suffixes = suffixes - self.calls: tuple[str, ...] = () - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls = (*self.calls, url) - if self.failing and url.endswith(self.suffixes): - return _transient_failure_response() - return _ok_response() - - -class _FailUntilClearedPut: - def __init__(self, status: int = 503, code: str | None = "SlowDown", raw_body: str | None = None) -> None: - self.failing = True - self.response: Final = _coded_failure_response(status, code, raw_body) - self.calls: tuple[tuple[str, str | None], ...] = () - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls = (*self.calls, (url, data)) - if self.failing: - return self.response - return _ok_response() - - -@pytest.mark.asyncio -async def test_async_send_batch_bounds_concurrent_uploads() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_concurrent_uploads=4, - ) - - put = _CountingPut(logger.s3_max_concurrent_uploads) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - - logger.log_queue = [_element({"i": i}, f"{i}") for i in range(40)] - - await logger.async_send_batch() - - assert put.peak == 4 - assert put.calls == 40 - - -@pytest.mark.asyncio -async def test_async_send_batch_uploads_single_jsonl_file() -> None: - import json - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_batch_file_upload=True, - ) - - put = _RecordingPut() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - - payloads = [{"id": "req-1"}, {"id": "req-2"}, {"id": "req-3"}] - logger.log_queue = [_element(payload, f"{i}") for i, payload in enumerate(payloads)] - - await logger.async_send_batch() - - assert len(put.calls) == 1 - url, data, headers = put.calls[0] - assert url.endswith(".jsonl") - assert data is not None - assert headers is not None - assert [json.loads(line) for line in data.splitlines()] == payloads - assert headers["Content-Type"] == "application/x-ndjson" - - -@pytest.mark.asyncio -async def test_flush_queue_preserves_events_added_during_upload() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - late_element = _element({"id": "late"}, "late") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _LateAppendingPut(logger, late_element) - - logger.log_queue = [_element({"id": "first"}, "first")] - - await logger.flush_queue() - - assert logger.log_queue == [late_element] - - -def _override_logger(**overrides: object) -> S3Logger: - return S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_callback_params_override=overrides, - ) - - -def test_env_backed_false_string_keeps_per_request_uploads() -> None: - assert _override_logger(s3_batch_file_upload="false").s3_batch_file_upload is False - assert _override_logger(s3_batch_file_upload="true").s3_batch_file_upload is True - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_batch_file_upload=True, - s3_callback_params_override={"s3_batch_file_upload": "false"}, - ) - assert logger.s3_batch_file_upload is True - - -@pytest.mark.parametrize("bad", [0, -3, "0", "abc", ""]) -def test_invalid_concurrency_falls_back_to_default(bad: object) -> None: - from litellm.constants import DEFAULT_S3_MAX_CONCURRENT_UPLOADS - - logger = _override_logger(s3_max_concurrent_uploads=bad) - - assert logger.s3_max_concurrent_uploads == DEFAULT_S3_MAX_CONCURRENT_UPLOADS - assert logger._upload_limiter._value == DEFAULT_S3_MAX_CONCURRENT_UPLOADS - - -def test_env_backed_concurrency_string_is_parsed() -> None: - logger = _override_logger(s3_max_concurrent_uploads="4") - - assert logger.s3_max_concurrent_uploads == 4 - assert logger._upload_limiter._value == 4 - - -@pytest.mark.parametrize("empty", [None, ""]) -def test_empty_config_concurrency_falls_back_to_constructor_value(empty: object) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_concurrent_uploads=4, - s3_callback_params_override={"s3_max_concurrent_uploads": empty}, - ) - - assert logger.s3_max_concurrent_uploads == 4 - assert logger._upload_limiter._value == 4 - - -def _coded_failure_response(status: int, code: str | None, raw_body: str | None = None) -> MagicMock: - body: Final = ( - raw_body if raw_body is not None else (f"{code}" if code is not None else "") - ) - response = MagicMock() - response.status_code = status - response.text = body - response.raise_for_status = MagicMock( - side_effect=httpx.HTTPStatusError(str(status), request=MagicMock(), response=response) - ) - return response - - -def _transient_failure_response(status: int = 503) -> MagicMock: - return _coded_failure_response(status, "SlowDown") - - -def _terminal_failure_response() -> MagicMock: - return _coded_failure_response(400, "EntityTooLarge") - - -@pytest.mark.asyncio -async def test_failed_uploads_stay_queued_for_next_flush() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - elements = [_element({"i": i}, f"{i}") for i in range(5)] - put = _FailOnSuffixPut(("test-2.json", "test-4.json")) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = list(elements) - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert [element.s3_object_key for element in logger.log_queue] == [ - elements[2].s3_object_key, - elements[4].s3_object_key, - ] - - put.failing = False - await logger.flush_queue() - - assert logger.log_queue == [] - - -@pytest.mark.asyncio -async def test_batch_file_upload_failure_keeps_whole_batch() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_batch_file_upload=True, - ) - - put = _FailUntilClearedPut() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - - elements = [_element({"i": i}, f"{i}") for i in range(3)] - logger.log_queue = list(elements) - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert len(put.calls) == 3 - assert len(logger.log_queue) == 1 - assert logger.log_queue[0].body == "\n".join(json.dumps(element.payload) for element in elements) - - -@pytest.mark.asyncio -async def test_events_appended_during_failed_flush_survive() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - late = _element({"id": "late"}, "late") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _LateAppendingPut(logger, late, fail_first=True) - - first = _element({"id": "first"}, "first") - logger.log_queue = [first] - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert [element.s3_object_key for element in logger.log_queue] == [first.s3_object_key, late.s3_object_key] - assert logger.log_queue[0].retrying_since is None - - logger.async_httpx_client.put.failed_key = None - await logger.flush_queue() - - assert logger.log_queue == [] - - -@pytest.mark.asyncio -async def test_batch_file_key_shape() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_path="logs", - s3_batch_file_upload=True, - ) - - put = _RecordingPut() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"id": "req-1"}, "0")] - - await logger.async_send_batch() - - ((url, _data, headers),) = put.calls - assert headers is not None - assert re.search(r".*/2025-09-14/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", url) - assert headers["Content-Disposition"].endswith('.jsonl"') - - -@pytest.mark.asyncio -async def test_batch_file_groups_raw_elements_by_key_parent() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_batch_file_upload=True, - ) - - put = _RecordingPut() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - - alpha = s3BatchLoggingElement( - s3_object_key="logs/alpha/2026-01-01/a.json", payload={"id": "a"}, s3_object_download_filename="a.json" - ) - beta = s3BatchLoggingElement( - s3_object_key="logs/beta/2026-01-01/b.json", payload={"id": "b"}, s3_object_download_filename="b.json" - ) - plain = s3BatchLoggingElement( - s3_object_key="logs/2026-01-01/c.json", payload={"id": "c"}, s3_object_download_filename="c.json" - ) - root = s3BatchLoggingElement( - s3_object_key="solo.json", payload={"id": "d"}, s3_object_download_filename="solo.json" - ) - logger.log_queue = [alpha, beta, plain, root] - - await logger.async_send_batch() - - assert len(put.calls) == 4 - by_parent = { - re.sub(r"(^|/)batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", "", url.split(".com/", 1)[-1]): (url, data) - for url, data, _headers in put.calls - } - assert sorted(by_parent) == ["", "logs/2026-01-01", "logs/alpha/2026-01-01", "logs/beta/2026-01-01"] - assert [line for line in by_parent[""][1].splitlines()] == [json.dumps({"id": "d"})] - assert [line for line in by_parent["logs/alpha/2026-01-01"][1].splitlines()] == [json.dumps({"id": "a"})] - assert [line for line in by_parent["logs/beta/2026-01-01"][1].splitlines()] == [json.dumps({"id": "b"})] - assert [line for line in by_parent["logs/2026-01-01"][1].splitlines()] == [json.dumps({"id": "c"})] - - -@pytest.mark.asyncio -async def test_failed_batch_file_is_requeued_and_resent_unchanged() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_batch_file_upload=True, - ) - - put = _FailUntilClearedPut() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": i}, f"{i}") for i in range(3)] - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert len(logger.log_queue) == 1 - assert logger.log_queue[0].body is not None - assert logger.log_queue[0].s3_object_key.endswith(".jsonl") - - put.failing = False - await logger.flush_queue() - - assert logger.log_queue == [] - assert len(put.calls) == 4 - assert put.calls[0] == put.calls[1] - - -@pytest.mark.asyncio -async def test_elements_appended_after_failed_batch_file_get_their_own_file() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_batch_file_upload=True, - ) - - put = _FailUntilClearedPut() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"id": "first"}, "first")] - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - late = _element({"id": "late"}, "late") - logger.log_queue.append(late) - - put.failing = False - await logger.flush_queue() - - assert logger.log_queue == [] - assert len(put.calls) == 5 - assert put.calls[0] == put.calls[1] - second_flush: Final = put.calls[3:] - assert put.calls[0] in second_flush - late_call: Final = next(call for call in second_flush if call != put.calls[0]) - assert late_call[0] != put.calls[0][0] - assert late_call[1] == json.dumps({"id": "late"}) - - -@pytest.mark.asyncio -async def test_batch_file_mode_disabled_when_s3_v2_is_cold_storage_logger(monkeypatch: pytest.MonkeyPatch) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_batch_file_upload=True, - ) - - put = _RecordingPut() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - - import litellm - - monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2") - logger.log_queue = [_element({"id": "req-1"}, "0")] - - await logger.async_send_batch() - - assert len(put.calls) == 1 - assert put.calls[0][0].endswith("test-0.json") - - monkeypatch.setattr(litellm, "cold_storage_custom_logger", None) - logger.log_queue = [_element({"id": "req-2"}, "1")] - - await logger.async_send_batch() - - assert len(put.calls) == 2 - assert put.calls[1][0].endswith(".jsonl") - - -class _FailOnSuffixCodedPut: - def __init__( - self, suffixes: tuple[str, ...], status: int, code: str | None = None, raw_body: str | None = None - ) -> None: - self.suffixes = suffixes - self.response: Final = _coded_failure_response(status, code, raw_body) - self.calls: tuple[str, ...] = () - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls = (*self.calls, url) - if url.endswith(self.suffixes): - return self.response - return _ok_response() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("status", "code", "raw_body", "puts_per_element"), - [ - pytest.param(403, "AccessDenied", None, 3, id="access-denied-403"), - pytest.param(403, None, None, 3, id="empty-403"), - pytest.param(403, None, "Forbidden", 3, id="html-403"), - pytest.param(400, "KMS.DisabledException", None, 1, id="kms-disabled-400"), - pytest.param(404, "NoSuchBucket", None, 1, id="no-such-bucket-404"), - ], -) -async def test_non_terminal_failure_is_requeued_and_delivered_on_recovery( - status: int, code: str | None, raw_body: str | None, puts_per_element: int -) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - elements = [_element({"i": i}, f"{i}") for i in range(5)] - put = _FailUntilClearedPut(status=status, code=code, raw_body=raw_body) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = list(elements) - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert len(logger.log_queue) == 5 - assert len(put.calls) == 5 * puts_per_element - - put.failing = False - await logger.flush_queue() - - assert logger.log_queue == [] - assert len(put.calls) == 5 * puts_per_element + 5 - landed: Final = frozenset( - element.s3_object_key - for element in elements - if any(call[0].endswith(element.s3_object_key) for call in put.calls[-5:]) - ) - assert landed == frozenset(element.s3_object_key for element in elements) - - -@pytest.mark.asyncio -async def test_persistent_500_stays_queued_through_a_dozen_failed_flushes() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - put = _FailUntilClearedPut(status=500, code="InternalError") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": i}, f"{i}") for i in range(5)] - - with patch("asyncio.sleep", new_callable=AsyncMock): - for _ in range(12): - await logger.flush_queue() - assert len(logger.log_queue) == 5 - - assert len(put.calls) == 12 * 15 - - put.failing = False - await logger.flush_queue() - - assert logger.log_queue == [] - assert len(put.calls) == 12 * 15 + 5 - - -@pytest.mark.asyncio -async def test_terminal_object_is_dropped_once_next_to_delivered_siblings_when_opted_in() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_drop_on_terminal_error=True, - ) - - elements = [_element({"i": i}, f"{i}") for i in range(5)] - put = _FailOnSuffixCodedPut(("test-1.json",), 400, "EntityTooLarge") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = list(elements) - - await logger.flush_queue() - - assert len(put.calls) == 5 - assert logger.log_queue == [] - - -@pytest.mark.asyncio -async def test_terminal_object_is_requeued_when_opted_out() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_drop_on_terminal_error=False, - ) - - put = _FailOnSuffixCodedPut(("test-1.json",), 400, "EntityTooLarge") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": i}, f"{i}") for i in range(5)] - - await logger.flush_queue() - - assert len(logger.log_queue) == 1 - assert logger.log_queue[0].s3_object_key.endswith("test-1.json") - assert sum(call.endswith("test-1.json") for call in put.calls) == 1 - assert len(put.calls) == 5 - - -@pytest.mark.asyncio -async def test_terminal_objects_are_requeued_when_every_upload_in_the_flush_fails() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_drop_on_terminal_error=True, - ) - - put = _FailUntilClearedPut(status=400, code="EntityTooLarge") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": i}, f"{i}") for i in range(5)] - - await logger.flush_queue() - - assert len(logger.log_queue) == 5 - - -@pytest.mark.asyncio -async def test_retrying_past_the_opted_in_budget_is_dropped_only_next_to_delivered_siblings() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_retry_age_seconds=60, - ) - - aged = _element({"id": "aged"}, "aged").model_copy(update={"retrying_since": _NOW - 120}) - fresh = _element({"id": "fresh"}, "fresh") - put = _FailOnSuffixCodedPut(("test-aged.json",), 503, "SlowDown") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [aged, fresh] - - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=_NOW), - ): - await logger.flush_queue() - - assert logger.log_queue == [] - assert len(put.calls) == 4 - - -@pytest.mark.asyncio -async def test_retrying_past_the_budget_stays_queued_when_the_whole_flush_fails() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_retry_age_seconds=60, - ) - - aged = _element({"id": "aged"}, "aged").model_copy(update={"retrying_since": _NOW - 120}) - put = _FailUntilClearedPut(status=503, code="SlowDown") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [aged] - - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=_NOW), - ): - await logger.flush_queue() - - assert len(logger.log_queue) == 1 - - -@pytest.mark.asyncio -async def test_overflow_after_a_failed_flush_trims_failed_first_and_counts_upload_failures_only( - caplog, -) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_queue_size=4, - ) - - late = tuple(_element({"id": f"late-{index}"}, f"late-{index}") for index in range(3)) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _AppendingFailingPut(logger, late) - logger.log_queue = [_element({"id": "first"}, "first"), _element({"id": "second"}, "second")] - - with ( - patch.object(logger, "handle_callback_failure") as mock_failure, - patch("asyncio.sleep", new_callable=AsyncMock), - pytest.raises(S3BatchUploadError), - ): - await logger.async_send_batch() - - assert [element.payload["id"] for element in logger.log_queue] == ["second", "late-0", "late-1", "late-2"] - failed_uploads: Final = 2 - assert mock_failure.call_count == failed_uploads - mock_failure.assert_called_with(callback_name="S3Logger") - assert "dropped 1 oldest events" in caplog.text - - -@pytest.mark.asyncio -async def test_default_logger_ages_out_elements_retrying_longer_than_an_hour(caplog) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - elements = [ - _element({"i": index}, f"{index}").model_copy(update={"retrying_since": _NOW - 7200}) for index in range(3) - ] - put = _FailOnSuffixPut(("test-1.json", "test-2.json")) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = list(elements) - - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=_NOW), - ): - await logger.flush_queue() - - assert logger.log_queue == [] - assert "uploads dropped" in caplog.text - - -@pytest.mark.asyncio -async def test_opted_out_logger_never_ages_out_long_retrying_elements(caplog) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_retry_age_seconds=0, - ) - - elements = [ - _element({"i": index}, f"{index}").model_copy(update={"retrying_since": _NOW - 7200}) for index in range(3) - ] - put = _FailOnSuffixPut(("test-1.json", "test-2.json")) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = list(elements) - - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=_NOW), - ): - await logger.flush_queue() - - assert [element.s3_object_key for element in logger.log_queue] == [ - elements[1].s3_object_key, - elements[2].s3_object_key, - ] - assert "uploads dropped" not in caplog.text - - put.failing = False - await logger.flush_queue() - - assert logger.log_queue == [] - landed: Final = frozenset(call_url.rsplit("/", 1)[-1] for call_url in put.calls) - assert landed == frozenset(f"test-{index}.json" for index in range(3)) - - -@pytest.mark.asyncio -async def test_queue_grows_past_the_cap_while_the_sink_fails_and_everything_lands() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_queue_size=5, - ) - - elements = [_element({"i": index}, f"{index}") for index in range(8)] - put = _FailUntilClearedPut() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = list(elements[:5]) - - with ( - patch.object(logger, "handle_callback_failure") as mock_failure, - patch("asyncio.sleep", new_callable=AsyncMock), - ): - await logger.flush_queue() - - assert len(logger.log_queue) == 5 - upload_failures: Final = 5 - assert mock_failure.call_count == upload_failures - - for element in elements[5:]: - logger.log_queue.append(element) - - put.failing = False - await logger.flush_queue() - - assert logger.log_queue == [] - landed: Final = frozenset(call[0].rsplit("/", 1)[-1] for call in put.calls[-8:]) - assert landed == frozenset(f"test-{index}.json" for index in range(8)) # calls are (url, data) pairs - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("status", "code"), - [ - pytest.param(404, "NoSuchKey", id="404"), - pytest.param(401, None, id="401"), - pytest.param(400, None, id="uncoded-400"), - ], -) -async def test_unlisted_status_gets_one_put_and_stays_queued(status: int, code: str | None) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - put = _FailUntilClearedPut(status=status, code=code) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": index}, f"{index}") for index in range(4)] - - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - await logger.flush_queue() - - assert len(put.calls) == 4 - mock_sleep.assert_not_awaited() - assert len(logger.log_queue) == 4 - - -class _SyncRecordingClient: - def __init__(self, response: httpx.Response) -> None: - self.response: Final = response - self.put_calls: list = [] # mutable-ok: call log appended once per PUT - - def put( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.put_calls.append(url) - return self.response - - -def test_sync_upload_404_is_single_attempt_without_sleep() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - sync_client: Final = _SyncRecordingClient(_coded_failure_response(404, "NoSuchKey")) - - with ( - patch("litellm.integrations.s3_v2.get_httpx_client", return_value=sync_client), - patch("time.sleep") as mock_sleep, - ): - logger.upload_data_to_s3(_element({"id": "sync-404"}, "sync-404")) - - assert len(sync_client.put_calls) == 1 - mock_sleep.assert_not_called() - - -@pytest.mark.parametrize( - ("status", "expected_puts", "expected_retries"), - [ - pytest.param(408, 1, 0, id="408-single"), - pytest.param(502, 1, 0, id="502-single"), - pytest.param(504, 1, 0, id="504-single"), - pytest.param(429, 3, 2, id="429-backoff"), - pytest.param(503, 3, 2, id="503-backoff"), - ], -) -def test_sync_upload_retry_set(status: int, expected_puts: int, expected_retries: int) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - sync_client: Final = _SyncRecordingClient(_coded_failure_response(status, "SlowDown")) - - with ( - patch("litellm.integrations.s3_v2.get_httpx_client", return_value=sync_client), - patch("time.sleep") as mock_sleep, - ): - logger.upload_data_to_s3(_element({"id": "sync"}, "sync")) - - assert len(sync_client.put_calls) == expected_puts - _assert_jittered_backoff(mock_sleep, expected_retries) - - -@pytest.mark.parametrize( - "failure", - [pytest.param(httpx.ConnectError, id="connect-error"), pytest.param(httpx.ReadTimeout, id="read-timeout")], -) -def test_sync_upload_retries_a_failed_attempt_through_production_http_handler( - failure: type[httpx.TransportError], caplog: pytest.LogCaptureFixture -) -> None: - """HTTPHandler.put turns an httpx timeout into litellm.Timeout, and each attempt carries its own deadline.""" - requests: Final[list[httpx.Request]] = [] # mutable-ok: the transport records every attempt - - def respond(request: httpx.Request) -> httpx.Response: - requests.append(request) - if len(requests) == 1: - raise failure("S3 attempt failed", request=request) - return httpx.Response(200, request=request) - - handler: Final = HTTPHandler() - handler.client = httpx.Client(transport=httpx.MockTransport(respond), timeout=None) - logger: Final = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - with ( - patch( # test-quality-ok: sync upload builds its HTTPHandler per call, there is no injection seam for it - "litellm.integrations.s3_v2.get_httpx_client", return_value=handler - ), - patch("time.sleep") as mock_sleep, - ): - logger.upload_data_to_s3(_element({"id": "sync-transient"}, "sync-transient")) - handler.client.close() - - assert len(requests) == 2 - assert requests[0].content == requests[1].content - assert all( - request.extensions["timeout"] == httpx.Timeout(S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS).as_dict() - for request in requests - ) - _assert_jittered_backoff(mock_sleep, 1) - assert "Error uploading to s3" not in caplog.text - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("status", "code"), - [ - pytest.param(503, "SlowDown", id="503"), - pytest.param(429, "SlowDown", id="429"), - pytest.param(500, "InternalError", id="500"), - pytest.param(403, "AccessDenied", id="access-denied-403"), - ], -) -async def test_retryable_statuses_back_off_three_attempts(status: int, code: str | None) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - put = _FailUntilClearedPut(status=status, code=code) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"id": "req"}, "req")] - - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - await logger.flush_queue() - - assert len(put.calls) == 3 - _assert_jittered_backoff(mock_sleep, 2) - assert len(logger.log_queue) == 1 - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("status", "code"), - [ - pytest.param(408, None, id="408"), - pytest.param(502, None, id="502"), - pytest.param(504, None, id="504"), - ], -) -async def test_unlisted_statuses_are_not_retried_in_call(status: int, code: str | None) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - put = _FailUntilClearedPut(status=status, code=code) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"id": "req"}, "req")] - - with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - await logger.flush_queue() - - assert len(put.calls) == 1 - assert mock_sleep.await_args_list == [] - assert len(logger.log_queue) == 1 - - -class _FirstFailThenOkPut: - def __init__(self, fail_suffix: str) -> None: - self.fail_suffix = fail_suffix - self.failed_once = False - self.calls: tuple[str, ...] = () - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls = (*self.calls, url) - if url.endswith(self.fail_suffix) and not self.failed_once: - self.failed_once = True - return _transient_failure_response() - return _ok_response() - - -@pytest.mark.asyncio -async def test_retry_finishes_before_the_next_first_attempt() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_concurrent_uploads=1, - ) - - put = _FirstFailThenOkPut("test-a.json") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"id": "a"}, "a"), _element({"id": "b"}, "b")] - - with patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))): - await logger.flush_queue() - - assert [call_url.rsplit("/", 1)[-1] for call_url in put.calls] == ["test-a.json", "test-a.json", "test-b.json"] - assert logger.log_queue == [] - - -@pytest.mark.asyncio -async def test_objects_in_backoff_are_bounded_by_the_slot_width() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_concurrent_uploads=2, - ) - - put = _FailUntilClearedPut(status=503) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": i}, f"{i}") for i in range(20)] - - sleeping: Final[list[int]] = [0] - peak: Final[list[int]] = [0] - - async def counting_sleep(delay: float) -> None: - sleeping[0] += 1 - peak[0] = max(peak[0], sleeping[0]) - for _ in range(10): - await _real_sleep(0) - sleeping[0] -= 1 - - with patch("asyncio.sleep", new=counting_sleep): - await logger.flush_queue() - - assert peak[0] <= 2, f"{peak[0]} objects slept at once, slot width is 2" - assert len(put.calls) == 60 - - -@pytest.mark.asyncio -async def test_subclass_returning_true_drains_the_queue() -> None: - class _TrueUploadLogger(S3Logger): - async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement) -> bool: - return True - - logger = _TrueUploadLogger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - logger.async_httpx_client = AsyncMock() - logger.log_queue = [_element({"id": "a"}, "a")] - - await logger.flush_queue() - - assert logger.log_queue == [] - logger.async_httpx_client.put.assert_not_called() - - -@pytest.mark.asyncio -async def test_failed_direct_upload_returns_false() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - put = _FailUntilClearedPut(status=500) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - test_element = _element({"id": "x"}, "x") - - with patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))): - outcome = await logger.async_upload_data_to_s3(test_element) - - assert outcome is False - assert len(put.calls) == 3 - - -@pytest.mark.asyncio -async def test_terminal_drop_of_one_element_does_not_drop_a_sibling_with_the_same_key() -> None: - class _TerminalForMarkerPut: - def __init__(self) -> None: - self.calls: tuple[str | None, ...] = () - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls = (*self.calls, data) - if data is not None and "terminal-marker" in data: - return _terminal_failure_response() - return _transient_failure_response() - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_drop_on_terminal_error=True, - ) - - put = _TerminalForMarkerPut() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - - shared_key = "2025-09-14/shared.json" - dropped = s3BatchLoggingElement( - s3_object_key=shared_key, payload={"m": "terminal-marker"}, s3_object_download_filename="shared.json" - ) - sibling = s3BatchLoggingElement( - s3_object_key=shared_key, payload={"m": "healthy"}, s3_object_download_filename="shared.json" - ) - - with patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))): - assert await logger._upload_outcome(dropped) == "dropped" - assert await logger._upload_outcome(sibling) == "retry" - - -def test_upload_semaphore_alias_is_the_limiter() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - assert logger._upload_semaphore is logger._upload_limiter - - -@pytest.mark.asyncio -async def test_overridden_upload_stays_bounded_by_the_configured_width() -> None: - class _InFlightUploadLogger(S3Logger): - def __init__(self, **kwargs: object) -> None: - super().__init__(**kwargs) - self.in_flight = 0 - self.peak = 0 - - async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement) -> bool: - self.in_flight += 1 - self.peak = max(self.peak, self.in_flight) - for _ in range(10): - await _real_sleep(0) - self.in_flight -= 1 - return True - - logger = _InFlightUploadLogger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_concurrent_uploads=4, - ) - logger.log_queue = [_element({"i": index}, f"{index}") for index in range(40)] - - await logger.flush_queue() - - assert logger.peak <= 4 - assert logger.log_queue == [] - - -@pytest.mark.asyncio -async def test_holding_the_semaphore_during_a_direct_upload_does_not_deadlock() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_concurrent_uploads=1, - ) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _RecordingPut() - element = _element({"id": "x"}, "x") - - async def held_upload() -> bool: - async with logger._upload_semaphore: - return await logger.async_upload_data_to_s3(element) - - assert await asyncio.wait_for(held_upload(), timeout=5) is True - - -@pytest.mark.asyncio -async def test_assigning_a_semaphore_changes_the_upload_width() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - logger._upload_semaphore = asyncio.Semaphore(3) - - put = _CountingPut(width=3) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": index}, f"{index}") for index in range(30)] - - await logger.flush_queue() - - assert put.peak == 3 - assert logger.log_queue == [] - - -@pytest.mark.asyncio -async def test_terminal_code_is_retried_like_base_when_the_drop_flag_is_off() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_drop_on_terminal_error=False, - ) - put = _StatusPut([_coded_failure_response(403, "InvalidRequest"), _ok_response()]) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - failures = AsyncMock() - logger.handle_callback_failure = failures - - with patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))): - assert await logger.async_upload_data_to_s3(_element({"id": "x"}, "x")) is True - - assert put.calls == 2 - failures.assert_not_called() - - dropping = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - put.calls = 0 - dropping.async_httpx_client = AsyncMock() - dropping.async_httpx_client.put = put - - with patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))): - assert await dropping.async_upload_data_to_s3(_element({"id": "x"}, "x")) is False - - assert put.calls == 1 - - -def test_sync_terminal_code_is_retried_like_base_when_the_drop_flag_is_off() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_drop_on_terminal_error=False, - ) - mock_sync_client = MagicMock() - mock_sync_client.put = MagicMock(side_effect=[_coded_failure_response(403, "InvalidRequest"), _ok_response()]) - failures = MagicMock() - logger.handle_callback_failure = failures - - with ( - patch("litellm.integrations.s3_v2.get_httpx_client", return_value=mock_sync_client), - patch("time.sleep"), - ): - logger.upload_data_to_s3(_element({"id": "x"}, "x")) - - assert mock_sync_client.put.call_count == 2 - failures.assert_not_called() - - dropping = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - mock_sync_client.put = MagicMock(side_effect=[_coded_failure_response(403, "InvalidRequest"), _ok_response()]) - - with ( - patch("litellm.integrations.s3_v2.get_httpx_client", return_value=mock_sync_client), - patch("time.sleep"), - ): - dropping.upload_data_to_s3(_element({"id": "x"}, "x")) - - assert mock_sync_client.put.call_count == 1 - - -def test_sync_retry_lines_stay_at_warning_level(caplog) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - mock_sync_client = MagicMock() - mock_sync_client.put = MagicMock( - side_effect=[_transient_failure_response(503), _transient_failure_response(503), _ok_response()] - ) - - with ( - caplog.at_level("WARNING"), - patch("litellm.integrations.s3_v2.get_httpx_client", return_value=mock_sync_client), - patch("time.sleep"), - ): - logger.upload_data_to_s3(_element({"id": "x"}, "x")) - - assert mock_sync_client.put.call_count == 3 - assert sum(1 for record in caplog.records if "retrying in" in record.getMessage()) == 2 - - -@pytest.mark.asyncio -async def test_direct_async_upload_logs_retry_lines_at_warning_level(caplog) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - put = _StatusPut([_transient_failure_response(503), _ok_response()]) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - - with caplog.at_level("WARNING"), patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))): - assert await logger.async_upload_data_to_s3(_element({"id": "x"}, "x")) is True - - assert put.calls == 2 - assert sum(1 for record in caplog.records if "retrying in" in record.getMessage()) == 1 - - -def _init_bypassed_logger() -> S3Logger: - from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM - - logger = S3Logger.__new__(S3Logger) - logger.iam_cache = BaseAWSLLM._shared_iam_cache - logger.s3_endpoint_url = None - logger.s3_bucket_name = "test-bucket" - logger.s3_region_name = "us-east-1" - logger.s3_use_virtual_hosted_style = False - logger.s3_verify = None - logger.s3_aws_access_key_id = "test-key" - logger.s3_aws_secret_access_key = "test-secret" - logger.s3_aws_session_token = None - logger.s3_aws_session_name = None - logger.s3_aws_profile_name = None - logger.s3_aws_role_name = None - logger.s3_aws_web_identity_token = None - logger.s3_aws_sts_endpoint = None - logger.s3_server_side_encryption = None - logger.s3_sse_kms_key_id = None - logger.s3_log_prompts_only = None - return logger - - -@pytest.mark.asyncio -async def test_init_bypassed_logger_retries_a_503_and_reports_a_404() -> None: - logger = _init_bypassed_logger() - put = _StatusPut([_transient_failure_response(503), _ok_response()]) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - - with patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))): - assert await logger.async_upload_data_to_s3(_element({"id": "x"}, "x")) is True - - assert put.calls == 2 - - put.calls = 0 - put.responses = [_coded_failure_response(404, None)] - with patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))): - assert await logger.async_upload_data_to_s3(_element({"id": "y"}, "y")) is False - - assert put.calls == 1 - - -def test_init_bypassed_sync_logger_retries_a_503_and_reports_a_404() -> None: - logger = _init_bypassed_logger() - mock_sync_client = MagicMock() - mock_sync_client.put = MagicMock(side_effect=[_transient_failure_response(503), _ok_response()]) - - with ( - patch("litellm.integrations.s3_v2.get_httpx_client", return_value=mock_sync_client), - patch("time.sleep"), - ): - logger.upload_data_to_s3(_element({"id": "x"}, "x")) - - assert mock_sync_client.put.call_count == 2 - retried_headers: Final = dict(mock_sync_client.put.call_args.kwargs["headers"]) - assert "X-Amz-Date" in retried_headers - - mock_sync_client.put = MagicMock(return_value=_coded_failure_response(404, None)) - with ( - patch("litellm.integrations.s3_v2.get_httpx_client", return_value=mock_sync_client), - patch("time.sleep"), - ): - logger.upload_data_to_s3(_element({"id": "y"}, "y")) - - assert mock_sync_client.put.call_count == 1 - failed_url: Final = str(mock_sync_client.put.call_args[0][0]) - assert "test-y.json" in failed_url - - -@pytest.mark.asyncio -async def test_subclass_with_base_style_upload_bounded_drains_the_queue() -> None: - class _BaseStyleLogger(S3Logger): - async def _upload_bounded(self, element: s3BatchLoggingElement) -> bool: - return True - - logger = _BaseStyleLogger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - logger.async_httpx_client = AsyncMock() - logger.log_queue = [_element({"id": "a"}, "a")] - - await logger.flush_queue() - - assert logger.log_queue == [] - logger.async_httpx_client.put.assert_not_called() - - -def test_bool_config_values_fall_back_to_the_default() -> None: - from litellm.integrations.s3 import ( - resolve_s3_max_concurrent_uploads, - resolve_s3_max_queue_size, - resolve_s3_max_retry_age_seconds, - ) - - assert resolve_s3_max_concurrent_uploads(True, 16) == 1 - assert resolve_s3_max_queue_size(True, 50000) == 50000 - assert resolve_s3_max_retry_age_seconds(True, 3600) == 3600 - - -def test_int_env_helper_falls_back_on_non_numeric(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.litellm_core_utils.env_utils import get_env_int - - monkeypatch.setenv("TEST_S3_INT_ENV", "abc") - assert get_env_int("TEST_S3_INT_ENV", 3) == 3 - monkeypatch.setenv("TEST_S3_INT_ENV", "7") - assert get_env_int("TEST_S3_INT_ENV", 3) == 7 - - -class _FailOncePerKeyPut: - def __init__(self) -> None: - self.failed: set[str] = set() - self.calls: tuple[str, ...] = () - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls = (*self.calls, url) - if url not in self.failed: - self.failed.add(url) - return _transient_failure_response() - return _ok_response() - - -class _SlowFailOncePerKeyPut: - def __init__(self, dumps_count) -> None: - self.failed: set[str] = set() - self.dumps_count = dumps_count - self.first_completed: int | None = None - self.calls: tuple[str, ...] = () - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls = (*self.calls, url) - await _real_sleep(0) - if self.first_completed is None: - self.first_completed = self.dumps_count() - if url not in self.failed: - self.failed.add(url) - return _transient_failure_response() - return _ok_response() - - -@pytest.mark.asyncio -async def test_peak_serialized_bodies_bounded_by_upload_width() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps as real_safe_dumps - - dumps_calls: list[object] = [] - - def counting_dumps(*args, **kwargs): - dumps_calls.append(args) - return real_safe_dumps(*args, **kwargs) - - put = _SlowFailOncePerKeyPut(lambda: len(dumps_calls)) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": index}, f"{index}") for index in range(64)] - - with ( - patch("litellm.integrations.s3_v2.safe_dumps", side_effect=counting_dumps), - patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))), - ): - await logger.flush_queue() - - assert put.first_completed is not None - assert put.first_completed <= logger.s3_max_concurrent_uploads - assert len(dumps_calls) == 64 - assert len(put.calls) == 128 - - -@pytest.mark.asyncio -async def test_send_batch_calls_upload_with_one_positional_arg() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - uploaded: list[str] = [] # mutable-ok: appended once per upload by the double - - async def mock_upload(batch_logging_element) -> str: - uploaded.append(batch_logging_element.s3_object_key) - return "delivered" - - logger.async_upload_data_to_s3 = mock_upload - logger.log_queue = [_element({"id": "a"}, "a"), _element({"id": "b"}, "b")] - - await logger.flush_queue() - - assert sorted(key.rsplit("/", 1)[-1] for key in uploaded) == ["test-a.json", "test-b.json"] - assert logger.log_queue == [] - - -@pytest.mark.asyncio -async def test_retries_serialize_the_body_once_per_element_per_flush() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps as real_safe_dumps - - dumps_calls: list[object] = [] - - def counting_dumps(*args, **kwargs): - dumps_calls.append(args) - return real_safe_dumps(*args, **kwargs) - - put = _FailUntilClearedPut(status=503) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": index}, f"{index}") for index in range(8)] - - with ( - patch("litellm.integrations.s3_v2.safe_dumps", side_effect=counting_dumps), - patch("asyncio.sleep", new=AsyncMock(side_effect=lambda delay: _real_sleep(0))), - ): - await logger.flush_queue() - - assert len(dumps_calls) == 8 - assert len(put.calls) == 24 - assert len(logger.log_queue) == 8 - - -@pytest.mark.asyncio -async def test_async_flush_logs_one_retry_warning(caplog) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - put = _FailOncePerKeyPut() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"i": index}, f"{index}") for index in range(8)] - - with caplog.at_level("WARNING"), patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert logger.log_queue == [] - assert sum(1 for record in caplog.records if "in-call retries" in record.getMessage()) == 1 - assert all("retrying in" not in record.getMessage() for record in caplog.records) - - -class _AppendingSuffixFailingPut: - def __init__(self, logger: S3Logger, element: s3BatchLoggingElement, fail_suffixes: tuple[str, ...]) -> None: - self.logger = logger - self.element = element - self.fail_suffixes = fail_suffixes - self.appended = False - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - if not self.appended: - self.appended = True - self.logger.log_queue.append(self.element) - if url.endswith(self.fail_suffixes): - return _transient_failure_response() - return _ok_response() - - -@pytest.mark.asyncio -async def test_failed_elements_stay_oldest_first_when_requeued() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_retry_age_seconds=3600, - ) - - late = _element({"id": "late"}, "late") - failed = _element({"id": "f2"}, "f2") - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _AppendingSuffixFailingPut(logger, late, ("test-f2.json",)) - logger.log_queue = [_element({"id": "f1"}, "f1"), failed] - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert [element.s3_object_key for element in logger.log_queue] == [failed.s3_object_key, late.s3_object_key] - assert logger.log_queue[0].retrying_since is not None - - -@pytest.mark.asyncio -async def test_overflow_prefers_arrivals_over_failed_elements_without_counting_the_trim(caplog) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_queue_size=1, - ) - - late = _element({"id": "late"}, "late") - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _AppendingSuffixFailingPut(logger, late, ("test-f2.json",)) - logger.log_queue = [_element({"id": "f1"}, "f1"), _element({"id": "f2"}, "f2")] - - with ( - patch.object(logger, "handle_callback_failure") as mock_failure, - patch("asyncio.sleep", new_callable=AsyncMock), - ): - await logger.flush_queue() - - assert [element.s3_object_key for element in logger.log_queue] == [late.s3_object_key] - failed_uploads: Final = 1 - assert mock_failure.call_count == failed_uploads - assert "dropped 1 oldest events" in caplog.text - - -@pytest.mark.asyncio -async def test_fresh_elements_upload_before_stale_retries_after_a_failed_flush() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_concurrent_uploads=1, - ) - - late = _element({"id": "late"}, "late") - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _AppendingSuffixFailingPut(logger, late, ("test-f2.json",)) - logger.log_queue = [_element({"id": "f1"}, "f1"), _element({"id": "f2"}, "f2")] - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - recovered = _FailOnSuffixPut(("never-matches",)) - logger.async_httpx_client.put = recovered - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert [call_url.rsplit("/", 1)[-1] for call_url in recovered.calls] == ["test-late.json", "test-f2.json"] - assert logger.log_queue == [] - - -@pytest.mark.asyncio -async def test_repeated_overflow_trims_oldest_across_failed_flushes(caplog) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_queue_size=3, - ) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _AppendingFailingPut(logger, (_element({"id": "d"}, "d"),)) - logger.log_queue = [_element({"id": name}, name) for name in ("a", "b", "c")] - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert [element.payload["id"] for element in logger.log_queue] == ["b", "c", "d"] - - logger.async_httpx_client.put = _AppendingFailingPut(logger, (_element({"id": "e"}, "e"),)) - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert [element.payload["id"] for element in logger.log_queue] == ["c", "d", "e"] - assert caplog.text.count("dropped 1 oldest events") == 2 - - -@pytest.mark.asyncio -async def test_retry_age_budget_drops_after_the_clock_set_by_a_partial_failure(caplog) -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_retry_age_seconds=1, - ) - - put = _FailOnSuffixCodedPut(("test-poison.json",), 503, "SlowDown") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"id": "poison"}, "poison"), _element({"id": "good"}, "good")] - - t0: Final = _NOW - with patch.object(logger, "handle_callback_failure") as mock_failure: - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=t0), - ): - await logger.flush_queue() - - assert [element.s3_object_key for element in logger.log_queue] == ["2025-09-14/test-poison.json"] - assert logger.log_queue[0].retrying_since == t0 - - logger.log_queue.append(_element({"id": "good-2"}, "good-2")) - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=t0 + 2), - ): - await logger.flush_queue() - - assert logger.log_queue == [] - assert "retrying longer than s3_max_retry_age_seconds=1" in caplog.text - poison_puts: Final = sum(1 for call_url in put.calls if call_url.endswith("test-poison.json")) - assert poison_puts == 6 - upload_failures: Final = 2 - assert mock_failure.call_count == upload_failures - - -@pytest.mark.asyncio -async def test_the_retry_clock_starts_at_the_first_partial_failure_not_first_seen() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_retry_age_seconds=1, - ) - - put = _FailUntilClearedPut(status=503, code="SlowDown") - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"id": "poison"}, "poison")] - - t0: Final = _NOW - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=t0), - ): - await logger.flush_queue() - - assert len(logger.log_queue) == 1 - assert logger.log_queue[0].retrying_since is None - - logger.log_queue.append(_element({"id": "good"}, "good")) - put.failing = False - failing_poison: Final = _FailOnSuffixCodedPut(("test-poison.json",), 503, "SlowDown") - logger.async_httpx_client.put = failing_poison - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=t0 + 500), - ): - await logger.flush_queue() - - assert [element.s3_object_key for element in logger.log_queue] == ["2025-09-14/test-poison.json"] - assert logger.log_queue[0].retrying_since == t0 + 500 - - -def test_sync_upload_retries_access_denied_403(caplog): - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-sync-403.json", - payload={"test": "sync-403"}, - s3_object_download_filename="test-sync-403.json", - ) - - mock_sync_client = MagicMock() - mock_sync_client.put = MagicMock(return_value=_coded_failure_response(403, "AccessDenied")) - - with ( - patch("litellm.integrations.s3_v2.get_httpx_client", return_value=mock_sync_client), - patch("time.sleep") as mock_sleep, - ): - logger.upload_data_to_s3(test_element) - - assert mock_sync_client.put.call_count == 3 - _assert_jittered_backoff(mock_sleep, 2) - assert "dropping object" not in caplog.text - - -def test_sync_upload_drops_terminal_object_once_and_logs_it_only_when_opted_in(caplog): - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_drop_on_terminal_error=True, - ) - - test_element = s3BatchLoggingElement( - s3_object_key="2025-09-14/test-sync-terminal.json", - payload={"test": "sync-terminal"}, - s3_object_download_filename="test-sync-terminal.json", - ) - - mock_sync_client = MagicMock() - mock_sync_client.put = MagicMock(return_value=_coded_failure_response(400, "EntityTooLarge")) - - with ( - patch("litellm.integrations.s3_v2.get_httpx_client", return_value=mock_sync_client), - patch("time.sleep") as mock_sleep, - ): - logger.upload_data_to_s3(test_element) - - assert mock_sync_client.put.call_count == 1 - mock_sleep.assert_not_called() - assert "dropping object" in caplog.text - - -@pytest.mark.asyncio -async def test_requeued_batch_file_keeps_the_earliest_member_retrying_since() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_batch_file_upload=True, - ) - - put = _FailUntilClearedPut() - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - - stale: Final = _NOW - 30 - retried = _element({"id": "retried"}, "retried").model_copy(update={"retrying_since": stale}) - fresh = _element({"id": "fresh"}, "fresh") - logger.log_queue = [retried, fresh] - - with ( - patch("asyncio.sleep", new_callable=AsyncMock), - patch("time.monotonic", return_value=_NOW), - ): - await logger.flush_queue() - - assert len(logger.log_queue) == 1 - assert logger.log_queue[0].s3_object_key.endswith(".jsonl") - assert logger.log_queue[0].retrying_since == stale - - -@pytest.mark.asyncio -async def test_an_unlisted_5xx_is_requeued_without_an_extra_attempt() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - put = _FailUntilClearedPut(status=507) - - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = put - logger.log_queue = [_element({"id": "req-507"}, "507")] - - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert len(logger.log_queue) == 1 - assert len(put.calls) == 1 - - -@pytest.mark.parametrize("configured", [0, "0", None, ""]) -def test_retry_age_resolution_disables_the_budget(configured: object) -> None: - from litellm.integrations.s3 import resolve_s3_max_retry_age_seconds - - assert resolve_s3_max_retry_age_seconds(configured, 3600) is None - - -@pytest.mark.parametrize("configured", ["abc", -5, True]) -def test_invalid_retry_age_resolution_falls_back_with_a_warning(configured: object, caplog) -> None: - from litellm.integrations.s3 import resolve_s3_max_retry_age_seconds - - assert resolve_s3_max_retry_age_seconds(configured, 3600) == 3600 - assert "s3_max_retry_age_seconds" in caplog.text - - -def test_retry_age_resolution_accepts_a_positive_int() -> None: - from litellm.integrations.s3 import resolve_s3_max_retry_age_seconds - - assert resolve_s3_max_retry_age_seconds(30, 3600) == 30 - - -def test_default_logger_sets_a_one_hour_retry_age_budget() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - ) - - assert logger.s3_max_retry_age_seconds == 3600 - - -def test_constructor_zero_disables_the_retry_age_budget() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_retry_age_seconds=0, - ) - - assert logger.s3_max_retry_age_seconds is None - - -def test_invalid_callback_params_retry_age_falls_back_to_the_default() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_callback_params_override={"s3_max_retry_age_seconds": "abc"}, - ) - - assert logger.s3_max_retry_age_seconds == 3600 - - -def test_callback_params_retry_age_wins_over_constructor() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_retry_age_seconds=30, - s3_callback_params_override={"s3_max_retry_age_seconds": 60}, - ) - - assert logger.s3_max_retry_age_seconds == 60 - - -def test_callback_params_drop_terminal_error_wins_over_constructor() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_drop_on_terminal_error=False, - s3_callback_params_override={"s3_drop_on_terminal_error": True}, - ) - - assert logger.s3_drop_on_terminal_error is True - - -def test_invalid_callback_params_drop_terminal_error_falls_back_to_constructor_value() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_drop_on_terminal_error=True, - s3_callback_params_override={"s3_drop_on_terminal_error": "banana"}, - ) - - assert logger.s3_drop_on_terminal_error is True - - -def test_callback_params_adaptive_concurrency_wins_over_constructor() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_adaptive_concurrency=False, - s3_callback_params_override={"s3_adaptive_concurrency": "true"}, - ) - - assert logger.s3_adaptive_concurrency is True - assert logger._upload_limiter._ceiling > logger._upload_limiter.limit - - -def test_invalid_callback_params_max_adaptive_concurrency_falls_back_to_default() -> None: - from litellm.constants import DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY - - logger = _override_logger(s3_adaptive_concurrency=True, s3_max_adaptive_concurrency="abc") - - assert logger.s3_max_adaptive_concurrency == DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY - assert logger._upload_limiter._ceiling == DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY - - -def test_callback_params_queue_size_wins_over_constructor() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_queue_size=7, - s3_callback_params_override={"s3_max_queue_size": 4}, - ) - - assert logger.s3_max_queue_size == 4 - assert logger.max_queue_size == 4 - - -def test_invalid_callback_params_queue_size_falls_back_to_constructor_value() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_queue_size=7, - s3_callback_params_override={"s3_max_queue_size": "abc"}, - ) - - assert logger.s3_max_queue_size == 7 - - -def test_invalid_constructor_queue_size_falls_back_to_default() -> None: - from litellm.integrations.custom_batch_logger import CustomBatchLogger - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_queue_size="abc", - ) - - assert logger.s3_max_queue_size == CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE - assert logger.max_queue_size == CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE - - -class _StatusPut: - def __init__(self, responses: "list[MagicMock | Exception]") -> None: - self.responses = responses - self.calls = 0 - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls += 1 - outcome = self.responses[min(self.calls - 1, len(self.responses) - 1)] - if isinstance(outcome, Exception): - raise outcome - return outcome - - -def _slow_down_response(status: int = 200) -> MagicMock: - response = _ok_response() if status == 200 else _transient_failure_response(status) - response.text = "SlowDown" - return response - - -@pytest.mark.asyncio -async def test_503_response_lowers_the_adaptive_limit() -> None: - logger = _override_logger(s3_adaptive_concurrency=True) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _StatusPut([_transient_failure_response(503), _ok_response()]) - - logger._upload_limiter._limit = 64 # mutable-ok: seed the AIMD state above the floor without replaying growth - with patch("asyncio.sleep", new_callable=AsyncMock): - logger.log_queue = [_element({"i": 0}, "0")] - await logger.flush_queue() - - assert logger._upload_limiter.limit == 32 - - -@pytest.mark.asyncio -async def test_429_response_lowers_the_adaptive_limit() -> None: - logger = _override_logger(s3_adaptive_concurrency=True) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _StatusPut([_transient_failure_response(429), _ok_response()]) - - logger._upload_limiter._limit = 64 # mutable-ok: seed the AIMD state above the floor without replaying growth - with patch("asyncio.sleep", new_callable=AsyncMock): - logger.log_queue = [_element({"i": 0}, "0")] - await logger.flush_queue() - - assert logger._upload_limiter.limit == 32 - - -@pytest.mark.asyncio -async def test_slow_down_body_code_lowers_the_adaptive_limit() -> None: - logger = _override_logger(s3_adaptive_concurrency=True) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _StatusPut([_slow_down_response()]) - - logger._upload_limiter._limit = 64 # mutable-ok: seed the AIMD state above the floor without replaying growth - logger.log_queue = [_element({"i": 0}, "0")] - await logger.async_send_batch() - - assert logger._upload_limiter.limit == 32 - - -@pytest.mark.asyncio -async def test_transport_error_lowers_the_adaptive_limit() -> None: - logger = _override_logger(s3_adaptive_concurrency=True) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _StatusPut( - [httpx.ConnectError("connect refused", request=MagicMock()), _ok_response()] - ) - - logger._upload_limiter._limit = 64 # mutable-ok: seed the AIMD state above the floor without replaying growth - with patch("asyncio.sleep", new_callable=AsyncMock): - logger.log_queue = [_element({"i": 0}, "0")] - await logger.flush_queue() - - assert logger._upload_limiter.limit == 32 - - -@pytest.mark.asyncio -async def test_fast_uploads_raise_the_adaptive_limit() -> None: - logger = _override_logger(s3_adaptive_concurrency=True) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _RecordingPut() - - before: Final = logger._upload_limiter.limit - logger.log_queue = [_element({"i": i}, f"{i}") for i in range(before)] - await logger.async_send_batch() - - assert logger._upload_limiter.limit > before - - -@pytest.mark.asyncio -async def test_configured_concurrency_is_the_fixed_limit_when_adaptive_is_off() -> None: - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_aws_access_key_id="test-key", - s3_aws_secret_access_key="test-secret", - s3_region_name="us-east-1", - s3_max_concurrent_uploads=64, - ) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _StatusPut([_transient_failure_response(503), _ok_response()]) - - assert logger._upload_limiter._value == 64 - - logger.log_queue = [_element({"i": 0}, "0")] - with patch("asyncio.sleep", new_callable=AsyncMock): - await logger.flush_queue() - - assert logger._upload_limiter._value == 64 - - -@pytest.mark.asyncio -async def test_the_limit_never_falls_below_the_configured_width() -> None: - logger = _override_logger(s3_adaptive_concurrency=True, s3_max_concurrent_uploads=8) - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _StatusPut( - [_transient_failure_response(503), _transient_failure_response(503), _transient_failure_response(503)] - ) - - with patch("asyncio.sleep", new_callable=AsyncMock): - logger.log_queue = [_element({"i": 0}, "0")] - await logger.flush_queue() - - assert logger._upload_limiter.limit == 8 - - -def test_default_upload_width_is_16() -> None: - from litellm.constants import DEFAULT_S3_MAX_CONCURRENT_UPLOADS - - logger = _override_logger() - - assert logger._upload_semaphore._value == DEFAULT_S3_MAX_CONCURRENT_UPLOADS - assert DEFAULT_S3_MAX_CONCURRENT_UPLOADS == 16 - - -@pytest.mark.asyncio -async def test_a_slow_put_does_not_lower_the_adaptive_limit() -> None: - logger = _override_logger(s3_adaptive_concurrency=True) - logger.async_httpx_client = AsyncMock() - - async def slow_put( - url: str, data: str | None = None, headers: dict[str, str] | None = None, timeout: httpx.Timeout | None = None - ) -> MagicMock: - await _real_sleep(0) - return _ok_response() - - logger.async_httpx_client.put = slow_put - - before: Final = logger._upload_limiter.limit - logger.log_queue = [_element({"i": 0}, "0")] - await logger.async_send_batch() - - assert logger._upload_limiter.limit >= before - - -class _FastOkPut: - def __init__(self) -> None: - self.calls = 0 - - async def __call__( - self, - url: str, - data: str | None = None, - headers: dict[str, str] | None = None, - timeout: httpx.Timeout | None = None, - ) -> MagicMock: - self.calls += 1 - await _real_sleep(0) - return _ok_response() - - -async def _timed_send_batch(size: int) -> float: - logger = _override_logger() - logger.async_httpx_client = AsyncMock() - logger.async_httpx_client.put = _FastOkPut() - logger.log_queue = [_element({"i": i}, f"{i}") for i in range(size)] - started = time.perf_counter() - await logger.async_send_batch() - return time.perf_counter() - started - - -@pytest.mark.asyncio -async def test_send_batch_time_grows_linearly_with_the_batch() -> None: - baseline: Final = await _timed_send_batch(2_000) - quadrupled: Final = await _timed_send_batch(8_000) - - assert quadrupled / baseline < 8, f"2k took {baseline:.3f}s, 8k took {quadrupled:.3f}s" - - -class QueueOnlyS3LoggerWithoutCredentialsOrPeriodicFlush(S3Logger): - def __init__(self) -> None: - CustomLogger.__init__(self) - self.log_queue: list[s3BatchLoggingElement] = [] - self.batch_size = 100 - self.s3_strip_base64_files = False - self.s3_use_team_prefix = False - self.s3_use_key_prefix = False - self.s3_path = "" - self.s3_log_prompts_only = None - self.s3_batch_file_upload = False - self._upload_semaphore = asyncio.Semaphore(1) - - -@pytest.mark.parametrize("stream", [False, True]) -@pytest.mark.parametrize("concurrent", [False, True]) -@pytest.mark.asyncio -async def test_repeated_failure_notifications_upload_once( - monkeypatch: pytest.MonkeyPatch, - stream: bool, - concurrent: bool, -) -> None: - sink: Final = QueueOnlyS3LoggerWithoutCredentialsOrPeriodicFlush() - upload: Final = AsyncMock(return_value=True) - monkeypatch.setattr(sink, "async_upload_data_to_s3", upload) - monkeypatch.setattr(litellm, "callbacks", []) - monkeypatch.setattr(litellm, "_async_failure_callback", [sink]) - logging_obj: Final = Logging( - model="openai/gpt-4o-mini", - messages=[], - stream=stream, - call_type="acompletion", - start_time=datetime.now(), - litellm_call_id="synthetic-request", - function_id="synthetic-function", - ) - error: Final = litellm.ServiceUnavailableError( - message="synthetic deployment-selection failure", - llm_provider="openai", - model="openai/gpt-4o-mini", - ) - - if concurrent: - await asyncio.gather( - *(logging_obj.async_failure_handler(error, "synthetic traceback") for _ in range(3)) - ) - else: - for _ in range(3): - await logging_obj.async_failure_handler(error, "synthetic traceback") - - assert len(sink.log_queue) == 1 - assert all(entry.payload["status"] == "failure" for entry in sink.log_queue) - await sink.async_send_batch() - await asyncio.sleep(0) - assert upload.await_count == 1 - - -@pytest.fixture() -def _vcr_outcome_gate(request, vcr): - install_live_call_probe(request, vcr) - yield - record_vcr_outcome(request, vcr) - -@pytest_asyncio.fixture(loop_scope="function") -async def drain_logging_worker(isolate_litellm_state: None) -> AsyncIterator[None]: - yield - await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS) - -LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS: Final = LOGGING_WORKER_MAX_TIME_PER_COROUTINE + 5.0 - -@pytest.fixture(scope="function", autouse=True) -def isolate_litellm_state(): - """ - Per-function isolation fixture. - - Resets litellm state to the true defaults captured at conftest import time, - then restores after the test. This prevents module-level mutations (e.g. - `litellm.num_retries = 3` at the top of test_langfuse_e2e_test.py) from - leaking across tests within the same xdist worker. - """ - from litellm.litellm_core_utils import litellm_logging as ll_logging - from litellm.proxy.management_helpers import audit_logs as ll_audit_logs - - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - ll_logging._in_memory_loggers.clear() - ll_audit_logs._audit_log_callback_cache.clear() - for attr in _LIST_ATTRS: - if attr in _DEFAULTS: - default = _DEFAULTS[attr] - setattr(litellm, attr, default.copy() if isinstance(default, list) else default) - for attr in _SCALAR_ATTRS: - if attr in _DEFAULTS: - setattr(litellm, attr, _DEFAULTS[attr]) - yield - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - ll_logging._in_memory_loggers.clear() - ll_audit_logs._audit_log_callback_cache.clear() - for attr in _LIST_ATTRS: - if attr in _DEFAULTS: - default = _DEFAULTS[attr] - setattr(litellm, attr, default.copy() if isinstance(default, list) else default) - for attr in _SCALAR_ATTRS: - if attr in _DEFAULTS: - setattr(litellm, attr, _DEFAULTS[attr]) - -_LIST_ATTRS = ( - "callbacks", - "success_callback", - "failure_callback", - "_async_success_callback", - "_async_failure_callback", - "service_callback", - "pre_call_rules", - "post_call_rules", -) - -_SCALAR_ATTRS = ( - "set_verbose", - "cache", - "num_retries", - "num_retries_per_request", - "turn_off_message_logging", - "redact_messages_in_exceptions", - "redact_user_api_key_info", - "s3_callback_params", - "s3_audit_callback_params", - "datadog_params", - "vector_store_registry", -) - -_DEFAULTS: Final = { - attr: getattr(litellm, attr).copy() if isinstance(getattr(litellm, attr), list) else getattr(litellm, attr) - for attr in (*_LIST_ATTRS, *_SCALAR_ATTRS) - if hasattr(litellm, attr) -} - -@pytest.fixture(scope="module") -def setup_and_teardown(): - """ - Module-scoped setup. Reloads litellm only in single-process mode - (skipped under xdist to avoid cross-worker interference). - """ - import litellm - - worker_id = os.environ.get("PYTEST_XDIST_WORKER", None) - if worker_id is None: - importlib.reload(litellm) - try: - if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - - importlib.reload(litellm.proxy.proxy_server) - except Exception: - pass - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - yield - -@pytest.fixture -def amazing_s3_retries(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(litellm, "num_retries", 3) - -class _FakeS3Paginator: - def __init__(self, objects): - self.objects = objects - - def paginate(self, Bucket): - keys = sorted(self.objects[Bucket]) - if not keys: - return [{}] - return [{"Contents": [{"Key": key} for key in keys]}] - -class _FakeS3Client: - def __init__(self): - self.objects = defaultdict(dict) - self.object_written = threading.Event() - - def clear(self): - self.objects.clear() - self.object_written.clear() - - def put_object(self, Bucket, Key, Body, **_kwargs): - self.objects[Bucket][Key] = Body - self.object_written.set() - return {"ResponseMetadata": {"HTTPStatusCode": 200}} - - def delete_object(self, Bucket, Key): - self.objects[Bucket].pop(Key, None) - return {"ResponseMetadata": {"HTTPStatusCode": 204}} - - def get_paginator(self, name): - assert name == "list_objects_v2" - return _FakeS3Paginator(self.objects) - - def list_objects(self, Bucket): - keys = sorted(self.objects[Bucket]) - return {"Contents": [{"Key": key, "LastModified": 0} for key in keys]} - -_FAKE_S3_CLIENT = _FakeS3Client() - -@pytest.fixture -def fake_s3_client(monkeypatch): - _FAKE_S3_CLIENT.clear() - - def fake_boto3_client(service_name, *args, **kwargs): - assert service_name == "s3" - return _FAKE_S3_CLIENT - - monkeypatch.setattr(boto3, "client", fake_boto3_client) - litellm.success_callback = [] - litellm.callbacks = [] - yield _FAKE_S3_CLIENT - litellm.success_callback = [] - litellm.callbacks = [] - -@pytest.mark.usefixtures( - "cancel_s3_periodic_flush_tasks", - "fake_s3_client", - "_vcr_outcome_gate", - "drain_logging_worker", - "isolate_litellm_state", - "setup_and_teardown", - "amazing_s3_retries", -) -@pytest.mark.asyncio -@pytest.mark.parametrize("sync_mode,streaming", [(True, True), (True, False), (False, True), (False, False)]) -@pytest.mark.flaky(retries=3, delay=1) -async def test_basic_s3_logging(sync_mode, streaming): - verbose_logger.setLevel(level=logging.DEBUG) - litellm.success_callback = ["s3"] - litellm.s3_callback_params = { - "s3_bucket_name": "load-testing-oct", - "s3_aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY", - "s3_aws_access_key_id": "os.environ/AWS_ACCESS_KEY_ID", - "s3_region_name": "us-west-2", - } - litellm.set_verbose = True - response_id = None - if sync_mode is True: - response = litellm.completion( - model="gpt-5-mini", - messages=[{"role": "user", "content": "This is a test"}], - mock_response="It's simple to use and easy to get started", - stream=streaming, - ) - if streaming: - for chunk in response: - response_id = chunk.id - else: - response_id = response.id - assert _FAKE_S3_CLIENT.object_written.wait(timeout=5) - else: - response = await litellm.acompletion( - model="gpt-5-mini", - messages=[{"role": "user", "content": "This is a test"}], - mock_response="It's simple to use and easy to get started", - stream=streaming, - ) - if streaming: - async for chunk in response: - response_id = chunk.id - else: - response_id = response.id - assert await asyncio.to_thread(_FAKE_S3_CLIENT.object_written.wait, 5) - - total_objects, all_s3_keys = list_all_s3_objects("load-testing-oct") - - assert any(response_id in key for key in all_s3_keys) - s3 = boto3.client("s3") - for key in all_s3_keys: - s3.delete_object(Bucket="load-testing-oct", Key=key) - -@pytest.mark.usefixtures( - "cancel_s3_periodic_flush_tasks", - "fake_s3_client", - "_vcr_outcome_gate", - "drain_logging_worker", - "isolate_litellm_state", - "setup_and_teardown", - "amazing_s3_retries", -) -@pytest.mark.asyncio -@pytest.mark.parametrize("streaming", [True]) -@pytest.mark.flaky(retries=3, delay=1) -async def test_basic_s3_v2_logging(streaming): - from litellm.integrations.s3_v2 import S3Logger - - litellm.s3_callback_params = { - "s3_bucket_name": "load-testing-oct", - "s3_aws_secret_access_key": "test-secret", - "s3_aws_access_key_id": "test-key", - "s3_region_name": "us-west-2", - } - - s3_v2_logger = S3Logger(s3_flush_interval=1) - litellm.callbacks = [s3_v2_logger] - - uploaded_keys: list = [] - uploaded = asyncio.Event() - - async def mock_upload(batch_logging_element): - uploaded_keys.append(batch_logging_element.s3_object_key) - uploaded.set() - - s3_v2_logger.async_upload_data_to_s3 = mock_upload - - litellm.set_verbose = True - response_id = None - response = await litellm.acompletion( - model="gpt-5-mini", - messages=[{"role": "user", "content": "This is a test"}], - mock_response="It's simple to use and easy to get started", - stream=streaming, - ) - if streaming: - async for chunk in response: - response_id = chunk.id - else: - response_id = response.id - - assert await asyncio.wait_for(uploaded.wait(), timeout=5) - - assert len(uploaded_keys) > 0, "S3 upload was never called" - assert any(response_id in key for key in uploaded_keys), ( - f"Expected response_id={response_id} in one of the uploaded S3 keys: {uploaded_keys}" - ) - -@pytest.mark.usefixtures( - "cancel_s3_periodic_flush_tasks", - "fake_s3_client", - "_vcr_outcome_gate", - "drain_logging_worker", - "isolate_litellm_state", - "setup_and_teardown", - "amazing_s3_retries", -) -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_basic_s3_v2_logging_failure(): - """Test that S3 v2 logger makes httpx PUT request when logging failures""" - from unittest.mock import AsyncMock, MagicMock - - from litellm.integrations.s3_v2 import S3Logger - - s3_v2_logger = S3Logger(s3_flush_interval=1) - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - - s3_v2_logger.async_httpx_client = AsyncMock() - s3_v2_logger.async_httpx_client.put.return_value = mock_response - - upload_called = False - uploaded = asyncio.Event() - - async def mock_upload(batch_logging_element): - nonlocal upload_called - upload_called = True - url = f"https://test-bucket.s3.us-west-2.amazonaws.com/{batch_logging_element.s3_object_key}" - headers = {"Content-Type": "application/json"} - data = '{"model": "gpt-5-mini"}' - - await s3_v2_logger.async_httpx_client.put(url=url, headers=headers, data=data) - uploaded.set() - - s3_v2_logger.async_upload_data_to_s3 = mock_upload - - litellm.callbacks = [s3_v2_logger] - litellm.s3_callback_params = { - "s3_bucket_name": "test-bucket", - "s3_aws_secret_access_key": "test-secret", - "s3_aws_access_key_id": "test-key", - "s3_region_name": "us-west-2", - } - litellm.set_verbose = True - - try: - await litellm.acompletion( - model="gpt-5-mini", - api_key="invalid-api-key", - messages=[{"role": "user", "content": "This is a test"}], - mock_response=Exception("forced failure for S3 logging test"), - ) - except Exception: - pass - - assert await asyncio.wait_for(uploaded.wait(), timeout=5) - - assert upload_called, "S3 upload method was not called" - - s3_v2_logger.async_httpx_client.put.assert_called() - - call_args = s3_v2_logger.async_httpx_client.put.call_args - assert call_args is not None - url = call_args[1]["url"] if "url" in call_args[1] else call_args[0][0] - - assert "test-bucket.s3.us-west-2.amazonaws.com" in url - - headers = call_args[1]["headers"] - assert headers["Content-Type"] == "application/json" - - data = call_args[1]["data"] - assert data is not None - assert '"model": "gpt-5-mini"' in data - -def list_all_s3_objects(bucket_name): - s3 = boto3.client("s3") - - all_s3_keys = [] - - paginator = s3.get_paginator("list_objects_v2") - total_objects = 0 - - for page in paginator.paginate(Bucket=bucket_name): - if "Contents" in page: - total_objects += len(page["Contents"]) - all_s3_keys.extend([obj["Key"] for obj in page["Contents"]]) - - return total_objects, all_s3_keys - -class TestS3Logger(S3Logger): - def __init__(self, *args, **kwargs): - self.recorded_requests = {} - self.logged_standard_logging_payload = None - super().__init__(*args, **kwargs) - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - self.recorded_requests[response_obj["id"]] = start_time - self.logged_standard_logging_payload = kwargs["standard_logging_object"] - return await super().async_log_success_event(kwargs, response_obj, start_time, end_time) - - -def _batch_line_items_payload() -> dict: - line = { - "batch_id": "bGl0ZWxsbV9wcm94eTtiYXRjaA", - "custom_id": "ok-1", - "status": "success", - "request": {"custom_id": "ok-1", "body": {"messages": [{"role": "user", "content": "hi"}]}}, - "response": {"custom_id": "ok-1", "response": {"status_code": 200, "body": {"id": "chatcmpl-1"}}}, - "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, - "response_cost": 0.005, - "model": "gpt-4o-mini", - "custom_llm_provider": "openai", - "metadata": {"user_api_key_hash": "hash-alice"}, - } - return { - "batch_id": "bGl0ZWxsbV9wcm94eTtiYXRjaA", - "finished_at": 1791494130, - "part": 0, - "lines": (line, {**line, "custom_id": "ok-2"}), - } - - -def _s3_logger_answering( - status: int, s3_path: str | None, content: bytes = b"" -) -> tuple[S3Logger, list[httpx.Request]]: - requests: list[httpx.Request] = [] - - def respond(request: httpx.Request) -> httpx.Response: - requests.append(request) - return httpx.Response(status, content=content, request=request) - - logger = S3Logger( - s3_bucket_name="test-bucket", - s3_region_name="us-east-1", - s3_path=s3_path, - s3_aws_access_key_id="AKIAIOSFODNN7EXAMPLE", - s3_aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - s3_flush_interval=3600, - ) - logger.async_httpx_client = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) - return logger, requests - - -@pytest.mark.asyncio -async def test_batch_line_items_are_one_ndjson_object_under_the_configured_path(): - logger, requests = _s3_logger_answering(200, "cold/") - payload = _batch_line_items_payload() - - await logger.async_log_batch_line_items(payload) - - (request,) = requests - assert request.method == "PUT" - assert request.url.path == "/cold/batch_line_items/2026-10-08/bGl0ZWxsbV9wcm94eTtiYXRjaA/part-00000.jsonl" - assert request.headers["Content-Type"] == "application/x-ndjson" - assert request.headers["Content-Disposition"] == 'inline; filename="bGl0ZWxsbV9wcm94eTtiYXRjaA-part-00000.jsonl"' - assert [json.loads(line) for line in request.content.decode().splitlines()] == list(payload["lines"]) - assert logger.log_queue == [] - - -@pytest.mark.asyncio -async def test_batch_line_items_without_an_s3_path_sit_at_the_bucket_root(): - logger, requests = _s3_logger_answering(200, None) - - await logger.async_log_batch_line_items(_batch_line_items_payload()) - - assert [request.url.path for request in requests] == [ - "/batch_line_items/2026-10-08/bGl0ZWxsbV9wcm94eTtiYXRjaA/part-00000.jsonl" - ] - - -@pytest.mark.asyncio -async def test_a_rejected_batch_line_items_upload_is_logged_with_the_batch_id(caplog): - logger, requests = _s3_logger_answering(400, "cold") - - with caplog.at_level(logging.ERROR, logger="LiteLLM"): - await logger.async_log_batch_line_items(_batch_line_items_payload()) - - assert len(requests) == 1 - assert any( - "bGl0ZWxsbV9wcm94eTtiYXRjaA" in record.getMessage() and "were not stored" in record.getMessage() - for record in caplog.records - ) - - -@pytest.mark.asyncio -async def test_a_batch_line_items_object_the_bucket_refuses_for_good_is_not_kept_in_memory(): - logger, requests = _s3_logger_answering(400, "cold", b"InvalidArgument") - - await logger.async_log_batch_line_items(_batch_line_items_payload()) - await logger.flush_queue() - - assert len(requests) == 1 - assert logger._flush_dropped == {} - - -@pytest.mark.asyncio -async def test_batch_line_items_are_serialized_and_hashed_off_the_event_loop(monkeypatch): - import threading - - import litellm.integrations.s3_v2 as s3_v2_module - - logger, requests = _s3_logger_answering(200, "cold/") - serialize = s3_v2_module.batch_line_items_ndjson - prepare = logger._prepare_put - threads = {} - - def _serialize(payload): - threads["serialize"] = threading.current_thread() - return serialize(payload) - - def _prepare(element): - threads["hash"] = threading.current_thread() - return prepare(element) - - monkeypatch.setattr(s3_v2_module, "batch_line_items_ndjson", _serialize) - monkeypatch.setattr(logger, "_prepare_put", _prepare) - - await logger.async_log_batch_line_items(_batch_line_items_payload()) - - assert len(requests) == 1 - assert set(threads) == {"serialize", "hash"} - assert threading.main_thread() not in threads.values() diff --git a/tests/unit/integrations/test_s3_v2_rust.py b/tests/unit/integrations/test_s3_v2_rust.py new file mode 100644 index 00000000000..3c19d63c615 --- /dev/null +++ b/tests/unit/integrations/test_s3_v2_rust.py @@ -0,0 +1,3105 @@ +import asyncio +import base64 +import contextlib +import copy +import hashlib +import json +import logging +import re +import time +from collections.abc import Iterator, Mapping +from datetime import datetime +from pathlib import Path +from typing import Final +from unittest.mock import patch +from urllib.parse import quote, unquote, urlsplit, urlunsplit + +import pytest +import requests +from tests.unit.integrations.s3_v3_rust_support import ( + DROP, + Received, + ScriptedBucket, + log_attempt, + log_payload, + rust_s3_logger, + scripted_bucket, +) + +import litellm +from litellm.integrations import s3_v3 +from litellm.integrations.s3_v3 import S3Logger +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.rust_bridge import _native +from litellm.types.integrations.s3_v2 import S3LoggerSettings +from litellm.types.integrations.custom_logger_v2 import Outcome + +pytestmark = pytest.mark.skipif(not hasattr(_native, "NativeS3Log"), reason="the built _native has no NativeS3Log") + +START: Final = datetime(2025, 9, 14, 9, 5, 7, 42) +_ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE" +_SECRET_KEY = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + + +@pytest.fixture(autouse=True) +def _isolated(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + for name in ("S3_PARTITION_GRANULARITY", "S3_LOG_PROMPTS_ONLY", "AWS_REGION", "AWS_REGION_NAME"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "cold_storage_custom_logger", None) + yield + + +@pytest.fixture +def bucket() -> Iterator[ScriptedBucket]: + with scripted_bucket() as scripted: + yield scripted + + +def _logger( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, **params: object +) -> S3Logger: # kwargs-ok: S3Logger keywords + logger: Final = rust_s3_logger( + monkeypatch, + **{ + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": bucket.url, + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + "s3_region_name": "us-east-1", + **params, + }, + ) + _ = logger.periodic_flush_task.cancel() + return logger + + +def _payload(response_id: str = "test", **fields: object) -> dict[str, object]: # kwargs-ok: payload fields + return {"id": response_id, "metadata": {}, "messages": [], **fields} + + +async def _upload(logger: S3Logger, payload: Mapping[str, object], start_time: datetime = START) -> None: + await log_payload(logger, payload, start_time) + await logger.flush_queue() + + +def _failures() -> object: + return patch.object(s3_v3._Host, "callback_failed") + + +def _uploaded(put: Received) -> dict[str, object]: + loaded: Final = json.loads(put.body) + assert isinstance(loaded, dict) + return loaded + + +def _assert_backoff_gaps(puts: list[Received]) -> None: + gaps: Final = [later.at - earlier.at for earlier, later in zip(puts, puts[1:])] + for attempt, gap in enumerate(gaps): + assert (1 << attempt) * 0.5 <= gap < (1 << attempt) * 1.5 + 0.25, (attempt, gap) + + +class TestS3V2UnitTests: + @pytest.mark.asyncio + async def test_s3_v2_endpoint_url(self, monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + logger: Final = _logger(monkeypatch, bucket) + await _upload(logger, _payload("endpoint")) + + (put,) = bucket.puts() + assert f"{bucket.url}{put.path}" == f"{bucket.url}/test-bucket/2025-09-14/time-09-05-07-000042_endpoint.json" + bucket.objects["/test-bucket/2025-09-14/download-test-key.json"] = b'{"downloaded": "data"}' + result: Final = await logger.get_proxy_server_request_from_cold_storage_with_object_key( + "2025-09-14/download-test-key.json" + ) + assert bucket.received[-1].path == "/test-bucket/2025-09-14/download-test-key.json" + assert result == {"downloaded": "data"} + + @pytest.mark.asyncio + async def test_s3_v2_put_url_encodes_spaces_in_object_key( + self, monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket + ) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_use_team_prefix=True) + await _upload(logger, _payload("test-key", metadata={"user_api_key_team_alias": "My Team"})) + + (put,) = bucket.puts() + s3_object_key: Final = "My Team/2025-09-14/time-09-05-07-000042_test-key.json" + raw_url: Final = f"{bucket.url}/test-bucket/{s3_object_key}" + expected_url: Final = requests.Request("PUT", raw_url).prepare().url + actual_url: Final = f"{bucket.url}{put.path}" + assert actual_url == expected_url + assert " " not in actual_url + + +@pytest.mark.asyncio +async def test_async_upload_retries_on_s3_503(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + bucket.reply("test-retry", (503, ""), (200, "")) + await _upload(_logger(monkeypatch, bucket), _payload("test-retry")) + + assert len(bucket.puts()) == 2 + _assert_backoff_gaps(bucket.puts()) + + +@pytest.mark.asyncio +async def test_async_upload_retries_on_s3_500(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + bucket.reply("test-retry-500", (500, ""), (200, "")) + await _upload(_logger(monkeypatch, bucket), _payload("test-retry-500")) + + assert len(bucket.puts()) == 2 + _assert_backoff_gaps(bucket.puts()) + + +@pytest.mark.asyncio +async def test_async_upload_exhausts_retries_on_persistent_503( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + bucket.reply("test-exhaust", (503, "")) + logger: Final = _logger(monkeypatch, bucket) + with _failures() as mock_failure: + await _upload(logger, _payload("test-exhaust")) + + assert len(bucket.puts()) == 3 + _assert_backoff_gaps(bucket.puts()) + mock_failure.assert_called_once_with() + + +@pytest.mark.asyncio +async def test_async_upload_retries_400_with_an_unknown_error_code( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + bucket.reply("test-retry-400", (400, "SomethingElse"), (200, "")) + logger: Final = _logger(monkeypatch, bucket) + await _upload(logger, _payload("test-retry-400")) + + assert len(bucket.puts()) == 1 + assert len(logger.sink.queued_keys()) == 1 + + +@pytest.mark.asyncio +async def test_async_upload_is_single_attempted_on_404_through_production_http_handler( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + bucket.reply("test-404", (404, "NoSuchBucket")) + logger: Final = _logger(monkeypatch, bucket) + await _upload(logger, _payload("test-404")) + + assert len(bucket.puts()) == 1 + assert len(logger.sink.queued_keys()) == 1 + + +@pytest.mark.asyncio +async def test_async_upload_access_denied_403_is_retried_and_then_requeued( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + bucket.reply("test-403-denied", (403, "AccessDenied")) + logger: Final = _logger(monkeypatch, bucket) + await _upload(logger, _payload("test-403-denied")) + + assert len(bucket.puts()) == 3 + _assert_backoff_gaps(bucket.puts()) + assert len(logger.sink.queued_keys()) == 1 + + +@pytest.mark.asyncio +async def test_async_upload_retries_429_through_production_http_handler( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + bucket.reply("429", (429, ""), (200, "")) + logger: Final = _logger(monkeypatch, bucket) + await _upload(logger, _payload("429")) + + puts: Final = bucket.puts() + assert len(puts) == 2 + assert puts[0].body == puts[1].body + _assert_backoff_gaps(puts) + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_async_upload_retries_transport_errors_then_succeeds( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + bucket.reply("transport", (DROP, ""), (200, "")) + logger: Final = _logger(monkeypatch, bucket) + await _upload(logger, _payload("transport")) + + assert logger.sink.queued_keys() == [] + assert len(bucket.objects) == 1 + + +@pytest.mark.asyncio +async def test_async_upload_exhausts_transport_error_retries_and_keeps_the_event_queued( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + bucket.reply("down", (DROP, "")) + logger: Final = _logger(monkeypatch, bucket) + await _upload(logger, _payload("down")) + + assert logger.sink.queued_keys() == ["2025-09-14/time-09-05-07-000042_down.json"] + + +@pytest.mark.asyncio +async def test_async_log_event_skips_when_standard_logging_object_missing( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket) + kwargs_without_slo: Final = {"call_type": "afile_delete", "model": None, "litellm_call_id": "test-call-id"} + + with _failures() as mock_failure: + await log_attempt(logger, kwargs_without_slo, START) + + assert not mock_failure.called + assert logger.sink.queued_keys() == [] + + +async def _stripped( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, payload: dict[str, object] +) -> dict[str, object]: + logger: Final = _logger(monkeypatch, bucket, s3_strip_base64_files=True) + await _upload(logger, {**_payload("strip"), **payload}) + (put,) = bucket.puts() + return _uploaded(put) + + +@pytest.mark.asyncio +async def test_strip_base64_removes_file_and_nontext_entries( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + payload: Final = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello world"}, + {"type": "image", "file": {"file_data": "data:image/png;base64,AAAA"}}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,BBBB"}}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Response"}, + {"type": "audio", "file": {"file_data": "data:audio/wav;base64,CCCC"}}, + ], + }, + ] + } + + stripped: Final = await _stripped(monkeypatch, bucket, payload) + + messages: Final = stripped["messages"] + assert isinstance(messages, list) + assert len(messages[0]["content"]) == 1 + assert messages[0]["content"][0]["text"] == "Hello world" + assert len(messages[1]["content"]) == 1 + assert messages[1]["content"][0]["text"] == "Response" + for msg in messages: + for content in msg["content"]: + assert "file" not in content + assert content.get("type") == "text" + + +@pytest.mark.asyncio +async def test_strip_base64_keeps_non_file_content(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + payload: Final = { + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "Just text"}, {"type": "text", "text": "Another message"}], + } + ] + } + + stripped: Final = await _stripped(monkeypatch, bucket, payload) + + assert stripped["messages"][0]["content"] == payload["messages"][0]["content"] + + +@pytest.mark.asyncio +async def test_strip_base64_handles_empty_or_missing_messages( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_strip_base64_files=True) + payload_no_messages: Final = {"id": "no-messages", "metadata": {}} + payload_empty: Final = {"id": "empty", "metadata": {}, "messages": []} + await log_payload(logger, payload_no_messages, START) + await log_payload(logger, payload_empty, START) + await logger.flush_queue() + + uploaded: Final = {body["id"]: body for body in (_uploaded(put) for put in bucket.puts())} + assert uploaded["no-messages"] == payload_no_messages + assert uploaded["empty"] == payload_empty + + +@pytest.mark.asyncio +async def test_strip_base64_mixed_nested_objects(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + payload: Final = { + "messages": [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Keep me"}, + {"type": "custom", "metadata": "ignore but non-text"}, + {"foo": "bar"}, + {"file": {"file_data": "data:application/pdf;base64,XXX"}}, + ], + "extra": {"trace_id": "123"}, + } + ] + } + + stripped: Final = await _stripped(monkeypatch, bucket, payload) + + content: Final = stripped["messages"][0]["content"] + assert len(content) == 2 + assert {"type": "text", "text": "Keep me"} in content + assert {"foo": "bar"} in content + assert stripped["messages"][0]["extra"]["trace_id"] == "123" + + +@pytest.mark.asyncio +async def test_strip_base64_recursive_redaction(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + payload: Final = { + "messages": [ + { + "content": [ + {"type": "text", "text": "normal text"}, + {"type": "text", "text": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg"}, + {"type": "text", "text": "Nested: {'data': 'data:application/pdf;base64,AAA...'}"}, + {"file": {"file_data": "data:application/pdf;base64,AAAA"}}, + {"metadata": {"preview": "data:audio/mp3;base64,AAAAA=="}}, + ] + } + ] + } + + result: Final = await _stripped(monkeypatch, bucket, payload) + content: Final = result["messages"][0]["content"] + + assert not any("file" in c for c in content) + for c in content: + if isinstance(c, dict): + s = json.dumps(c).lower() + assert "base64," not in s, f"Found real base64 blob in: {s}" + + +async def _queued_key(logger: S3Logger, payload: Mapping[str, object], start_time: datetime = START) -> str: + await log_payload(logger, payload, start_time) + (key,) = logger.sink.queued_keys() + return key + + +@pytest.mark.parametrize( + "use_team_prefix,use_key_prefix,team_alias,key_alias,expected_prefix", + [ + (False, False, "teamA", "keyA", ""), + (True, False, "teamA", "keyA", "teamA/"), + (False, True, "teamA", "keyA", "keyA/"), + (True, True, "teamA", "keyA", "teamA/keyA/"), + (True, True, None, "keyA", "keyA/"), + (True, True, "teamA", None, "teamA/"), + (True, True, None, None, ""), + ], +) +@pytest.mark.asyncio +async def test_s3_object_key_prefix_combinations( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + use_team_prefix: bool, + use_key_prefix: bool, + team_alias: str | None, + key_alias: str | None, + expected_prefix: str, +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_use_team_prefix=use_team_prefix, s3_use_key_prefix=use_key_prefix) + payload: Final = _payload( + "abc123", + metadata={"user_api_key_team_alias": team_alias, "user_api_key_alias": key_alias}, + messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + ) + + key: Final = await _queued_key(logger, payload) + + assert key == f"{expected_prefix}2025-09-14/time-09-05-07-000042_abc123.json" + + +@pytest.mark.asyncio +async def test_prefix_priority_and_path_construction(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_use_team_prefix=True, s3_use_key_prefix=True) + payload: Final = _payload( + "xyz999", metadata={"user_api_key_team_alias": "Team-Alpha", "user_api_key_alias": "API-12345"} + ) + + key: Final = await _queued_key(logger, payload) + + assert key.startswith("Team-Alpha/API-12345/2025-09-14/") + assert "//" not in key + + +@pytest.mark.asyncio +async def test_prefix_absent_when_flags_disabled(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_use_team_prefix=False, s3_use_key_prefix=False) + payload: Final = _payload( + "no-prefix", metadata={"user_api_key_team_alias": "team-x", "user_api_key_alias": "key-x"} + ) + + key: Final = await _queued_key(logger, payload) + + assert key == "2025-09-14/time-09-05-07-000042_no-prefix.json" + + +@pytest.mark.asyncio +async def test_combined_prefix_reflects_in_s3_object_key( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_use_team_prefix=True, s3_use_key_prefix=True) + payload: Final = _payload( + "int-test", metadata={"user_api_key_team_alias": "myteam", "user_api_key_alias": "apikey"} + ) + + key: Final = await _queued_key(logger, payload) + + assert "myteam/apikey/" in key, f"Expected both prefixes in key: {key}" + + +@pytest.mark.asyncio +async def test_s3_object_key_sanitizes_slashes_and_colons_in_file_name( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_path="LiteLLMAPPLogs", s3_use_team_prefix=True) + payload: Final = _payload( + "arn:aws:bedrock:us-east-1:123456789012:model-invocation-job/gl18r6skk9yy", + metadata={"user_api_key_team_alias": "myteam"}, + ) + + key: Final = await _queued_key(logger, payload, datetime(2026, 2, 11, 0, 35, 18, 391582)) + + assert key == ( + "LiteLLMAPPLogs/myteam/2026-02-11/" + "time-00-35-18-391582_arn_aws_bedrock_us-east-1_123456789012_model-invocation-job_gl18r6skk9yy.json" + ) + + +@pytest.mark.parametrize( + "response_id", + [ + "s3://example-batch-bucket/litellm-bedrock-files/input.jsonl", + "gs://example-batch-bucket/litellm-vertex-files/input.jsonl", + ], +) +@pytest.mark.asyncio +async def test_s3_object_key_has_no_colon_for_cloud_uri_file_ids( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, response_id: str +) -> None: + logger: Final = _logger(monkeypatch, bucket) + + key: Final = await _queued_key(logger, _payload(response_id), datetime(2026, 9, 7, 4, 51, 6, 685889)) + + filename: Final = key.rsplit("/", 1)[-1] + assert ":" not in filename + assert filename.endswith("_input.jsonl.json") + + +@pytest.mark.asyncio +async def test_create_s3_batch_logging_element_flat_key_for_arn_response_id( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_use_team_prefix=False, s3_use_key_prefix=False) + payload: Final = _payload("arn:aws:bedrock:us-east-1:123456789012:model-invocation-job/gl18r6skk9yy") + + key: Final = await _queued_key(logger, payload, datetime(2026, 2, 11, 0, 35, 18, 391582)) + + file_segment: Final = key.split("2026-02-11/", 1)[1] + assert "/" not in file_segment, f"Expected flat file under date segment, got: {key}" + assert file_segment.endswith("model-invocation-job_gl18r6skk9yy.json") + + +def _expected_content_md5(payload: dict[str, object]) -> str: + json_string: Final = safe_dumps(payload) + return base64.b64encode(hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()).decode() + + +@pytest.mark.asyncio +async def test_async_upload_sets_content_md5_header(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + payload: Final = _payload("test-md5", test="content-md5") + await _upload(_logger(monkeypatch, bucket), payload) + + (put,) = bucket.puts() + assert put.headers["content-md5"] == _expected_content_md5(payload) + assert "x-amz-server-side-encryption" not in put.headers + + +@pytest.mark.asyncio +async def test_async_upload_sets_server_side_encryption_header_when_configured( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + await _upload(_logger(monkeypatch, bucket, s3_server_side_encryption="aws:kms"), _payload("test-sse")) + + (put,) = bucket.puts() + assert put.headers["x-amz-server-side-encryption"] == "aws:kms" + + +async def _global_params_put( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, params: Mapping[str, object] +) -> Received: + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_endpoint_url": bucket.url, + "s3_region_name": "us-east-1", + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + **params, + }, + ) + logger: Final = rust_s3_logger(monkeypatch) + await _upload(logger, _payload("global")) + (put,) = bucket.puts() + return put + + +@pytest.mark.asyncio +async def test_s3_server_side_encryption_read_from_callback_params( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + put: Final = await _global_params_put( + monkeypatch, bucket, {"s3_bucket_name": "from-global", "s3_server_side_encryption": "aws:kms"} + ) + assert put.path.startswith("/from-global/") + assert put.headers["x-amz-server-side-encryption"] == "aws:kms" + + +@pytest.mark.asyncio +async def test_async_upload_sets_sse_kms_key_id_header_when_configured( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger( + monkeypatch, + bucket, + s3_server_side_encryption="aws:kms", + s3_sse_kms_key_id="arn:aws:kms:us-east-1:111122223333:key/test-key-id", + ) + await _upload(logger, _payload("test-kms")) + + (put,) = bucket.puts() + assert put.headers["x-amz-server-side-encryption"] == "aws:kms" + assert put.headers["x-amz-server-side-encryption-aws-kms-key-id"] == ( + "arn:aws:kms:us-east-1:111122223333:key/test-key-id" + ) + + +@pytest.mark.asyncio +async def test_async_upload_omits_kms_key_id_header_when_not_configured( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + await _upload(_logger(monkeypatch, bucket, s3_server_side_encryption="aws:kms"), _payload("test-no-kms")) + + (put,) = bucket.puts() + assert put.headers["x-amz-server-side-encryption"] == "aws:kms" + assert "x-amz-server-side-encryption-aws-kms-key-id" not in put.headers + + +@pytest.mark.asyncio +async def test_s3_sse_kms_key_id_read_from_callback_params( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + put: Final = await _global_params_put( + monkeypatch, + bucket, + { + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "aws:kms", + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", + }, + ) + assert put.headers["x-amz-server-side-encryption-aws-kms-key-id"] == ( + "arn:aws:kms:us-east-1:111122223333:key/test-key-id" + ) + + +@pytest.mark.asyncio +async def test_async_upload_infers_aws_kms_when_only_key_id_set( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_sse_kms_key_id="arn:aws:kms:us-east-1:111122223333:key/test-key-id") + await _upload(logger, _payload("test-kms-only")) + + (put,) = bucket.puts() + assert put.headers["x-amz-server-side-encryption"] == "aws:kms" + assert put.headers["x-amz-server-side-encryption-aws-kms-key-id"] == ( + "arn:aws:kms:us-east-1:111122223333:key/test-key-id" + ) + + +@pytest.mark.asyncio +async def test_s3_sse_kms_key_id_read_from_audit_override_params( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "normal-logs-bucket"}) + logger: Final = rust_s3_logger( + monkeypatch, + s3_bucket_name="audit-logs-bucket", + s3_sse_kms_key_id="arn:aws:kms:us-east-1:111122223333:key/audit-key-id", + s3_endpoint_url=bucket.url, + s3_region_name="us-east-1", + s3_aws_access_key_id="test-key", + s3_aws_secret_access_key="test-secret", + ) + await _upload(logger, _payload("audit")) + + (put,) = bucket.puts() + assert put.path.startswith("/audit-logs-bucket/") + assert put.headers["x-amz-server-side-encryption-aws-kms-key-id"] == ( + "arn:aws:kms:us-east-1:111122223333:key/audit-key-id" + ) + + +@pytest.mark.asyncio +async def test_kms_key_id_dropped_when_algorithm_is_not_kms( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + put: Final = await _global_params_put( + monkeypatch, + bucket, + { + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "AES256", + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", + }, + ) + assert put.headers["x-amz-server-side-encryption"] == "AES256" + assert "x-amz-server-side-encryption-aws-kms-key-id" not in put.headers + + +@pytest.mark.asyncio +async def test_non_string_algorithm_is_dropped_and_valid_key_id_is_rescued( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + put: Final = await _global_params_put( + monkeypatch, + bucket, + { + "s3_bucket_name": "from-global", + "s3_server_side_encryption": True, + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", + }, + ) + assert put.headers["x-amz-server-side-encryption"] == "aws:kms" + assert put.headers["x-amz-server-side-encryption-aws-kms-key-id"] == ( + "arn:aws:kms:us-east-1:111122223333:key/test-key-id" + ) + + +@pytest.mark.asyncio +async def test_non_string_key_id_is_dropped_and_valid_algorithm_is_kept( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + put: Final = await _global_params_put( + monkeypatch, + bucket, + {"s3_bucket_name": "from-global", "s3_server_side_encryption": "aws:kms", "s3_sse_kms_key_id": 12345}, + ) + assert put.headers["x-amz-server-side-encryption"] == "aws:kms" + assert "x-amz-server-side-encryption-aws-kms-key-id" not in put.headers + + +def _signature_for(signer_cls: type, url: str, method: str, body: bytes | None, headers: Mapping[str, str]) -> str: + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + + signed_header_names: Final = headers["authorization"].split("SignedHeaders=")[1].split(", ")[0] + request: Final = AWSRequest( + method=method, + url=url, + data=body, + headers={name: headers[name] for name in signed_header_names.split(";") if name in headers}, + ) + request.context["timestamp"] = headers["x-amz-date"] + signer: Final = signer_cls(Credentials(_ACCESS_KEY, _SECRET_KEY), "s3", "us-east-1") + canonical_request: Final = signer.canonical_request(request) + return signer.signature(signer.string_to_sign(request, canonical_request), request) + + +def _as_s3_canonicalizes(url: str) -> str: + split: Final = urlsplit(url) + return urlunsplit(split._replace(path=quote(unquote(split.path), safe="/~"))) + + +def _assert_signed_for_s3_canonicalization( + url: str, method: str, body: bytes | None, headers: Mapping[str, str] +) -> None: + from botocore.auth import S3SigV4Auth, SigV4Auth + + assert url == _as_s3_canonicalizes(url) + sent_signature: Final = headers["authorization"].split("Signature=")[1].strip() + assert sent_signature == _signature_for(S3SigV4Auth, url, method, body, headers) + assert sent_signature != _signature_for(SigV4Auth, url, method, body, headers) + + +def _logger_for_signing( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, **params: object +) -> S3Logger: # kwargs-ok: S3Logger keywords + return rust_s3_logger( + monkeypatch, + s3_bucket_name="logs-bucket", + s3_endpoint_url=bucket.url, + s3_aws_access_key_id=_ACCESS_KEY, + s3_aws_secret_access_key=_SECRET_KEY, + s3_region_name="us-east-1", + **params, + ) + + +_KEY_WITH_SPACE = "LOGS/LLM AI Projects/2026-08-04/time-13-01-00-abc.json" +_RESERVED_CHAR_KEYS = ( + "2026-08-21/time-05-29-36_resp_bGl0ZWxsbTpjdXN0b20=.json", + "session=logs/2026-08-21/time-05-29-36_abc.json", + "a+b/2026-08-21/time-05-29-36_abc.json", + "a&b/2026-08-21/time-05-29-36_abc.json", + "a#b/2026-08-21/time-05-29-36_abc.json", + "a?b/2026-08-21/time-05-29-36_abc.json", + "a%b/2026-08-21/time-05-29-36_abc.json", + _KEY_WITH_SPACE, +) + + +@pytest.mark.asyncio +async def test_async_upload_signs_object_key_with_space_the_way_s3_does( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger_for_signing(monkeypatch, bucket, s3_path="LOGS", s3_use_team_prefix=True) + await _upload( + logger, _payload("abc", metadata={"user_api_key_team_alias": "LLM AI Projects"}), datetime(2026, 8, 4, 13, 1) + ) + + (put,) = bucket.puts() + assert unquote(put.path) == "/logs-bucket/LOGS/LLM AI Projects/2026-08-04/time-13-01-00-000000_abc.json" + _assert_signed_for_s3_canonicalization(f"{bucket.url}{put.path}", "PUT", put.body, put.headers) + + +@pytest.mark.asyncio +async def test_download_signs_object_key_with_space_the_way_s3_does( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + bucket.objects[f"/logs-bucket/{quote(_KEY_WITH_SPACE)}"] = b'{"downloaded": "data"}' + logger: Final = _logger_for_signing(monkeypatch, bucket) + + assert await logger.get_proxy_server_request_from_cold_storage_with_object_key(_KEY_WITH_SPACE) == { + "downloaded": "data" + } + + get: Final = bucket.received[-1] + _assert_signed_for_s3_canonicalization(f"{bucket.url}{get.path}", "GET", None, get.headers) + + +@pytest.mark.parametrize("s3_object_key", _RESERVED_CHAR_KEYS) +@pytest.mark.asyncio +async def test_download_percent_encodes_reserved_characters_in_object_key( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, s3_object_key: str +) -> None: + bucket.objects[f"/logs-bucket/{quote(s3_object_key, safe='/')}"] = b'{"downloaded": "data"}' + logger: Final = _logger_for_signing(monkeypatch, bucket) + + assert await logger.get_proxy_server_request_from_cold_storage_with_object_key(s3_object_key) == { + "downloaded": "data" + } + + get: Final = bucket.received[-1] + assert get.path == f"/logs-bucket/{quote(s3_object_key, safe='/')}" + _assert_signed_for_s3_canonicalization(f"{bucket.url}{get.path}", "GET", None, get.headers) + + +def _chat_payload() -> dict[str, object]: + return { + "id": "chatcmpl-prompts-only", + "messages": [{"role": "user", "content": "Reply with exactly the word PINEAPPLE."}], + "response": {"choices": [{"message": {"role": "assistant", "content": "PINEAPPLE"}}]}, + "metadata": {"user_api_key_team_alias": None}, + } + + +async def _uploaded_chat_body( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, params: Mapping[str, object], outcome: Outcome +) -> dict[str, object]: + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": bucket.url, + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + "s3_region_name": "us-east-1", + **params, + }, + ) + logger: Final = rust_s3_logger(monkeypatch) + payload: Final = _chat_payload() + original: Final = copy.deepcopy(payload) + await log_attempt(logger, {"standard_logging_object": payload}, datetime(2026, 7, 30, 12, 0, 0), outcome) + assert payload == original, "the caller's standard_logging_object must not be mutated" + await logger.flush_queue() + (put,) = bucket.puts() + return _uploaded(put) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event_name", ["async_log_success_event", "async_log_failure_event"]) +async def test_prompts_only_drops_response_but_keeps_messages_in_uploaded_object( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, event_name: str +) -> None: + outcome: Final = Outcome.success if event_name == "async_log_success_event" else Outcome.failure + body: Final = await _uploaded_chat_body(monkeypatch, bucket, {"s3_log_prompts_only": True}, outcome) + + assert body["messages"] == _chat_payload()["messages"] + assert body["response"] is None + assert body["id"] == "chatcmpl-prompts-only" + + +@pytest.mark.asyncio +async def test_prompts_only_default_off_keeps_response_in_uploaded_object( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + body: Final = await _uploaded_chat_body(monkeypatch, bucket, {}, Outcome.success) + + assert body["response"] == _chat_payload()["response"] + assert body["messages"] == _chat_payload()["messages"] + + +@pytest.mark.asyncio +async def test_prompts_only_explicit_false_in_params_beats_env_var( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setenv("S3_LOG_PROMPTS_ONLY", "true") + body: Final = await _uploaded_chat_body(monkeypatch, bucket, {"s3_log_prompts_only": False}, Outcome.success) + + assert body["response"] == _chat_payload()["response"] + + +@pytest.mark.asyncio +async def test_prompts_only_env_var_applies_when_param_unset( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setenv("S3_LOG_PROMPTS_ONLY", "true") + body: Final = await _uploaded_chat_body(monkeypatch, bucket, {}, Outcome.success) + + assert body["response"] is None + + +_PARTITION_START: Final = datetime(2026, 9, 29, 14, 5, 9, 123456) +_PARTITION_ID: Final = "chatcmpl-partition" +_DAILY_KEY: Final = f"logs/team-a/key-a/2026-09-29/time-14-05-09-123456_{_PARTITION_ID}.json" +_HOURLY_KEY: Final = f"logs/team-a/key-a/2026-09-29/14/time-14-05-09-123456_{_PARTITION_ID}.json" + + +def _partition_payload(response_id: str = _PARTITION_ID) -> dict[str, object]: + return _payload(response_id, metadata={"user_api_key_team_alias": "team-a", "user_api_key_alias": "key-a"}) + + +def _partition_logger( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, callback_params: Mapping[str, object] +) -> S3Logger: + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "test-bucket", + "s3_region_name": "us-east-1", + "s3_path": "logs", + "s3_endpoint_url": bucket.url, + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + "s3_use_team_prefix": True, + "s3_use_key_prefix": True, + **callback_params, + }, + ) + return rust_s3_logger(monkeypatch) + + +@pytest.mark.parametrize( + ("callback_params", "expected_key"), + [ + ({}, _DAILY_KEY), + ({"s3_partition_granularity": None}, _DAILY_KEY), + ({"s3_partition_granularity": "day"}, _DAILY_KEY), + ({"s3_partition_granularity": "hour"}, _HOURLY_KEY), + ], +) +@pytest.mark.asyncio +async def test_partition_granularity_sets_request_log_folder( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, callback_params: Mapping[str, object], expected_key: str +) -> None: + logger: Final = _partition_logger(monkeypatch, bucket, callback_params) + + assert await _queued_key(logger, _partition_payload(), _PARTITION_START) == expected_key + + +@pytest.mark.parametrize("invalid", ["hourly", "HOUR", "1", 1, True]) +@pytest.mark.asyncio +async def test_invalid_partition_granularity_warns_and_keeps_daily_folder( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, invalid: object +) -> None: + logger: Final = _partition_logger(monkeypatch, bucket, {"s3_partition_granularity": invalid}) + await log_payload(logger, _partition_payload(), _PARTITION_START) + await log_payload(logger, _partition_payload(), _PARTITION_START) + + assert logger.sink.queued_keys() == [_DAILY_KEY, _DAILY_KEY] + + +@pytest.mark.asyncio +async def test_partition_granularity_reads_admin_ui_env_var_below_callback_params( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setenv("S3_PARTITION_GRANULARITY", "hour") + + from_env: Final = await _queued_key( + _partition_logger(monkeypatch, bucket, {}), _partition_payload(), _PARTITION_START + ) + from_params: Final = await _queued_key( + _partition_logger(monkeypatch, bucket, {"s3_partition_granularity": "day"}), + _partition_payload(), + _PARTITION_START, + ) + + assert from_env == _HOURLY_KEY + assert from_params == _DAILY_KEY + + +@pytest.mark.asyncio +async def test_partition_granularity_constructor_argument_and_os_environ_reference( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setenv("MY_S3_PARTITION", "hour") + + from_secret: Final = _partition_logger( + monkeypatch, bucket, {"s3_partition_granularity": "os.environ/MY_S3_PARTITION"} + ) + + assert await _queued_key(from_secret, _partition_payload(), _PARTITION_START) == _HOURLY_KEY + + +@pytest.mark.asyncio +async def test_hourly_partition_long_key_keeps_hour_folder_within_s3_limit( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + logger: Final = _partition_logger(monkeypatch, bucket, {"s3_partition_granularity": "hour", "s3_path": "p" * 1100}) + + key: Final = await _queued_key(logger, _partition_payload("r" * 600), _PARTITION_START) + + assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert re.search(r"/2026-09-29/14/[0-9a-f]{64}\.json$", key) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("granularity", "hour_folder"), [("hour", True), ("day", False), (None, False)]) +async def test_audit_log_key_follows_audit_callback_params_partition_granularity( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, granularity: str | None, hour_folder: bool +) -> None: + logger: Final = rust_s3_logger( + monkeypatch, + s3_bucket_name="audit-bucket", + s3_path="audit", + s3_partition_granularity=granularity, + s3_endpoint_url=bucket.url, + ) + + await logger.async_log_audit_log_event({"id": "audit-1"}) + + (key,) = logger.sink.queued_keys() + match: Final = re.fullmatch( + r"audit/audit_logs/\d{4}-\d{2}-\d{2}/(?:(\d{2})/)?(\d{2})-\d{2}-\d{2}_audit-1\.json", key + ) + assert match is not None, key + assert (match.group(1) is not None) is hour_folder + if hour_folder: + assert match.group(1) == match.group(2) + + +@pytest.mark.asyncio +async def test_hourly_batch_file_upload_writes_one_file_per_hour_folder( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _partition_logger( + monkeypatch, bucket, {"s3_partition_granularity": "hour", "s3_batch_file_upload": True} + ) + await log_payload(logger, _partition_payload("before"), datetime(2026, 9, 29, 13, 59, 59)) + await log_payload(logger, _partition_payload("after"), datetime(2026, 9, 29, 14, 0, 1)) + + await logger.flush_queue() + + by_folder: Final = { + re.sub( + r"/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", "", unquote(put.path).removeprefix("/test-bucket/") + ): put.body + for put in bucket.puts() + } + assert sorted(by_folder) == ["logs/team-a/key-a/2026-09-29/13", "logs/team-a/key-a/2026-09-29/14"] + assert [json.loads(line)["id"] for line in by_folder["logs/team-a/key-a/2026-09-29/13"].splitlines()] == ["before"] + assert [json.loads(line)["id"] for line in by_folder["logs/team-a/key-a/2026-09-29/14"].splitlines()] == ["after"] + + +def _cold_storage_key() -> str: + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + return StandardLoggingPayloadSetup._generate_cold_storage_object_key( + start_time=_PARTITION_START, response_id=_PARTITION_ID + ) + + +def _cold_logger(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, params: Mapping[str, object]) -> S3Logger: + monkeypatch.setattr( + litellm, "s3_callback_params", {"s3_bucket_name": "test-bucket", "s3_endpoint_url": bucket.url, **params} + ) + return rust_s3_logger(monkeypatch) + + +@pytest.mark.parametrize("granularity", [None, "day", "hour"]) +@pytest.mark.asyncio +async def test_cold_storage_object_key_matches_the_uploaded_request_log_key( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, granularity: str | None +) -> None: + monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2") + logger: Final = _cold_logger(monkeypatch, bucket, {"s3_path": "coldlogs", "s3_partition_granularity": granularity}) + uploaded: Final = await _queued_key(logger, _payload(_PARTITION_ID), _PARTITION_START) + + monkeypatch.setattr(litellm, "callbacks", [logger]) + cold_key: Final = _cold_storage_key() + + assert cold_key == uploaded + assert ("/2026-09-29/14/" in cold_key) is (granularity == "hour") + + +@pytest.mark.asyncio +async def test_cold_storage_key_matches_upload_when_env_var_changes_mid_request( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2") + logger: Final = _cold_logger(monkeypatch, bucket, {"s3_path": "coldlogs"}) + monkeypatch.setattr(litellm, "callbacks", [logger]) + + cold_key: Final = _cold_storage_key() + monkeypatch.setenv("S3_PARTITION_GRANULARITY", "hour") + uploaded: Final = await _queued_key( + logger, _payload(_PARTITION_ID, metadata={"cold_storage_object_key": cold_key}), _PARTITION_START + ) + + assert cold_key == uploaded == f"coldlogs/2026-09-29/time-14-05-09-123456_{_PARTITION_ID}.json" + + +@pytest.mark.asyncio +async def test_hour_upload_ignores_a_cold_storage_key_owned_by_another_logger( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setattr(litellm, "cold_storage_custom_logger", "gcs_bucket") + logger: Final = _cold_logger(monkeypatch, bucket, {"s3_partition_granularity": "hour"}) + cold_key: Final = _cold_storage_key() + uploaded: Final = await _queued_key( + logger, _payload(_PARTITION_ID, metadata={"cold_storage_object_key": cold_key}), _PARTITION_START + ) + + assert cold_key == f"2026-09-29/time-14-05-09-123456_{_PARTITION_ID}.json" + assert uploaded == f"2026-09-29/14/time-14-05-09-123456_{_PARTITION_ID}.json" + + +def _key(response_id: str) -> str: + return f"2025-09-14/time-09-05-07-000042_{response_id}.json" + + +async def _log_all(logger: S3Logger, payloads: list[dict[str, object]]) -> None: + for payload in payloads: + await log_payload(logger, payload, START) + + +async def _log_once_a_put_arrives(logger: S3Logger, bucket: ScriptedBucket, payloads: list[dict[str, object]]) -> None: + while not bucket.puts(): + await asyncio.sleep(0.005) + await _log_all(logger, payloads) + + +def _put_calls(bucket: ScriptedBucket) -> list[tuple[str, bytes]]: + return [(put.path, put.body) for put in bucket.puts()] + + +@pytest.mark.asyncio +async def test_async_send_batch_bounds_concurrent_uploads( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_concurrent_uploads=4) + bucket.delay = 0.05 + await _log_all(logger, [_payload(f"elem-{i}") for i in range(40)]) + + await logger.flush_queue() + + assert bucket.peak == 4 + assert len(bucket.puts()) == 40 + + +@pytest.mark.asyncio +async def test_async_send_batch_uploads_single_jsonl_file( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_batch_file_upload=True) + payloads: Final[list[dict[str, object]]] = [{"id": "req-1"}, {"id": "req-2"}, {"id": "req-3"}] + await _log_all(logger, payloads) + + await logger.flush_queue() + + (put,) = bucket.puts() + assert put.path.endswith(".jsonl") + assert [json.loads(line) for line in put.body.decode().splitlines()] == payloads + assert put.headers["content-type"] == "application/x-ndjson" + + +@pytest.mark.asyncio +async def test_flush_queue_preserves_events_added_during_upload( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket) + bucket.delay = 0.2 + await log_payload(logger, _payload("first"), START) + + await asyncio.gather(logger.flush_queue(), _log_once_a_put_arrives(logger, bucket, [_payload("late")])) + + assert logger.sink.queued_keys() == [_key("late")] + + +@pytest.mark.parametrize(("configured", "batch_file"), [("false", False), ("true", True)]) +@pytest.mark.asyncio +async def test_env_backed_false_string_keeps_per_request_uploads( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, configured: str, batch_file: bool +) -> None: + await _upload(_logger(monkeypatch, bucket, s3_batch_file_upload=configured), _payload("req-1")) + + (put,) = bucket.puts() + assert put.path.endswith(".jsonl") is batch_file + + +@pytest.mark.parametrize("bad", [0, -3, "0", "abc", ""]) +@pytest.mark.asyncio +async def test_invalid_concurrency_falls_back_to_default( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, bad: object +) -> None: + from litellm.constants import DEFAULT_S3_MAX_CONCURRENT_UPLOADS + + logger: Final = _logger(monkeypatch, bucket, s3_max_concurrent_uploads=bad) + bucket.delay = 0.1 + await _log_all(logger, [_payload(f"elem-{i}") for i in range(2 * DEFAULT_S3_MAX_CONCURRENT_UPLOADS)]) + + await logger.flush_queue() + + assert bucket.peak == DEFAULT_S3_MAX_CONCURRENT_UPLOADS + + +@pytest.mark.asyncio +async def test_env_backed_concurrency_string_is_parsed(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_concurrent_uploads="4") + bucket.delay = 0.05 + await _log_all(logger, [_payload(f"elem-{i}") for i in range(12)]) + + await logger.flush_queue() + + assert bucket.peak == 4 + + +@pytest.mark.asyncio +async def test_failed_uploads_stay_queued_for_next_flush( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("_elem-2.json", (503, "SlowDown")) + bucket.reply("_elem-4.json", (503, "SlowDown")) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(5)]) + + await logger.flush_queue() + + assert logger.sink.queued_keys() == [_key("elem-2"), _key("elem-4")] + + bucket.script.clear() + await logger.flush_queue() + + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_batch_file_upload_failure_keeps_whole_batch( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_batch_file_upload=True) + bucket.reply(".jsonl", (503, "SlowDown")) + payloads: Final[list[dict[str, object]]] = [{"id": f"elem-{i}", "i": i} for i in range(3)] + await _log_all(logger, payloads) + + await logger.flush_queue() + + assert len(bucket.puts()) == 3 + assert len(logger.sink.queued_keys()) == 1 + assert bucket.puts()[-1].body.decode() == "\n".join(json.dumps(payload) for payload in payloads) + + +@pytest.mark.asyncio +async def test_events_appended_during_failed_flush_survive( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("_first.json", (503, "SlowDown")) + await log_payload(logger, _payload("first"), START) + + await asyncio.gather(logger.flush_queue(), _log_once_a_put_arrives(logger, bucket, [_payload("late")])) + + assert logger.sink.queued_keys() == [_key("first"), _key("late")] + + bucket.script.clear() + await logger.flush_queue() + + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_batch_file_key_shape(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + await _upload(_logger(monkeypatch, bucket, s3_path="logs", s3_batch_file_upload=True), {"id": "req-1"}) + + (put,) = bucket.puts() + assert re.search(r".*/2025-09-14/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", put.path) + assert put.headers["content-disposition"].endswith('.jsonl"') + + +@pytest.mark.asyncio +async def test_failed_batch_file_is_requeued_and_resent_unchanged( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_batch_file_upload=True) + bucket.reply(".jsonl", (503, "SlowDown")) + await _log_all(logger, [{"id": f"elem-{i}", "i": i} for i in range(3)]) + + await logger.flush_queue() + + (queued,) = logger.sink.queued_keys() + assert queued.endswith(".jsonl") + + bucket.script.clear() + await logger.flush_queue() + + assert logger.sink.queued_keys() == [] + calls: Final = _put_calls(bucket) + assert len(calls) == 4 + assert calls[0] == calls[1] + + +@pytest.mark.asyncio +async def test_elements_appended_after_failed_batch_file_get_their_own_file( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_batch_file_upload=True) + bucket.reply(".jsonl", (503, "SlowDown")) + await log_payload(logger, {"id": "first"}, START) + + await logger.flush_queue() + + await log_payload(logger, {"id": "late"}, START) + bucket.script.clear() + await logger.flush_queue() + + assert logger.sink.queued_keys() == [] + calls: Final = _put_calls(bucket) + assert len(calls) == 5 + assert calls[0] == calls[1] + second_flush: Final = calls[3:] + assert calls[0] in second_flush + late_call: Final = next(call for call in second_flush if call != calls[0]) + assert late_call[0] != calls[0][0] + assert late_call[1].decode() == json.dumps({"id": "late"}) + + +@pytest.mark.asyncio +async def test_batch_file_mode_disabled_when_s3_v2_is_cold_storage_logger( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2") + logger: Final = _logger(monkeypatch, bucket, s3_batch_file_upload=True) + await _upload(logger, {"id": "req-1"}) + + assert len(bucket.puts()) == 1 + assert bucket.puts()[0].path.endswith("_req-1.json") + + monkeypatch.setattr(litellm, "cold_storage_custom_logger", None) + batch_logger: Final = _logger(monkeypatch, bucket, s3_batch_file_upload=True) + await _upload(batch_logger, {"id": "req-2"}) + + assert len(bucket.puts()) == 2 + assert bucket.puts()[1].path.endswith(".jsonl") + + +@pytest.mark.parametrize( + ("owner_when_built", "owner_after", "suffix"), + [("s3_v2", None, "_req-1.json"), (None, "s3_v2", ".jsonl")], +) +@pytest.mark.asyncio +async def test_the_cold_storage_owner_is_fixed_when_the_logger_is_built( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + owner_when_built: str | None, + owner_after: str | None, + suffix: str, +) -> None: + monkeypatch.setattr(litellm, "cold_storage_custom_logger", owner_when_built) + logger: Final = _logger(monkeypatch, bucket, s3_batch_file_upload=True) + monkeypatch.setattr(litellm, "cold_storage_custom_logger", owner_after) + + await _upload(logger, {"id": "req-1"}) + + (put,) = bucket.puts() + assert put.path.endswith(suffix) + + +@pytest.mark.parametrize( + ("environment", "region"), + [ + ({"AWS_REGION": "ap-south-1"}, "ap-south-1"), + ({"AWS_REGION_NAME": "eu-central-1"}, "eu-central-1"), + ({"AWS_REGION": "ap-south-1", "AWS_REGION_NAME": "eu-central-1"}, "ap-south-1"), + ({}, "us-west-2"), + ], +) +@pytest.mark.asyncio +async def test_the_environment_region_is_read_when_the_logger_is_built( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, environment: Mapping[str, str], region: str +) -> None: + for name, value in environment.items(): + monkeypatch.setenv(name, value) + logger: Final = _logger(monkeypatch, bucket, s3_region_name=None) + monkeypatch.setenv("AWS_REGION", "sa-east-1") + monkeypatch.setenv("AWS_REGION_NAME", "sa-east-1") + + await _upload(logger, {"id": "req-1"}) + + (put,) = bucket.puts() + assert f"/{region}/s3/aws4_request" in put.headers["authorization"] + + +@pytest.mark.asyncio +async def test_a_region_param_is_not_replaced_by_the_environment_region( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setenv("AWS_REGION", "ap-south-1") + logger: Final = _logger(monkeypatch, bucket) + + await _upload(logger, {"id": "req-1"}) + + (put,) = bucket.puts() + assert "/us-east-1/s3/aws4_request" in put.headers["authorization"] + + +def _coded_body(code: str | None, raw_body: str | None = None) -> str: + return raw_body if raw_body is not None else (f"{code}" if code is not None else "") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("status", "code", "raw_body", "puts_per_element"), + [ + pytest.param(403, "AccessDenied", None, 3, id="access-denied-403"), + pytest.param(403, None, None, 3, id="empty-403"), + pytest.param(403, None, "Forbidden", 3, id="html-403"), + pytest.param(400, "KMS.DisabledException", None, 1, id="kms-disabled-400"), + pytest.param(404, "NoSuchBucket", None, 1, id="no-such-bucket-404"), + ], +) +async def test_non_terminal_failure_is_requeued_and_delivered_on_recovery( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + status: int, + code: str | None, + raw_body: str | None, + puts_per_element: int, +) -> None: + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("/test-bucket/", (status, _coded_body(code, raw_body))) + keys: Final = [_key(f"elem-{i}") for i in range(5)] + await _log_all(logger, [_payload(f"elem-{i}") for i in range(5)]) + + await logger.flush_queue() + + assert len(logger.sink.queued_keys()) == 5 + assert len(bucket.puts()) == 5 * puts_per_element + + bucket.script.clear() + await logger.flush_queue() + + assert logger.sink.queued_keys() == [] + assert len(bucket.puts()) == 5 * puts_per_element + 5 + landed: Final = frozenset(key for key in keys if any(put.path.endswith(key) for put in bucket.puts()[-5:])) + assert landed == frozenset(keys) + + +@pytest.mark.asyncio +async def test_persistent_500_stays_queued_through_a_dozen_failed_flushes( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("/test-bucket/", (500, "InternalError")) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(5)]) + + for _ in range(12): + await logger.flush_queue() + assert len(logger.sink.queued_keys()) == 5 + + assert len(bucket.puts()) == 12 * 15 + + bucket.script.clear() + await logger.flush_queue() + + assert logger.sink.queued_keys() == [] + assert len(bucket.puts()) == 12 * 15 + 5 + + +@pytest.mark.asyncio +async def test_terminal_object_is_dropped_once_next_to_delivered_siblings_when_opted_in( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_drop_on_terminal_error=True) + bucket.reply("_elem-1.json", (400, "EntityTooLarge")) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(5)]) + + await logger.flush_queue() + + assert len(bucket.puts()) == 5 + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_terminal_object_is_requeued_when_opted_out( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_drop_on_terminal_error=False) + bucket.reply("_elem-1.json", (400, "EntityTooLarge")) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(5)]) + + await logger.flush_queue() + + (queued,) = logger.sink.queued_keys() + assert queued.endswith("_elem-1.json") + assert sum(put.path.endswith("_elem-1.json") for put in bucket.puts()) == 1 + assert len(bucket.puts()) == 5 + + +@pytest.mark.asyncio +async def test_terminal_objects_are_requeued_when_every_upload_in_the_flush_fails( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_drop_on_terminal_error=True) + bucket.reply("/test-bucket/", (400, "EntityTooLarge")) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(5)]) + + await logger.flush_queue() + + assert len(logger.sink.queued_keys()) == 5 + + +@pytest.mark.asyncio +async def test_overflow_after_a_failed_flush_trims_failed_first_and_counts_upload_failures_only( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_queue_size=4) + bucket.reply("/test-bucket/", (503, "SlowDown")) + await _log_all(logger, [_payload("first"), _payload("second")]) + + with _failures() as mock_failure: + await asyncio.gather( + logger.flush_queue(), + _log_once_a_put_arrives(logger, bucket, [_payload(f"late-{index}") for index in range(3)]), + ) + + assert logger.sink.queued_keys() == [_key("second"), _key("late-0"), _key("late-1"), _key("late-2")] + failed_uploads: Final = 2 + assert mock_failure.call_count == failed_uploads + + +@pytest.mark.asyncio +async def test_queue_grows_past_the_cap_while_the_sink_fails_and_everything_lands( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_queue_size=5) + bucket.reply("/test-bucket/", (503, "SlowDown")) + await _log_all(logger, [_payload(f"elem-{index}") for index in range(5)]) + + with _failures() as mock_failure: + await logger.flush_queue() + + assert len(logger.sink.queued_keys()) == 5 + upload_failures: Final = 5 + assert mock_failure.call_count == upload_failures + + await _log_all(logger, [_payload(f"elem-{index}") for index in range(5, 8)]) + + bucket.script.clear() + await logger.flush_queue() + + assert logger.sink.queued_keys() == [] + landed: Final = frozenset(put.path.rsplit("_", 1)[-1] for put in bucket.puts()[-8:]) + assert landed == frozenset(f"elem-{index}.json" for index in range(8)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("status", "code"), + [ + pytest.param(404, "NoSuchKey", id="404"), + pytest.param(401, None, id="401"), + pytest.param(400, None, id="uncoded-400"), + ], +) +async def test_unlisted_status_gets_one_put_and_stays_queued( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, status: int, code: str | None +) -> None: + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("/test-bucket/", (status, _coded_body(code))) + await _log_all(logger, [_payload(f"elem-{index}") for index in range(4)]) + + started: Final = time.monotonic() + await logger.flush_queue() + + assert len(bucket.puts()) == 4 + assert time.monotonic() - started < 0.5 + assert len(logger.sink.queued_keys()) == 4 + + +_SLOW_DOWN: Final = (503, "SlowDown") + + +def _names(puts: list[Received]) -> list[str]: + return [put.path.rsplit("_", 1)[-1] for put in puts] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("status", "code"), + [ + pytest.param(503, "SlowDown", id="503"), + pytest.param(429, "SlowDown", id="429"), + pytest.param(500, "InternalError", id="500"), + pytest.param(403, "AccessDenied", id="access-denied-403"), + ], +) +async def test_retryable_statuses_back_off_three_attempts( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, status: int, code: str | None +) -> None: + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("/test-bucket/", (status, _coded_body(code))) + + await _upload(logger, _payload("req")) + + assert len(bucket.puts()) == 3 + _assert_backoff_gaps(bucket.puts()) + assert len(logger.sink.queued_keys()) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("status", "code"), + [ + pytest.param(408, None, id="408"), + pytest.param(502, None, id="502"), + pytest.param(504, None, id="504"), + ], +) +async def test_unlisted_statuses_are_not_retried_in_call( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, status: int, code: str | None +) -> None: + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("/test-bucket/", (status, _coded_body(code))) + + started: Final = time.monotonic() + await _upload(logger, _payload("req")) + + assert len(bucket.puts()) == 1 + assert time.monotonic() - started < 0.5 + assert len(logger.sink.queued_keys()) == 1 + + +@pytest.mark.asyncio +async def test_retry_finishes_before_the_next_first_attempt( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_concurrent_uploads=1) + bucket.reply("_a.json", _SLOW_DOWN, (200, "")) + await _log_all(logger, [_payload("a"), _payload("b")]) + + await logger.flush_queue() + + assert _names(bucket.puts()) == ["a.json", "a.json", "b.json"] + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_objects_in_backoff_are_bounded_by_the_slot_width( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_concurrent_uploads=2) + bucket.reply("/test-bucket/", _SLOW_DOWN) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(20)]) + + await logger.flush_queue() + + first_and_last: Final[dict[str, list[float]]] = {} + for put in bucket.puts(): + span = first_and_last.setdefault(put.path, [put.at, put.at]) + span[1] = put.at + peak: Final = max( + sum(1 for start, end in first_and_last.values() if start <= moment <= end) + for moment, _ in first_and_last.values() + ) + assert peak <= 2, f"{peak} objects slept at once, slot width is 2" + assert len(bucket.puts()) == 60 + + +@pytest.mark.asyncio +async def test_terminal_code_is_retried_like_base_when_the_drop_flag_is_off( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_drop_on_terminal_error=False) + bucket.reply("_x.json", (403, "InvalidRequest"), (200, "")) + + with _failures() as failures: + await _upload(logger, _payload("x")) + + assert len(bucket.puts()) == 2 + failures.assert_not_called() + + dropping: Final = _logger(monkeypatch, bucket) + bucket.script.clear() + bucket.reply("_x.json", (403, "InvalidRequest"), (200, "")) + before: Final = len(bucket.puts()) + + await _upload(dropping, _payload("x")) + + assert len(bucket.puts()) - before == 1 + + +@pytest.mark.asyncio +async def test_send_batch_calls_upload_with_one_positional_arg( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket) + await _log_all(logger, [_payload("a"), _payload("b")]) + + await logger.flush_queue() + + assert sorted(_names(bucket.puts())) == ["a.json", "b.json"] + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_failed_elements_stay_oldest_first_when_requeued( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_retry_age_seconds=3600) + bucket.reply("_f2.json", _SLOW_DOWN) + await _log_all(logger, [_payload("f1"), _payload("f2")]) + + await asyncio.gather(logger.flush_queue(), _log_once_a_put_arrives(logger, bucket, [_payload("late")])) + + assert logger.sink.queued_keys() == [_key("f2"), _key("late")] + + +@pytest.mark.asyncio +async def test_overflow_prefers_arrivals_over_failed_elements_without_counting_the_trim( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_queue_size=1) + bucket.reply("_f2.json", _SLOW_DOWN) + await _log_all(logger, [_payload("f1"), _payload("f2")]) + + with _failures() as mock_failure: + await asyncio.gather(logger.flush_queue(), _log_once_a_put_arrives(logger, bucket, [_payload("late")])) + + assert logger.sink.queued_keys() == [_key("late")] + failed_uploads: Final = 1 + assert mock_failure.call_count == failed_uploads + + +@pytest.mark.asyncio +async def test_fresh_elements_upload_before_stale_retries_after_a_failed_flush( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_concurrent_uploads=1) + bucket.reply("_f2.json", _SLOW_DOWN) + await _log_all(logger, [_payload("f1"), _payload("f2")]) + + await asyncio.gather(logger.flush_queue(), _log_once_a_put_arrives(logger, bucket, [_payload("late")])) + + bucket.script.clear() + before: Final = len(bucket.puts()) + await logger.flush_queue() + + assert _names(bucket.puts()[before:]) == ["late.json", "f2.json"] + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_repeated_overflow_trims_oldest_across_failed_flushes( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_queue_size=3) + bucket.reply("/test-bucket/", _SLOW_DOWN) + await _log_all(logger, [_payload(name) for name in ("a", "b", "c")]) + + await asyncio.gather(logger.flush_queue(), _log_once_a_put_arrives(logger, bucket, [_payload("d")])) + + assert logger.sink.queued_keys() == [_key("b"), _key("c"), _key("d")] + + seen: Final = len(bucket.puts()) + + async def log_e_once_the_next_flush_puts() -> None: + while len(bucket.puts()) == seen: + await asyncio.sleep(0.005) + await log_payload(logger, _payload("e"), START) + + await asyncio.gather(logger.flush_queue(), log_e_once_the_next_flush_puts()) + + assert logger.sink.queued_keys() == [_key("c"), _key("d"), _key("e")] + + +@pytest.mark.asyncio +async def test_retry_age_budget_drops_after_the_clock_set_by_a_partial_failure( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_retry_age_seconds=1) + bucket.reply("_poison.json", _SLOW_DOWN) + await _log_all(logger, [_payload("poison"), _payload("good")]) + + with _failures() as mock_failure: + await logger.flush_queue() + + assert logger.sink.queued_keys() == [_key("poison")] + + await asyncio.sleep(1.1) + await log_payload(logger, _payload("good-2"), START) + await logger.flush_queue() + + assert logger.sink.queued_keys() == [] + poison_puts: Final = sum(1 for put in bucket.puts() if put.path.endswith("_poison.json")) + assert poison_puts == 6 + upload_failures: Final = 2 + assert mock_failure.call_count == upload_failures + + +@pytest.mark.asyncio +async def test_the_retry_clock_starts_at_the_first_partial_failure_not_first_seen( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_retry_age_seconds=1) + bucket.reply("_poison.json", _SLOW_DOWN) + await log_payload(logger, _payload("poison"), START) + + await logger.flush_queue() + + assert len(logger.sink.queued_keys()) == 1 + + await asyncio.sleep(1.1) + await log_payload(logger, _payload("good"), START) + await logger.flush_queue() + + assert logger.sink.queued_keys() == [_key("poison")] + + +@pytest.mark.asyncio +async def test_an_unlisted_5xx_is_requeued_without_an_extra_attempt( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("/test-bucket/", (507, "")) + + await _upload(logger, _payload("req-507")) + + assert len(logger.sink.queued_keys()) == 1 + assert len(bucket.puts()) == 1 + + +async def _peak_width(logger: S3Logger, bucket: ScriptedBucket, events: int) -> int: + bucket.peak = 0 + bucket.delay = 0.1 + await _log_all(logger, [_payload(f"width-{index}") for index in range(events)]) + await logger.flush_queue() + bucket.delay = 0.0 + return bucket.peak + + +@pytest.mark.asyncio +async def test_configured_concurrency_is_the_fixed_limit_when_adaptive_is_off( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_max_concurrent_uploads=64) + + assert await _peak_width(logger, bucket, 128) == 64 + + bucket.reply("_elem-0.json", _SLOW_DOWN, (200, "")) + await _upload(logger, _payload("elem-0")) + + assert await _peak_width(logger, bucket, 128) == 64 + + +@pytest.mark.asyncio +async def test_default_upload_width_is_16(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + from litellm.constants import DEFAULT_S3_MAX_CONCURRENT_UPLOADS + + logger: Final = _logger(monkeypatch, bucket) + + assert await _peak_width(logger, bucket, 2 * DEFAULT_S3_MAX_CONCURRENT_UPLOADS) == DEFAULT_S3_MAX_CONCURRENT_UPLOADS + assert DEFAULT_S3_MAX_CONCURRENT_UPLOADS == 16 + + +async def _timed_send_batch(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, size: int) -> float: + logger: Final = _logger(monkeypatch, bucket) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(size)]) + started: Final = time.perf_counter() + await logger.flush_queue() + return time.perf_counter() - started + + +@pytest.mark.asyncio +async def test_send_batch_time_grows_linearly_with_the_batch( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + baseline: Final = await _timed_send_batch(monkeypatch, bucket, 2_000) + quadrupled: Final = await _timed_send_batch(monkeypatch, bucket, 8_000) + + assert quadrupled / baseline < 8, f"2k took {baseline:.3f}s, 8k took {quadrupled:.3f}s" + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("concurrent", [False, True]) +@pytest.mark.asyncio +async def test_repeated_failure_notifications_upload_once( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + stream: bool, + concurrent: bool, +) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + + sink: Final = _logger(monkeypatch, bucket) + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "_async_failure_callback", [sink]) + logging_obj: Final = Logging( + model="openai/gpt-4o-mini", + messages=[], + stream=stream, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="synthetic-request", + function_id="synthetic-function", + ) + error: Final = litellm.ServiceUnavailableError( + message="synthetic deployment-selection failure", + llm_provider="openai", + model="openai/gpt-4o-mini", + ) + + if concurrent: + await asyncio.gather(*(logging_obj.async_failure_handler(error, "synthetic traceback") for _ in range(3))) + else: + for _ in range(3): + await logging_obj.async_failure_handler(error, "synthetic traceback") + + assert len(sink.sink.queued_keys()) == 1 + await sink.flush_queue() + await asyncio.sleep(0) + assert len(bucket.puts()) == 1 + assert all(_uploaded(put)["status"] == "failure" for put in bucket.puts()) + + +@pytest.fixture +def _s3_v2_callback_params(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + + +async def _wait_for_a_queued_key(logger: S3Logger) -> list[str]: + for _ in range(500): + keys = logger.sink.queued_keys() + if keys: + return keys + await asyncio.sleep(0.01) + return [] + + +@pytest.mark.usefixtures("_s3_v2_callback_params") +@pytest.mark.asyncio +@pytest.mark.parametrize("streaming", [True]) +async def test_basic_s3_v2_logging(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, streaming: bool) -> None: + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "load-testing-oct", + "s3_endpoint_url": bucket.url, + "s3_aws_secret_access_key": "test-secret", + "s3_aws_access_key_id": "test-key", + "s3_region_name": "us-west-2", + }, + ) + + s3_v2_logger: Final = _logger_from_globals(monkeypatch) + monkeypatch.setattr(litellm, "callbacks", [s3_v2_logger]) + + response_id = None + response = await litellm.acompletion( + model="gpt-5-mini", + messages=[{"role": "user", "content": "This is a test"}], + mock_response="It's simple to use and easy to get started", + stream=streaming, + ) + if streaming: + async for chunk in response: + response_id = chunk.id + else: + response_id = response.id + + assert await _wait_for_a_queued_key(s3_v2_logger) + await s3_v2_logger.flush_queue() + + uploaded_keys: Final = [unquote(put.path) for put in bucket.puts()] + assert len(uploaded_keys) > 0, "S3 upload was never called" + assert response_id is not None + assert any(response_id in key for key in uploaded_keys), ( + f"Expected response_id={response_id} in one of the uploaded S3 keys: {uploaded_keys}" + ) + + +def _logger_from_globals(monkeypatch: pytest.MonkeyPatch) -> S3Logger: + logger: Final = rust_s3_logger(monkeypatch) + _ = logger.periodic_flush_task.cancel() + return logger + + +@pytest.mark.usefixtures("_s3_v2_callback_params") +@pytest.mark.asyncio +async def test_basic_s3_v2_logging_failure(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket) -> None: + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": bucket.url, + "s3_aws_secret_access_key": "test-secret", + "s3_aws_access_key_id": "test-key", + "s3_region_name": "us-west-2", + }, + ) + s3_v2_logger: Final = _logger_from_globals(monkeypatch) + monkeypatch.setattr(litellm, "callbacks", [s3_v2_logger]) + + with contextlib.suppress(Exception): + await litellm.acompletion( + model="gpt-5-mini", + api_key="invalid-api-key", + messages=[{"role": "user", "content": "This is a test"}], + mock_response=Exception("forced failure for S3 logging test"), + ) + + assert await _wait_for_a_queued_key(s3_v2_logger) + await s3_v2_logger.flush_queue() + + (put,) = bucket.puts() + assert put.path.startswith("/test-bucket/") + assert put.headers["content-type"] == "application/json" + assert '"model": "gpt-5-mini"' in put.body.decode() + + +def _batch_line_items_payload() -> dict[str, object]: + line = { + "batch_id": "bGl0ZWxsbV9wcm94eTtiYXRjaA", + "custom_id": "ok-1", + "status": "success", + "request": {"custom_id": "ok-1", "body": {"messages": [{"role": "user", "content": "hi"}]}}, + "response": {"custom_id": "ok-1", "response": {"status_code": 200, "body": {"id": "chatcmpl-1"}}}, + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + "response_cost": 0.005, + "model": "gpt-4o-mini", + "custom_llm_provider": "openai", + "metadata": {"user_api_key_hash": "hash-alice"}, + } + return { + "batch_id": "bGl0ZWxsbV9wcm94eTtiYXRjaA", + "finished_at": 1791494130, + "part": 0, + "lines": (line, {**line, "custom_id": "ok-2"}), + } + + +def _s3_logger_answering( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, status: int, s3_path: str | None, content: bytes = b"" +) -> S3Logger: + bucket.reply("/test-bucket/", (status, content.decode())) + return _logger( + monkeypatch, + bucket, + s3_path=s3_path, + s3_aws_access_key_id=_ACCESS_KEY, + s3_aws_secret_access_key=_SECRET_KEY, + s3_flush_interval=3600, + ) + + +@pytest.mark.asyncio +async def test_batch_line_items_are_one_ndjson_object_under_the_configured_path( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _s3_logger_answering(monkeypatch, bucket, 200, "cold/") + payload: Final = _batch_line_items_payload() + + await logger.async_log_batch_line_items(payload) + + (request,) = bucket.received + assert request.method == "PUT" + assert request.path == "/test-bucket/cold/batch_line_items/2026-10-08/bGl0ZWxsbV9wcm94eTtiYXRjaA/part-00000.jsonl" + assert request.headers["content-type"] == "application/x-ndjson" + assert request.headers["content-disposition"] == 'inline; filename="bGl0ZWxsbV9wcm94eTtiYXRjaA-part-00000.jsonl"' + assert [json.loads(line) for line in request.body.decode().splitlines()] == list(payload["lines"]) + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_batch_line_items_without_an_s3_path_sit_at_the_bucket_root( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _s3_logger_answering(monkeypatch, bucket, 200, None) + + await logger.async_log_batch_line_items(_batch_line_items_payload()) + + assert [request.path for request in bucket.received] == [ + "/test-bucket/batch_line_items/2026-10-08/bGl0ZWxsbV9wcm94eTtiYXRjaA/part-00000.jsonl" + ] + + +@pytest.mark.asyncio +async def test_a_rejected_batch_line_items_upload_is_logged_with_the_batch_id( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, caplog: pytest.LogCaptureFixture +) -> None: + import logging + + logger: Final = _s3_logger_answering(monkeypatch, bucket, 400, "cold") + + with caplog.at_level(logging.ERROR, logger="LiteLLM"): + await logger.async_log_batch_line_items(_batch_line_items_payload()) + + assert len(bucket.received) == 1 + assert any( + "bGl0ZWxsbV9wcm94eTtiYXRjaA" in record.getMessage() and "were not stored" in record.getMessage() + for record in caplog.records + ) + + +@pytest.mark.asyncio +async def test_a_batch_line_items_object_the_bucket_refuses_for_good_is_not_kept_in_memory( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _s3_logger_answering( + monkeypatch, bucket, 400, "cold", b"InvalidArgument" + ) + + await logger.async_log_batch_line_items(_batch_line_items_payload()) + await logger.flush_queue() + + assert len(bucket.received) == 1 + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_batch_line_items_are_serialized_and_hashed_off_the_event_loop( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + import threading + + logger: Final = _s3_logger_answering(monkeypatch, bucket, 200, "cold/") + serialize: Final = s3_v3.batch_line_items_ndjson + threads: Final[dict[str, threading.Thread]] = {} + + def _serialize(payload: object) -> str: + threads["serialize"] = threading.current_thread() + return serialize(payload) + + monkeypatch.setattr(s3_v3, "batch_line_items_ndjson", _serialize) + + await logger.async_log_batch_line_items(_batch_line_items_payload()) + + assert len(bucket.received) == 1 + assert set(threads) == {"serialize"} + assert threading.main_thread() not in threads.values() + + +_BOUNDING_START: Final = datetime(2026, 8, 24, 6, 18, 41, 948021) + + +def _oversized_response_id() -> str: + return "resp_" + "A" * 1100 + + +def _alias_metadata(team_alias: str | None = None, key_alias: str | None = None) -> dict[str, object]: + metadata: Final[dict[str, object]] = {} + if team_alias is not None: + metadata["user_api_key_team_alias"] = team_alias + if key_alias is not None: + metadata["user_api_key_alias"] = key_alias + return metadata + + +async def _bounded_key( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + response_id: str, + s3_path: str = "", + team_alias: str | None = None, + key_alias: str | None = None, + start_time: datetime = _BOUNDING_START, +) -> str: + logger: Final = _logger( + monkeypatch, + bucket, + s3_path=s3_path or None, + s3_use_team_prefix=team_alias is not None, + s3_use_key_prefix=key_alias is not None, + ) + return await _queued_key(logger, _payload(response_id, metadata=_alias_metadata(team_alias, key_alias)), start_time) + + +@pytest.mark.asyncio +async def test_s3_object_key_bounds_the_file_segment_when_the_full_key_fits( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + file_name: Final = "time-01-14-19-015434_resp_" + "A" * 300 + + key: Final = await _bounded_key( + monkeypatch, bucket, "resp_" + "A" * 300, "requests", start_time=datetime(2026, 9, 12, 1, 14, 19, 15434) + ) + + file_segment: Final = key.rsplit("/", 1)[1] + assert len(key.encode("utf-8")) < MAX_S3_OBJECT_KEY_BYTES + assert len(file_segment.encode("utf-8")) <= 255 + assert file_segment.startswith("time-01-14-19-015434_resp_") + assert file_segment.endswith(f"_{hashlib.sha256(file_name.encode('utf-8')).hexdigest()}.json") + + +@pytest.mark.asyncio +async def test_s3_object_key_file_segment_limit_counts_utf8_bytes( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + key: Final = await _bounded_key( + monkeypatch, bucket, "resp_" + "日" * 100, "requests", start_time=datetime(2026, 9, 12, 1, 14, 19, 15434) + ) + + file_segment: Final = key.rsplit("/", 1)[1] + assert len(file_segment.encode("utf-8")) <= 255 + assert "�" not in file_segment + + +@pytest.mark.asyncio +async def test_s3_object_key_is_bounded_for_oversized_response_id( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + file_name: Final = f"time-06-18-41-948021_{_oversized_response_id()}" + + key: Final = await _bounded_key( + monkeypatch, bucket, _oversized_response_id(), "input", team_alias="DefaultTeamProd" + ) + + assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert key.startswith("input/DefaultTeamProd/2026-08-24/time-06-18-41-948021_resp_") + assert key.endswith(f"_{hashlib.sha256(file_name.encode('utf-8')).hexdigest()}.json") + + +@pytest.mark.parametrize( + "s3_path,team_alias,key_alias", + [ + ("input", None, None), + ("a" * 900, None, None), + ("input", "team-" + "b" * 900, None), + ("c" * 600, "team-" + "d" * 600, "key-" + "e" * 600), + ("", "ssss/" * 199 + "ssss", None), + ], +) +@pytest.mark.asyncio +async def test_s3_object_key_is_bounded_for_long_paths_and_aliases( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + s3_path: str, + team_alias: str | None, + key_alias: str | None, +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + key: Final = await _bounded_key(monkeypatch, bucket, _oversized_response_id(), s3_path, team_alias, key_alias) + + assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert key.endswith(".json") + assert "/2026-08-24/" in key or key.startswith("2026-08-24/") + assert "/" not in key.rsplit("2026-08-24/", 1)[1] + + +@pytest.mark.asyncio +async def test_s3_object_key_trimmed_prefixes_stay_distinct_per_operator( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + keys: Final = [ + await _bounded_key(monkeypatch, bucket, _oversized_response_id(), "input", "team-" + "b" * 1000 + suffix) + for suffix in ("-one", "-two") + ] + + assert keys[0] != keys[1] + assert all(key.startswith("input/team-" + "b" * 900) for key in keys) + assert all(len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES for key in keys) + + +@pytest.mark.asyncio +async def test_s3_object_key_bounded_prefix_never_splits_a_multibyte_character( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + s3_path: Final = "日本語" * 200 + + key: Final = await _bounded_key(monkeypatch, bucket, _oversized_response_id(), s3_path, "チーム" * 200) + + assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert key.startswith(s3_path[:100]) + assert "�" not in key + + +@pytest.mark.asyncio +async def test_s3_object_key_stays_unique_for_ids_sharing_a_head( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + keys: Final = { + await _bounded_key(monkeypatch, bucket, f"{_oversized_response_id()}{suffix}", "input") + for suffix in ("first", "second", "third") + } + + assert len(keys) == 3 + + +@pytest.mark.asyncio +async def test_s3_object_key_bounding_matches_the_documented_layout( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + file_name: Final = f"time-06-18-41-948021_{_oversized_response_id()}" + + key: Final = await _bounded_key(monkeypatch, bucket, _oversized_response_id(), "input", "team") + + digest: Final = hashlib.sha256(file_name.encode("utf-8")).hexdigest() + assert key == f"input/team/2026-08-24/{file_name[:64]}_{digest}.json" + + +@pytest.mark.asyncio +async def test_s3_object_key_keeps_the_configured_prefix_when_only_the_id_overflows( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + prefix: Final = "team-" + "b" * 934 + "/" + + key: Final = await _bounded_key(monkeypatch, bucket, _oversized_response_id(), "", prefix[:-1]) + + assert key.startswith(prefix + "2026-08-24/") + assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES + + +@pytest.mark.asyncio +async def test_s3_object_key_spends_the_whole_budget_when_the_prefix_must_be_trimmed( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + key: Final = await _bounded_key(monkeypatch, bucket, "abc", "p" * 400 + "/" + "q" * 600) + + assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES + assert key.startswith("p" * 400 + "/" + "q" * 500) + + +@pytest.mark.asyncio +async def test_s3_object_key_keeps_a_single_segment_path_as_far_as_it_fits( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + key: Final = await _bounded_key(monkeypatch, bucket, "chatcmpl-xyz", "a" * 1050) + + assert len(key.encode("utf-8")) == MAX_S3_OBJECT_KEY_BYTES + assert key.startswith("a" * 900) + + +@pytest.mark.asyncio +async def test_create_s3_batch_logging_element_bounds_key_and_keeps_full_response_id( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + logger: Final = _logger(monkeypatch, bucket, s3_use_team_prefix=True, s3_use_key_prefix=True) + response_id: Final = _oversized_response_id() + + await _upload( + logger, _payload(response_id, metadata=_alias_metadata("DefaultTeamProd", "prod-key")), _BOUNDING_START + ) + + (put,) = bucket.puts() + s3_object_key: Final = unquote(put.path).removeprefix("/test-bucket/") + assert len(s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert s3_object_key.startswith("DefaultTeamProd/prod-key/2026-08-24/") + assert _uploaded(put)["id"] == response_id + + +async def _download_filenames( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, response_ids: list[str] +) -> list[str]: + logger: Final = _logger(monkeypatch, bucket) + await _log_all_at(logger, [_payload(response_id) for response_id in response_ids], _BOUNDING_START) + await logger.flush_queue() + names: Final = [ + re.fullmatch(r'inline; filename="(.*)"', put.headers["content-disposition"]) for put in bucket.puts() + ] + return [name.group(1) for name in names if name is not None] + + +async def _log_all_at(logger: S3Logger, payloads: list[dict[str, object]], start_time: datetime) -> None: + for payload in payloads: + await log_payload(logger, payload, start_time) + + +@pytest.mark.asyncio +async def test_s3_object_download_filename_is_bounded_for_oversized_response_id( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES + + (file_name,) = await _download_filenames(monkeypatch, bucket, [_oversized_response_id()]) + + assert len(file_name.encode("utf-8")) <= MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES + assert file_name.startswith("time-2026-08-24T06-18-41-948021_resp_") + assert file_name.endswith(".json") + + +@pytest.mark.asyncio +async def test_s3_object_download_filenames_stay_distinct_when_shortened( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + file_names: Final = set( + await _download_filenames( + monkeypatch, bucket, [_oversized_response_id() + suffix for suffix in ("first", "second", "third")] + ) + ) + + assert len(file_names) == 3 + + +@pytest.mark.asyncio +async def test_s3_object_download_filename_short_id_is_unchanged( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + (file_name,) = await _download_filenames(monkeypatch, bucket, ["resp_abc123"]) + + assert file_name == "time-2026-08-24T06-18-41-948021_resp_abc123.json" + + +@pytest.mark.asyncio +async def test_create_s3_batch_logging_element_bounds_the_download_filename( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES + + (file_name,) = await _download_filenames(monkeypatch, bucket, [_oversized_response_id()]) + + assert len(file_name.encode("utf-8")) <= MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES + + +@pytest.mark.asyncio +async def test_audit_log_object_key_is_bounded_for_a_long_configured_path( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.constants import MAX_S3_OBJECT_KEY_BYTES + + logger: Final = _logger(monkeypatch, bucket, s3_path="audit-archive/" + "z" * 1100) + + await logger.async_log_audit_log_event({"id": "1a4f7bd0-6f1e-4d0a-9b3c-9f2e1d5a7c88"}) + + (key,) = logger.sink.queued_keys() + assert len(key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES + assert key.startswith("audit-archive/" + "z" * 900) + + +@pytest.mark.asyncio +async def test_s3_object_download_filename_drops_characters_that_break_the_header( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + (file_name,) = await _download_filenames(monkeypatch, bucket, ['resp_a"b/c']) + + assert file_name == "time-2026-08-24T06-18-41-948021_resp_a_b_c.json" + + +@pytest.mark.asyncio +async def test_s3_callback_params_override_uses_alternate_dict( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "normal-bucket"}) + logger: Final = rust_s3_logger( + monkeypatch, + s3_bucket_name="audit-bucket", + s3_path="audit-prefix", + s3_region_name="us-west-2", + s3_endpoint_url=bucket.url, + s3_aws_access_key_id="test-key", + s3_aws_secret_access_key="test-secret", + ) + _ = logger.periodic_flush_task.cancel() + + await _upload(logger, _payload("override")) + + (put,) = bucket.puts() + assert put.path.startswith("/audit-bucket/") + assert put.path.startswith("/audit-bucket/audit-prefix/") + assert "/us-west-2/s3/aws4_request" in put.headers["authorization"] + + +@pytest.mark.asyncio +async def test_s3_callback_params_override_does_not_mutate_inputs( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setenv("MY_AUDIT_BUCKET", "resolved-bucket") + override: Final = { + "s3_bucket_name": "os.environ/MY_AUDIT_BUCKET", + "s3_endpoint_url": bucket.url, + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + "s3_region_name": "us-east-1", + } + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "os.environ/MY_AUDIT_BUCKET"}) + logger: Final = rust_s3_logger(monkeypatch, **override) + _ = logger.periodic_flush_task.cancel() + + await _upload(logger, _payload("override")) + + assert bucket.puts()[0].path.startswith("/resolved-bucket/") + assert override["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" + assert litellm.s3_callback_params["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" + + +@pytest.mark.asyncio +async def test_s3_callback_params_override_none_falls_back_to_global( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "from-global", + "s3_endpoint_url": bucket.url, + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + "s3_region_name": "us-east-1", + }, + ) + logger: Final = _logger_from_globals(monkeypatch) + + await _upload(logger, _payload("global")) + + assert bucket.puts()[0].path.startswith("/from-global/") + + +_SIGV4_ACCESS_KEY: Final = re.compile(r"Credential=(AKIA\d+)/") + + +@pytest.fixture +def rotating_profile(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str: + import sys + import textwrap + import uuid + + counter: Final = tmp_path / "generation" + script: Final = tmp_path / "rotate_credentials.py" + script.write_text( + textwrap.dedent( + f""" + import json, sys + from datetime import datetime, timedelta, timezone + from pathlib import Path + + counter = Path({str(counter)!r}) + generation = int(counter.read_text()) if counter.exists() else 0 + counter.write_text(str(generation + 1)) + expiry = (datetime.now(timezone.utc) + timedelta(minutes=12)).strftime("%Y-%m-%dT%H:%M:%SZ") + json.dump( + {{ + "Version": 1, + "AccessKeyId": f"AKIA{{generation}}", + "SecretAccessKey": f"secret-{{generation}}", + "SessionToken": f"token-{{generation}}", + "Expiration": expiry, + }}, + sys.stdout, + ) + """ + ) + ) + profile: Final = f"rotating-{uuid.uuid4().hex}" + (tmp_path / "config").write_text(f"[profile {profile}]\ncredential_process = {sys.executable} {script}\n") + monkeypatch.setenv("AWS_CONFIG_FILE", str(tmp_path / "config")) + return profile + + +def _generation(request: Received) -> tuple[str, str]: + access_key: Final = _SIGV4_ACCESS_KEY.search(request.headers["authorization"]) + assert access_key is not None + return access_key.group(1).removeprefix("AKIA"), request.headers["x-amz-security-token"].removeprefix("token-") + + +def _profile_logger(monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, profile: str) -> S3Logger: + logger: Final = rust_s3_logger( + monkeypatch, + s3_bucket_name="test-bucket", + s3_endpoint_url=bucket.url, + s3_region_name="us-east-1", + s3_aws_profile_name=profile, + s3_flush_interval=3600, + ) + _ = logger.periodic_flush_task.cancel() + return logger + + +_SIGNATURE_DOES_NOT_MATCH: Final = "SignatureDoesNotMatch" + + +@pytest.mark.asyncio +async def test_async_upload_signs_with_one_frozen_credential_snapshot( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, rotating_profile: str +) -> None: + await _upload(_profile_logger(monkeypatch, bucket, rotating_profile), _payload("frozen")) + + assert len(bucket.puts()) == 1 + access_key_generation, token_generation = _generation(bucket.puts()[0]) + assert access_key_generation == token_generation + + +@pytest.mark.asyncio +async def test_async_upload_retries_403_with_fresh_credentials_and_signature( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, rotating_profile: str +) -> None: + bucket.reply("_test-403.json", (403, _SIGNATURE_DOES_NOT_MATCH), (200, "")) + logger: Final = _profile_logger(monkeypatch, bucket, rotating_profile) + + await _upload(logger, _payload("test-403")) + + requests: Final = bucket.puts() + assert len(requests) == 2 + first_key, first_token = _generation(requests[0]) + second_key, second_token = _generation(requests[1]) + assert first_key == first_token + assert second_key == second_token + assert int(second_key) > int(first_key) + assert requests[1].headers["authorization"] != requests[0].headers["authorization"] + _assert_backoff_gaps(requests) + assert logger.sink.queued_keys() == [] + + +@pytest.mark.asyncio +async def test_async_upload_exhausts_403_retries_through_production_http_handler( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, rotating_profile: str +) -> None: + bucket.reply("_test-403-exhausted.json", (403, _SIGNATURE_DOES_NOT_MATCH)) + logger: Final = _profile_logger(monkeypatch, bucket, rotating_profile) + + with _failures() as failure: + await _upload(logger, _payload("test-403-exhausted")) + + assert len(bucket.puts()) == 3 + _assert_backoff_gaps(bucket.puts()) + failure.assert_called_once_with() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transient_status", [500, 503]) +async def test_async_upload_recovers_from_transient_5xx_through_production_http_handler( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, rotating_profile: str, transient_status: int +) -> None: + bucket.reply(f"_test-{transient_status}.json", (transient_status, _SIGNATURE_DOES_NOT_MATCH), (200, "")) + logger: Final = _profile_logger(monkeypatch, bucket, rotating_profile) + + with _failures() as failure: + await _upload(logger, _payload(f"test-{transient_status}")) + + requests: Final = bucket.puts() + assert len(requests) == 2 + assert all(request.method == "PUT" for request in requests) + assert requests[0].path == requests[1].path + assert requests[0].body == requests[1].body + _assert_backoff_gaps(requests) + failure.assert_not_called() + assert logger.sink.queued_keys() == [] + + +_CONSTRUCTOR_BASE: Final[S3LoggerSettings] = { + "s3_bucket_name": "test-bucket", + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + "s3_region_name": "us-east-1", +} + + +def _constructed( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + settings: S3LoggerSettings, + override: Mapping[str, object] | None = None, + *, + configured: Mapping[str, object] | None = None, + periodic: bool = False, +) -> S3Logger: + monkeypatch.setattr(litellm, "s3_callback_params", configured) + logger: Final = S3Logger(override, **(_CONSTRUCTOR_BASE | {"s3_endpoint_url": bucket.url} | settings)) + if not periodic: + _ = logger.periodic_flush_task.cancel() + return logger + + +@pytest.mark.asyncio +async def test_constructor_keywords_build_a_logger_that_uploads_under_them( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _constructed(monkeypatch, bucket, {"s3_path": "from-constructor"}) + + await _upload(logger, _payload("ctor")) + + assert isinstance(logger, S3Logger) + assert logger.s3_path == "from-constructor" + assert [put.path for put in bucket.puts()] == [f"/test-bucket/from-constructor/{_key('ctor')}"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("configured", "override", "expected"), + [ + pytest.param(None, {"s3_path": "configured"}, "configured", id="override-wins"), + pytest.param({"s3_path": "global"}, None, "global", id="global-wins"), + pytest.param({"s3_path": "global"}, {}, "constructor", id="empty-override-is-opt-in"), + pytest.param(None, {"s3_path": ""}, "constructor", id="blank-falls-back"), + ], +) +async def test_callback_params_text_settings_win_over_constructor( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + configured: Mapping[str, object] | None, + override: Mapping[str, object] | None, + expected: str, +) -> None: + logger: Final = _constructed(monkeypatch, bucket, {"s3_path": "constructor"}, override, configured=configured) + + await _upload(logger, _payload("text")) + + assert [put.path for put in bucket.puts()] == [f"/test-bucket/{expected}/{_key('text')}"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("constructor", "configured", "expected"), + [ + pytest.param("hour", "day", "hour", id="constructor-wins"), + pytest.param(None, "hour", "hour", id="configured-when-constructor-unset"), + ], +) +async def test_constructor_partition_granularity_wins_over_callback_params( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, constructor: str | None, configured: str, expected: str +) -> None: + logger: Final = _constructed( + monkeypatch, bucket, {"s3_partition_granularity": constructor}, {"s3_partition_granularity": configured} + ) + + assert logger.resolve_partition_granularity() == expected + + +@pytest.mark.asyncio +async def test_constructor_prompts_only_wins_over_callback_params( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + monkeypatch.delenv("S3_LOG_PROMPTS_ONLY", raising=False) + logger: Final = _constructed(monkeypatch, bucket, {"s3_log_prompts_only": True}, {"s3_log_prompts_only": False}) + + await _upload(logger, _payload("prompts", response={"choices": [{"message": {"content": "secret answer"}}]})) + + (put,) = bucket.puts() + assert b"secret answer" not in put.body + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("constructor", "override", "kept"), + [ + pytest.param(7, {"s3_max_queue_size": 4}, 4, id="callback-params-win"), + pytest.param(4, {"s3_max_queue_size": "abc"}, 4, id="invalid-callback-params-fall-back-to-constructor"), + pytest.param(4, {}, 4, id="constructor-only"), + ], +) +async def test_queue_size_resolves_callback_params_then_constructor( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + constructor: int, + override: Mapping[str, object], + kept: int, +) -> None: + logger: Final = _constructed(monkeypatch, bucket, {"s3_max_queue_size": constructor}, override) + bucket.reply("/test-bucket/", _SLOW_DOWN) + await _log_all(logger, [_payload("first"), _payload("second")]) + + with _failures(): + await asyncio.gather( + logger.flush_queue(), + _log_once_a_put_arrives(logger, bucket, [_payload(f"late-{index}") for index in range(3)]), + ) + + assert len(logger.sink.queued_keys()) == kept + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("constructor", "override", "still_queued"), + [ + pytest.param(0, {}, True, id="constructor-zero-disables-the-budget"), + pytest.param(None, {}, True, id="constructor-none-disables-the-budget"), + pytest.param(1, {"s3_max_retry_age_seconds": "abc"}, False, id="invalid-callback-params-fall-back"), + pytest.param(3600, {"s3_max_retry_age_seconds": 1}, False, id="callback-params-win"), + pytest.param(1, {"s3_max_retry_age_seconds": ""}, False, id="blank-callback-params-fall-back"), + ], +) +async def test_retry_age_resolves_callback_params_then_constructor( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + constructor: int | None, + override: Mapping[str, object], + still_queued: bool, +) -> None: + logger: Final = _constructed(monkeypatch, bucket, {"s3_max_retry_age_seconds": constructor}, override) + bucket.reply("_poison.json", _SLOW_DOWN) + await _log_all(logger, [_payload("poison"), _payload("good")]) + + with _failures(): + await logger.flush_queue() + await asyncio.sleep(1.1) + await log_payload(logger, _payload("good-2"), START) + await logger.flush_queue() + + assert logger.sink.queued_keys() == ([_key("poison")] if still_queued else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("constructor", "override", "dropped"), + [ + pytest.param(False, {"s3_drop_on_terminal_error": True}, True, id="callback-params-win"), + pytest.param(False, {}, False, id="constructor-opts-out"), + pytest.param(False, {"s3_drop_on_terminal_error": "banana"}, True, id="invalid-callback-params-drop"), + ], +) +async def test_drop_on_terminal_error_resolves_callback_params_then_constructor( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + constructor: bool, + override: Mapping[str, object], + dropped: bool, +) -> None: + logger: Final = _constructed(monkeypatch, bucket, {"s3_drop_on_terminal_error": constructor}, override) + bucket.reply("_elem-1.json", (400, "EntityTooLarge")) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(5)]) + + with _failures(): + await logger.flush_queue() + + assert logger.sink.queued_keys() == ([] if dropped else [_key("elem-1")]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("empty", [None, ""]) +async def test_empty_callback_params_concurrency_falls_back_to_constructor_value( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, empty: object +) -> None: + logger: Final = _constructed( + monkeypatch, bucket, {"s3_max_concurrent_uploads": 4}, {"s3_max_concurrent_uploads": empty} + ) + + assert await _peak_width(logger, bucket, 12) == 4 + + +@pytest.mark.asyncio +async def test_callback_params_concurrency_wins_over_constructor( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _constructed( + monkeypatch, bucket, {"s3_max_concurrent_uploads": 4}, {"s3_max_concurrent_uploads": 2} + ) + + assert await _peak_width(logger, bucket, 12) == 2 + + +@pytest.mark.asyncio +async def test_constructor_batch_size_flushes_audit_logs_once_the_queue_reaches_it( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _constructed(monkeypatch, bucket, {"s3_batch_size": 2}) + + await logger.async_log_audit_log_event({"id": "audit-1"}) + + assert len(logger.sink.queued_keys()) == 1 + assert bucket.puts() == [] + + await logger.async_log_audit_log_event({"id": "audit-2"}) + + assert logger.sink.queued_keys() == [] + assert len(bucket.puts()) == 2 + + +@pytest.mark.asyncio +async def test_constructor_flush_interval_drives_the_periodic_flush( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + logger: Final = _constructed(monkeypatch, bucket, {"s3_flush_interval": 1}, periodic=True) + try: + await log_payload(logger, _payload("tick"), START) + for _ in range(100): + if bucket.puts(): + break + await asyncio.sleep(0.05) + + assert [put.path for put in bucket.puts()] == [f"/test-bucket/{_key('tick')}"] + finally: + _ = logger.periodic_flush_task.cancel() + + +@pytest.mark.asyncio +async def test_the_declared_subscription_matches_the_rust_manifest( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.integrations.custom_logger_v2.rust import WireManifest + + logger: Final = _logger(monkeypatch, bucket) + manifest: Final = WireManifest.model_validate_json(logger.rust.manifest()) + + assert {(s.hook, s.reads, s.writes, s.may_abort, s.parallel, s.awaited) for s in manifest.subscriptions} == { + (hook, s.reads, s.writes, s.may_abort, s.parallel, s.awaited) for hook, s in S3Logger.subscriptions.items() + } + assert logger.selection is not None + assert manifest.selection.name == logger.selection.name + assert frozenset(manifest.selection.aliases) == logger.selection.aliases + assert manifest.selection.activation == logger.selection.activation + assert frozenset(manifest.selection.opt_out_by) == logger.selection.opt_out_by + assert manifest.selection.honors_no_log == logger.selection.honors_no_log + assert not manifest.has_request_state + + +@pytest.mark.asyncio +async def test_the_s3_v2_callback_name_resolves_to_the_rust_logger_class( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry + + logger: Final = _logger(monkeypatch, bucket) + + assert CustomLoggerRegistry.get_class_type_for_custom_logger_name("s3_v2") is S3Logger + assert CustomLoggerRegistry.get_callback_str_from_class_type(type(logger)) == "s3_v2" + + +class _Records(logging.Handler): + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.records: Final[list[logging.LogRecord]] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.records.append(record) + + +@pytest.mark.asyncio +async def test_a_partially_failed_flush_logs_s3_batch_upload_error_like_main( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm._logging import verbose_logger + from litellm.integrations.s3_v2 import S3BatchUploadError + + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("_elem-1.json", _SLOW_DOWN) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(3)]) + records: Final = _Records() + verbose_logger.addHandler(records) + try: + await logger.flush_queue() + finally: + verbose_logger.removeHandler(records) + + failures: Final = [record for record in records.records if record.exc_info is not None] + assert len(failures) == 1 + record: Final = failures[0] + assert record.name == "LiteLLM" + assert record.levelno == logging.ERROR + assert record.getMessage() == "CustomLogger: async_send_batch raised; preserving 3 events in queue for retry" + assert record.exc_info is not None + error: Final = record.exc_info[1] + assert isinstance(error, S3BatchUploadError) + assert (error.failed, error.total) == (1, 3) + assert str(error) == "1 of 3 S3 uploads failed; transient failures kept in queue for the next flush" + assert "S3BatchUploadError: 1 of 3 S3 uploads failed" in logging.Formatter().formatException(record.exc_info) + + +@pytest.mark.asyncio +async def test_the_native_flush_returns_a_partial_failure_and_logs_no_record_of_its_own( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm._logging import verbose_logger + + logger: Final = _logger(monkeypatch, bucket) + bucket.reply("_elem-1.json", _SLOW_DOWN) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(3)]) + records: Final = _Records() + verbose_logger.addHandler(records) + try: + failure: Final = await logger.sink.flush() + finally: + verbose_logger.removeHandler(records) + + assert failure == (3, 1, 3) + assert [record for record in records.records if "async_send_batch raised" in record.getMessage()] == [] + bucket.script.clear() + assert await logger.sink.flush() is None + + +@pytest.mark.asyncio +async def test_a_partially_failed_flush_from_an_audit_log_logs_s3_batch_upload_error( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket +) -> None: + from litellm._logging import verbose_logger + from litellm.integrations.s3_v2 import S3BatchUploadError + + logger: Final = _constructed(monkeypatch, bucket, {"s3_batch_size": 2}) + bucket.reply("_audit-1.json", _SLOW_DOWN) + records: Final = _Records() + verbose_logger.addHandler(records) + try: + for audit_id in ("audit-1", "audit-2"): + await logger.async_log_audit_log_event({"id": audit_id}) + finally: + verbose_logger.removeHandler(records) + + (record,) = [record for record in records.records if record.exc_info is not None] + assert record.getMessage() == "CustomLogger: async_send_batch raised; preserving 2 events in queue for retry" + assert record.exc_info is not None + assert isinstance(record.exc_info[1], S3BatchUploadError) + + +def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [record.getMessage() for record in caplog.records if record.levelno == logging.WARNING] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("params", "warning"), + [ + pytest.param( + {"s3_adaptive_concurrency": "maybe"}, + "s3 logging: s3_adaptive_concurrency='maybe' is not a boolean, keeping the fixed upload width", + id="flag", + ), + pytest.param( + {"s3_max_queue_size": True}, + "s3 logging: s3_max_queue_size=True is a boolean, not an integer, using 50000", + id="bool_integer", + ), + pytest.param( + {"s3_max_retry_age_seconds": "soon"}, + "s3 logging: s3_max_retry_age_seconds='soon' is not an integer, falling back to 3600", + id="retry_age", + ), + pytest.param( + {"s3_server_side_encryption": 5}, + "s3 logging: ignoring s3_server_side_encryption because it has invalid type int; expected a string", + id="sse_type", + ), + pytest.param( + {"s3_server_side_encryption": "AES256", "s3_sse_kms_key_id": "key"}, + "s3 logging: ignoring s3_sse_kms_key_id because s3_server_side_encryption is AES256; " + "set it to aws:kms to encrypt with the KMS key", + id="sse_conflict", + ), + ], +) +async def test_invalid_settings_warn_on_the_litellm_logger_when_the_logger_is_built( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + caplog: pytest.LogCaptureFixture, + params: Mapping[str, object], + warning: str, +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + _ = _logger(monkeypatch, bucket, **params) + + assert warning in _warnings(caplog) + + +@pytest.mark.asyncio +async def test_an_unknown_partition_granularity_warns_once_per_value( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, caplog: pytest.LogCaptureFixture +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_partition_granularity="HOUR") + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await log_payload(logger, _payload("first"), START) + await log_payload(logger, _payload("second"), START) + + assert ( + _warnings(caplog).count("s3 logging: s3_partition_granularity='HOUR' is not one of day, hour, using day") == 1 + ) + + +@pytest.mark.asyncio +async def test_an_invalid_prompts_only_value_warns_on_every_log( + monkeypatch: pytest.MonkeyPatch, bucket: ScriptedBucket, caplog: pytest.LogCaptureFixture +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_log_prompts_only="maybe") + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await log_payload(logger, _payload("first"), START) + await log_payload(logger, _payload("second"), START) + + assert ( + _warnings(caplog).count("s3 logging: s3_log_prompts_only='maybe' is not a boolean, logging prompts only") == 2 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("params", "shown"), + [ + pytest.param({}, "3600", id="default"), + pytest.param({"s3_max_retry_age_seconds": 0}, "None", id="unbounded"), + ], +) +async def test_the_drop_warning_prints_the_retry_age_as_python_does( + monkeypatch: pytest.MonkeyPatch, + bucket: ScriptedBucket, + caplog: pytest.LogCaptureFixture, + params: Mapping[str, object], + shown: str, +) -> None: + logger: Final = _logger(monkeypatch, bucket, s3_drop_on_terminal_error=True, **params) + bucket.reply("_elem-1.json", (400, "EntityTooLarge")) + await _log_all(logger, [_payload(f"elem-{i}") for i in range(2)]) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await logger.flush_queue() + + assert ( + "s3 logging: 1 uploads dropped this flush: 1 rejected as terminal by S3, " + f"0 retrying longer than s3_max_retry_age_seconds={shown}" + ) in _warnings(caplog) diff --git a/tests/unit/integrations/test_s3_v3_native.py b/tests/unit/integrations/test_s3_v3_native.py new file mode 100644 index 00000000000..86e14d3fea9 --- /dev/null +++ b/tests/unit/integrations/test_s3_v3_native.py @@ -0,0 +1,1525 @@ +import asyncio +import base64 +import copy +import json +import re +import threading +import time +from collections.abc import Iterator, Mapping +from contextlib import nullcontext +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from types import SimpleNamespace +from typing import Final +from unittest.mock import patch +from urllib.parse import parse_qs + +import pytest +from pydantic import BaseModel + +import litellm +from litellm.integrations import s3_v3 +from litellm.integrations.custom_logger_v2.derivations import DERIVED_ATTEMPT_FIELDS +from litellm.integrations.custom_logger_v2.enforcement import FieldView +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.host import request_scope +from litellm.integrations.custom_logger_v2.legacy import KwargsFields +from litellm.integrations.custom_logger_v2.field_sources import HookArgs, with_sources +from litellm.integrations.custom_logger_v2.selection import RequestSelection +from litellm.integrations.s3_v3 import S3Logger +from litellm.litellm_core_utils.litellm_logging import LOGGING_KEYS, V2_ASYNC_LOGGING_LINE +from litellm.llms.bedrock import base_aws_llm +from litellm.types.integrations.custom_logger_v2 import Outcome + +COMPARED_HEADERS: Final = ( + "content-type", + "content-md5", + "x-amz-content-sha256", + "content-language", + "content-disposition", + "cache-control", + "x-amz-server-side-encryption", + "x-amz-server-side-encryption-aws-kms-key-id", + "x-amz-security-token", +) +START: Final = datetime(2026, 11, 13, 9, 5, 7, 42) +GOLDEN_FILE: Final = Path(__file__).parent / "s3_v2_golden" / "python_s3_v2.json" + + +@dataclass(frozen=True) +class Golden: + test: str + values: Mapping[str, object] + consumed: set[str] = field(default_factory=set) # mutable-ok: the labels this test read + + def expected(self, label: str) -> object: + key: Final = f"{self.test}::{label}" + self.consumed.add(key) + return self.values[key] + + def unconsumed(self) -> list[str]: + return sorted(key for key in self.values if key.startswith(f"{self.test}::") and key not in self.consumed) + + +@pytest.fixture(scope="session") +def _golden_values() -> Mapping[str, object]: + return json.loads(GOLDEN_FILE.read_text(encoding="utf-8")) + + +@pytest.fixture +def golden(request: pytest.FixtureRequest, _golden_values: Mapping[str, object]) -> Iterator[Golden]: + recorded: Final = Golden(request.node.name, _golden_values) + yield recorded + assert recorded.unconsumed() == [] + + +@dataclass +class Received: + method: str + path: str + headers: Mapping[str, str] + body: bytes + + +@dataclass +class FakeBucket: + received: list[Received] = field(default_factory=list) + objects: dict[str, bytes] = field(default_factory=dict) + failures: list[tuple[str, int, str]] = field(default_factory=list) + delays: list[tuple[str, float]] = field(default_factory=list) + port: int = 0 + in_flight: int = 0 + peak: int = 0 + lock: threading.Lock = field(default_factory=threading.Lock) + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def puts(self) -> list[Received]: + return [request for request in self.received if request.method == "PUT"] + + +def _handler(bucket: FakeBucket) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_: object) -> None: + return + + def _reply(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_PUT(self) -> None: + body: Final = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + with bucket.lock: + bucket.received.append( + Received("PUT", self.path, {k.lower(): v for k, v in self.headers.items()}, body) + ) + bucket.in_flight += 1 + bucket.peak = max(bucket.peak, bucket.in_flight) + try: + self._answer_put(body) + finally: + with bucket.lock: + bucket.in_flight -= 1 + + def _answer_put(self, body: bytes) -> None: + delay: Final = next((d[1] for d in bucket.delays if re.search(d[0], self.path)), 0.0) + if delay: + time.sleep(delay) + failure: Final = next((f for f in bucket.failures if re.search(f[0], self.path)), None) + if failure is not None and failure[1] == 0: + self.close_connection = True + return + if failure is not None: + self._reply(failure[1], failure[2].encode()) + return + bucket.objects[self.path] = body + self._reply(200, b"") + + def do_GET(self) -> None: + bucket.received.append(Received("GET", self.path, {k.lower(): v for k, v in self.headers.items()}, b"")) + stored: Final = bucket.objects.get(self.path) + self._reply(404, b"NoSuchKey") if stored is None else self._reply(200, stored) + + return Handler + + +@pytest.fixture +def bucket() -> Iterator[FakeBucket]: + fake: Final = FakeBucket() + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), _handler(fake)) + fake.port = server.server_address[1] + threading.Thread(target=server.serve_forever, daemon=True).start() + yield fake + server.shutdown() + + +@pytest.fixture(autouse=True) +def _isolated(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.delenv("S3_PARTITION_GRANULARITY", raising=False) + monkeypatch.delenv("S3_LOG_PROMPTS_ONLY", raising=False) + monkeypatch.setattr(litellm, "cold_storage_custom_logger", None) + yield + + +def _params(endpoint: str, extra: Mapping[str, object]) -> dict[str, object]: + return { + "s3_bucket_name": "logs", + "s3_region_name": "us-east-1", + "s3_endpoint_url": endpoint, + "s3_aws_access_key_id": "AKIDEXAMPLE", + "s3_aws_secret_access_key": "secret", + "s3_aws_session_token": "session-token", + **extra, + } + + +async def _logger(monkeypatch: pytest.MonkeyPatch, bucket: FakeBucket, extra: Mapping[str, object]) -> S3Logger: + monkeypatch.setattr(litellm, "s3_callback_params", _params(bucket.url, extra)) + return S3Logger() + + +def _attempt_ended_view(kwargs: Mapping[str, object], start_time: datetime) -> FieldView: + args: Final = HookArgs(request=kwargs, start_time=start_time, end_time=start_time, outcome=Outcome.success) + return FieldView(with_sources(KwargsFields(kwargs, LOGGING_KEYS), DERIVED_ATTEMPT_FIELDS, args)) + + +async def _log_native(logger: S3Logger, kwargs: Mapping[str, object], start_time: datetime = START) -> None: + line: Final = V2_ASYNC_LOGGING_LINE.set(True) + try: + await logger.on_attempt_ended( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # generated from the native manifest with its declared reads + _attempt_ended_view(kwargs, start_time), + SimpleNamespace(call=None, state=None), + ) + finally: + V2_ASYNC_LOGGING_LINE.reset(line) + + +async def _log(logger: S3Logger, payload: object) -> None: + await _log_native(logger, {"standard_logging_object": copy.deepcopy(payload)}) + + +def _signed_headers(request: Received) -> str: + match: Final = re.search(r"SignedHeaders=([^,]+)", request.headers["authorization"]) + assert match is not None + return match.group(1) + + +def _botocore_authorization( + request: Received, + port: int, + credentials: tuple[str, str, str] = ("AKIDEXAMPLE", "secret", "session-token"), + region: str = "us-east-1", +) -> str: + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + + signed: Final = _signed_headers(request).split(";") + unsigned: Final = { + name: value + for name, value in request.headers.items() + if name in signed and name not in ("host", "x-amz-date", "x-amz-security-token") + } + aws_request: Final = AWSRequest( + method=request.method, url=f"http://127.0.0.1:{port}{request.path}", data=request.body, headers=unsigned + ) + at: Final = datetime.strptime(request.headers["x-amz-date"], "%Y%m%dT%H%M%SZ") + with patch("botocore.auth.get_current_datetime", return_value=at): + S3SigV4Auth(Credentials(*credentials), "s3", region).add_auth(aws_request) + return aws_request.headers["Authorization"] + + +def _json_differences(python_value: object, rust_value: object, path: str = "$") -> list[str]: + if isinstance(python_value, dict) and isinstance(rust_value, dict): + return [ + difference + for key in dict.fromkeys([*python_value, *rust_value]) + for difference in _json_differences( + python_value.get(key, ""), rust_value.get(key, ""), f"{path}.{key}" + ) + ] + if isinstance(python_value, list) and isinstance(rust_value, list) and len(python_value) == len(rust_value): + return [ + difference + for index, (python_item, rust_item) in enumerate(zip(python_value, rust_value, strict=True)) + for difference in _json_differences(python_item, rust_item, f"{path}[{index}]") + ] + return [] if python_value == rust_value else [f"{path}: python {python_value!r} != rust {rust_value!r}"] + + +def _as_text(body: bytes) -> object: + return json.loads(body, parse_float=str, parse_int=str, parse_constant=str) + + +def _body_differences(python_body: bytes, rust_body: bytes) -> str: + try: + differences: Final = _json_differences(_as_text(python_body), _as_text(rust_body)) + except ValueError: + return "the bodies are not both JSON" + at: Final = next( + (index for index, pair in enumerate(zip(python_body, rust_body)) if pair[0] != pair[1]), + min(len(python_body), len(rust_body)), + ) + around: Final = slice(max(at - 40, 0), at + 40) + return "; ".join(differences) or f"at byte {at}: python {python_body[around]!r} != rust {rust_body[around]!r}" + + +def _body(record: Mapping[str, object]) -> bytes: + if "body_base64" in record: + return base64.b64decode(str(record["body_base64"])) + return str(record["body"]).encode("utf-8") + + +def _expected_puts(golden: Golden) -> list[dict[str, object]]: + recorded: Final = golden.expected("puts") + assert isinstance(recorded, list) + return recorded + + +LIVE_BODY_FIELDS: Final = ( + "id", + "litellm_call_id", + "trace_id", + "startTime", + "endTime", + "completionStartTime", + "response_time", +) +LIVE_RESPONSE_FIELDS: Final = ("id", "created") +LIVE_HEADERS: Final = ("content-md5", "x-amz-content-sha256", "content-disposition") +LIVE_KEY: Final = re.compile(r"\d{4}-\d\d-\d\d/time-\d\d-\d\d-\d\d-\d{6}_[^/]+\.json$") +TRACEBACK_FRAME: Final = re.compile(r'^ File "(?P[^"]+)", line \d+, in (?P.+)$') +ENVIRONMENT_ROOTS: Final = ( + (re.compile(r"^.*/litellm/"), "litellm/"), + (re.compile(r"^.*/site-packages/"), "/"), + (re.compile(r"^.*/lib/python3\.\d+/"), "/"), +) + + +def _portable_path(path: str) -> str: + return next((root.sub(name, path) for root, name in ENVIRONMENT_ROOTS if root.match(path)), path) + + +def _portable_traceback_line(line: str) -> str | None: + frame: Final = TRACEBACK_FRAME.match(line) + if frame is not None: + return f"{_portable_path(frame['path'])} in {frame['function']}" + return None if line.startswith(" ") else line + + +def _portable_traceback(text: str) -> str: + return "\n".join(line for line in map(_portable_traceback_line, text.splitlines()) if line is not None) + + +def _with_portable_traceback(body: object) -> object: + if not isinstance(body, dict): + return body + error: Final = body.get("error_information") + if not isinstance(error, dict) or not isinstance(error.get("traceback"), str): + return body + return {**body, "error_information": {**error, "traceback": _portable_traceback(error["traceback"])}} + + +def _with_portable_put(put: object) -> object: + if not isinstance(put, dict): + return put + return {**put, "body_json": _with_portable_traceback(put.get("body_json"))} + + +def _live_record(request: Received) -> dict[str, object]: + body: Final = json.loads(request.body) + assert isinstance(body, dict) + for name in LIVE_BODY_FIELDS: + if name in body: + body[name] = "" + response: Final = body.get("response") + if isinstance(response, dict): + for name in LIVE_RESPONSE_FIELDS: + if name in response: + response[name] = "" + return { + "path": LIVE_KEY.sub("/time-_.json", request.path), + "headers": {name: "" if name in LIVE_HEADERS else request.headers.get(name) for name in COMPARED_HEADERS}, + "signed_headers": _signed_headers(request), + "body_keys": list(body), + "body_json": _with_portable_traceback(body), + } + + +def _assert_same_live_puts(golden: Golden, bucket: FakeBucket, count: int) -> None: + python_puts: Final = golden.expected("live-puts") + rust_puts: Final = [_live_record(put) for put in bucket.puts()] + assert isinstance(python_puts, list) + assert len(python_puts) == len(rust_puts) == count + for python_put, rust_put in zip(python_puts, rust_puts, strict=True): + assert rust_put == _with_portable_put(python_put) + for put in bucket.puts(): + assert put.headers["authorization"] == _botocore_authorization(put, bucket.port) + + +def _assert_same_puts(golden: Golden, bucket: FakeBucket, count: int) -> None: + python_puts: Final = _expected_puts(golden) + rust_puts: Final = sorted(bucket.puts(), key=lambda r: r.path) + assert len(python_puts) == len(rust_puts) == count + for python_put, rust_put in zip(python_puts, rust_puts, strict=True): + assert rust_put.path == python_put["path"] + assert rust_put.body == _body(python_put), _body_differences(_body(python_put), rust_put.body) + for name in COMPARED_HEADERS: + assert rust_put.headers.get(name) == python_put["headers"][name], name + assert _signed_headers(rust_put) == python_put["signed_headers"] + assert rust_put.headers["authorization"] == _botocore_authorization(rust_put, bucket.port) + + +class _Model(BaseModel): + name: str + when: datetime + + +class _Unprintable: + def __str__(self) -> str: + raise RuntimeError("no text") + + +class _Opaque: + def __str__(self) -> str: + return "opaque\x00object" + + +def _payload(response_id: str = "chatcmpl-1", **overrides: object) -> dict[str, object]: + return { + "id": response_id, + "call_type": "acompletion", + "startTime": START.timestamp(), + "endTime": START.timestamp() + 1.25, + "response_cost": 0.000123, + "metadata": { + "user_api_key_team_alias": "team-α", + "user_api_key_alias": "key/one", + "user_api_key_hash": "88dc28", + }, + "messages": [{"role": "user", "content": "hello «world» 😀"}], + "response": {"choices": [{"message": {"content": "hi"}}]}, + **overrides, + } + + +def _odd_payload() -> dict[str, object]: + looped: dict[str, object] = {"name": "loop"} + looped["self"] = looped + deep: object = "bottom" + for _ in range(120): + deep = [deep] + moment: Final = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + amount: Final = Decimal("1.50") + return _payload( + "odd", + repeated={"first": moment, "second": moment, "amounts": [amount, amount]}, + numbers=(1, 2.5, 1e16, 1e-5, -0.0, float("nan"), float("inf"), 10**30, True), + members={3, 1, 2}, + model=_Model(name="m", when=START), + opaque=_Opaque(), + nul="a\x00b", + looped=looped, + deep=deep, + keys={"a\x00": 1, "a": 2, 3: "dropped"}, + ) + + +PARAM_GRID: Final = [ + pytest.param({}, id="defaults"), + pytest.param({"s3_partition_granularity": "hour", "s3_path": "base/logs/"}, id="hourly-path"), + pytest.param({"s3_use_team_prefix": True, "s3_use_key_prefix": "false"}, id="team-and-key-prefix"), + pytest.param({"s3_log_prompts_only": "yes"}, id="prompts-only"), + pytest.param({"s3_sse_kms_key_id": "kms-1"}, id="sse-kms"), + pytest.param({"s3_server_side_encryption": "AES256", "s3_sse_kms_key_id": "k"}, id="sse-aes-drops-key"), + pytest.param({"s3_strip_base64_files": True}, id="strip-base64"), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("extra", PARAM_GRID) +async def test_both_loggers_put_the_same_objects( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, extra: Mapping[str, object] +) -> None: + logger = await _logger(monkeypatch, bucket, extra) + payloads: Final = [ + _payload(), + _payload("x" * 400), + _payload("cmpl:with/odd chars=+&#?%"), + _payload( + "with-file", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "see data:image/png;base64,AAAA== inline"}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBER"}}, + ], + }, + "not a dict", + ], + ), + _odd_payload(), + ] + for payload in payloads: + await _log(logger, payload) + await logger.flush_queue() + _assert_same_puts(golden, bucket, len(payloads)) + + +@pytest.mark.asyncio +async def test_a_missing_payload_logs_nothing( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {}) + await _log(logger, None) + await logger.flush_queue() + _assert_same_puts(golden, bucket, 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "payload", + [ + pytest.param(_payload(metadata=None), id="metadata-none"), + pytest.param(_payload(metadata={"user_api_key_team_alias": 7}), id="alias-not-text"), + pytest.param({k: v for k, v in _payload().items() if k != "id"}, id="no-id"), + pytest.param(_payload(id=5), id="id-not-text"), + pytest.param(_payload(metadata={"user_api_key_team_alias": ["team-a"]}), id="alias-a-list"), + pytest.param(_payload(metadata={"user_api_key_team_alias": {"team": "a"}}), id="alias-a-dict"), + pytest.param(_payload(metadata={"user_api_key_team_alias": 10**40}), id="alias-past-i128"), + pytest.param(_payload(metadata={"user_api_key_team_alias": _Unprintable()}), id="alias-whose-str-raises"), + pytest.param(["not", "a", "mapping"], id="payload-not-a-mapping"), + ], +) +async def test_both_loggers_skip_and_count_the_same_bad_payloads( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, payload: object +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_use_team_prefix": True}) + with patch.object(s3_v3._Host, "callback_failed") as rust_failures: + await _log(logger, payload) + await logger.flush_queue() + await asyncio.sleep(0.2) + assert _expected_puts(golden) == bucket.puts() == [] + assert golden.expected("queue") == [] + assert logger.sink.queued_keys() == [] + assert golden.expected("failures") == rust_failures.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("granularity", ["day", "hour"]) +async def test_a_cold_storage_key_is_reused_when_it_matches( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, granularity: str +) -> None: + monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2") + logger = await _logger(monkeypatch, bucket, {"s3_path": "cold"}) + date: Final = "2026-11-13/" if granularity == "day" else "2026-11-13/09/" + matching: Final = f"cold/{date}time-09-05-07-000042_chatcmpl-1.json" + for response_id, key in (("chatcmpl-1", matching), ("chatcmpl-2", "elsewhere/x.json")): + await _log(logger, _payload(response_id, metadata={"cold_storage_object_key": key})) + await logger.flush_queue() + _assert_same_puts(golden, bucket, 2) + assert {r.path for r in bucket.puts()} == { + f"/logs/{matching}", + "/logs/cold/2026-11-13/time-09-05-07-000042_chatcmpl-2.json", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failures", "extra"), + [ + pytest.param([("_a\\.json", 503, "")], {}, id="partial-503-keeps-the-failed-one"), + pytest.param([(".", 500, "")], {}, id="bucket-down-keeps-everything"), + pytest.param([("_a\\.json", 400, "EntityTooLarge")], {}, id="terminal-is-dropped"), + pytest.param( + [("_a\\.json", 400, "EntityTooLarge")], + {"s3_drop_on_terminal_error": "false"}, + id="terminal-kept-when-dropping-is-off", + ), + pytest.param([(".", 503, "")], {"s3_max_queue_size": 2}, id="queue-cap-trims-oldest"), + pytest.param([("_a\\.json", 403, "AccessDenied")], {}, id="403-retried"), + ], +) +async def test_both_loggers_keep_and_drop_the_same_failed_uploads( + golden: Golden, + bucket: FakeBucket, + monkeypatch: pytest.MonkeyPatch, + failures: list[tuple[str, int, str]], + extra: Mapping[str, object], +) -> None: + bucket.failures.extend(failures) + logger = await _logger(monkeypatch, bucket, extra) + for response_id in ("a", "b", "c"): + await _log(logger, _payload(response_id)) + await logger.flush_queue() + python_queue: Final = golden.expected("queue") + assert logger.sink.queued_keys() == python_queue + assert len(bucket.puts()) == golden.expected("put-count") + bucket.failures.clear() + await logger.flush_queue() + assert logger.sink.queued_keys() == golden.expected("queue-after-recovery") == [] + assert sorted(r.path for r in bucket.puts()) == golden.expected("put-paths") + + +@pytest.mark.asyncio +async def test_batch_files_hold_the_same_lines( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_batch_file_upload": True, "s3_use_team_prefix": True}) + for response_id, team in (("a", "t1"), ("b", "t2"), ("c", "t1")): + await _log(logger, _payload(response_id, metadata={"user_api_key_team_alias": team})) + await logger.flush_queue() + name: Final = re.compile(r"/batch_\d\d-\d\d-\d\d_[0-9a-f]{32}\.jsonl$") + python_puts: Final = golden.expected("puts") + rust_puts: Final = bucket.puts() + assert isinstance(python_puts, list) + assert len(python_puts) == len(rust_puts) == 2 + for python_put, rust_put in zip(python_puts, sorted(rust_puts, key=lambda r: r.path), strict=True): + assert name.sub("", rust_put.path) == python_put["path"] + assert rust_put.body == _body(python_put), _body_differences(_body(python_put), rust_put.body) + assert rust_put.headers["content-type"] == python_put["headers"]["content-type"] == "application/x-ndjson" + + +@pytest.mark.asyncio +async def test_audit_events_and_line_items_land_at_the_same_keys( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_path": "audit-base"}) + audit: Final = {"id": "audit-1", "action": "created", "before_value": None, "updated_at": "2026-11-13"} + await logger.async_log_audit_log_event(audit) # pyright: ignore[reportArgumentType] # a partial audit payload is enough here + await logger.flush_queue() + line_items: Final = { + "batch_id": "batch/1", + "part": 3, + "finished_at": START.timestamp(), + "lines": [{"custom_id": "x", "status": "ok", "text": "ünïcode"}], + } + await logger.async_log_batch_line_items(line_items) # pyright: ignore[reportArgumentType] # a partial line items payload is enough here + clock: Final = re.compile(r"/\d\d-\d\d-\d\d_") + date: Final = re.compile(r"/\d{4}-\d\d-\d\d/") + python_puts: Final = golden.expected("puts") + rust_puts: Final = bucket.puts() + assert isinstance(python_puts, list) + assert len(python_puts) == len(rust_puts) == 2 + for python_put, rust_put in zip(python_puts, rust_puts, strict=True): + assert date.sub("/", clock.sub("/", rust_put.path)) == python_put["path"] + assert rust_put.body == _body(python_put), _body_differences(_body(python_put), rust_put.body) + for name in COMPARED_HEADERS: + assert rust_put.headers.get(name) == python_put["headers"][name], name + + +@pytest.mark.asyncio +async def test_cold_storage_reads_return_the_same_object( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {}) + await _log(logger, _payload()) + await logger.flush_queue() + key: Final = "2026-11-13/time-09-05-07-000042_chatcmpl-1.json" + python_object: Final = golden.expected("object") + rust_object: Final = await logger.get_proxy_server_request_from_cold_storage_with_object_key(key) + assert python_object is not None + assert rust_object == python_object + assert await logger.get_proxy_server_request_from_cold_storage_with_object_key("missing.json") is None + python_get: Final = golden.expected("get") + assert isinstance(python_get, dict) + rust_get: Final = bucket.received[-1] + assert rust_get.headers["x-amz-content-sha256"] == python_get["headers"]["x-amz-content-sha256"] + assert _signed_headers(rust_get) == python_get["signed_headers"] + + +@pytest.mark.asyncio +async def test_the_native_logger_answers_the_python_helpers( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_path": "p", "s3_partition_granularity": "hour"}) + assert logger.s3_path == golden.expected("s3_path") == "p" + assert logger.resolve_partition_granularity() == golden.expected("granularity") == "hour" + selection: Final = type(logger).selection + assert selection is not None + assert selection.name == "s3_v2" + assert json.loads(logger.sink.callback().manifest())["subscriptions"][0]["hook"] == "attempt_ended" + + +@pytest.mark.asyncio +async def test_stripping_base64_rewrites_the_shared_payload_the_same_way( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_strip_base64_files": True}) + original: Final = _payload( + "with-file", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "see data:image/png;base64,AAAA== inline"}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBER"}}, + ], + } + ], + ) + rust_payload: Final = copy.deepcopy(original) + await _log_native(logger, {"standard_logging_object": rust_payload}) + assert rust_payload == golden.expected("payload") != original + await logger.flush_queue() + _assert_same_puts(golden, bucket, 1) + + +@pytest.mark.asyncio +async def test_a_payload_changed_before_the_flush_is_stored_as_it_is_at_the_flush( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {}) + payload: Final = _payload() + await _log_native(logger, {"standard_logging_object": payload}) + payload["metadata"]["late"] = "written after the event" # pyright: ignore[reportIndexIssue] # metadata is the dict _payload builds + payload["response_cost"] = 99.0 + await logger.flush_queue() + _assert_same_puts(golden, bucket, 1) + stored: Final = json.loads(bucket.puts()[0].body) + assert stored["metadata"]["late"] == "written after the event" + assert stored["response_cost"] == 0.000123 + assert logger.sink.stashed() == 0 + + +def test_both_loggers_need_a_running_event_loop(golden: Golden, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "s3_callback_params", _params("http://127.0.0.1:9", {})) + with pytest.raises(RuntimeError, match="no running event loop") as raised: + S3Logger() + assert str(raised.value) == golden.expected("error") + + +@pytest.mark.asyncio +async def test_the_native_logger_flushes_on_its_own_after_the_interval( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(s3_v3, "DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 0.05) + logger = await _logger(monkeypatch, bucket, {}) + await _log_native(logger, {"standard_logging_object": _payload()}) + for _ in range(100): + if bucket.puts(): + break + await asyncio.sleep(0.02) + assert [r.path for r in bucket.puts()] == ["/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("environment", "region"), + [ + pytest.param({"AWS_REGION_NAME": "eu-central-1"}, "eu-central-1", id="region-name"), + pytest.param({"AWS_REGION_NAME": "eu-central-1", "AWS_REGION": "ap-south-1"}, "ap-south-1", id="region-wins"), + pytest.param({}, "us-west-2", id="default"), + pytest.param({"AWS_REGION": "Bad_Region"}, None, id="invalid"), + ], +) +async def test_without_a_region_param_both_sign_with_the_same_environment_region( + golden: Golden, + bucket: FakeBucket, + monkeypatch: pytest.MonkeyPatch, + environment: Mapping[str, str], + region: str | None, +) -> None: + for name in ("AWS_REGION_NAME", "AWS_REGION"): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + logger = await _logger(monkeypatch, bucket, {"s3_region_name": None}) + with patch.object(s3_v3._Host, "callback_failed") as rust_failures: + await _log(logger, _payload()) + await logger.flush_queue() + await asyncio.sleep(0.2) + python_puts: Final = golden.expected("put-count") + python_failure_count: Final = golden.expected("failures") + if region is None: + assert python_puts == len(bucket.puts()) == 0 + assert python_failure_count == rust_failures.call_count == 1 + return + assert python_puts == len(bucket.puts()) == 1 + put: Final = bucket.puts()[0] + assert put.headers["authorization"] == _botocore_authorization(put, bucket.port, region=region) + + +@dataclass +class FakeSts: + calls: list[Mapping[str, list[str]]] = field(default_factory=list) + port: int = 0 + + +STS_REPLY: Final = ( + '' + "AKIDSTS" + "sts-secretsts-token" + "2099-01-01T00:00:00Z" + "subject" + "arn:aws:sts::123456789012:assumed-role/logs/s3" + "AROA:s310" + "" +) + + +@pytest.fixture +def sts() -> Iterator[FakeSts]: + fake: Final = FakeSts() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_: object) -> None: + return + + def do_POST(self) -> None: + body: Final = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + fake.calls.append(parse_qs(body.decode())) + reply: Final = STS_REPLY.encode() + self.send_response(200) + self.send_header("Content-Type", "text/xml") + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + fake.port = server.server_address[1] + threading.Thread(target=server.serve_forever, daemon=True).start() + yield fake + server.shutdown() + + +def _sts_session(sts: FakeSts, values: object) -> object: + if not isinstance(values, list): + return values + return [str(value).replace(f"s3-{sts.port}", "s3-") for value in values] + + +@pytest.mark.asyncio +async def test_web_identity_assumes_the_role_with_the_same_session_policy( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, sts: FakeSts +) -> None: + for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE", "AWS_EXTERNAL_ID"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("S3_V3_TEST_OIDC_TOKEN", "oidc-token-value") + identity: Final = { + "s3_aws_access_key_id": None, + "s3_aws_secret_access_key": None, + "s3_aws_session_token": None, + "s3_aws_web_identity_token": "oidc/env/S3_V3_TEST_OIDC_TOKEN", + "s3_aws_role_name": "arn:aws:iam::123456789012:role/logs", + "s3_aws_session_name": f"s3-{sts.port}", + "s3_aws_sts_endpoint": f"http://127.0.0.1:{sts.port}", + } + logger = await _logger(monkeypatch, bucket, identity) + await _log(logger, _payload()) + await logger.flush_queue() + compared: Final = ("Action", "RoleArn", "RoleSessionName", "WebIdentityToken", "DurationSeconds", "Policy") + assert len(sts.calls) == 1 + python_call: Final = golden.expected("sts-call") + assert isinstance(python_call, dict) + rust_call: Final = sts.calls[-1] + for name in compared: + assert _sts_session(sts, rust_call.get(name)) == python_call[name], name + assert rust_call["WebIdentityToken"] == ["oidc-token-value"] + assert json.loads(rust_call["Policy"][0]) == json.loads(str(golden.expected("policy"))) + python_put: Final = golden.expected("put") + assert isinstance(python_put, dict) + assert python_put["headers"]["x-amz-security-token"] == "sts-token" + put: Final = bucket.puts()[0] + assert put.headers["x-amz-security-token"] == "sts-token" + assert put.headers["authorization"] == _botocore_authorization( + put, bucket.port, credentials=("AKIDSTS", "sts-secret", "sts-token") + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("token", [True, 7, "abc", ""]) +async def test_a_web_identity_token_botocore_rejects_fails_both_before_sts( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, sts: FakeSts, token: object +) -> None: + for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE", "AWS_EXTERNAL_ID"): + monkeypatch.delenv(name, raising=False) + reference: Final = "s3-v3-test/oidc-token" + + def secret(name: str, default_value: object = None) -> object: + return token if name == reference else default_value + + monkeypatch.setattr(base_aws_llm, "get_secret", secret) + monkeypatch.setattr(s3_v3, "get_secret", secret) + identity: Final = { + "s3_aws_access_key_id": None, + "s3_aws_secret_access_key": None, + "s3_aws_session_token": None, + "s3_aws_web_identity_token": reference, + "s3_aws_role_name": "arn:aws:iam::123456789012:role/logs", + "s3_aws_session_name": f"s3-rejected-{sts.port}", + "s3_aws_sts_endpoint": f"http://127.0.0.1:{sts.port}", + } + logger = await _logger(monkeypatch, bucket, identity) + with patch.object(s3_v3._Host, "callback_failed") as rust_failures: + await _log(logger, _payload()) + await logger.flush_queue() + assert sts.calls == [] + assert _expected_puts(golden) == bucket.puts() == [] + assert golden.expected("failures") == rust_failures.call_count + assert rust_failures.call_count == 1 + assert logger.sink.queued_keys() == golden.expected("queue") + python_error: Final = golden.expected("error") + with pytest.raises(ValueError, match=r"^Parameter validation failed:") as rust_error: + s3_v3._Host().web_identity_token(reference) + assert str(rust_error.value) == python_error + + +@pytest.mark.asyncio +@pytest.mark.parametrize("global_disable", [False, True]) +@pytest.mark.parametrize("no_log", [None, True, "true"]) +@pytest.mark.parametrize("disabled_header", [None, "s3_v2", "langfuse, S3_V2"]) +async def test_the_native_opt_out_matches_should_run_callback( + golden: Golden, + bucket: FakeBucket, + monkeypatch: pytest.MonkeyPatch, + global_disable: bool, + no_log: object, + disabled_header: str | None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy import proxy_server + + monkeypatch.setattr(litellm, "global_disable_no_log_param", global_disable) + monkeypatch.setattr(litellm, "allow_dynamic_callback_disabling", True) + monkeypatch.setattr(proxy_server, "premium_user", True) + logger = await _logger(monkeypatch, bucket, {}) + logging_obj: Final = Logging( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + litellm_call_id="call-1", + start_time=START, + function_id="fn-1", + ) + litellm_params: Final = { + **({} if no_log is None else {"no-log": no_log}), + "proxy_server_request": { + "headers": {} if disabled_header is None else {"x-litellm-disable-callbacks": disabled_header} + }, + } + expected: Final = golden.expected("should_run_callback") + assert isinstance(expected, bool) + kwargs: Final = { + "litellm_params": litellm_params, + "standard_callback_dynamic_params": logging_obj.standard_callback_dynamic_params, + "standard_logging_object": _payload(), + } + assert s3_v3.legacy_should_log(kwargs) is expected + await _log_native(logger, kwargs) + await logger.flush_queue() + assert len(bucket.puts()) == int(expected) + + +async def _until_queued(logger: S3Logger) -> None: + for _ in range(100): + if logger.sink.queued_keys(): + return + await asyncio.sleep(0.02) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("proxy_scope", [False, True]) +@pytest.mark.parametrize("global_disable", [False, True]) +@pytest.mark.parametrize("no_log", [False, True]) +async def test_an_acompletion_reaches_both_loggers_alike( + golden: Golden, + bucket: FakeBucket, + monkeypatch: pytest.MonkeyPatch, + global_disable: bool, + no_log: bool, + proxy_scope: bool, +) -> None: + monkeypatch.setattr(litellm, "global_disable_no_log_param", global_disable) + logger = await _logger(monkeypatch, bucket, {"s3_use_team_prefix": True}) + monkeypatch.setattr(litellm, "callbacks", [logger]) + with request_scope(RequestSelection(no_log=no_log and not global_disable)) if proxy_scope else nullcontext(): + await litellm.acompletion( + model="gpt-4o", + messages=[{"role": "user", "content": "hello «world»"}], + mock_response="hi there", + metadata={"user_api_key_team_alias": "team-a"}, + **({"no-log": True} if no_log else {}), + ) + await _until_queued(logger) + await logger.flush_queue() + expected: Final = 0 if no_log and not global_disable else 1 + _assert_same_live_puts(golden, bucket, expected) + assert all(put.path.startswith("/logs/team-a/") for put in bucket.puts()) + + +class _Mutating: + def __init__(self, target: dict[str, object]) -> None: + self.target: Final = target + + def __str__(self) -> str: + self.target[f"grown-{len(self.target)}"] = True + return "mutating" + + +@pytest.mark.asyncio +async def test_a_payload_that_changes_while_it_is_rendered_is_kept_for_retry_by_both_loggers( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {}) + racing: Final = _payload("racing") + metadata: Final = racing["metadata"] + assert isinstance(metadata, dict) + metadata["mutating"] = _Mutating(metadata) # pyright: ignore[reportUnknownArgumentType] # the metadata dict _payload builds + with patch.object(s3_v3._Host, "callback_failed") as rust_failures: + await _log_native(logger, {"standard_logging_object": racing}) + await _log(logger, _payload("steady")) + await logger.flush_queue() + await asyncio.sleep(0.2) + assert logger.sink.queued_keys() == golden.expected("queue") + assert len(logger.sink.queued_keys()) == 1 + assert golden.expected("failures") == rust_failures.call_count + assert rust_failures.call_count == 1 + _assert_same_puts(golden, bucket, 1) + assert not logger.periodic_flush_task.done() + + +class _FlakySink: + def __init__(self, failures: tuple[BaseException, ...]) -> None: + self.failures: Final = list(failures) + self.flushed = 0 + + async def flush(self) -> None: + self.flushed += 1 + if self.failures: + raise self.failures.pop(0) + + +class _Panic(BaseException): + pass + + +@pytest.mark.asyncio +async def test_the_periodic_flush_keeps_running_after_a_flush_fails() -> None: + sink: Final = _FlakySink((RuntimeError("bucket gone"), _Panic("a PanicException is a BaseException too"))) + task: Final = asyncio.create_task(s3_v3._periodic_flush(sink, 0.01)) # pyright: ignore[reportArgumentType] # the loop only calls flush + for _ in range(100): + if sink.flushed >= 3: + break + await asyncio.sleep(0.01) + assert sink.flushed >= 3 + assert not task.done() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +async def _settled(logger: S3Logger) -> None: + for _ in range(3): + await asyncio.sleep(0.3) + await logger.flush_queue() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_a_sync_completion_reaches_both_loggers_alike( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + logger = await _logger(monkeypatch, bucket, {}) + monkeypatch.setattr(litellm, "callbacks", [logger]) + + def call() -> None: + response = litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "hello"}], + mock_response="hi there", + stream=stream, + ) + if stream: + for _ in response: # pyright: ignore[reportGeneralTypeIssues] # a stream when stream=True + pass + + await asyncio.to_thread(call) + await _settled(logger) + _assert_same_puts(golden, bucket, 0) + + +@pytest.mark.asyncio +async def test_a_sync_line_delivery_is_skipped_by_the_native_logger( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {}) + await logger.run_rust( + Hook.attempt_ended, + frozenset({Field.deprecated_kwargs}), + SimpleNamespace(deprecated_kwargs={"standard_logging_object": _payload()}), + SimpleNamespace(call=None, state=None), # pyright: ignore[reportArgumentType] # run_rust reads only call and state + ) + assert logger.sink.queued_keys() == [] + assert logger.sink.stashed() == 0 + + +@pytest.mark.asyncio +async def test_readiness_and_the_callbacks_view_see_the_native_logger_as_s3_v2( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.proxy_server import _hidden_runtime_callback_names + + logger = await _logger(monkeypatch, bucket, {}) + python_view: Final = golden.expected("callbacks-view") + assert python_view == {"str-names-s3-logger": True, "hidden": True} + assert "S3Logger" in str(logger) + monkeypatch.setattr(litellm, "callbacks", [logger]) + name: Final = litellm.logging_callback_manager._get_callback_string(logger) + assert name in _hidden_runtime_callback_names(frozenset({"s3_v2"})) + + +def _put_paths(bucket: FakeBucket) -> list[str]: + return [request.path for request in bucket.puts()] + + +def _queues(golden: Golden, logger: S3Logger) -> tuple[object, list[str]]: + python_queue: Final = golden.expected("queue") + return python_queue, logger.sink.queued_keys() + + +@pytest.mark.asyncio +async def test_a_failed_acompletion_reaches_both_loggers_alike( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_use_key_prefix": True}) + monkeypatch.setattr(litellm, "callbacks", [logger]) + with pytest.raises(litellm.RateLimitError): + await litellm.acompletion( + model="gpt-4o", + messages=[{"role": "user", "content": "hello"}], + mock_response="litellm.RateLimitError", + metadata={"user_api_key_alias": "key-a"}, + ) + await _settled(logger) + _assert_same_live_puts(golden, bucket, 1) + assert _put_paths(bucket)[0].startswith("/logs/key-a/") + assert json.loads(bucket.puts()[0].body)["status"] == "failure" + + +@pytest.mark.asyncio +async def test_a_streamed_acompletion_reaches_both_loggers_alike( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {}) + monkeypatch.setattr(litellm, "callbacks", [logger]) + response: Final = await litellm.acompletion( + model="gpt-4o", + messages=[{"role": "user", "content": "hello"}], + mock_response="hi there, streamed", + stream=True, + ) + async for _ in response: # pyright: ignore[reportGeneralTypeIssues] # a stream when stream=True + pass + await _settled(logger) + _assert_same_live_puts(golden, bucket, 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("wait", "kept"), [pytest.param(0.0, True, id="young"), pytest.param(1.2, False, id="expired")] +) +async def test_both_loggers_drop_an_upload_retrying_past_the_max_retry_age( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, wait: float, kept: bool +) -> None: + bucket.failures.append(("_a\\.json", 404, "")) + logger = await _logger(monkeypatch, bucket, {"s3_max_retry_age_seconds": 1}) + for response_id in ("a", "b"): + await _log(logger, _payload(response_id)) + await logger.flush_queue() + await asyncio.sleep(wait) + await _log(logger, _payload("c")) + await logger.flush_queue() + python_queue, rust_queue = _queues(golden, logger) + assert rust_queue == python_queue + assert bool(rust_queue) is kept + + +@pytest.mark.asyncio +async def test_both_loggers_upload_in_the_same_order_with_requeued_items_last( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + bucket.failures.extend([("_a\\.json", 404, ""), ("_b\\.json", 404, "")]) + logger = await _logger(monkeypatch, bucket, {"s3_max_concurrent_uploads": 1}) + for response_id in ("a", "b", "c"): + await _log(logger, _payload(response_id)) + await logger.flush_queue() + python_queue, rust_queue = _queues(golden, logger) + assert rust_queue == python_queue + bucket.failures.clear() + for response_id in ("d", "e"): + await _log(logger, _payload(response_id)) + await logger.flush_queue() + assert _put_paths(bucket) == golden.expected("put-paths") + assert [path.rsplit("_", 1)[-1] for path in _put_paths(bucket)] == [ + "a.json", + "b.json", + "c.json", + "d.json", + "e.json", + "a.json", + "b.json", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("adaptive", "peak"), [pytest.param(True, 3, id="adaptive"), pytest.param(False, 1, id="fixed")] +) +async def test_both_loggers_widen_the_adaptive_upload_limit_alike( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, adaptive: bool, peak: int +) -> None: + bucket.delays.append((".", 0.05)) + logger = await _logger( + monkeypatch, + bucket, + {"s3_max_concurrent_uploads": 1, "s3_adaptive_concurrency": adaptive, "s3_max_adaptive_concurrency": 3}, + ) + for index in range(10): + await _log(logger, _payload(f"r{index}")) + await logger.flush_queue() + _assert_same_puts(golden, bucket, 10) + assert bucket.peak == golden.expected("peak") == peak + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("batch_size", "puts"), [(1, 2), (2, 2), (3, 0)]) +async def test_audit_events_flush_at_the_same_batch_size( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, batch_size: int, puts: int +) -> None: + monkeypatch.setattr(s3_v3, "DEFAULT_S3_BATCH_SIZE", batch_size) + logger = await _logger(monkeypatch, bucket, {}) + for number in (1, 2): + audit = {"id": f"audit-{number}", "action": "created", "updated_at": "2026-11-13"} + await logger.async_log_audit_log_event(audit) # pyright: ignore[reportArgumentType] # a partial audit payload is enough here + python_bodies: Final = golden.expected("bodies") + assert isinstance(python_bodies, list) + assert len(bucket.puts()) == len(python_bodies) == puts + assert sorted(r.body.decode() for r in bucket.puts()) == python_bodies + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["dropped-connection", "timeout"]) +async def test_both_loggers_keep_uploads_that_fail_in_transport( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, failure: str +) -> None: + monkeypatch.setattr(s3_v3, "S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS", 0.3) + if failure == "timeout": + bucket.delays.append(("_a\\.json", 1.0)) + else: + bucket.failures.append(("_a\\.json", 0, "")) + logger = await _logger(monkeypatch, bucket, {}) + with patch.object(s3_v3._Host, "callback_failed") as rust_failures: + for response_id in ("a", "b"): + await _log(logger, _payload(response_id)) + await logger.flush_queue() + await asyncio.sleep(0.2) + python_queue, rust_queue = _queues(golden, logger) + assert rust_queue == python_queue + assert [key.rsplit("_", 1)[-1] for key in rust_queue] == ["a.json"] + assert golden.expected("failures") == rust_failures.call_count + assert rust_failures.call_count == 1 + attempts: Final = 3 if failure == "timeout" else 6 + python_paths: Final = golden.expected("put-paths") + assert isinstance(python_paths, list) + assert [str(path).rsplit("_", 1)[-1] for path in python_paths].count("a.json") == attempts + assert [path.rsplit("_", 1)[-1] for path in _put_paths(bucket)].count("a.json") == attempts + + +_NO_METADATA: Final = object() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "puts"), + [ + pytest.param(_NO_METADATA, 1, id="missing"), + pytest.param(None, 1, id="none"), + pytest.param("text", 0, id="text"), + pytest.param(["listed"], 0, id="list"), + ], +) +async def test_cold_storage_handles_odd_metadata_like_python( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, metadata: object, puts: int +) -> None: + monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2") + logger = await _logger(monkeypatch, bucket, {}) + payload: Final = _payload() + if metadata is _NO_METADATA: + del payload["metadata"] + else: + payload["metadata"] = metadata + with patch.object(s3_v3._Host, "callback_failed") as rust_failures: + await _log(logger, payload) + await logger.flush_queue() + await asyncio.sleep(0.2) + _assert_same_puts(golden, bucket, puts) + assert golden.expected("failures") == rust_failures.call_count + assert rust_failures.call_count == 1 - puts + assert logger.sink.stashed() == 0 + + +AWARE_START: Final = datetime(2026, 11, 13, 23, 30, 0, 5, tzinfo=timezone(timedelta(hours=5))) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("start_time", "payload_start", "logging_start"), + [ + pytest.param(AWARE_START, AWARE_START.timestamp(), AWARE_START, id="aware"), + pytest.param(START, None, START, id="payload-without-start"), + pytest.param(START, START.timestamp(), START + timedelta(seconds=3), id="handler-start-differs"), + ], +) +async def test_both_loggers_name_the_object_from_the_same_start_time( + golden: Golden, + bucket: FakeBucket, + monkeypatch: pytest.MonkeyPatch, + start_time: datetime, + payload_start: float | None, + logging_start: datetime, +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_partition_granularity": "hour"}) + payload: Final = _payload() + if payload_start is None: + del payload["startTime"] + else: + payload["startTime"] = payload_start + await _log_native( + logger, {"standard_logging_object": copy.deepcopy(payload), "start_time": logging_start}, start_time + ) + await logger.flush_queue() + _assert_same_puts(golden, bucket, 1) + stamp: Final = start_time.strftime("%Y-%m-%d/%H/time-%H-%M-%S-%f") + assert _put_paths(bucket) == [f"/logs/{stamp}_chatcmpl-1.json"] + + +@pytest.mark.asyncio +async def test_cold_storage_reads_without_a_region_send_nothing_like_python( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_region_name": None}) + bucket.objects["/logs/k.json"] = b'{"stored": 1}' + python_object: Final = golden.expected("object") + rust_object: Final = await logger.get_proxy_server_request_from_cold_storage_with_object_key("k.json") + assert rust_object is python_object is None + assert golden.expected("received") == [] + assert bucket.received == [] + + +@pytest.mark.asyncio +async def test_params_json_cannot_hold_still_build_the_native_logger( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + from botocore.config import Config + + logger = await _logger( + monkeypatch, bucket, {"s3_config": Config(retries={"max_attempts": 1}), "s3_max_concurrent_uploads": 10**30} + ) + await _log(logger, _payload()) + await logger.flush_queue() + _assert_same_puts(golden, bucket, 1) + + +class _AliasLookupFails(dict[str, object]): + def get(self, key: str, default: object = None) -> object: + if key == "user_api_key_team_alias": + raise RuntimeError("the alias lookup failed") + return super().get(key, default) + + +@pytest.mark.asyncio +async def test_the_attempt_ended_hook_only_enqueues_while_s3_is_slow( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(s3_v3, "DEFAULT_S3_BATCH_SIZE", 1) + logger = await _logger(monkeypatch, bucket, {}) + bucket.delays.append((".", 3.0)) + started: Final = time.monotonic() + await _log_native(logger, {"standard_logging_object": _payload()}) + hook_seconds: Final = time.monotonic() - started + monkeypatch.setattr(litellm, "callbacks", [logger]) + await litellm.acompletion(model="gpt-4o", messages=[{"role": "user", "content": "hello"}], mock_response="hi there") + for _ in range(100): + if len(logger.sink.queued_keys()) == 2: + break + await asyncio.sleep(0.02) + await asyncio.sleep(0.2) + assert hook_seconds < 1.0 + assert golden.expected("queue-length") == 1 + assert len(logger.sink.queued_keys()) == 2 + assert _expected_puts(golden) == bucket.puts() == [] + + +@pytest.mark.asyncio +async def test_a_payload_whose_alias_lookup_fails_is_counted_and_released( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_use_team_prefix": True}) + with patch.object(s3_v3._Host, "callback_failed") as rust_failures: + await _log(logger, _payload(metadata=_AliasLookupFails(team="t"))) + await logger.flush_queue() + await asyncio.sleep(0.2) + _assert_same_puts(golden, bucket, 0) + assert golden.expected("failures") == rust_failures.call_count + assert rust_failures.call_count == 1 + assert logger.sink.stashed() == 0 + + +@pytest.mark.asyncio +async def test_prompts_only_adds_a_null_response_to_a_payload_without_one_like_python( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_log_prompts_only": True}) + payload: Final = {key: value for key, value in _payload().items() if key != "response"} + await _log(logger, payload) + await logger.flush_queue() + _assert_same_puts(golden, bucket, 1) + assert list(json.loads(bucket.puts()[0].body))[-1] == "response" + + +@pytest.mark.asyncio +async def test_the_dispatched_native_strips_the_messages_before_the_payload_is_copied( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.litellm_core_utils import litellm_logging + + logger = await _logger(monkeypatch, bucket, {"s3_strip_base64_files": True}) + payload: Final = _payload( + "stripped", + messages=[ + {"role": "user", "content": [{"type": "text", "text": "see data:image/png;base64,AAAA== inline"}]}, + "not a message", + ], + ) + kwargs: Final = {"standard_logging_object": copy.deepcopy(payload)} + await litellm_logging._run_callbacks( + Hook.attempt_ended, + [logger], + kwargs, + lambda _callback, _kwargs: None, + args=HookArgs(request=kwargs, start_time=START, end_time=START, outcome=Outcome.success), + ) + await _until_queued(logger) + await logger.flush_queue() + _assert_same_puts(golden, bucket, 1) + assert json.loads(bucket.puts()[0].body)["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "see [BASE64_REDACTED]"}]} + ] + + +class _FalsyAlias: + def __bool__(self) -> bool: + return False + + def __str__(self) -> str: + return "falsy" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "alias", + [ + pytest.param([], id="empty-list"), + pytest.param({}, id="empty-dict"), + pytest.param(set(), id="empty-set"), + pytest.param(_FalsyAlias(), id="falsy-object"), + pytest.param(0.0, id="zero-float"), + pytest.param("", id="empty-text"), + ], +) +async def test_a_falsy_alias_of_any_type_adds_no_prefix_like_python( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, alias: object +) -> None: + logger = await _logger(monkeypatch, bucket, {"s3_use_team_prefix": True, "s3_use_key_prefix": True}) + await _log(logger, _payload(metadata={"user_api_key_team_alias": alias})) + await logger.flush_queue() + _assert_same_puts(golden, bucket, 1) + assert _put_paths(bucket) == ["/logs/2026-11-13/time-09-05-07-000042_chatcmpl-1.json"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_excluded_payload_fields_are_left_out_like_python( + golden: Golden, bucket: FakeBucket, monkeypatch: pytest.MonkeyPatch, failed: bool +) -> None: + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["response", "metadata", "messages"]) + logger = await _logger(monkeypatch, bucket, {"s3_use_team_prefix": True}) + monkeypatch.setattr(litellm, "callbacks", [logger]) + with pytest.raises(litellm.RateLimitError) if failed else nullcontext(): + await litellm.acompletion( + model="gpt-4o", + messages=[{"role": "user", "content": "hello"}], + mock_response="litellm.RateLimitError" if failed else "hi there", + metadata={"user_api_key_team_alias": "team-a"}, + ) + await _settled(logger) + _assert_same_live_puts(golden, bucket, 1) + + +RECORDED_TRACEBACK: Final = ( + ' File "/home/user/wt/rust-s3/litellm/main.py", line 956, in acompletion\n' + " init_response: Final = await loop.run_in_executor(None, func_with_context)\n" + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" + ' File "/usr/local/lib/python3.13/concurrent/futures/thread.py", line 59, in run\n' + " result = self.fn(*self.args, **self.kwargs)\n" + ' File "/home/user/wt/rust-s3/litellm/main.py", line 1235, in mock_completion\n' + " _handle_mock_potential_exceptions(\n" + " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n" + " ...<3 lines>...\n" + " )\n" +) + + +@pytest.mark.parametrize( + ("other", "same"), + [ + ( + ' File "/home/runner/work/litellm/litellm/litellm/main.py", line 970, in acompletion\n' + " init_response: Final = await loop.run_in_executor(None, func_with_context)\n" + ' File "/opt/hostedtoolcache/Python/3.12.16/x64/lib/python3.12/concurrent/futures/thread.py",' + " line 58, in run\n" + " result = self.fn(*self.args, **self.kwargs)\n" + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" + ' File "/home/runner/work/litellm/litellm/litellm/main.py", line 1240, in mock_completion\n' + " _handle_mock_potential_exceptions(\n", + True, + ), + ( + ' File "/home/runner/work/litellm/litellm/litellm/main.py", line 956, in acompletion\n' + ' File "/usr/local/lib/python3.13/concurrent/futures/thread.py", line 59, in run\n' + ' File "/home/runner/work/litellm/litellm/litellm/utils.py", line 1235, in mock_completion\n', + False, + ), + ( + ' File "/home/runner/work/litellm/litellm/litellm/main.py", line 956, in acompletion\n' + ' File "/usr/local/lib/python3.13/concurrent/futures/thread.py", line 59, in run\n' + ' File "/home/runner/work/litellm/litellm/litellm/main.py", line 1235, in completion\n', + False, + ), + ( + ' File "/home/runner/work/litellm/litellm/litellm/main.py", line 956, in acompletion\n' + ' File "/home/runner/work/litellm/litellm/litellm/main.py", line 1235, in mock_completion\n', + False, + ), + ], + ids=["other-machine-and-python", "other-litellm-file", "other-function", "missing-frame"], +) +def test_a_recorded_traceback_matches_only_the_same_frames_from_another_environment(other: str, same: bool) -> None: + assert (_portable_traceback(other) == _portable_traceback(RECORDED_TRACEBACK)) is same + assert _portable_traceback(RECORDED_TRACEBACK) == ( + "litellm/main.py in acompletion\n/concurrent/futures/thread.py in run\nlitellm/main.py in mock_completion" + ) diff --git a/tests/unit/integrations/test_s3_v3_strip_writer.py b/tests/unit/integrations/test_s3_v3_strip_writer.py new file mode 100644 index 00000000000..3678624ca09 --- /dev/null +++ b/tests/unit/integrations/test_s3_v3_strip_writer.py @@ -0,0 +1,445 @@ +import asyncio +import copy +import datetime +import hashlib +import json +from collections.abc import Iterator, Mapping, Sequence +from typing import Final +from unittest.mock import MagicMock + +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.field_sources import HookArgs +from litellm.integrations.custom_logger_v2.manifest import build_manifest +from litellm.integrations.s3_v3 import STRIP_CALLBACK_NAME, S3Logger, S3StripBase64Files +from litellm.litellm_core_utils import litellm_logging +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.rust_bridge import _native +from litellm.types.integrations.custom_logger_v2 import Outcome +from litellm.types.utils import ModelResponse +from tests.unit.integrations.s3_v3_rust_support import ScriptedBucket, scripted_bucket + +pytestmark = pytest.mark.skipif(not hasattr(_native, "NativeS3Log"), reason="the built _native has no NativeS3Log") + +START: Final = datetime.datetime(2025, 9, 14, 9, 5, 7, 42) + +RECORDED_AT_86B70FEF16: Final = { + "stripped": ( + "/test-bucket/2025-09-14/time-09-05-07-000042_stripped.json", + "31e903035e2185abea6cac867139417cb42a37a0da6b7d4efea47b9d1c24f1e1", + ), + "excluded": ( + "/test-bucket/2025-09-14/time-09-05-07-000042_excluded.json", + "5958206c8eee87b3221e6ee760b48a0ead92343768adbf8f6f80295bd4d51329", + ), + "failed": ( + "/test-bucket/2025-09-14/time-09-05-07-000042_failed.json", + "04dcb95630cb5b61fbf96560be30411b5fcacebbb4f4ed03aeaccc2e0687c95f", + ), +} + +STRIPPED_MESSAGES: Final = [ + {"role": "user", "content": [{"type": "text", "text": "see [BASE64_REDACTED]"}]}, + {"role": "assistant", "content": "plain [BASE64_REDACTED]", "name": "[BASE64_REDACTED]"}, +] + + +def _messages() -> list[object]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "see data:image/png;base64,AAAA== inline"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,BBBB"}}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBER"}}, + ], + }, + {"role": "assistant", "content": "plain data:audio/wav;base64,CCCC tail", "name": "data:image/gif;base64,DDDD"}, + "not a message", + ] + + +def _payload(response_id: str, status: str = "success") -> dict[str, object]: + return { + "id": response_id, + "status": status, + "metadata": {}, + "response": {"text": "ok"}, + "messages": _messages(), + } + + +class _Reader(CustomLogger): + def __init__(self, name: str) -> None: + super().__init__() + self.name: Final = name + self.seen: list[object] = [] + + def saw(self, kwargs: object) -> None: + payload: Final = kwargs.get("standard_logging_object") if isinstance(kwargs, Mapping) else None + self.seen.append(copy.deepcopy(payload.get("messages")) if isinstance(payload, Mapping) else None) + + async def async_log_success_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.saw(kwargs) + + +class _SyncReader(_Reader): + def log_success_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.saw(kwargs) + + +@pytest.fixture(autouse=True) +def _isolated(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + for name in ("S3_PARTITION_GRANULARITY", "S3_LOG_PROMPTS_ONLY", "AWS_REGION", "AWS_REGION_NAME"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "cold_storage_custom_logger", None) + monkeypatch.setattr(litellm, "s3_callback_params", None) + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", None) + for name in ( + "callbacks", + "input_callback", + "success_callback", + "failure_callback", + "_async_success_callback", + "_async_failure_callback", + ): + monkeypatch.setattr(litellm, name, []) + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", []) + yield + + +@pytest.fixture +def bucket() -> Iterator[ScriptedBucket]: + with scripted_bucket() as scripted: + yield scripted + + +def _params(bucket: ScriptedBucket, strip: bool, region: str = "us-east-1") -> dict[str, object]: + return { + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": bucket.url, + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + "s3_region_name": region, + **({"s3_strip_base64_files": True} if strip else {}), + } + + +def _s3(bucket: ScriptedBucket, strip: bool = True, region: str = "us-east-1") -> S3Logger: + logger: Final = S3Logger(_params(bucket, strip, region)) + _ = logger.periodic_flush_task.cancel() + return logger + + +def _registered(*callbacks: object) -> list[object]: + for callback in callbacks: + assert isinstance(callback, CustomLogger | S3Logger) + litellm.logging_callback_manager.add_litellm_async_success_callback(callback) + return list(litellm._async_success_callback) + + +def _deliver(callback: object, kwargs: object) -> None: + if isinstance(callback, _Reader): + callback.saw(kwargs) + + +def _prepared(_callback: object, _kwargs: object) -> None: + return None + + +async def _run_line(callbacks: Sequence[object], kwargs: dict[str, object], outcome: Outcome = Outcome.success) -> None: + if outcome is Outcome.success: + await litellm_logging._run_callbacks( + Hook.log_prepare, + callbacks, + kwargs, + _prepared, + args=HookArgs(request=kwargs, start_time=START, end_time=START), + ) + await litellm_logging._run_callbacks( + Hook.attempt_ended, + callbacks, + kwargs, + _deliver, + args=HookArgs(request=kwargs, start_time=START, end_time=START, outcome=outcome), + ) + + +async def _flushed(logger: S3Logger) -> None: + for _ in range(100): + if logger.sink.queued_keys(): + break + await asyncio.sleep(0.02) + await logger.flush_queue() + + +def _recorded(bucket: ScriptedBucket, case: str) -> bytes: + (put,) = bucket.puts() + assert (put.path, hashlib.sha256(put.body).hexdigest()) == RECORDED_AT_86B70FEF16[case] + return put.body + + +@pytest.mark.asyncio +async def test_strip_on_success_patches_readers_before_and_after_s3_and_uploads_the_recorded_object( + bucket: ScriptedBucket, +) -> None: + before: Final = _Reader("before") + after: Final = _Reader("after") + s3: Final = _s3(bucket) + callbacks: Final = _registered(before, s3, after) + assert callbacks == [before, s3.v2_companion, s3, after] + kwargs: Final[dict[str, object]] = {"standard_logging_object": _payload("stripped"), "litellm_params": {}} + + await _run_line(callbacks, kwargs) + await _flushed(s3) + + assert before.seen == after.seen == [STRIPPED_MESSAGES] + body: Final = _recorded(bucket, "stripped") + assert json.loads(body)["messages"] == STRIPPED_MESSAGES + + +@pytest.mark.asyncio +async def test_the_object_matches_the_attempt_ended_strip_alone(bucket: ScriptedBucket) -> None: + patched: Final = _s3(bucket) + await _run_line(_registered(patched), {"standard_logging_object": _payload("same"), "litellm_params": {}}) + await _flushed(patched) + unpatched: Final = _s3(bucket) + await _run_line([unpatched], {"standard_logging_object": _payload("same"), "litellm_params": {}}) + await _flushed(unpatched) + + first, second = bucket.puts() + assert (first.path, first.body) == (second.path, second.body) + + +@pytest.mark.asyncio +async def test_strip_on_with_excluded_fields_uploads_the_recorded_object( + bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["response"]) + after: Final = _Reader("after") + s3: Final = _s3(bucket) + kwargs: Final[dict[str, object]] = {"standard_logging_object": _payload("excluded"), "litellm_params": {}} + + await _run_line(_registered(s3, after), kwargs) + await _flushed(s3) + + assert after.seen == [STRIPPED_MESSAGES] + body: Final = json.loads(_recorded(bucket, "excluded")) + assert "response" not in body + assert body["messages"] == STRIPPED_MESSAGES + + +@pytest.mark.asyncio +async def test_strip_on_failure_still_uploads_the_recorded_stripped_object(bucket: ScriptedBucket) -> None: + s3: Final = _s3(bucket) + kwargs: Final[dict[str, object]] = { + "standard_logging_object": _payload("failed", status="failure"), + "litellm_params": {}, + } + + await _run_line(_registered(s3), kwargs, Outcome.failure) + await _flushed(s3) + + assert json.loads(_recorded(bucket, "failed"))["messages"] == STRIPPED_MESSAGES + + +@pytest.mark.asyncio +async def test_strip_off_registers_no_writer_and_readers_see_the_original_messages(bucket: ScriptedBucket) -> None: + before: Final = _Reader("before") + s3: Final = _s3(bucket, strip=False) + callbacks: Final = _registered(before, s3) + kwargs: Final[dict[str, object]] = {"standard_logging_object": _payload("kept"), "litellm_params": {}} + + await _run_line(callbacks, kwargs) + await _flushed(s3) + + assert s3.v2_companion is None + assert callbacks == [before, s3] + assert before.seen == [_messages()] + assert json.loads(bucket.puts()[0].body)["messages"] == _messages() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "litellm_params", + [ + pytest.param({"no-log": True}, id="no-log"), + pytest.param({"proxy_server_request": {"headers": {"x-litellm-disable-callbacks": "s3_v2"}}}, id="opt-out"), + ], +) +async def test_a_request_s3_skips_gets_no_patch_and_nothing_is_uploaded( + bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, object] +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(litellm, "allow_dynamic_callback_disabling", True) + monkeypatch.setattr(proxy_server, "premium_user", True) + before: Final = _Reader("before") + after: Final = _Reader("after") + s3: Final = _s3(bucket) + kwargs: Final[dict[str, object]] = { + "standard_logging_object": _payload("skipped"), + "litellm_params": litellm_params, + } + + await _run_line(_registered(before, s3, after), kwargs) + await s3.flush_queue() + + assert before.seen == after.seen == [_messages()] + assert bucket.puts() == [] + + +@pytest.mark.asyncio +async def test_the_writer_patches_nothing_on_the_sync_line(bucket: ScriptedBucket) -> None: + s3: Final = _s3(bucket) + writer: Final = s3.v2_companion + assert isinstance(writer, S3StripBase64Files) + payload: Final = _payload("sync") + kwargs: Final = {"standard_logging_object": payload, "litellm_params": {}} + + await asyncio.to_thread( + litellm_logging._run_callbacks_sync, + Hook.log_prepare, + [writer, s3], + kwargs, + lambda _callback, _kwargs: None, + args=HookArgs(request=kwargs, start_time=START, end_time=START), + ) + + assert payload["messages"] == _messages() + + +@pytest.mark.asyncio +async def test_an_acompletion_shows_every_reader_the_messages_s3_stores( + bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + before: Final = _Reader("before") + after: Final = _Reader("after") + s3: Final = _s3(bucket) + monkeypatch.setattr(litellm, "callbacks", [before, s3, after]) + + await litellm.acompletion( + model="gpt-4o", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "see data:image/png;base64,AAAA== inline"}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBER"}}, + ], + } + ], + mock_response="hi there", + ) + for _ in range(100): + if after.seen: + break + await asyncio.sleep(0.02) + await _flushed(s3) + + assert litellm._async_success_callback == [before, s3.v2_companion, s3, after] + expected: Final = [{"role": "user", "content": [{"type": "text", "text": "see [BASE64_REDACTED]"}]}] + assert before.seen == after.seen == [expected] + assert json.loads(bucket.puts()[0].body)["messages"] == expected + + +def test_a_sync_success_leaves_the_messages_for_readers_on_the_sync_line(bucket: ScriptedBucket) -> None: + async def built() -> S3Logger: + return _s3(bucket) + + s3: Final = asyncio.run(built()) + before: Final = _SyncReader("before") + litellm.logging_callback_manager.add_litellm_success_callback(before) + litellm.logging_callback_manager.add_litellm_success_callback(s3) + logging_obj: Final = Logging( + model="gpt-4o", + messages=[{"role": "user", "content": "see data:image/png;base64,AAAA== inline"}], + stream=False, + call_type="completion", + start_time=START, + litellm_call_id="call-sync", + function_id="fn-sync", + ) + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) + + logging_obj.success_handler(result=ModelResponse(id="served"), start_time=START, end_time=START) + + assert litellm.success_callback == [before, s3.v2_companion, s3] + assert before.seen == [[{"role": "user", "content": "see data:image/png;base64,AAAA== inline"}]] + assert bucket.puts() == [] + + +@pytest.mark.asyncio +async def test_registering_s3_again_adds_no_second_writer(bucket: ScriptedBucket) -> None: + s3: Final = _s3(bucket) + _ = _registered(s3, s3) + litellm.logging_callback_manager.add_litellm_success_callback(s3) + litellm.logging_callback_manager.add_litellm_success_callback(s3) + + assert litellm._async_success_callback == litellm.success_callback == [s3.v2_companion, s3] + + +@pytest.mark.asyncio +async def test_reinitialising_proxy_callbacks_keeps_one_writer_right_before_s3( + bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(ProxyLogging, "_add_proxy_hooks", lambda _self, _router=None: None) + monkeypatch.setattr(litellm, "s3_callback_params", _params(bucket, strip=True)) + reader: Final = _Reader("reader") + monkeypatch.setattr(litellm, "callbacks", [reader, "s3_v2"]) + proxy_logging: Final = ProxyLogging(user_api_key_cache=MagicMock()) + + proxy_logging._init_litellm_callbacks(llm_router=None) + proxy_logging._init_litellm_callbacks(llm_router=None) + + (s3,) = (entry for entry in litellm.callbacks if isinstance(entry, S3Logger)) + _ = s3.periodic_flush_task.cancel() + lists: Final = ( + litellm.callbacks, + litellm.success_callback, + litellm.failure_callback, + litellm._async_success_callback, + litellm._async_failure_callback, + ) + assert [[entry for entry in listed if entry is not proxy_logging.service_logging_obj] for listed in lists] == [ + [reader, s3.v2_companion, s3] + ] * len(lists) + + +@pytest.mark.asyncio +async def test_removing_s3_removes_its_writer(bucket: ScriptedBucket) -> None: + dropped: Final = _s3(bucket) + removed: Final = _s3(bucket, region="eu-west-1") + reader: Final = _Reader("reader") + registered: Final = _registered(dropped, reader, removed) + assert registered == [dropped.v2_companion, dropped, reader, removed.v2_companion, removed] + + _ = litellm.logging_callback_manager.drop_callbacks((dropped,)) + litellm.logging_callback_manager.remove_callback_from_all_lists(removed) + + assert litellm._async_success_callback == [reader] + + +@pytest.mark.asyncio +async def test_the_writer_manifest_builds_as_a_blocking_log_prepare_writer(bucket: ScriptedBucket) -> None: + s3: Final = _s3(bucket) + writer: Final = s3.v2_companion + assert isinstance(writer, S3StripBase64Files) + + manifest: Final = build_manifest(writer) + subscription: Final = manifest.subscriptions[Hook.log_prepare] + + assert manifest.selection.name == STRIP_CALLBACK_NAME + assert manifest.selection.honors_no_log is False + assert set(manifest.subscriptions) == {Hook.log_prepare} + assert subscription.writes == frozenset({Field.standard_logging_payload}) + assert subscription.reads == frozenset({Field.deprecated_kwargs, Field.standard_logging_payload}) + assert build_manifest(s3).selection.honors_no_log is True + assert [entry["hook"] for entry in json.loads(s3.sink.callback().manifest())["subscriptions"]] == ["attempt_ended"] diff --git a/tests/unit/litellm_core_utils/test_aws_partition.py b/tests/unit/litellm_core_utils/test_aws_partition.py index 91fff1996ee..84e2b2a9fd6 100644 --- a/tests/unit/litellm_core_utils/test_aws_partition.py +++ b/tests/unit/litellm_core_utils/test_aws_partition.py @@ -7,7 +7,6 @@ from urllib.parse import urlparse import pytest import litellm -from litellm.integrations.s3_v2 import S3Logger from litellm.litellm_core_utils.aws_partition import ( AwsPartition, contains_aws_arn, @@ -100,14 +99,6 @@ def _agentcore_model(region: str) -> str: return f"agentcore/{get_aws_arn_prefix(region)}bedrock-agentcore:{region}:111122223333:runtime/my-agent" -def _s3_object_url(region: str) -> str: - logger = S3Logger.__new__(S3Logger) - logger.s3_endpoint_url = None - logger.s3_bucket_name = "audit-bucket" - logger.s3_region_name = region - return logger._build_object_url("2025-01-01/key.json") - - ENDPOINT_BUILDERS: Final = { "bedrock_runtime_default": lambda region: BaseAWSLLM()._select_default_endpoint_url("runtime", region), "bedrock_agent_default": lambda region: BaseAWSLLM()._select_default_endpoint_url("agent", region), @@ -158,7 +149,6 @@ ENDPOINT_BUILDERS: Final = { api_base=None, litellm_params={"aws_region_name": region}, ), - "s3_object_url": _s3_object_url, } diff --git a/tests/unit/llms/bedrock/test_web_identity_session_policy.py b/tests/unit/llms/bedrock/test_web_identity_session_policy.py index 9d304b45e09..43ca0485584 100644 --- a/tests/unit/llms/bedrock/test_web_identity_session_policy.py +++ b/tests/unit/llms/bedrock/test_web_identity_session_policy.py @@ -43,7 +43,6 @@ import pytest from botocore.session import get_session from pydantic import TypeAdapter -from litellm.integrations.s3_v2 import S3Logger from litellm.integrations.sqs import SQSLogger from litellm.llms.aws_polly.text_to_speech.transformation import AWSPollyTextToSpeechConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM @@ -55,7 +54,11 @@ from litellm.llms.bedrock.files.transformation import BedrockFilesConfig from litellm.llms.bedrock.model_listing import BedrockModelLister from litellm.llms.bedrock.passthrough.transformation import BedrockAgentRuntimeAWS from litellm.llms.bedrock.realtime.handler import BedrockRealtime -from litellm.llms.bedrock.web_identity_session_policy import WebIdentitySessionPolicy, _SessionPolicyStatement +from litellm.llms.bedrock.web_identity_session_policy import ( + S3_LOGGING_SESSION_POLICY, + WebIdentitySessionPolicy, + _SessionPolicyStatement, +) from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.opensearch.vector_stores.transformation import OpenSearchVectorStoreConfig from litellm.llms.s3_vectors.vector_stores.transformation import S3VectorsVectorStoreConfig @@ -433,9 +436,12 @@ class TestSessionPolicyGrantsEveryBedrockRoute: assert len(_captured_policy_document()) <= _STS_SESSION_POLICY_PLAINTEXT_LIMIT -def _s3_logger() -> S3Logger: - with patch("asyncio.create_task"): - return S3Logger(s3_bucket_name="litellm-logs", s3_region_name="us-east-1") +class _S3LoggingSession(BaseAWSLLM): + _web_identity_session_policy = S3_LOGGING_SESSION_POLICY + + +def _s3_logger() -> _S3LoggingSession: + return _S3LoggingSession() def _sqs_logger() -> SQSLogger: diff --git a/tests/unit/proxy/management_helpers/test_audit_log_callbacks.py b/tests/unit/proxy/management_helpers/test_audit_log_callbacks.py index 0dff25965f4..e6323bd876b 100644 --- a/tests/unit/proxy/management_helpers/test_audit_log_callbacks.py +++ b/tests/unit/proxy/management_helpers/test_audit_log_callbacks.py @@ -22,6 +22,7 @@ from litellm.proxy.management_helpers.audit_logs import ( is_audit_logging_enabled, ) from litellm.types.utils import StandardAuditLogPayload +from tests.unit.integrations.s3_v3_rust_support import scripted_bucket @pytest.fixture(autouse=True) @@ -313,16 +314,25 @@ class TestAuditLogTaskDoneCallback: class TestS3LoggerAuditLogEvent: - @pytest.mark.asyncio - async def test_queues_audit_log_with_correct_s3_key(self): - with patch("litellm.integrations.s3_v2.S3Logger.__init__", return_value=None): - from litellm.integrations.s3_v2 import S3Logger + @staticmethod + def _logger(monkeypatch: pytest.MonkeyPatch, endpoint: str, s3_path: str | None): + from litellm.integrations.s3_v2 import S3Logger - logger = S3Logger() - logger.s3_path = "my-prefix" - logger.log_queue = [] - logger.batch_size = 100 - logger.max_queue_size = 100 + monkeypatch.setattr(litellm, "s3_callback_params", None) + return S3Logger( + s3_bucket_name="audit-bucket", + s3_region_name="us-east-1", + s3_endpoint_url=endpoint, + s3_aws_access_key_id="test-key", + s3_aws_secret_access_key="test-secret", + s3_path=s3_path, + s3_batch_size=100, + ) + + @pytest.mark.asyncio + async def test_queues_audit_log_with_correct_s3_key(self, monkeypatch: pytest.MonkeyPatch): + with scripted_bucket() as bucket: + logger = self._logger(monkeypatch, bucket.url, "my-prefix") audit_log = StandardAuditLogPayload( id="audit-123", @@ -338,25 +348,26 @@ class TestS3LoggerAuditLogEvent: await logger.async_log_audit_log_event(audit_log) - assert len(logger.log_queue) == 1 - element = logger.log_queue[0] - assert element.s3_object_key.startswith("my-prefix/audit_logs/") - assert "audit-123" in element.s3_object_key - assert element.s3_object_key.endswith(".json") - assert element.s3_object_download_filename == "audit-audit-123.json" - assert element.payload["id"] == "audit-123" - assert element.payload["action"] == "created" + queued = logger.sink.queued_keys() + assert len(queued) == 1 + s3_object_key = queued[0] + assert s3_object_key.startswith("my-prefix/audit_logs/") + assert "audit-123" in s3_object_key + assert s3_object_key.endswith(".json") + + await logger.flush_queue() + + put = bucket.puts()[0] + assert put.path == f"/audit-bucket/{s3_object_key}" + assert put.headers["content-disposition"] == 'inline; filename="audit-audit-123.json"' + payload = json.loads(put.body) + assert payload["id"] == "audit-123" + assert payload["action"] == "created" @pytest.mark.asyncio - async def test_s3_key_format_no_path(self): - with patch("litellm.integrations.s3_v2.S3Logger.__init__", return_value=None): - from litellm.integrations.s3_v2 import S3Logger - - logger = S3Logger() - logger.s3_path = None - logger.log_queue = [] - logger.batch_size = 100 - logger.max_queue_size = 100 + async def test_s3_key_format_no_path(self, monkeypatch: pytest.MonkeyPatch): + with scripted_bucket() as bucket: + logger = self._logger(monkeypatch, bucket.url, None) audit_log = StandardAuditLogPayload( id="audit-456", @@ -372,10 +383,10 @@ class TestS3LoggerAuditLogEvent: await logger.async_log_audit_log_event(audit_log) - assert len(logger.log_queue) == 1 - element = logger.log_queue[0] - assert element.s3_object_key.startswith("audit_logs/") - assert "audit-456" in element.s3_object_key + queued = logger.sink.queued_keys() + assert len(queued) == 1 + assert queued[0].startswith("audit_logs/") + assert "audit-456" in queued[0] class TestS3AuditCallbackParamsDecoupling: diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py index 1565820cbbd..8fd97ea5bce 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py @@ -14,6 +14,8 @@ from unittest.mock import AsyncMock, patch import pytest +import litellm + from litellm.integrations.s3_v2 import S3Logger from litellm.proxy._types import SpendLogsMetadata from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler @@ -22,6 +24,7 @@ from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, ) from litellm.types.utils import StandardLoggingMetadata +from tests.unit.integrations.s3_v3_rust_support import scripted_bucket class TestColdStorageObjectKeyIntegration: @@ -172,24 +175,29 @@ class TestColdStorageObjectKeyIntegration: ) @pytest.mark.asyncio - @patch("asyncio.create_task") # Mock asyncio.create_task to avoid event loop issues - async def test_s3_logger_supports_object_key_retrieval(self, mock_create_task): + async def test_s3_logger_supports_object_key_retrieval(self, monkeypatch: pytest.MonkeyPatch): """ Test: S3Logger supports retrieval using provided object key. This test verifies that the S3Logger can retrieve objects using the object key directly without generating it from request_id and start_time. """ - # Create S3Logger instance - s3_logger = S3Logger(s3_bucket_name="test-bucket") + monkeypatch.setattr(litellm, "s3_callback_params", None) + with scripted_bucket() as bucket: + bucket.objects["/test-bucket/test/path/to/object.json"] = b'{"test": "data"}' + s3_logger = S3Logger( + s3_bucket_name="test-bucket", + s3_region_name="us-east-1", + s3_endpoint_url=bucket.url, + s3_aws_access_key_id="test-key", + s3_aws_secret_access_key="test-secret", + ) - # Mock the _download_object_from_s3 method - with patch.object( - s3_logger, "_download_object_from_s3", return_value={"test": "data"} - ) as mock_download: result = await s3_logger.get_proxy_server_request_from_cold_storage_with_object_key( object_key="test/path/to/object.json" ) assert result == {"test": "data"} - mock_download.assert_called_once_with("test/path/to/object.json") + assert [(request.method, request.path) for request in bucket.received] == [ + ("GET", "/test-bucket/test/path/to/object.json") + ]