mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(s3-log): replace the python s3_v2 logger with a rust callback (#51539)
The s3_v2 S3Logger is ported to a Rust callback, callbacks-s3-log, behind the CustomLoggerV2 contract, with the bridge constructor NativeS3Log and the Python glue in litellm/integrations/s3_v3.py. It is now the only implementation: the Python logger is deleted, s3_v2.py re-exports the Rust-backed S3Logger, and LITELLM_NATIVE_S3_V2 is removed. The callback keeps its name, list position and settings, so no config changes. The original s3_v2 tests are replicated against the Rust callback as <original>_rust.py, and Python's outputs for the differential suite are recorded in tests/unit/integrations/s3_v2_golden/python_s3_v2.json.
This commit is contained in:
parent
74fd46a59f
commit
8e92894ff2
55 changed files with 15926 additions and 7622 deletions
|
|
@ -7,7 +7,7 @@ from collections import Counter
|
|||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, chain, count
|
||||
from typing import Final, Literal, TypeAlias
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -23,6 +23,9 @@ from litellm.types.utils import (
|
|||
StandardBatchLineItemUsage,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.s3_v3 import S3Logger
|
||||
|
||||
BatchFileProvider: TypeAlias = Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"]
|
||||
|
||||
_VERTEX_AI_UNKNOWN_CUSTOM_ID: Final = "unknown" # what a Vertex AI batch row that carries no custom_id is read as
|
||||
|
|
@ -238,9 +241,11 @@ async def _deliver(
|
|||
)
|
||||
|
||||
|
||||
def _resolve_logger(callback: litellm.CALLBACK_TYPES) -> CustomLogger | None:
|
||||
def _resolve_logger(callback: "litellm.CALLBACK_TYPES | S3Logger") -> "CustomLogger | S3Logger | None":
|
||||
from litellm.integrations.s3_v3 import S3Logger
|
||||
|
||||
match callback:
|
||||
case CustomLogger():
|
||||
case CustomLogger() | S3Logger():
|
||||
return callback
|
||||
case "s3_v2" | "gcs_bucket":
|
||||
return init_custom_logger_compatible_class(callback, internal_usage_cache=None, llm_router=None)
|
||||
|
|
@ -248,7 +253,7 @@ def _resolve_logger(callback: litellm.CALLBACK_TYPES) -> CustomLogger | None:
|
|||
return None
|
||||
|
||||
|
||||
def _logger_turned_off_message_logging(logger: CustomLogger) -> bool:
|
||||
def _logger_turned_off_message_logging(logger: object) -> bool:
|
||||
return (
|
||||
getattr(logger, "turn_off_message_logging", False) is True
|
||||
or getattr(logger, "message_logging", True) is not True
|
||||
|
|
|
|||
|
|
@ -130,7 +130,6 @@ S3_PREFIX_DIGEST_CHARS: Final = 16
|
|||
# s3 allows 2048 bytes of combined metadata headers, which Content-Disposition counts against
|
||||
MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
|
||||
S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY"
|
||||
S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
|
||||
MAX_FILE_LIST_LIMIT: Final = 10000
|
||||
ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
|
||||
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import X_LITELLM_DISABLE_CALLBACKS
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
|
||||
from litellm.litellm_core_utils.llm_request_utils import (
|
||||
get_proxy_server_request_headers,
|
||||
)
|
||||
|
|
@ -45,7 +46,7 @@ class EnterpriseCallbackControls:
|
|||
if callback.lower() in disabled_callbacks:
|
||||
verbose_logger.debug(f"Not logging to {callback} because it is disabled via {X_LITELLM_DISABLE_CALLBACKS}")
|
||||
return True
|
||||
elif isinstance(callback, CustomLogger):
|
||||
elif isinstance(callback, CustomLogger | CustomLoggerV2):
|
||||
# get the string name of the callback
|
||||
callback_str = CustomLoggerRegistry.get_callback_str_from_class_type(callback.__class__)
|
||||
if callback_str is not None and callback_str.lower() in disabled_callbacks:
|
||||
|
|
|
|||
|
|
@ -1,78 +0,0 @@
|
|||
"""
|
||||
Adaptive in-flight concurrency limiter (AIMD, Vector ARC style).
|
||||
|
||||
Grows the limit additively after `limit` consecutive clean completions and
|
||||
halves it only on an explicit throttle signal (429, 503, SlowDown, or a
|
||||
transport error out of the PUT). With floor == ceiling it degenerates to a
|
||||
fixed-width semaphore.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PutSample:
|
||||
throttled: bool
|
||||
|
||||
|
||||
class AdaptiveConcurrencyLimiter:
|
||||
"""AIMD in-flight limiter used as `async with limiter:`."""
|
||||
|
||||
def __init__(self, initial: int, floor: int, ceiling: int) -> None:
|
||||
if not 1 <= floor <= ceiling:
|
||||
raise ValueError(f"adaptive limiter bounds must satisfy 1 <= floor <= ceiling, got {floor}..{ceiling}")
|
||||
self._limit: int = min(max(initial, floor), ceiling)
|
||||
self._floor: Final[int] = floor
|
||||
self._ceiling: Final[int] = ceiling
|
||||
self._clean_streak: int = 0
|
||||
self._in_flight: int = 0
|
||||
self._waiters: deque[asyncio.Future[None]] = deque() # mutable-ok: waiters queue up behind a full limit
|
||||
|
||||
@property
|
||||
def limit(self) -> int:
|
||||
return self._limit
|
||||
|
||||
async def __aenter__(self) -> "AdaptiveConcurrencyLimiter":
|
||||
if self._in_flight < self._limit:
|
||||
self._in_flight += 1
|
||||
return self
|
||||
waiter: Final = asyncio.get_running_loop().create_future()
|
||||
self._waiters.append(waiter)
|
||||
try:
|
||||
await waiter
|
||||
except asyncio.CancelledError:
|
||||
if waiter.done() and not waiter.cancelled():
|
||||
self._in_flight -= 1
|
||||
self._grant()
|
||||
else:
|
||||
with suppress(ValueError):
|
||||
self._waiters.remove(waiter)
|
||||
raise
|
||||
return self
|
||||
|
||||
def _grant(self) -> None:
|
||||
while self._in_flight < self._limit and self._waiters:
|
||||
waiter = self._waiters.popleft()
|
||||
if waiter.done():
|
||||
continue
|
||||
self._in_flight += 1
|
||||
waiter.set_result(None)
|
||||
|
||||
async def __aexit__(self, *_: object) -> None:
|
||||
self._in_flight -= 1
|
||||
self._grant()
|
||||
|
||||
def record(self, sample: PutSample) -> None:
|
||||
if sample.throttled:
|
||||
self._limit = max(self._floor, self._limit // 2)
|
||||
self._clean_streak = 0
|
||||
return
|
||||
self._clean_streak += 1
|
||||
if self._clean_streak >= self._limit and self._limit < self._ceiling:
|
||||
self._limit += 1
|
||||
self._clean_streak = 0
|
||||
self._grant()
|
||||
|
|
@ -17,7 +17,6 @@ from litellm.constants import (
|
|||
MAX_S3_OBJECT_KEY_FILENAME_BYTES,
|
||||
S3_BOUNDED_OBJECT_KEY_HEAD_BYTES,
|
||||
S3_LOG_PROMPTS_ONLY_ENV_VAR,
|
||||
S3_PARTITION_GRANULARITY_ENV_VAR,
|
||||
S3_PREFIX_DIGEST_CHARS,
|
||||
)
|
||||
from litellm.types.integrations.s3_v2 import S3PartitionGranularity
|
||||
|
|
@ -39,18 +38,6 @@ def resolve_s3_log_prompts_only(configured: object, environ: Mapping[str, str] |
|
|||
return True
|
||||
|
||||
|
||||
def resolve_s3_partition_granularity(
|
||||
configured: object, environ: Mapping[str, str] | None = None
|
||||
) -> S3PartitionGranularity:
|
||||
env: Final = os.environ if environ is None else environ
|
||||
raw: Final = env.get(S3_PARTITION_GRANULARITY_ENV_VAR) if configured is None else configured
|
||||
if raw == "hour":
|
||||
return "hour"
|
||||
if raw is not None and raw not in ("", "day"):
|
||||
verbose_logger.warning("s3 logging: s3_partition_granularity=%r is not one of day, hour, using day", raw)
|
||||
return "day"
|
||||
|
||||
|
||||
def _resolve_positive_int(setting: str, configured: object, fallback: int, *, reject_bool: bool) -> int:
|
||||
if configured is None or configured == "":
|
||||
return fallback
|
||||
|
|
@ -70,10 +57,6 @@ def _resolve_positive_int(setting: str, configured: object, fallback: int, *, re
|
|||
return bound
|
||||
|
||||
|
||||
def resolve_s3_max_concurrent_uploads(configured: object, fallback: int) -> int:
|
||||
return _resolve_positive_int("s3_max_concurrent_uploads", configured, fallback, reject_bool=False)
|
||||
|
||||
|
||||
def resolve_s3_max_queue_size(configured: object, fallback: int) -> int:
|
||||
return _resolve_positive_int("s3_max_queue_size", configured, fallback, reject_bool=True)
|
||||
|
||||
|
|
@ -103,48 +86,6 @@ def resolve_s3_max_retry_age_seconds(configured: object, fallback: int | None) -
|
|||
return bound or None
|
||||
|
||||
|
||||
def resolve_s3_max_adaptive_concurrency(configured: object, fallback: int) -> int:
|
||||
return _resolve_positive_int("s3_max_adaptive_concurrency", configured, fallback, reject_bool=True)
|
||||
|
||||
|
||||
def resolve_s3_drop_on_terminal_error(configured: object) -> bool:
|
||||
if configured is None or configured == "":
|
||||
return True
|
||||
try:
|
||||
return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
|
||||
except ValidationError:
|
||||
verbose_logger.warning(
|
||||
"s3 logging: s3_drop_on_terminal_error=%r is not a boolean, dropping terminal-failed uploads",
|
||||
configured,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def resolve_s3_adaptive_concurrency(configured: object) -> bool:
|
||||
if configured is None or configured == "":
|
||||
return False
|
||||
try:
|
||||
return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
|
||||
except ValidationError:
|
||||
verbose_logger.warning(
|
||||
"s3 logging: s3_adaptive_concurrency=%r is not a boolean, keeping the fixed upload width",
|
||||
configured,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def resolve_s3_batch_file_upload(configured: object) -> bool:
|
||||
if configured is None or configured == "":
|
||||
return False
|
||||
try:
|
||||
return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
|
||||
except ValidationError:
|
||||
verbose_logger.warning(
|
||||
"s3 logging: s3_batch_file_upload=%r is not a boolean, keeping per-request objects", configured
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def prompts_only_payload(payload: StandardLoggingPayload) -> StandardLoggingPayload:
|
||||
return {**payload, "response": None}
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
481
litellm/integrations/s3_v3.py
Normal file
481
litellm/integrations/s3_v3.py
Normal file
|
|
@ -0,0 +1,481 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Callable, Collection, Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from typing_extensions import Unpack
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
COMPLETION_HTTP_FALLBACK_SECONDS,
|
||||
DEFAULT_S3_BATCH_SIZE,
|
||||
DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
|
||||
DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
|
||||
DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
|
||||
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
|
||||
S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
|
||||
)
|
||||
from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name
|
||||
from litellm.integrations.custom_batch_logger import CustomBatchLogger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_logger_v2.call import CallContext
|
||||
from litellm.integrations.custom_logger_v2.derivations import (
|
||||
cold_storage_object_key,
|
||||
log_id,
|
||||
logged_key_identity,
|
||||
mapping,
|
||||
payload_copy,
|
||||
)
|
||||
from litellm.integrations.custom_logger_v2.enums import Field, Hook
|
||||
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
|
||||
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on
|
||||
from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch
|
||||
from litellm.integrations.custom_logger_v2.rust import RustLogger
|
||||
from litellm.integrations.custom_logger_v2.selection import Selection
|
||||
from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds
|
||||
from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document
|
||||
from litellm.secret_managers.main import get_secret, get_secret_str
|
||||
from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity
|
||||
from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.rust_bridge._native import NativeS3Log
|
||||
|
||||
CALLBACK_NAME: Final = "s3_v2"
|
||||
STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files"
|
||||
_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4
|
||||
_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600
|
||||
|
||||
|
||||
class S3BatchUploadError(Exception):
|
||||
def __init__(self, failed: int, total: int) -> None:
|
||||
self.failed: Final = failed
|
||||
self.total: Final = total
|
||||
super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush")
|
||||
|
||||
|
||||
def _raise_batch_upload_error(failed: int, total: int) -> None:
|
||||
raise S3BatchUploadError(failed=failed, total=total)
|
||||
|
||||
|
||||
def batch_upload_failed(failure: tuple[int, int, int] | None) -> None:
|
||||
if failure is None:
|
||||
return
|
||||
queued, failed, total = failure
|
||||
try:
|
||||
_raise_batch_upload_error(failed, total)
|
||||
except S3BatchUploadError:
|
||||
verbose_logger.exception(
|
||||
"CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued
|
||||
)
|
||||
|
||||
|
||||
def _cold_storage_is_s3_v2() -> bool:
|
||||
return litellm.cold_storage_custom_logger == "s3_v2"
|
||||
|
||||
|
||||
def _region_from_environment() -> str | None:
|
||||
region_name: Final = get_secret_str("AWS_REGION_NAME", None)
|
||||
region: Final = get_secret_str("AWS_REGION", None)
|
||||
return region if region is not None else region_name
|
||||
|
||||
|
||||
class _Host:
|
||||
def __init__(self) -> None:
|
||||
self.failures: Final = CustomLogger()
|
||||
|
||||
def callback_failed(self) -> None:
|
||||
self.failures.handle_callback_failure(callback_name="S3Logger")
|
||||
|
||||
def web_identity_token(self, reference: str) -> str:
|
||||
if reference.startswith("os.environ/") or reference in os.environ:
|
||||
raise ValueError("Invalid web identity token reference.")
|
||||
match _read_secret(get_secret, reference):
|
||||
case None:
|
||||
raise ValueError("OIDC token could not be retrieved from secret manager.")
|
||||
case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH:
|
||||
raise ValueError(
|
||||
"Parameter validation failed:\nInvalid length for parameter WebIdentityToken, "
|
||||
f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}"
|
||||
)
|
||||
case str() as token:
|
||||
return token
|
||||
case other:
|
||||
raise ValueError(
|
||||
"Parameter validation failed:\nInvalid type for parameter WebIdentityToken, "
|
||||
f"value: {other}, type: {type(other)}, valid types: {str}"
|
||||
)
|
||||
|
||||
|
||||
def _read_secret(read: Callable[[str], object], reference: str) -> object:
|
||||
return read(reference)
|
||||
|
||||
|
||||
def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust
|
||||
configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None)
|
||||
source: Final[Mapping[str, object] | Mapping[object, object]] = (
|
||||
params_source if params_source is not None else (configured or {})
|
||||
)
|
||||
return {
|
||||
key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value)
|
||||
for key, value in source.items()
|
||||
if isinstance(key, str)
|
||||
}
|
||||
|
||||
|
||||
def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]:
|
||||
return {
|
||||
"batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE,
|
||||
"max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
|
||||
"max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
|
||||
"upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
|
||||
"download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS,
|
||||
"download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
|
||||
"web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY),
|
||||
}
|
||||
|
||||
|
||||
def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float:
|
||||
return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS
|
||||
|
||||
|
||||
_CONFIGURED_FIRST: Final = frozenset(
|
||||
{
|
||||
"s3_bucket_name",
|
||||
"s3_path",
|
||||
"s3_region_name",
|
||||
"s3_api_version",
|
||||
"s3_endpoint_url",
|
||||
"s3_aws_access_key_id",
|
||||
"s3_aws_secret_access_key",
|
||||
"s3_aws_session_token",
|
||||
"s3_aws_session_name",
|
||||
"s3_aws_profile_name",
|
||||
"s3_aws_role_name",
|
||||
"s3_aws_web_identity_token",
|
||||
"s3_aws_sts_endpoint",
|
||||
"s3_config",
|
||||
"s3_server_side_encryption",
|
||||
"s3_sse_kms_key_id",
|
||||
}
|
||||
)
|
||||
_CONSTRUCTOR_ENABLES: Final = frozenset(
|
||||
{
|
||||
"s3_use_team_prefix",
|
||||
"s3_strip_base64_files",
|
||||
"s3_use_key_prefix",
|
||||
"s3_use_virtual_hosted_style",
|
||||
"s3_adaptive_concurrency",
|
||||
"s3_batch_file_upload",
|
||||
}
|
||||
)
|
||||
_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"})
|
||||
_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"})
|
||||
_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"})
|
||||
_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = {
|
||||
"s3_use_ssl": True,
|
||||
"s3_drop_on_terminal_error": True,
|
||||
"s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
|
||||
"s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS,
|
||||
"s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
|
||||
"s3_batch_size": DEFAULT_S3_BATCH_SIZE,
|
||||
}
|
||||
|
||||
|
||||
def _blank(value: object) -> bool:
|
||||
return value is None or value == ""
|
||||
|
||||
|
||||
def _constructor_wins(key: str, configured: object, value: object) -> bool:
|
||||
return (
|
||||
(key in _CONFIGURED_FIRST and not configured)
|
||||
or (key in _CONFIGURED_UNLESS_NONE and configured is None)
|
||||
or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured))
|
||||
or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None)
|
||||
)
|
||||
|
||||
|
||||
_KEEP_CONFIGURED: Final = object()
|
||||
|
||||
|
||||
def _constructor_value(key: str, configured: object, value: object) -> object:
|
||||
if _constructor_wins(key, configured, value):
|
||||
return value
|
||||
if key in _CONSTRUCTOR_ENABLES and value:
|
||||
return True
|
||||
if key == "s3_max_queue_size" and value is not None:
|
||||
return resolve_s3_max_queue_size(
|
||||
configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE)
|
||||
)
|
||||
if key == "s3_max_retry_age_seconds":
|
||||
fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS)
|
||||
resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback)
|
||||
return 0 if resolved is None else resolved
|
||||
return _KEEP_CONFIGURED
|
||||
|
||||
|
||||
def _constructed_params(
|
||||
params_source: Mapping[str, object] | None, settings: S3LoggerSettings
|
||||
) -> dict[str, object]: # mutable-ok: handed to Rust
|
||||
params: Final = _resolved_params(params_source)
|
||||
overrides: Final = {
|
||||
key: replaced
|
||||
for key, value in settings.items()
|
||||
if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None)
|
||||
and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED
|
||||
}
|
||||
return {**params, **overrides}
|
||||
|
||||
|
||||
class _Base64Stripper(CustomLogger):
|
||||
def strip(self, payload: StandardLoggingPayload) -> None:
|
||||
_ = self._strip_base64_from_messages_sync(payload)
|
||||
|
||||
def stripped(self, messages: list[object]) -> list[dict[str, object]]:
|
||||
return self._process_messages(messages=[_shallow_copy(message) for message in messages])
|
||||
|
||||
|
||||
def _shallow_copy(message: object) -> object:
|
||||
entries: Final = mapping(message)
|
||||
return copy.copy(entries) if entries is not None and isinstance(message, dict) else message
|
||||
|
||||
|
||||
_STRIPPER: Final = _Base64Stripper()
|
||||
|
||||
|
||||
def legacy_should_log(kwargs: Mapping[str, object]) -> bool:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
if litellm.global_disable_no_log_param:
|
||||
return True
|
||||
litellm_params: Final = kwargs.get("litellm_params", {})
|
||||
lookup: Final = getattr(litellm_params, "get", None)
|
||||
if callable(lookup) and lookup("no-log", False) is True:
|
||||
return False
|
||||
return not litellm_logging.callback_disabled_dynamically(
|
||||
callback=CALLBACK_NAME,
|
||||
litellm_params=litellm_params,
|
||||
standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check
|
||||
)
|
||||
|
||||
|
||||
def _payload_messages(payload: Mapping[str, object]) -> list[object]:
|
||||
raw: Final = payload.get("messages")
|
||||
return [*raw] if isinstance(raw, list) else []
|
||||
|
||||
|
||||
class S3StripBase64Files(CustomLoggerV2):
|
||||
selection = Selection(name=STRIP_CALLBACK_NAME)
|
||||
|
||||
def __init__(self, s3_logger_key: str) -> None:
|
||||
self.s3_logger_key: Final = s3_logger_key
|
||||
|
||||
@on(
|
||||
Hook.log_prepare,
|
||||
reads={Field.deprecated_kwargs, Field.standard_logging_payload},
|
||||
writes={Field.standard_logging_payload},
|
||||
)
|
||||
async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None:
|
||||
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
|
||||
|
||||
kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None))
|
||||
payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None))
|
||||
if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs):
|
||||
return None
|
||||
messages: Final = _payload_messages(payload)
|
||||
if not messages:
|
||||
return None
|
||||
return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)})
|
||||
|
||||
|
||||
async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
batch_upload_failed(await sink.flush())
|
||||
except (asyncio.CancelledError, KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does
|
||||
verbose_logger.exception("CustomLogger: periodic flush failed: %s", e)
|
||||
|
||||
|
||||
def _excluding_fields(kwargs: object) -> Mapping[str, object] | None:
|
||||
excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
|
||||
details: Final = mapping(kwargs)
|
||||
logged: Final = None if details is None else mapping(details.get("standard_logging_object"))
|
||||
if details is None or logged is None or not isinstance(excluded, Collection) or not excluded:
|
||||
return None
|
||||
if logged.get("status") == "failure":
|
||||
return None
|
||||
return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}}
|
||||
|
||||
|
||||
def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object:
|
||||
source: Final = mapping(kwargs) if details is None else details
|
||||
return None if source is None else source.get("standard_logging_object")
|
||||
|
||||
|
||||
def _legacy_kwargs(view: object) -> object:
|
||||
return getattr(view, Field.deprecated_kwargs.value, None)
|
||||
|
||||
|
||||
_read_attribute: Final[Callable[[object, str], object]] = getattr
|
||||
|
||||
|
||||
class _ExcludedView:
|
||||
def __init__(self, view: object, details: Mapping[str, object]) -> None:
|
||||
args: Final = HookArgs(request=details)
|
||||
self.view: Final = view
|
||||
self.fields: Final[Mapping[str, object]] = {
|
||||
Field.deprecated_kwargs.value: details,
|
||||
Field.standard_logging_payload_ref.value: payload_copy(args),
|
||||
Field.log_id.value: log_id(args),
|
||||
Field.cold_storage_object_key.value: cold_storage_object_key(args),
|
||||
Field.logged_key_identity.value: logged_key_identity(args),
|
||||
}
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
if name not in self.fields:
|
||||
return _read_attribute(self.view, name)
|
||||
value: Final = self.fields[name]
|
||||
if value is None:
|
||||
raise AttributeError(name)
|
||||
return value
|
||||
|
||||
|
||||
_ATTEMPT_ENDED_READS: Final = frozenset(
|
||||
{
|
||||
Field.deprecated_kwargs,
|
||||
Field.attempt_timing,
|
||||
Field.log_id,
|
||||
Field.cold_storage_object_key,
|
||||
Field.standard_logging_payload_ref,
|
||||
Field.logged_key_identity,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class S3Logger(RustLogger):
|
||||
selection = Selection(name=CALLBACK_NAME, honors_no_log=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
s3_callback_params_override: Mapping[str, object] | None = None,
|
||||
**settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords
|
||||
) -> None:
|
||||
from litellm.rust_bridge._native import NativeS3Log
|
||||
|
||||
host: Final = _Host()
|
||||
params: Final = _constructed_params(s3_callback_params_override, settings)
|
||||
sink: Final = NativeS3Log(
|
||||
params,
|
||||
_constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)),
|
||||
host,
|
||||
_cold_storage_is_s3_v2(),
|
||||
None if params.get("s3_region_name") else _region_from_environment(),
|
||||
)
|
||||
self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name")
|
||||
self.sink: Final = sink
|
||||
self.s3_host: Final = host
|
||||
self.periodic_flush_task: Final = asyncio.create_task(
|
||||
_periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS)))
|
||||
)
|
||||
super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None})
|
||||
self.v2_companion: Final = (
|
||||
S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self))
|
||||
if sink.strip_base64_files
|
||||
else None
|
||||
)
|
||||
|
||||
@on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True)
|
||||
async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None:
|
||||
return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx)
|
||||
|
||||
async def run_rust(
|
||||
self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object]
|
||||
) -> Patch | Abort | None:
|
||||
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
|
||||
|
||||
kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None)
|
||||
if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict
|
||||
return None
|
||||
payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict
|
||||
if payload is not None and not isinstance(payload, Mapping):
|
||||
return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload
|
||||
try:
|
||||
raw: Final = _legacy_kwargs(view)
|
||||
details: Final = _excluding_fields(raw)
|
||||
logged: Final = _logged_payload(raw, details)
|
||||
if self.sink.strip_base64_files and isinstance(logged, Mapping):
|
||||
_STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does
|
||||
return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx)
|
||||
except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on
|
||||
return self._failed(e)
|
||||
|
||||
def _failed(self, error: Exception) -> None:
|
||||
verbose_logger.error("s3 Layer Error - %s", error, exc_info=error)
|
||||
self.s3_host.callback_failed()
|
||||
|
||||
@property
|
||||
def s3_path(self) -> str | None:
|
||||
return self.sink.s3_path
|
||||
|
||||
@property
|
||||
def s3_bucket_name(self) -> str | None:
|
||||
return self.sink.s3_bucket_name
|
||||
|
||||
def resolve_partition_granularity(self) -> S3PartitionGranularity:
|
||||
return "hour" if self.sink.partition_granularity() == "hour" else "day"
|
||||
|
||||
async def flush_queue(self) -> None:
|
||||
batch_upload_failed(await self.sink.flush())
|
||||
|
||||
async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None:
|
||||
try:
|
||||
audit_log_id: Final = audit_log.get("id", "unknown")
|
||||
batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}"))
|
||||
except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger
|
||||
verbose_logger.exception("S3 audit log error: %s", e)
|
||||
|
||||
async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
|
||||
object_name: Final = batch_line_items_object_name(payload)
|
||||
s3_path: Final = self.s3_path
|
||||
s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name
|
||||
body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload)
|
||||
delivered: Final = await self.sink.upload_line_items(
|
||||
s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:])
|
||||
)
|
||||
if not delivered:
|
||||
verbose_logger.error(
|
||||
"s3 logging: the %d line items of part %d of batch %s were not stored at %s",
|
||||
len(payload["lines"]),
|
||||
payload["part"],
|
||||
payload["batch_id"],
|
||||
s3_object_key,
|
||||
)
|
||||
|
||||
async def _download_object_from_s3(self, s3_object_key: str) -> object:
|
||||
try:
|
||||
downloaded: Final = await self.sink.download(s3_object_key)
|
||||
return None if downloaded is None else json.loads(downloaded)
|
||||
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
|
||||
verbose_logger.exception("Error downloading from S3: %s", e)
|
||||
return None
|
||||
|
||||
async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object:
|
||||
try:
|
||||
return await self._download_object_from_s3(object_key)
|
||||
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
|
||||
verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e)
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"S3BatchUploadError",
|
||||
"S3Logger",
|
||||
"S3StripBase64Files",
|
||||
"legacy_should_log",
|
||||
]
|
||||
|
|
@ -46,7 +46,7 @@ from litellm.integrations.opik.opik import OpikLogger
|
|||
from litellm.integrations.pointfive import PointFiveLogger
|
||||
from litellm.integrations.posthog import PostHogLogger
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
from litellm.integrations.s3_v3 import S3Logger
|
||||
from litellm.integrations.sqs import SQSLogger
|
||||
from litellm.integrations.ternary.ternary_logger import TernaryLogger
|
||||
from litellm.integrations.vantage.vantage_logger import VantageLogger
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import time
|
|||
import traceback
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime as dt_object
|
||||
from enum import Enum
|
||||
|
|
@ -305,7 +306,6 @@ from ..integrations.pointfive import PointFiveLogger
|
|||
from ..integrations.posthog import PostHogLogger
|
||||
from ..integrations.prompt_layer import PromptLayerLogger
|
||||
from ..integrations.s3 import S3Logger
|
||||
from ..integrations.s3_v2 import S3Logger as S3V2Logger
|
||||
from ..integrations.supabase import Supabase
|
||||
from ..integrations.traceloop import TraceloopLogger
|
||||
from ..integrations.zerobus import ZerobusLogger
|
||||
|
|
@ -768,6 +768,16 @@ def _callback_control_params(litellm_params: object) -> dict[str, object]:
|
|||
return litellm_params.model_dump() if isinstance(litellm_params, BaseModel) else {}
|
||||
|
||||
|
||||
def callback_disabled_dynamically(
|
||||
callback: object, litellm_params: object, standard_callback_dynamic_params: StandardCallbackDynamicParams
|
||||
) -> bool:
|
||||
return EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically(
|
||||
callback=callback,
|
||||
litellm_params=_callback_control_params(litellm_params),
|
||||
standard_callback_dynamic_params=standard_callback_dynamic_params,
|
||||
)
|
||||
|
||||
|
||||
LOGGING_KEYS: Final[Mapping[Field, str]] = MappingProxyType(
|
||||
{
|
||||
Field.model: "model",
|
||||
|
|
@ -783,6 +793,9 @@ def _holds_v2(callbacks: Iterable[object]) -> bool:
|
|||
return any(isinstance(callback, CustomLoggerV2) for callback in callbacks)
|
||||
|
||||
|
||||
V2_ASYNC_LOGGING_LINE: Final[ContextVar[bool]] = ContextVar("v2_async_logging_line", default=False)
|
||||
|
||||
|
||||
_HOOK_DERIVED_FIELDS: Final[Mapping[Hook, Mapping[Field, FieldSource]]] = MappingProxyType(
|
||||
{Hook.log_prepare: v2_derive.DERIVED_LOGGING_FIELDS, Hook.attempt_ended: v2_derive.DERIVED_ATTEMPT_FIELDS}
|
||||
)
|
||||
|
|
@ -839,15 +852,19 @@ async def _run_callbacks(
|
|||
args: HookArgs | None = None,
|
||||
) -> None:
|
||||
if include_v2 and _holds_v2(callbacks):
|
||||
_ = kwargs_after(
|
||||
await run_subscribers(
|
||||
hook_subscribers(hook, callbacks, admit=Admit.every_entry),
|
||||
_logging_fields(hook, kwargs, args, owner),
|
||||
binding,
|
||||
call=_v2_call(owner, args),
|
||||
scope=None if owner is None else owner.v2_logging_scope(),
|
||||
line: Final = V2_ASYNC_LOGGING_LINE.set(True)
|
||||
try:
|
||||
_ = kwargs_after(
|
||||
await run_subscribers(
|
||||
hook_subscribers(hook, callbacks, admit=Admit.every_entry),
|
||||
_logging_fields(hook, kwargs, args, owner),
|
||||
binding,
|
||||
call=_v2_call(owner, args),
|
||||
scope=None if owner is None else owner.v2_logging_scope(),
|
||||
)
|
||||
)
|
||||
)
|
||||
finally:
|
||||
V2_ASYNC_LOGGING_LINE.reset(line)
|
||||
return
|
||||
for callback in callbacks:
|
||||
if isinstance(callback, CustomLoggerV2):
|
||||
|
|
@ -3283,9 +3300,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return False
|
||||
|
||||
# Check for dynamically disabled callbacks via headers
|
||||
if EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically(
|
||||
if callback_disabled_dynamically(
|
||||
callback=callback,
|
||||
litellm_params=_callback_control_params(litellm_params),
|
||||
litellm_params=litellm_params,
|
||||
standard_callback_dynamic_params=self.standard_callback_dynamic_params,
|
||||
):
|
||||
verbose_logger.debug(
|
||||
|
|
@ -6519,6 +6536,8 @@ def build_custom_logger_compatible_class(
|
|||
_in_memory_loggers.append(_gcs_bucket_logger)
|
||||
return _gcs_bucket_logger
|
||||
elif logging_integration == "s3_v2":
|
||||
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, S3V2Logger):
|
||||
return callback
|
||||
|
|
@ -7341,6 +7360,8 @@ def get_custom_logger_compatible_class(
|
|||
if isinstance(callback, GCSBucketLogger):
|
||||
return callback
|
||||
elif logging_integration == "s3_v2":
|
||||
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, S3V2Logger):
|
||||
return callback
|
||||
|
|
@ -8046,6 +8067,8 @@ class StandardLoggingPayloadSetup:
|
|||
)
|
||||
if custom_logger and hasattr(custom_logger, "s3_path") and getattr(custom_logger, "s3_path"):
|
||||
s3_path = getattr(custom_logger, "s3_path")
|
||||
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
|
||||
|
||||
if isinstance(custom_logger, S3V2Logger):
|
||||
partition_granularity = custom_logger.resolve_partition_granularity()
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,16 @@ _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
|
|||
_callback_lists_lock: Final = threading.RLock()
|
||||
|
||||
|
||||
def _v2_companion(callback: object) -> CustomLoggerV2 | None:
|
||||
companion: Final[object] = getattr(callback, "v2_companion", None)
|
||||
return companion if isinstance(companion, CustomLoggerV2) else None
|
||||
|
||||
|
||||
def _with_v2_companions(callbacks: Iterable[object]) -> tuple[object, ...]:
|
||||
listed: Final = tuple(callbacks)
|
||||
return (*listed, *(companion for callback in listed if (companion := _v2_companion(callback)) is not None))
|
||||
|
||||
|
||||
def _instance_attributes(target: object) -> Mapping[str, object]:
|
||||
return getattr(target, "__dict__", {})
|
||||
|
||||
|
|
@ -292,16 +302,48 @@ class LoggingCallbackManager:
|
|||
parent_list=parent_list,
|
||||
)
|
||||
elif isinstance(callback, CustomLoggerV2):
|
||||
v2_key: Final = self._get_custom_logger_key(callback)
|
||||
if not any(
|
||||
isinstance(existing, CustomLoggerV2) and self._get_custom_logger_key(existing) == v2_key
|
||||
for existing in parent_list
|
||||
):
|
||||
parent_list.append(callback) # pyright: ignore[reportArgumentType] # the event lists also hold V2 callbacks
|
||||
listed: Final = self._listed_v2(callback, parent_list)
|
||||
if listed is None:
|
||||
self._append_v2(callback, parent_list)
|
||||
self._place_v2_companion(callback if listed is None else listed, parent_list)
|
||||
|
||||
elif callable(callback):
|
||||
self._add_callback_function_to_list(callback=callback, parent_list=parent_list)
|
||||
|
||||
def _append_v2(
|
||||
self, callback: CustomLoggerV2, parent_list: list[CustomLogger | Callable[..., object] | str]
|
||||
) -> None:
|
||||
parent_list.append(callback) # pyright: ignore[reportArgumentType] # the event lists also hold V2 callbacks
|
||||
|
||||
def _listed_v2(self, callback: CustomLoggerV2, parent_list: Iterable[object]) -> CustomLoggerV2 | None:
|
||||
v2_key: Final = self._get_custom_logger_key(callback)
|
||||
return next(
|
||||
(
|
||||
existing
|
||||
for existing in parent_list
|
||||
if isinstance(existing, CustomLoggerV2) and self._get_custom_logger_key(existing) == v2_key
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def _place_v2_companion(
|
||||
self, callback: CustomLoggerV2, parent_list: list[CustomLogger | Callable[..., object] | str]
|
||||
) -> None:
|
||||
companion: Final = _v2_companion(callback)
|
||||
if companion is None or self._listed_v2(companion, parent_list) is not None:
|
||||
return
|
||||
if not self._check_callback_list_size(parent_list):
|
||||
return
|
||||
self._append_v2(companion, parent_list)
|
||||
position: Final = next(index for index, existing in enumerate(parent_list) if existing is callback)
|
||||
parent_list.insert(position, parent_list.pop())
|
||||
|
||||
def place_v2_companions(self, parent_list: list[CustomLogger | Callable[..., object] | str]) -> None:
|
||||
with _callback_lists_lock:
|
||||
for callback in tuple(parent_list):
|
||||
if isinstance(callback, CustomLoggerV2):
|
||||
self._place_v2_companion(callback, parent_list)
|
||||
|
||||
def _add_callback_function_to_list(
|
||||
self,
|
||||
callback: Callable[..., object],
|
||||
|
|
@ -364,7 +406,7 @@ class LoggingCallbackManager:
|
|||
callback_lists_lock: Final = _callback_lists_lock
|
||||
|
||||
def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]:
|
||||
dropped_ids: Final = frozenset(map(id, callbacks))
|
||||
dropped_ids: Final = frozenset(map(id, _with_v2_companions(callbacks)))
|
||||
owner_ids: Final = frozenset(map(id, method_owners))
|
||||
|
||||
def keep(callback: object) -> bool:
|
||||
|
|
@ -444,6 +486,7 @@ class LoggingCallbackManager:
|
|||
Remove a callback object from every callback list it may have been
|
||||
promoted into, so a re-initialized callback leaves no stale instance behind.
|
||||
"""
|
||||
companion: Final = None if require_self else _v2_companion(obj)
|
||||
with _callback_lists_lock:
|
||||
for callback_list in (
|
||||
litellm.callbacks,
|
||||
|
|
@ -454,6 +497,8 @@ class LoggingCallbackManager:
|
|||
litellm._async_failure_callback,
|
||||
):
|
||||
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
|
||||
if companion is not None:
|
||||
callback_list[:] = [callback for callback in callback_list if callback is not companion]
|
||||
|
||||
def get_active_additional_logging_utils_from_custom_logger(
|
||||
self,
|
||||
|
|
@ -536,7 +581,7 @@ class LoggingCallbackManager:
|
|||
if not isinstance(callback, str)
|
||||
)
|
||||
|
||||
def _get_callback_string(self, callback: CustomLogger | Callable[..., object] | str) -> str:
|
||||
def _get_callback_string(self, callback: CustomLogger | CustomLoggerV2 | Callable[..., object] | str) -> str:
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
from litellm.litellm_core_utils.custom_logger_registry import (
|
||||
CustomLoggerRegistry,
|
||||
|
|
@ -547,7 +592,7 @@ class LoggingCallbackManager:
|
|||
return callback
|
||||
elif isinstance(callback, OpenTelemetry) and callback.callback_name is not None:
|
||||
return callback.callback_name
|
||||
elif isinstance(callback, CustomLogger):
|
||||
elif isinstance(callback, CustomLogger | CustomLoggerV2):
|
||||
# Try to get the string representation from the registry
|
||||
callback_str: Final = CustomLoggerRegistry.get_callback_str_from_class_type(type(callback))
|
||||
return callback_str if callback_str is not None else type(callback).__name__
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ Functions to create audit logs for LiteLLM Proxy
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Awaitable, Sequence
|
||||
from collections.abc import Awaitable, Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final, TypeVar
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -21,7 +21,10 @@ from litellm.proxy._types import (
|
|||
from litellm.repositories.table_repositories import AuditLogRepository
|
||||
from litellm.types.utils import StandardAuditLogPayload
|
||||
|
||||
_audit_log_callback_cache: Final[dict[str, CustomLogger]] = {}
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.s3_v3 import S3Logger
|
||||
|
||||
_audit_log_callback_cache: Final[dict[str, "CustomLogger | S3Logger"]] = {}
|
||||
ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY: Final = "allow_litellm_changed_by_header"
|
||||
|
||||
|
||||
|
|
@ -62,7 +65,7 @@ def get_audit_log_changed_by(
|
|||
return user_api_key_dict.user_id or litellm_proxy_admin_name
|
||||
|
||||
|
||||
def _resolve_audit_log_callback(name: str) -> CustomLogger | None:
|
||||
def _resolve_audit_log_callback(name: str) -> "CustomLogger | S3Logger | None":
|
||||
"""Resolve a string callback name to a CustomLogger instance, with caching.
|
||||
|
||||
For "s3_v2" with `litellm.s3_audit_callback_params` set, constructs a
|
||||
|
|
@ -72,11 +75,12 @@ def _resolve_audit_log_callback(name: str) -> CustomLogger | None:
|
|||
if name in _audit_log_callback_cache:
|
||||
return _audit_log_callback_cache[name]
|
||||
|
||||
instance: CustomLogger | None
|
||||
if name == "s3_v2" and getattr(litellm, "s3_audit_callback_params", None) is not None:
|
||||
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
|
||||
instance: CustomLogger | S3Logger | None
|
||||
audit_params: Final[Mapping[str, object] | None] = getattr(litellm, "s3_audit_callback_params", None)
|
||||
if name == "s3_v2" and audit_params is not None:
|
||||
from litellm.integrations import s3_v3
|
||||
|
||||
instance = S3V2Logger(s3_callback_params_override=litellm.s3_audit_callback_params)
|
||||
instance = s3_v3.S3Logger(s3_callback_params_override=audit_params)
|
||||
else:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
_init_custom_logger_compatible_class,
|
||||
|
|
@ -196,14 +200,14 @@ async def _dispatch_audit_log_to_callbacks(
|
|||
|
||||
for callback in litellm.audit_log_callbacks:
|
||||
try:
|
||||
resolved: CustomLogger | None = callback if isinstance(callback, CustomLogger) else None
|
||||
resolved: CustomLogger | S3Logger | None = callback if isinstance(callback, CustomLogger) else None
|
||||
if isinstance(callback, str):
|
||||
resolved = _resolve_audit_log_callback(callback)
|
||||
if resolved is None:
|
||||
verbose_proxy_logger.warning("Could not resolve audit log callback: %s", callback)
|
||||
continue
|
||||
|
||||
if isinstance(resolved, CustomLogger):
|
||||
if resolved is not None:
|
||||
track_audit_task(
|
||||
asyncio.create_task(
|
||||
resolved.async_log_audit_log_event(payload),
|
||||
|
|
|
|||
|
|
@ -7446,7 +7446,7 @@ class ProxyConfig:
|
|||
if key == "request_timeout":
|
||||
litellm.request_timeout_explicitly_set = True
|
||||
if key in {"s3_audit_callback_params", "s3_callback_params"}:
|
||||
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
|
||||
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
_in_memory_loggers,
|
||||
)
|
||||
|
|
@ -16441,12 +16441,13 @@ async def _routed_count_deployment(
|
|||
if llm_router is None:
|
||||
return None
|
||||
try:
|
||||
return await llm_router.select(model, dict(_responses_routing_kwargs(responses_request)))
|
||||
routed: Final = await llm_router.select(model, dict(_responses_routing_kwargs(responses_request)))
|
||||
except Exception:
|
||||
verbose_proxy_logger.exception(
|
||||
"litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment"
|
||||
)
|
||||
return None
|
||||
return routed if isinstance(routed, Mapping) else None
|
||||
|
||||
|
||||
def _with_chat_timeout(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
|
|||
|
|
@ -2514,6 +2514,7 @@ class ProxyLogging:
|
|||
# Replace string entries in litellm.callbacks with initialized instances
|
||||
for idx, initialized_callback in string_callbacks_to_replace.items():
|
||||
litellm.callbacks[idx] = initialized_callback
|
||||
litellm.logging_callback_manager.place_v2_companions(litellm.callbacks)
|
||||
|
||||
# Fan ``litellm.callbacks`` (the "all events" registry) out into the
|
||||
# success/failure event lists eagerly, at startup. ``completion()`` does
|
||||
|
|
|
|||
35
litellm/rust/Cargo.lock
generated
35
litellm/rust/Cargo.lock
generated
|
|
@ -3739,6 +3739,34 @@ dependencies = [
|
|||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-callbacks-s3-log"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"litellm-auth-aws",
|
||||
"litellm-callbacks",
|
||||
"litellm-http",
|
||||
"litellm-python-compat",
|
||||
"litellm-s3",
|
||||
"litellm-secret-value",
|
||||
"litellm-tracing",
|
||||
"md-5",
|
||||
"rand 0.8.7",
|
||||
"reqwest",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-circuit-breaker"
|
||||
version = "0.1.0"
|
||||
|
|
@ -4451,7 +4479,10 @@ dependencies = [
|
|||
"litellm-auth-azure",
|
||||
"litellm-cache",
|
||||
"litellm-cache-response",
|
||||
"litellm-callbacks",
|
||||
"litellm-callbacks-legacy-python",
|
||||
"litellm-callbacks-python",
|
||||
"litellm-callbacks-s3-log",
|
||||
"litellm-circuit-breaker",
|
||||
"litellm-core-utils",
|
||||
"litellm-error",
|
||||
|
|
@ -4691,11 +4722,15 @@ dependencies = [
|
|||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"hyper 1.12.0",
|
||||
"litellm-auth-aws",
|
||||
"litellm-http",
|
||||
"reqwest",
|
||||
"rstest",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ litellm-host-native = { path = "crates/host-native" }
|
|||
litellm-callbacks = { path = "crates/callbacks" }
|
||||
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
|
||||
litellm-callbacks-python = { path = "crates/callbacks-python" }
|
||||
litellm-callbacks-s3-log = { path = "crates/callbacks-s3-log" }
|
||||
litellm-framer = { path = "crates/framer" }
|
||||
litellm-auth = { path = "crates/auth" }
|
||||
litellm-auth-types = { path = "crates/auth-types" }
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ serde.workspace = true
|
|||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
url.workspace = true
|
||||
veil.workspace = true
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ use std::{future::Future, pin::Pin, sync::Arc};
|
|||
use aws_credential_types::Credentials;
|
||||
use aws_credential_types::provider::ProvideCredentials;
|
||||
use aws_sigv4::http_request::{
|
||||
SignableBody, SignableRequest, SignatureLocation, SigningInstructions, SigningParams,
|
||||
SigningSettings, sign,
|
||||
PercentEncodingMode, SignableBody, SignableRequest, SignatureLocation, SigningInstructions,
|
||||
SigningParams, SigningSettings, UriPathNormalizationMode, sign,
|
||||
};
|
||||
use aws_sigv4::sign::v4;
|
||||
use aws_smithy_runtime_api::client::identity::Identity;
|
||||
|
|
@ -38,10 +38,15 @@ pub struct AwsAuthService {
|
|||
|
||||
impl Default for AwsAuthService {
|
||||
fn default() -> Self {
|
||||
Self::new(Arc::new(NativeProviderLoader))
|
||||
Self::new(Arc::new(NativeProviderLoader {
|
||||
token: None,
|
||||
policy: None,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
pub type WebIdentityTokenResolver = Arc<dyn Fn(&str) -> Result<String, String> + Send + Sync>;
|
||||
|
||||
impl AwsAuthService {
|
||||
fn new(loader: Arc<dyn ProviderLoader>) -> Self {
|
||||
Self {
|
||||
|
|
@ -53,6 +58,16 @@ impl AwsAuthService {
|
|||
loader,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn for_web_identity(
|
||||
resolver: WebIdentityTokenResolver,
|
||||
session_policy: Option<String>,
|
||||
) -> Self {
|
||||
Self::new(Arc::new(NativeProviderLoader {
|
||||
token: Some(resolver),
|
||||
policy: session_policy,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
struct CredentialExpiry;
|
||||
|
|
@ -81,11 +96,19 @@ trait ProviderLoader: Send + Sync {
|
|||
fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_>;
|
||||
}
|
||||
|
||||
struct NativeProviderLoader;
|
||||
struct NativeProviderLoader {
|
||||
token: Option<WebIdentityTokenResolver>,
|
||||
policy: Option<String>,
|
||||
}
|
||||
|
||||
impl ProviderLoader for NativeProviderLoader {
|
||||
fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> {
|
||||
Box::pin(build_provider(config, flow))
|
||||
Box::pin(build_provider(
|
||||
config,
|
||||
flow,
|
||||
self.token.clone(),
|
||||
self.policy.clone(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -271,6 +294,16 @@ impl AwsAuthService {
|
|||
self.fresh_credentials(&resolved, &flow).await
|
||||
}
|
||||
|
||||
pub async fn invalidate_credentials(
|
||||
&self,
|
||||
config: AwsAuthConfig,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) {
|
||||
self.credentials
|
||||
.invalidate(&config.with_environment(env_lookup))
|
||||
.await;
|
||||
}
|
||||
|
||||
/// The credentials botocore's default chain finds in `region`, without LiteLLM's Bedrock settings
|
||||
/// (`AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, `AWS_SESSION_NAME`) that `resolve_credentials` also reads.
|
||||
pub async fn default_chain_credentials(&self, region: &str) -> Result<Credentials, Error> {
|
||||
|
|
@ -347,6 +380,8 @@ impl AwsAuthFlow {
|
|||
async fn build_provider(
|
||||
resolved: AwsAuthConfig,
|
||||
flow: AwsAuthFlow,
|
||||
token_resolver: Option<WebIdentityTokenResolver>,
|
||||
policy: Option<String>,
|
||||
) -> Result<SharedCredentialsProvider, Error> {
|
||||
match flow {
|
||||
AwsAuthFlow::SessionToken {
|
||||
|
|
@ -417,8 +452,10 @@ async fn build_provider(
|
|||
role,
|
||||
session_name,
|
||||
} => Ok(SharedCredentialsProvider::new(WebIdentityProvider {
|
||||
policy,
|
||||
client: aws_sdk_sts::Client::new(&sdk_config(resolved).await),
|
||||
token,
|
||||
token_resolver,
|
||||
role,
|
||||
session_name,
|
||||
})),
|
||||
|
|
@ -448,14 +485,27 @@ async fn sdk_config(config: AwsAuthConfig) -> aws_types::SdkConfig {
|
|||
loader.load().await
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct WebIdentityProvider {
|
||||
client: aws_sdk_sts::Client,
|
||||
token: SecretValue,
|
||||
token_resolver: Option<WebIdentityTokenResolver>,
|
||||
policy: Option<String>,
|
||||
role: String,
|
||||
session_name: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for WebIdentityProvider {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("WebIdentityProvider")
|
||||
.field("token", &self.token)
|
||||
.field("policy", &self.policy)
|
||||
.field("role", &self.role)
|
||||
.field("session_name", &self.session_name)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProvideCredentials for WebIdentityProvider {
|
||||
fn provide_credentials<'a>(
|
||||
&'a self,
|
||||
|
|
@ -464,12 +514,31 @@ impl ProvideCredentials for WebIdentityProvider {
|
|||
Self: 'a,
|
||||
{
|
||||
aws_credential_types::provider::future::ProvideCredentials::new(async {
|
||||
let response = self
|
||||
let token = match &self.token_resolver {
|
||||
Some(resolve) => {
|
||||
let resolve = Arc::clone(resolve);
|
||||
let reference = self.token.expose().to_owned();
|
||||
tokio::task::spawn_blocking(move || resolve(&reference))
|
||||
.await
|
||||
.map_err(|error| error.to_string())
|
||||
.and_then(|token| token)
|
||||
.map_err(|message| {
|
||||
CredentialsError::provider_error(std::io::Error::other(message))
|
||||
})?
|
||||
}
|
||||
None => self.token.expose().to_owned(),
|
||||
};
|
||||
let request = self
|
||||
.client
|
||||
.assume_role_with_web_identity()
|
||||
.role_arn(&self.role)
|
||||
.role_session_name(&self.session_name)
|
||||
.web_identity_token(self.token.expose())
|
||||
.web_identity_token(token);
|
||||
let request = match &self.policy {
|
||||
Some(policy) => request.policy(policy).duration_seconds(3600),
|
||||
None => request,
|
||||
};
|
||||
let response = request
|
||||
.send()
|
||||
.await
|
||||
.map_err(CredentialsError::provider_error)?;
|
||||
|
|
@ -605,6 +674,32 @@ pub fn sign_headers(
|
|||
.collect()
|
||||
}
|
||||
|
||||
pub fn sign_s3_headers(
|
||||
method: &str,
|
||||
url: &str,
|
||||
body: &[u8],
|
||||
headers: &[(String, String)],
|
||||
scope: SigningScope<'_>,
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
let mut settings = SigningSettings::default();
|
||||
settings.percent_encoding_mode = PercentEncodingMode::Single;
|
||||
settings.uri_path_normalization_mode = UriPathNormalizationMode::Disabled;
|
||||
let instructions = sign_request(
|
||||
method,
|
||||
url,
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str(), value.as_str())),
|
||||
body,
|
||||
settings,
|
||||
scope,
|
||||
)?;
|
||||
Ok(instructions
|
||||
.headers()
|
||||
.map(|(name, value)| (name.to_owned(), value.to_owned()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn presign_url(
|
||||
url: &str,
|
||||
expires_in: Duration,
|
||||
|
|
|
|||
35
litellm/rust/crates/callbacks-s3-log/Cargo.toml
Normal file
35
litellm/rust/crates/callbacks-s3-log/Cargo.toml
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
[package]
|
||||
name = "litellm-callbacks-s3-log"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-auth-aws.workspace = true
|
||||
litellm-callbacks.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-python-compat.workspace = true
|
||||
litellm-s3.workspace = true
|
||||
litellm-secret-value.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
|
||||
base64.workspace = true
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
md-5 = "0.11.0"
|
||||
rand.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["sync", "time", "rt"] }
|
||||
tracing.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-http = { workspace = true, features = ["test-support"] }
|
||||
rstest.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] }
|
||||
wiremock.workspace = true
|
||||
367
litellm/rust/crates/callbacks-s3-log/src/key.rs
Normal file
367
litellm/rust/crates/callbacks-s3-log/src/key.rs
Normal file
|
|
@ -0,0 +1,367 @@
|
|||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use litellm_callbacks::Moment;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::settings::{Granularity, Settings};
|
||||
|
||||
const MAX_KEY_BYTES: usize = 1024;
|
||||
const MAX_FILE_NAME_BYTES: usize = 255;
|
||||
const MAX_DOWNLOAD_NAME_BYTES: usize = 1024;
|
||||
const BOUNDED_HEAD_BYTES: i64 = 64;
|
||||
const MIN_BOUNDED_FILE_NAME_BYTES: usize = 64;
|
||||
const PREFIX_DIGEST_CHARS: usize = 16;
|
||||
const EXTENSION: &str = ".json";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Stamp {
|
||||
pub year: i64,
|
||||
pub month: u32,
|
||||
pub day: u32,
|
||||
pub hour: u32,
|
||||
pub minute: u32,
|
||||
pub second: u32,
|
||||
pub micros: u32,
|
||||
}
|
||||
|
||||
impl From<[i64; 7]> for Stamp {
|
||||
fn from(parts: [i64; 7]) -> Self {
|
||||
let [year, month, day, hour, minute, second, micros] = parts;
|
||||
let part = |value: i64| u32::try_from(value).unwrap_or_default();
|
||||
Self {
|
||||
year,
|
||||
month: part(month),
|
||||
day: part(day),
|
||||
hour: part(hour),
|
||||
minute: part(minute),
|
||||
second: part(second),
|
||||
micros: part(micros),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&Moment> for Stamp {
|
||||
fn from(moment: &Moment) -> Self {
|
||||
Self {
|
||||
year: i64::from(moment.year),
|
||||
month: u32::from(moment.month),
|
||||
day: u32::from(moment.day),
|
||||
hour: u32::from(moment.hour),
|
||||
minute: u32::from(moment.minute),
|
||||
second: u32::from(moment.second),
|
||||
micros: moment.microsecond,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Stamp {
|
||||
pub fn utc(time: SystemTime) -> Self {
|
||||
let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default();
|
||||
let seconds = elapsed.as_secs();
|
||||
let (year, month, day) = civil_from_days((seconds / 86_400) as i64);
|
||||
let of_day = seconds % 86_400;
|
||||
Self {
|
||||
year,
|
||||
month,
|
||||
day,
|
||||
hour: (of_day / 3600) as u32,
|
||||
minute: (of_day / 60 % 60) as u32,
|
||||
second: (of_day % 60) as u32,
|
||||
micros: elapsed.subsec_micros(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn date(&self) -> String {
|
||||
format!("{:04}-{:02}-{:02}", self.year, self.month, self.day)
|
||||
}
|
||||
|
||||
pub fn clock(&self) -> String {
|
||||
format!("{:02}-{:02}-{:02}", self.hour, self.minute, self.second)
|
||||
}
|
||||
|
||||
fn clock_micros(&self) -> String {
|
||||
format!("{}-{:06}", self.clock(), self.micros)
|
||||
}
|
||||
}
|
||||
|
||||
fn civil_from_days(days: i64) -> (i64, u32, u32) {
|
||||
let shifted = days + 719_468;
|
||||
let era = shifted.div_euclid(146_097);
|
||||
let day_of_era = shifted.rem_euclid(146_097);
|
||||
let year_of_era =
|
||||
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
|
||||
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
|
||||
let month_index = (5 * day_of_year + 2) / 153;
|
||||
let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32;
|
||||
let month = if month_index < 10 {
|
||||
month_index + 3
|
||||
} else {
|
||||
month_index - 9
|
||||
} as u32;
|
||||
let year = year_of_era + era * 400 + i64::from(month <= 2);
|
||||
(year, month, day)
|
||||
}
|
||||
|
||||
pub fn logging_id(start: &Stamp, id: &str) -> String {
|
||||
format!("time-{}_{id}", start.clock_micros())
|
||||
}
|
||||
|
||||
pub fn object_key(
|
||||
path: &str,
|
||||
prefix: &str,
|
||||
start: &Stamp,
|
||||
file_name: &str,
|
||||
granularity: Granularity,
|
||||
) -> String {
|
||||
let sanitized = file_name.replace(['/', ':'], "_");
|
||||
let configured = if path.is_empty() {
|
||||
prefix.to_owned()
|
||||
} else {
|
||||
format!("{}/{prefix}", path.trim_end_matches('/'))
|
||||
};
|
||||
let date = match granularity {
|
||||
Granularity::Day => format!("{}/", start.date()),
|
||||
Granularity::Hour => format!("{}/{:02}/", start.date(), start.hour),
|
||||
};
|
||||
let file_name_budget = MAX_FILE_NAME_BYTES - EXTENSION.len();
|
||||
let key = format!("{configured}{date}{sanitized}{EXTENSION}");
|
||||
if key.len() <= MAX_KEY_BYTES && sanitized.len() <= file_name_budget {
|
||||
return key;
|
||||
}
|
||||
let budget = MAX_KEY_BYTES - date.len() - EXTENSION.len();
|
||||
if configured.len() + MIN_BOUNDED_FILE_NAME_BYTES <= budget {
|
||||
let bounded = bounded_name(
|
||||
file_name,
|
||||
&sanitized,
|
||||
file_name_budget.min(budget - configured.len()) as i64,
|
||||
);
|
||||
return format!("{configured}{date}{bounded}{EXTENSION}");
|
||||
}
|
||||
let shortest = bounded_name(file_name, &sanitized, MIN_BOUNDED_FILE_NAME_BYTES as i64);
|
||||
let prefix = bounded_prefix(&configured, budget as i64 - shortest.len() as i64);
|
||||
format!("{prefix}{date}{shortest}{EXTENSION}")
|
||||
}
|
||||
|
||||
pub fn download_name(start: &Stamp, id: &str) -> String {
|
||||
let stamp = format!("time-{}T{}", start.date(), start.clock_micros());
|
||||
let name = format!("{stamp}_{id}");
|
||||
let sanitized = format!("{stamp}_{}", id.replace(['/', '"'], "_"));
|
||||
let budget = MAX_DOWNLOAD_NAME_BYTES - EXTENSION.len();
|
||||
if sanitized.len() <= budget {
|
||||
return format!("{sanitized}{EXTENSION}");
|
||||
}
|
||||
format!(
|
||||
"{}{EXTENSION}",
|
||||
bounded_name(&name, &sanitized, budget as i64)
|
||||
)
|
||||
}
|
||||
|
||||
fn bounded_name(name: &str, sanitized: &str, max_bytes: i64) -> String {
|
||||
let digest = hex_sha256(name);
|
||||
let head = truncate(
|
||||
sanitized,
|
||||
BOUNDED_HEAD_BYTES.min(max_bytes - digest.len() as i64 - 1),
|
||||
);
|
||||
if head.is_empty() {
|
||||
digest
|
||||
} else {
|
||||
format!("{head}_{digest}")
|
||||
}
|
||||
}
|
||||
|
||||
fn bounded_prefix(prefix: &str, max_bytes: i64) -> String {
|
||||
let digest = format!("{}/", &hex_sha256(prefix)[..PREFIX_DIGEST_CHARS]);
|
||||
if max_bytes < digest.len() as i64 {
|
||||
return String::new();
|
||||
}
|
||||
let head = truncate(prefix, max_bytes - digest.len() as i64 - 1).trim_end_matches('/');
|
||||
if head.is_empty() {
|
||||
digest
|
||||
} else {
|
||||
format!("{head}/{digest}")
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate(value: &str, max_bytes: i64) -> &str {
|
||||
let Ok(max_bytes) = usize::try_from(max_bytes) else {
|
||||
return "";
|
||||
};
|
||||
if max_bytes == 0 {
|
||||
return "";
|
||||
}
|
||||
if value.len() <= max_bytes {
|
||||
return value;
|
||||
}
|
||||
let end = (0..=max_bytes)
|
||||
.rev()
|
||||
.find(|end| value.is_char_boundary(*end))
|
||||
.unwrap_or(0);
|
||||
&value[..end]
|
||||
}
|
||||
|
||||
pub fn hex_sha256(value: impl AsRef<[u8]>) -> String {
|
||||
hex(&Sha256::digest(value.as_ref()))
|
||||
}
|
||||
|
||||
pub fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
|
||||
pub fn parent(key: &str) -> &str {
|
||||
key.rsplit_once('/').map_or("", |(parent, _)| parent)
|
||||
}
|
||||
|
||||
pub fn quote(key: &str) -> String {
|
||||
let mut encoded = String::with_capacity(key.len());
|
||||
for byte in key.bytes() {
|
||||
if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') {
|
||||
encoded.push(byte as char);
|
||||
} else {
|
||||
encoded.push_str(&format!("%{byte:02X}"));
|
||||
}
|
||||
}
|
||||
encoded
|
||||
}
|
||||
|
||||
pub fn dns_suffix(region: Option<&str>) -> &'static str {
|
||||
let Some(region) = region.filter(|region| !region.is_empty()) else {
|
||||
return "amazonaws.com";
|
||||
};
|
||||
[
|
||||
("cn-", "amazonaws.com.cn"),
|
||||
("us-gov-", "amazonaws.com"),
|
||||
("us-isob-", "sc2s.sgov.gov"),
|
||||
("us-isof-", "csp.hci.ic.gov"),
|
||||
("us-iso-", "c2s.ic.gov"),
|
||||
("eu-isoe-", "cloud.adc-e.uk"),
|
||||
]
|
||||
.into_iter()
|
||||
.find(|(prefix, _)| region.starts_with(prefix))
|
||||
.map_or("amazonaws.com", |(_, suffix)| suffix)
|
||||
}
|
||||
|
||||
pub fn object_url(settings: &Settings, key: &str) -> String {
|
||||
let encoded = quote(key);
|
||||
match (&settings.endpoint_url, &settings.bucket_name) {
|
||||
(Some(endpoint), Some(bucket)) if settings.use_virtual_hosted_style => {
|
||||
let host = endpoint.replace("https://", "").replace("http://", "");
|
||||
let scheme = if endpoint.starts_with("https://") {
|
||||
"https://"
|
||||
} else {
|
||||
"http://"
|
||||
};
|
||||
format!("{scheme}{bucket}.{host}/{encoded}")
|
||||
}
|
||||
(Some(endpoint), Some(bucket)) => format!("{endpoint}/{bucket}/{encoded}"),
|
||||
_ => {
|
||||
let region = settings.url_region();
|
||||
let suffix = dns_suffix(
|
||||
settings
|
||||
.region_name
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_str),
|
||||
);
|
||||
format!(
|
||||
"https://{}.s3.{region}.{suffix}/{encoded}",
|
||||
settings.bucket_name.as_deref().unwrap_or("None")
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn stamp() -> Stamp {
|
||||
Stamp::from([2026, 11, 13, 9, 5, 7, 42])
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("", "", Granularity::Day, "2026-11-13/time-09-05-07-000042_abc.json")]
|
||||
#[case(
|
||||
"logs/",
|
||||
"team/",
|
||||
Granularity::Hour,
|
||||
"logs/team/2026-11-13/09/time-09-05-07-000042_abc.json"
|
||||
)]
|
||||
#[case(
|
||||
"a/b",
|
||||
"",
|
||||
Granularity::Day,
|
||||
"a/b/2026-11-13/time-09-05-07-000042_abc.json"
|
||||
)]
|
||||
fn keys_are_laid_out_as_python_lays_them_out(
|
||||
#[case] path: &str,
|
||||
#[case] prefix: &str,
|
||||
#[case] granularity: Granularity,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let name = logging_id(&stamp(), "abc");
|
||||
assert_eq!(
|
||||
object_key(path, prefix, &stamp(), &name, granularity),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_long_id_is_bounded_with_its_digest() {
|
||||
let name = logging_id(&stamp(), &"x".repeat(400));
|
||||
let key = object_key("p", "", &stamp(), &name, Granularity::Day);
|
||||
let file = key.rsplit('/').next().unwrap();
|
||||
assert!(file.len() <= MAX_FILE_NAME_BYTES);
|
||||
assert!(file.ends_with(&format!("_{}.json", hex_sha256(&name))));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn utc_stamps_split_the_clock() {
|
||||
let time = UNIX_EPOCH + std::time::Duration::from_micros(1_763_024_707_000_042);
|
||||
assert_eq!(Stamp::utc(time), Stamp::from([2025, 11, 13, 9, 5, 7, 42]));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn keys_are_quoted_like_urllib() {
|
||||
assert_eq!(quote("a b/c=d+e~f.json"), "a%20b/c%3Dd%2Be~f.json");
|
||||
assert_eq!(quote("é"), "%C3%A9");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(
|
||||
"cn-north-1",
|
||||
"https://audit-bucket.s3.cn-north-1.amazonaws.com.cn/2025-01-01/key.json"
|
||||
)]
|
||||
#[case(
|
||||
"cn-northwest-1",
|
||||
"https://audit-bucket.s3.cn-northwest-1.amazonaws.com.cn/2025-01-01/key.json"
|
||||
)]
|
||||
#[case(
|
||||
"us-east-1",
|
||||
"https://audit-bucket.s3.us-east-1.amazonaws.com/2025-01-01/key.json"
|
||||
)]
|
||||
#[case(
|
||||
"us-gov-west-1",
|
||||
"https://audit-bucket.s3.us-gov-west-1.amazonaws.com/2025-01-01/key.json"
|
||||
)]
|
||||
fn object_urls_follow_the_partition_of_the_region(
|
||||
#[case] region: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let params =
|
||||
serde_json::json!({"s3_bucket_name": "audit-bucket", "s3_region_name": region});
|
||||
let settings = Settings::from_params(
|
||||
params.as_object().unwrap(),
|
||||
&crate::settings::Constants::default(),
|
||||
);
|
||||
assert_eq!(object_url(&settings, "2025-01-01/key.json"), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn download_names_replace_slashes_and_quotes() {
|
||||
assert_eq!(
|
||||
download_name(&stamp(), "a/b\"c"),
|
||||
"time-2026-11-13T09-05-07-000042_a_b_c.json"
|
||||
);
|
||||
}
|
||||
}
|
||||
11
litellm/rust/crates/callbacks-s3-log/src/lib.rs
Normal file
11
litellm/rust/crates/callbacks-s3-log/src/lib.rs
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
mod key;
|
||||
mod limiter;
|
||||
mod pyjson;
|
||||
mod settings;
|
||||
mod sink;
|
||||
mod upload;
|
||||
|
||||
pub use key::{Stamp, download_name, logging_id, object_key, object_url};
|
||||
pub use settings::{Constants, Environment, Granularity, Settings};
|
||||
pub use sink::{BatchUploadFailure, Host, NAME, S3Log, S3LogParts, system_clock};
|
||||
pub use upload::{Backoff, Clock, Outcome, panic_message, python_backoff};
|
||||
304
litellm/rust/crates/callbacks-s3-log/src/limiter.rs
Normal file
304
litellm/rust/crates/callbacks-s3-log/src/limiter.rs
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
use std::{
|
||||
collections::VecDeque,
|
||||
sync::{Arc, Mutex, PoisonError},
|
||||
};
|
||||
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, oneshot};
|
||||
|
||||
pub enum Limiter {
|
||||
Fixed(Arc<Semaphore>),
|
||||
Adaptive(Arc<Adaptive>),
|
||||
}
|
||||
|
||||
pub enum Permit {
|
||||
Fixed(#[allow(dead_code)] OwnedSemaphorePermit),
|
||||
Adaptive(Arc<Adaptive>),
|
||||
}
|
||||
|
||||
impl Drop for Permit {
|
||||
fn drop(&mut self) {
|
||||
if let Self::Adaptive(limiter) = self {
|
||||
limiter.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Limiter {
|
||||
pub fn new(width: usize, adaptive: bool, ceiling: usize) -> Self {
|
||||
if adaptive {
|
||||
Self::Adaptive(Arc::new(Adaptive::new(width, width, width.max(ceiling))))
|
||||
} else {
|
||||
Self::Fixed(Arc::new(Semaphore::new(width.min(Semaphore::MAX_PERMITS))))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn acquire(&self) -> Permit {
|
||||
match self {
|
||||
Self::Fixed(semaphore) => Permit::Fixed(
|
||||
Arc::clone(semaphore)
|
||||
.acquire_owned()
|
||||
.await
|
||||
.unwrap_or_else(|_| unreachable!("the upload semaphore is never closed")),
|
||||
),
|
||||
Self::Adaptive(limiter) => {
|
||||
limiter.acquire().await;
|
||||
Permit::Adaptive(Arc::clone(limiter))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record(&self, throttled: bool) {
|
||||
if let Self::Adaptive(limiter) = self {
|
||||
limiter.record(throttled);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn limit(&self) -> Option<usize> {
|
||||
match self {
|
||||
Self::Fixed(_) => None,
|
||||
Self::Adaptive(limiter) => Some(limiter.state().limit),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct State {
|
||||
limit: usize,
|
||||
clean_streak: usize,
|
||||
in_flight: usize,
|
||||
waiters: VecDeque<oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
pub struct Adaptive {
|
||||
floor: usize,
|
||||
ceiling: usize,
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
impl Adaptive {
|
||||
fn new(initial: usize, floor: usize, ceiling: usize) -> Self {
|
||||
Self {
|
||||
floor,
|
||||
ceiling,
|
||||
state: Mutex::new(State {
|
||||
limit: initial.max(floor).min(ceiling),
|
||||
clean_streak: 0,
|
||||
in_flight: 0,
|
||||
waiters: VecDeque::new(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn state(&self) -> std::sync::MutexGuard<'_, State> {
|
||||
self.state.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
async fn acquire(&self) {
|
||||
let waiter = {
|
||||
let mut state = self.state();
|
||||
if state.in_flight < state.limit {
|
||||
state.in_flight += 1;
|
||||
return;
|
||||
}
|
||||
let (sender, receiver) = oneshot::channel();
|
||||
state.waiters.push_back(sender);
|
||||
receiver
|
||||
};
|
||||
let _ = waiter.await;
|
||||
}
|
||||
|
||||
fn grant(state: &mut State) {
|
||||
while state.in_flight < state.limit {
|
||||
let Some(waiter) = state.waiters.pop_front() else {
|
||||
return;
|
||||
};
|
||||
if waiter.send(()).is_ok() {
|
||||
state.in_flight += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
let mut state = self.state();
|
||||
state.in_flight = state.in_flight.saturating_sub(1);
|
||||
Self::grant(&mut state);
|
||||
}
|
||||
|
||||
fn record(&self, throttled: bool) {
|
||||
let mut state = self.state();
|
||||
if throttled {
|
||||
state.limit = self.floor.max(state.limit / 2);
|
||||
state.clean_streak = 0;
|
||||
return;
|
||||
}
|
||||
state.clean_streak += 1;
|
||||
if state.clean_streak >= state.limit && state.limit < self.ceiling {
|
||||
state.limit += 1;
|
||||
state.clean_streak = 0;
|
||||
Self::grant(&mut state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
fn the_limit_grows_after_a_clean_streak_and_halves_on_a_throttle() {
|
||||
let limiter = Limiter::new(2, true, 8);
|
||||
for _ in 0..2 {
|
||||
limiter.record(false);
|
||||
}
|
||||
assert_eq!(limiter.limit(), Some(3));
|
||||
for _ in 0..3 {
|
||||
limiter.record(false);
|
||||
}
|
||||
assert_eq!(limiter.limit(), Some(4));
|
||||
limiter.record(true);
|
||||
assert_eq!(limiter.limit(), Some(2));
|
||||
limiter.record(true);
|
||||
assert_eq!(limiter.limit(), Some(2));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_bound_past_the_semaphore_maximum_acts_as_no_bound() {
|
||||
let limiter = Limiter::new(usize::MAX, false, 0);
|
||||
let _first = limiter.acquire().await;
|
||||
let _second = limiter.acquire().await;
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_full_limit_admits_the_next_waiter_when_a_permit_drops() {
|
||||
let limiter = Arc::new(Limiter::new(1, true, 1));
|
||||
let first = limiter.acquire().await;
|
||||
let waiting = {
|
||||
let limiter = Arc::clone(&limiter);
|
||||
tokio::spawn(async move {
|
||||
let _permit = limiter.acquire().await;
|
||||
})
|
||||
};
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!waiting.is_finished());
|
||||
drop(first);
|
||||
waiting.await.unwrap();
|
||||
}
|
||||
|
||||
fn adaptive(initial: usize, floor: usize, ceiling: usize) -> Arc<Limiter> {
|
||||
Arc::new(Limiter::Adaptive(Arc::new(Adaptive::new(
|
||||
initial, floor, ceiling,
|
||||
))))
|
||||
}
|
||||
|
||||
fn in_flight(limiter: &Limiter) -> usize {
|
||||
match limiter {
|
||||
Limiter::Fixed(_) => 0,
|
||||
Limiter::Adaptive(adaptive) => adaptive.state().in_flight,
|
||||
}
|
||||
}
|
||||
|
||||
fn record(limiter: &Limiter, throttled: bool, times: usize) {
|
||||
for _ in 0..times {
|
||||
limiter.record(throttled);
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(4, 1, 16, &[(false, 4)], 5)]
|
||||
#[case(4, 1, 16, &[(false, 3)], 4)]
|
||||
#[case(16, 1, 16, &[(true, 1)], 8)]
|
||||
#[case(4, 4, 16, &[(true, 1)], 4)]
|
||||
#[case(15, 1, 16, &[(false, 1000)], 16)]
|
||||
#[case(4, 1, 32, &[(false, 3), (true, 1), (false, 1)], 2)]
|
||||
fn the_limit_follows_the_samples(
|
||||
#[case] initial: usize,
|
||||
#[case] floor: usize,
|
||||
#[case] ceiling: usize,
|
||||
#[case] samples: &[(bool, usize)],
|
||||
#[case] expected: usize,
|
||||
) {
|
||||
let limiter = adaptive(initial, floor, ceiling);
|
||||
for &(throttled, times) in samples {
|
||||
record(&limiter, throttled, times);
|
||||
}
|
||||
assert_eq!(limiter.limit(), Some(expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn growing_the_limit_wakes_a_waiting_acquirer() {
|
||||
let limiter = adaptive(1, 1, 4);
|
||||
let holder = limiter.acquire().await;
|
||||
let waiting = {
|
||||
let limiter = Arc::clone(&limiter);
|
||||
tokio::spawn(async move {
|
||||
let _permit = limiter.acquire().await;
|
||||
})
|
||||
};
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!waiting.is_finished());
|
||||
limiter.record(false);
|
||||
waiting.await.unwrap();
|
||||
drop(holder);
|
||||
assert_eq!(in_flight(&limiter), 0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn releasing_a_slot_wakes_exactly_one_waiter() {
|
||||
let limiter = adaptive(1, 1, 4);
|
||||
let holder = limiter.acquire().await;
|
||||
let acquired = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let (release, released) = tokio::sync::watch::channel(false);
|
||||
let waiters: Vec<_> = (0..3)
|
||||
.map(|_| {
|
||||
let limiter = Arc::clone(&limiter);
|
||||
let acquired = Arc::clone(&acquired);
|
||||
let mut released = released.clone();
|
||||
tokio::spawn(async move {
|
||||
let _permit = limiter.acquire().await;
|
||||
acquired.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let _ = released.wait_for(|released| *released).await;
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
tokio::task::yield_now().await;
|
||||
drop(holder);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(acquired.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
for _ in 0..3 {
|
||||
limiter.record(false);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(acquired.load(std::sync::atomic::Ordering::SeqCst), 3);
|
||||
release.send(true).unwrap();
|
||||
for waiter in waiters {
|
||||
waiter.await.unwrap();
|
||||
}
|
||||
assert_eq!(in_flight(&limiter), 0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_cancelled_waiter_is_skipped_and_leaves_in_flight_at_zero() {
|
||||
let limiter = adaptive(1, 1, 1);
|
||||
let holder = limiter.acquire().await;
|
||||
let spawn_waiter = |limiter: Arc<Limiter>| {
|
||||
tokio::spawn(async move {
|
||||
let _permit = limiter.acquire().await;
|
||||
})
|
||||
};
|
||||
let doomed = spawn_waiter(Arc::clone(&limiter));
|
||||
let next = spawn_waiter(Arc::clone(&limiter));
|
||||
tokio::task::yield_now().await;
|
||||
doomed.abort();
|
||||
assert!(doomed.await.unwrap_err().is_cancelled());
|
||||
drop(holder);
|
||||
next.await.unwrap();
|
||||
assert_eq!(in_flight(&limiter), 0);
|
||||
}
|
||||
}
|
||||
214
litellm/rust/crates/callbacks-s3-log/src/pyjson.rs
Normal file
214
litellm/rust/crates/callbacks-s3-log/src/pyjson.rs
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
use serde_json::Value;
|
||||
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
|
||||
pub fn write_str(out: &mut Vec<u8>, text: &str) {
|
||||
out.push(b'"');
|
||||
let mut units = [0u16; 2];
|
||||
for character in text.chars() {
|
||||
match character {
|
||||
'"' => out.extend_from_slice(b"\\\""),
|
||||
'\\' => out.extend_from_slice(b"\\\\"),
|
||||
'\n' => out.extend_from_slice(b"\\n"),
|
||||
'\r' => out.extend_from_slice(b"\\r"),
|
||||
'\t' => out.extend_from_slice(b"\\t"),
|
||||
'\u{8}' => out.extend_from_slice(b"\\b"),
|
||||
'\u{c}' => out.extend_from_slice(b"\\f"),
|
||||
' '..='~' => out.push(character as u8),
|
||||
_ => {
|
||||
for unit in character.encode_utf16(&mut units) {
|
||||
write_unit(out, *unit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push(b'"');
|
||||
}
|
||||
|
||||
fn write_unit(out: &mut Vec<u8>, unit: u16) {
|
||||
out.extend_from_slice(b"\\u");
|
||||
for shift in [12, 8, 4, 0] {
|
||||
out.push(HEX[usize::from((unit >> shift) & 0xf)]);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn float_repr(value: f64) -> String {
|
||||
if value.is_nan() {
|
||||
return "NaN".to_owned();
|
||||
}
|
||||
if value.is_infinite() {
|
||||
return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned();
|
||||
}
|
||||
let shortest = format!("{value:e}");
|
||||
let length = shortest
|
||||
.split('e')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.chars()
|
||||
.filter(char::is_ascii_digit)
|
||||
.count();
|
||||
let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1));
|
||||
let (negative, scientific) = match scientific.strip_prefix('-') {
|
||||
Some(rest) => (true, rest),
|
||||
None => (false, scientific.as_str()),
|
||||
};
|
||||
let (mantissa, exponent) = scientific.split_once('e').unwrap_or((scientific, "0"));
|
||||
let exponent: i32 = exponent.parse().unwrap_or(0);
|
||||
let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect();
|
||||
let mut text = String::with_capacity(digits.len() + 8);
|
||||
if negative {
|
||||
text.push('-');
|
||||
}
|
||||
let point = exponent + 1;
|
||||
if (-3..=16).contains(&point) {
|
||||
let count = digits.len() as i32;
|
||||
if point <= 0 {
|
||||
text.push_str("0.");
|
||||
text.extend(std::iter::repeat_n('0', (-point) as usize));
|
||||
text.push_str(&digits);
|
||||
} else if point >= count {
|
||||
text.push_str(&digits);
|
||||
text.extend(std::iter::repeat_n('0', (point - count) as usize));
|
||||
text.push_str(".0");
|
||||
} else {
|
||||
let (whole, fraction) = digits.split_at(point as usize);
|
||||
text.push_str(whole);
|
||||
text.push('.');
|
||||
text.push_str(fraction);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
let (first, rest) = digits.split_at(1);
|
||||
text.push_str(first);
|
||||
if !rest.is_empty() {
|
||||
text.push('.');
|
||||
text.push_str(rest);
|
||||
}
|
||||
text.push('e');
|
||||
text.push(if exponent < 0 { '-' } else { '+' });
|
||||
text.push_str(&format!("{:02}", exponent.unsigned_abs()));
|
||||
text
|
||||
}
|
||||
|
||||
pub fn write_value(out: &mut Vec<u8>, value: &Value) {
|
||||
match value {
|
||||
Value::Null => out.extend_from_slice(b"null"),
|
||||
Value::Bool(true) => out.extend_from_slice(b"true"),
|
||||
Value::Bool(false) => out.extend_from_slice(b"false"),
|
||||
Value::Number(number) => match (number.as_i64(), number.as_u64(), number.as_f64()) {
|
||||
(Some(integer), _, _) => out.extend_from_slice(integer.to_string().as_bytes()),
|
||||
(None, Some(integer), _) => out.extend_from_slice(integer.to_string().as_bytes()),
|
||||
(None, None, Some(float)) => out.extend_from_slice(float_repr(float).as_bytes()),
|
||||
(None, None, None) => out.extend_from_slice(number.to_string().as_bytes()),
|
||||
},
|
||||
Value::String(text) => write_str(out, text),
|
||||
Value::Array(items) => {
|
||||
out.push(b'[');
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.extend_from_slice(b", ");
|
||||
}
|
||||
write_value(out, item);
|
||||
}
|
||||
out.push(b']');
|
||||
}
|
||||
Value::Object(entries) => {
|
||||
out.push(b'{');
|
||||
for (index, (key, item)) in entries.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.extend_from_slice(b", ");
|
||||
}
|
||||
write_str(out, key);
|
||||
out.extend_from_slice(b": ");
|
||||
write_value(out, item);
|
||||
}
|
||||
out.push(b'}');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn dumps(value: &Value) -> String {
|
||||
let mut out = Vec::new();
|
||||
write_value(&mut out, value);
|
||||
String::from_utf8(out).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn py_str(value: &Value) -> String {
|
||||
match value {
|
||||
Value::Null => "None".to_owned(),
|
||||
Value::Bool(true) => "True".to_owned(),
|
||||
Value::Bool(false) => "False".to_owned(),
|
||||
Value::String(text) => text.clone(),
|
||||
Value::Number(number) => match number.as_f64() {
|
||||
Some(float) if number.is_f64() => float_repr(float),
|
||||
_ => number.to_string(),
|
||||
},
|
||||
Value::Array(_) | Value::Object(_) => dumps(value),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn truthy(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Null => false,
|
||||
Value::Bool(flag) => *flag,
|
||||
Value::Number(number) => number.as_f64().is_some_and(|float| float != 0.0),
|
||||
Value::String(text) => !text.is_empty(),
|
||||
Value::Array(items) => !items.is_empty(),
|
||||
Value::Object(entries) => !entries.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case(0.0, "0.0")]
|
||||
#[case(-0.0, "-0.0")]
|
||||
#[case(1.0, "1.0")]
|
||||
#[case(0.1, "0.1")]
|
||||
#[case(1.5e-5, "1.5e-05")]
|
||||
#[case(1e-4, "0.0001")]
|
||||
#[case(1e-5, "1e-05")]
|
||||
#[case(1e15, "1000000000000000.0")]
|
||||
#[case(1e16, "1e+16")]
|
||||
#[case(1.2345678901234568e17, "1.2345678901234568e+17")]
|
||||
#[case(123.456, "123.456")]
|
||||
#[case(1e100, "1e+100")]
|
||||
#[case(-2.5e-300, "-2.5e-300")]
|
||||
#[case(f64::NAN, "NaN")]
|
||||
#[case(f64::INFINITY, "Infinity")]
|
||||
#[case(f64::NEG_INFINITY, "-Infinity")]
|
||||
#[case(0.30000000000000004, "0.30000000000000004")]
|
||||
#[case(0.07872390747070312, "0.07872390747070312")]
|
||||
#[case(1794700894.4414062, "1794700894.4414062")]
|
||||
fn floats_print_as_python_repr_prints_them(#[case] value: f64, #[case] expected: &str) {
|
||||
assert_eq!(float_repr(value), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("plain", "\"plain\"")]
|
||||
#[case("quote\" back\\", "\"quote\\\" back\\\\\"")]
|
||||
#[case("\n\r\t\u{8}\u{c}\u{1}\u{7f}", "\"\\n\\r\\t\\b\\f\\u0001\\u007f\"")]
|
||||
#[case("café", "\"caf\\u00e9\"")]
|
||||
#[case("😀", "\"\\ud83d\\ude00\"")]
|
||||
fn strings_are_escaped_as_ensure_ascii_escapes_them(
|
||||
#[case] text: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let mut out = Vec::new();
|
||||
write_str(&mut out, text);
|
||||
assert_eq!(String::from_utf8(out).unwrap(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn containers_use_the_default_separators_and_keep_key_order() {
|
||||
assert_eq!(
|
||||
dumps(&json!({"b": [1, 2.5, null, true], "a": {"x": "y"}})),
|
||||
"{\"b\": [1, 2.5, null, true], \"a\": {\"x\": \"y\"}}"
|
||||
);
|
||||
}
|
||||
}
|
||||
656
litellm/rust/crates/callbacks-s3-log/src/settings.rs
Normal file
656
litellm/rust/crates/callbacks-s3-log/src/settings.rs
Normal file
|
|
@ -0,0 +1,656 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_auth_aws::AwsAuthConfig;
|
||||
use litellm_secret_value::SecretValue;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::pyjson::{py_str, truthy};
|
||||
|
||||
fn py_repr(value: &Value) -> String {
|
||||
litellm_python_compat::repr::repr(&litellm_python_compat::json::from_json(value.clone()))
|
||||
}
|
||||
|
||||
fn py_optional(value: Option<u64>) -> String {
|
||||
value.map_or_else(|| "None".to_owned(), |value| value.to_string())
|
||||
}
|
||||
|
||||
fn py_type_name(value: &Value) -> &'static str {
|
||||
match value {
|
||||
Value::Null => "NoneType",
|
||||
Value::Bool(_) => "bool",
|
||||
Value::Number(number) if number.is_f64() => "float",
|
||||
Value::Number(_) => "int",
|
||||
Value::String(_) => "str",
|
||||
Value::Array(_) => "list",
|
||||
Value::Object(_) => "dict",
|
||||
}
|
||||
}
|
||||
|
||||
pub const PARTITION_GRANULARITY_ENV_VAR: &str = "S3_PARTITION_GRANULARITY";
|
||||
pub const LOG_PROMPTS_ONLY_ENV_VAR: &str = "S3_LOG_PROMPTS_ONLY";
|
||||
const DEFAULT_MAX_QUEUE_SIZE: usize = 50_000;
|
||||
const DEFAULT_MAX_RETRY_AGE_SECONDS: u64 = 3600;
|
||||
const DEFAULT_REGION: &str = "us-west-2";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Deserialize)]
|
||||
pub struct Constants {
|
||||
pub batch_size: usize,
|
||||
pub max_concurrent_uploads: usize,
|
||||
pub max_adaptive_concurrency: usize,
|
||||
pub upload_attempt_timeout_seconds: f64,
|
||||
pub download_read_timeout_seconds: f64,
|
||||
pub download_connect_timeout_seconds: f64,
|
||||
pub web_identity_session_policy: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for Constants {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
batch_size: 512,
|
||||
max_concurrent_uploads: 16,
|
||||
max_adaptive_concurrency: 200,
|
||||
upload_attempt_timeout_seconds: 30.0,
|
||||
download_read_timeout_seconds: 600.0,
|
||||
download_connect_timeout_seconds: 5.0,
|
||||
web_identity_session_policy: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Granularity {
|
||||
#[default]
|
||||
Day,
|
||||
Hour,
|
||||
}
|
||||
|
||||
impl Granularity {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Day => "day",
|
||||
Self::Hour => "hour",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Settings {
|
||||
pub bucket_name: Option<String>,
|
||||
pub region_name: Option<Value>,
|
||||
pub endpoint_url: Option<String>,
|
||||
pub path: Option<String>,
|
||||
pub verify: Option<Value>,
|
||||
pub auth: AwsAuthConfig,
|
||||
pub use_team_prefix: bool,
|
||||
pub use_key_prefix: bool,
|
||||
pub strip_base64_files: bool,
|
||||
pub use_virtual_hosted_style: bool,
|
||||
pub log_prompts_only: Option<Value>,
|
||||
pub partition_granularity: Option<Value>,
|
||||
pub server_side_encryption: Option<String>,
|
||||
pub sse_kms_key_id: Option<String>,
|
||||
pub max_concurrent_uploads: usize,
|
||||
pub max_queue_size: usize,
|
||||
pub max_retry_age: Option<Duration>,
|
||||
pub drop_on_terminal_error: bool,
|
||||
pub adaptive_concurrency: bool,
|
||||
pub max_adaptive_concurrency: usize,
|
||||
pub batch_file_upload: bool,
|
||||
pub batch_size: usize,
|
||||
pub attempt_timeout: Duration,
|
||||
pub download_read_timeout: Duration,
|
||||
pub web_identity_session_policy: Option<String>,
|
||||
}
|
||||
|
||||
pub type Environment = dyn Fn(&str) -> Option<String> + Send + Sync;
|
||||
|
||||
fn text(params: &Map<String, Value>, name: &str) -> Option<String> {
|
||||
params.get(name).filter(|value| truthy(value)).map(py_str)
|
||||
}
|
||||
|
||||
fn secret(params: &Map<String, Value>, name: &str) -> Option<SecretValue> {
|
||||
text(params, name).map(SecretValue::new)
|
||||
}
|
||||
|
||||
fn present(params: &Map<String, Value>, name: &str) -> Option<Value> {
|
||||
params.get(name).filter(|value| !value.is_null()).cloned()
|
||||
}
|
||||
|
||||
fn is_blank(value: &Value) -> bool {
|
||||
matches!(value, Value::String(text) if text.is_empty())
|
||||
}
|
||||
|
||||
pub fn lax_bool(value: &Value) -> Option<bool> {
|
||||
match value {
|
||||
Value::Bool(flag) => Some(*flag),
|
||||
Value::Number(number) => number
|
||||
.as_f64()
|
||||
.and_then(|float| {
|
||||
[(0.0, false), (1.0, true)]
|
||||
.into_iter()
|
||||
.find(|(bound, _)| *bound == float)
|
||||
})
|
||||
.map(|(_, flag)| flag),
|
||||
Value::String(text) => match text.trim().to_ascii_lowercase().as_str() {
|
||||
"0" | "off" | "f" | "false" | "n" | "no" => Some(false),
|
||||
"1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn lax_int(value: &Value) -> Option<i128> {
|
||||
match value {
|
||||
Value::Bool(flag) => Some(i128::from(*flag)),
|
||||
Value::Number(number) => {
|
||||
if let Some(integer) = number.as_i64() {
|
||||
return Some(i128::from(integer));
|
||||
}
|
||||
if let Some(integer) = number.as_u64() {
|
||||
return Some(i128::from(integer));
|
||||
}
|
||||
let float = number.as_f64()?;
|
||||
(float.is_finite() && float.fract() == 0.0 && float.abs() <= i64::MAX as f64)
|
||||
.then_some(float as i128)
|
||||
}
|
||||
Value::String(text) => int_text(text.trim()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn int_text(text: &str) -> Option<i128> {
|
||||
let (whole, fraction) = text.split_once('.').unwrap_or((text, ""));
|
||||
if !fraction.chars().all(|character| character == '0') {
|
||||
return None;
|
||||
}
|
||||
if text.contains('.') && fraction.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let (sign, digits) = match whole.strip_prefix('-') {
|
||||
Some(rest) => (-1, rest),
|
||||
None => (1, whole.strip_prefix('+').unwrap_or(whole)),
|
||||
};
|
||||
let well_formed = !digits.is_empty()
|
||||
&& !digits.starts_with('_')
|
||||
&& !digits.ends_with('_')
|
||||
&& !digits.contains("__")
|
||||
&& digits
|
||||
.chars()
|
||||
.all(|character| character.is_ascii_digit() || character == '_');
|
||||
if !well_formed {
|
||||
return None;
|
||||
}
|
||||
digits
|
||||
.replace('_', "")
|
||||
.parse::<i128>()
|
||||
.ok()
|
||||
.map(|number| sign * number)
|
||||
}
|
||||
|
||||
fn strip_text(value: &Value) -> Value {
|
||||
match value {
|
||||
Value::String(text) => Value::String(text.trim().to_owned()),
|
||||
other => other.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn positive_int(
|
||||
setting: &str,
|
||||
configured: Option<&Value>,
|
||||
fallback: usize,
|
||||
reject_bool: bool,
|
||||
) -> usize {
|
||||
let Some(configured) = configured.filter(|value| !value.is_null() && !is_blank(value)) else {
|
||||
return fallback;
|
||||
};
|
||||
if reject_bool && configured.is_boolean() {
|
||||
tracing::warn!(
|
||||
"s3 logging: {setting}={} is a boolean, not an integer, using {fallback}",
|
||||
py_repr(configured)
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
match lax_int(&strip_text(configured)) {
|
||||
None => {
|
||||
tracing::warn!(
|
||||
"s3 logging: {setting}={} is not an integer, using {fallback}",
|
||||
py_repr(configured)
|
||||
);
|
||||
fallback
|
||||
}
|
||||
Some(bound) if bound < 1 => {
|
||||
tracing::warn!(
|
||||
"s3 logging: {setting}={} must be at least 1, using {fallback}",
|
||||
py_repr(configured)
|
||||
);
|
||||
fallback
|
||||
}
|
||||
Some(bound) => usize::try_from(bound).unwrap_or(usize::MAX),
|
||||
}
|
||||
}
|
||||
|
||||
fn retry_age(configured: &Value, fallback: Option<u64>) -> Option<u64> {
|
||||
if configured.is_null() || is_blank(configured) {
|
||||
return None;
|
||||
}
|
||||
if configured.is_boolean() {
|
||||
tracing::warn!(
|
||||
"s3 logging: s3_max_retry_age_seconds={} is a boolean, not an integer, falling back to {}",
|
||||
py_repr(configured),
|
||||
py_optional(fallback)
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
match lax_int(&strip_text(configured)) {
|
||||
None => {
|
||||
tracing::warn!(
|
||||
"s3 logging: s3_max_retry_age_seconds={} is not an integer, falling back to {}",
|
||||
py_repr(configured),
|
||||
py_optional(fallback)
|
||||
);
|
||||
fallback
|
||||
}
|
||||
Some(bound) if bound < 0 => {
|
||||
tracing::warn!(
|
||||
"s3 logging: s3_max_retry_age_seconds={} must be at least 0, falling back to {}",
|
||||
py_repr(configured),
|
||||
py_optional(fallback)
|
||||
);
|
||||
fallback
|
||||
}
|
||||
Some(0) => None,
|
||||
Some(bound) => Some(u64::try_from(bound).unwrap_or(u64::MAX)),
|
||||
}
|
||||
}
|
||||
|
||||
fn flag(
|
||||
setting: &str,
|
||||
configured: Option<&Value>,
|
||||
unset: bool,
|
||||
invalid: bool,
|
||||
consequence: &str,
|
||||
) -> bool {
|
||||
let Some(configured) = configured.filter(|value| !value.is_null() && !is_blank(value)) else {
|
||||
return unset;
|
||||
};
|
||||
lax_bool(&strip_text(configured)).unwrap_or_else(|| {
|
||||
tracing::warn!(
|
||||
"s3 logging: {setting}={} is not a boolean, {consequence}",
|
||||
py_repr(configured)
|
||||
);
|
||||
invalid
|
||||
})
|
||||
}
|
||||
|
||||
fn sse_value(name: &str, value: Option<&Value>) -> Option<String> {
|
||||
match value {
|
||||
None => None,
|
||||
Some(Value::String(text)) => Some(text.clone()),
|
||||
Some(other) => {
|
||||
tracing::warn!(
|
||||
"s3 logging: ignoring {name} because it has invalid type {}; expected a string",
|
||||
py_type_name(other)
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resolve_sse(
|
||||
algorithm: Option<&Value>,
|
||||
key_id: Option<&Value>,
|
||||
) -> (Option<String>, Option<String>) {
|
||||
let valid_sse = sse_value("s3_server_side_encryption", algorithm);
|
||||
let valid_key_id = sse_value("s3_sse_kms_key_id", key_id).filter(|key| !key.is_empty());
|
||||
let Some(algorithm) = valid_sse
|
||||
.filter(|algorithm| !algorithm.is_empty())
|
||||
.or_else(|| valid_key_id.as_ref().map(|_| "aws:kms".to_owned()))
|
||||
else {
|
||||
return (None, None);
|
||||
};
|
||||
if valid_key_id.is_some() && !algorithm.starts_with("aws:kms") {
|
||||
tracing::warn!(
|
||||
"s3 logging: ignoring s3_sse_kms_key_id because s3_server_side_encryption is {algorithm}; set it to aws:kms to encrypt with the KMS key"
|
||||
);
|
||||
return (Some(algorithm), None);
|
||||
}
|
||||
(Some(algorithm), valid_key_id)
|
||||
}
|
||||
|
||||
pub fn resolve_partition_granularity(raw: Option<&Value>) -> Granularity {
|
||||
match raw {
|
||||
Some(Value::String(text)) if text == "hour" => Granularity::Hour,
|
||||
None => Granularity::Day,
|
||||
Some(Value::String(text)) if text.is_empty() || text == "day" => Granularity::Day,
|
||||
Some(other) => {
|
||||
tracing::warn!(
|
||||
"s3 logging: s3_partition_granularity={} is not one of day, hour, using day",
|
||||
py_repr(other)
|
||||
);
|
||||
Granularity::Day
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
pub fn from_params(params: &Map<String, Value>, constants: &Constants) -> Self {
|
||||
let truthy_param = |name: &str| params.get(name).is_some_and(truthy);
|
||||
let (server_side_encryption, sse_kms_key_id) = resolve_sse(
|
||||
params
|
||||
.get("s3_server_side_encryption")
|
||||
.filter(|value| truthy(value)),
|
||||
params
|
||||
.get("s3_sse_kms_key_id")
|
||||
.filter(|value| truthy(value)),
|
||||
);
|
||||
let max_concurrent_uploads = positive_int(
|
||||
"s3_max_concurrent_uploads",
|
||||
Some(
|
||||
params
|
||||
.get("s3_max_concurrent_uploads")
|
||||
.filter(|value| !value.is_null() && !is_blank(value))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| Value::from(constants.max_concurrent_uploads)),
|
||||
)
|
||||
.as_ref(),
|
||||
constants.max_concurrent_uploads,
|
||||
false,
|
||||
);
|
||||
let max_queue_size = positive_int(
|
||||
"s3_max_queue_size",
|
||||
params.get("s3_max_queue_size"),
|
||||
DEFAULT_MAX_QUEUE_SIZE,
|
||||
true,
|
||||
);
|
||||
let max_retry_age = match params.get("s3_max_retry_age_seconds") {
|
||||
Some(value) if !value.is_null() && !is_blank(value) => {
|
||||
retry_age(value, Some(DEFAULT_MAX_RETRY_AGE_SECONDS))
|
||||
}
|
||||
_ => Some(DEFAULT_MAX_RETRY_AGE_SECONDS),
|
||||
};
|
||||
Self {
|
||||
bucket_name: text(params, "s3_bucket_name"),
|
||||
region_name: params
|
||||
.get("s3_region_name")
|
||||
.filter(|value| truthy(value))
|
||||
.cloned(),
|
||||
endpoint_url: text(params, "s3_endpoint_url"),
|
||||
path: text(params, "s3_path"),
|
||||
verify: present(params, "s3_verify"),
|
||||
auth: AwsAuthConfig {
|
||||
access_key_id: secret(params, "s3_aws_access_key_id"),
|
||||
secret_access_key: secret(params, "s3_aws_secret_access_key"),
|
||||
session_token: secret(params, "s3_aws_session_token"),
|
||||
region_name: text(params, "s3_region_name"),
|
||||
session_name: text(params, "s3_aws_session_name"),
|
||||
profile_name: text(params, "s3_aws_profile_name"),
|
||||
role_name: text(params, "s3_aws_role_name"),
|
||||
web_identity_token: secret(params, "s3_aws_web_identity_token"),
|
||||
sts_endpoint: text(params, "s3_aws_sts_endpoint"),
|
||||
external_id: None,
|
||||
},
|
||||
use_team_prefix: truthy_param("s3_use_team_prefix"),
|
||||
use_key_prefix: truthy_param("s3_use_key_prefix"),
|
||||
strip_base64_files: truthy_param("s3_strip_base64_files"),
|
||||
use_virtual_hosted_style: truthy_param("s3_use_virtual_hosted_style"),
|
||||
log_prompts_only: present(params, "s3_log_prompts_only"),
|
||||
partition_granularity: present(params, "s3_partition_granularity"),
|
||||
server_side_encryption,
|
||||
sse_kms_key_id,
|
||||
max_concurrent_uploads,
|
||||
max_queue_size,
|
||||
max_retry_age: max_retry_age.map(Duration::from_secs),
|
||||
drop_on_terminal_error: flag(
|
||||
"s3_drop_on_terminal_error",
|
||||
params.get("s3_drop_on_terminal_error"),
|
||||
true,
|
||||
true,
|
||||
"dropping terminal-failed uploads",
|
||||
),
|
||||
adaptive_concurrency: flag(
|
||||
"s3_adaptive_concurrency",
|
||||
params.get("s3_adaptive_concurrency"),
|
||||
false,
|
||||
false,
|
||||
"keeping the fixed upload width",
|
||||
),
|
||||
max_adaptive_concurrency: positive_int(
|
||||
"s3_max_adaptive_concurrency",
|
||||
params.get("s3_max_adaptive_concurrency"),
|
||||
constants.max_adaptive_concurrency,
|
||||
true,
|
||||
),
|
||||
batch_file_upload: flag(
|
||||
"s3_batch_file_upload",
|
||||
params.get("s3_batch_file_upload"),
|
||||
false,
|
||||
false,
|
||||
"keeping per-request objects",
|
||||
),
|
||||
batch_size: constants.batch_size,
|
||||
attempt_timeout: Duration::from_secs_f64(
|
||||
constants.upload_attempt_timeout_seconds.max(0.0),
|
||||
),
|
||||
download_read_timeout: Duration::from_secs_f64(
|
||||
constants.download_read_timeout_seconds.max(0.0),
|
||||
),
|
||||
web_identity_session_policy: constants.web_identity_session_policy.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn partition_granularity(&self, environment: &Environment) -> Granularity {
|
||||
resolve_partition_granularity(self.partition_granularity_raw(environment).as_ref())
|
||||
}
|
||||
|
||||
pub fn partition_granularity_raw(&self, environment: &Environment) -> Option<Value> {
|
||||
match &self.partition_granularity {
|
||||
Some(value) => Some(value.clone()),
|
||||
None => environment(PARTITION_GRANULARITY_ENV_VAR).map(Value::String),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn log_prompts_only(&self, environment: &Environment) -> bool {
|
||||
let raw = match &self.log_prompts_only {
|
||||
Some(value) => Some(value.clone()),
|
||||
None => environment(LOG_PROMPTS_ONLY_ENV_VAR).map(Value::String),
|
||||
};
|
||||
let Some(raw) = raw.filter(|value| !is_blank(value)) else {
|
||||
return false;
|
||||
};
|
||||
lax_bool(&strip_text(&raw)).unwrap_or_else(|| {
|
||||
tracing::warn!(
|
||||
"s3 logging: s3_log_prompts_only={} is not a boolean, logging prompts only",
|
||||
py_repr(&raw)
|
||||
);
|
||||
true
|
||||
})
|
||||
}
|
||||
|
||||
pub fn signing_region(
|
||||
&self,
|
||||
from_environment: impl FnOnce() -> Result<Option<String>, String>,
|
||||
) -> Result<String, String> {
|
||||
let region = match &self.region_name {
|
||||
Some(Value::String(region)) => region.clone(),
|
||||
Some(other) => return Err(invalid_region(&py_str(other))),
|
||||
None => from_environment()?.unwrap_or_else(|| DEFAULT_REGION.to_owned()),
|
||||
};
|
||||
if valid_region(®ion) {
|
||||
Ok(region)
|
||||
} else {
|
||||
Err(invalid_region(®ion))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn url_region(&self) -> String {
|
||||
self.region_name
|
||||
.as_ref()
|
||||
.map_or_else(|| "None".to_owned(), py_str)
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_region(region: &str) -> String {
|
||||
format!(
|
||||
"Invalid AWS region format: {region}. Region names must contain only lowercase letters, digits, and hyphens."
|
||||
)
|
||||
}
|
||||
|
||||
fn valid_region(region: &str) -> bool {
|
||||
!region.is_empty()
|
||||
&& region
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn settings(params: Value) -> Settings {
|
||||
Settings::from_params(params.as_object().unwrap(), &Constants::default())
|
||||
}
|
||||
|
||||
fn no_env(_: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(json!(true), Some(1))]
|
||||
#[case(json!(2.0), Some(2))]
|
||||
#[case(json!(2.5), None)]
|
||||
#[case(json!(" 4 "), Some(4))]
|
||||
#[case(json!("4.0"), Some(4))]
|
||||
#[case(json!("+4"), Some(4))]
|
||||
#[case(json!("1_000"), Some(1000))]
|
||||
#[case(json!("0x10"), None)]
|
||||
#[case(json!("1e3"), None)]
|
||||
#[case(json!("4.5"), None)]
|
||||
#[case(json!("007"), Some(7))]
|
||||
#[case(json!(1e20), None)]
|
||||
fn integers_parse_as_pydantic_parses_them(
|
||||
#[case] value: Value,
|
||||
#[case] expected: Option<i128>,
|
||||
) {
|
||||
assert_eq!(lax_int(&strip_text(&value)), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(json!(0), Some(false))]
|
||||
#[case(json!(2), None)]
|
||||
#[case(json!(1.0), Some(true))]
|
||||
#[case(json!(0.5), None)]
|
||||
#[case(json!(" yes "), Some(true))]
|
||||
#[case(json!("TRUE"), Some(true))]
|
||||
#[case(json!("off"), Some(false))]
|
||||
#[case(json!("1.0"), None)]
|
||||
#[case(json!("2"), None)]
|
||||
fn booleans_parse_as_pydantic_parses_them(
|
||||
#[case] value: Value,
|
||||
#[case] expected: Option<bool>,
|
||||
) {
|
||||
assert_eq!(lax_bool(&strip_text(&value)), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn defaults_match_the_python_constructor() {
|
||||
let settings = settings(json!({}));
|
||||
assert_eq!(settings.bucket_name, None);
|
||||
assert_eq!(settings.path, None);
|
||||
assert_eq!(settings.max_concurrent_uploads, 16);
|
||||
assert_eq!(settings.max_queue_size, 50_000);
|
||||
assert_eq!(settings.max_retry_age, Some(Duration::from_secs(3600)));
|
||||
assert!(settings.drop_on_terminal_error);
|
||||
assert!(!settings.adaptive_concurrency);
|
||||
assert_eq!(settings.max_adaptive_concurrency, 200);
|
||||
assert!(!settings.batch_file_upload);
|
||||
assert_eq!(settings.partition_granularity(&no_env), Granularity::Day);
|
||||
assert!(!settings.log_prompts_only(&no_env));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn prefix_flags_take_python_truthiness() {
|
||||
let settings = settings(json!({"s3_use_team_prefix": "false", "s3_use_key_prefix": 0}));
|
||||
assert!(settings.use_team_prefix);
|
||||
assert!(!settings.use_key_prefix);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(json!({"s3_max_retry_age_seconds": 0}), None)]
|
||||
#[case(json!({"s3_max_retry_age_seconds": "60"}), Some(60))]
|
||||
#[case(json!({"s3_max_retry_age_seconds": -1}), Some(3600))]
|
||||
#[case(json!({"s3_max_retry_age_seconds": true}), Some(3600))]
|
||||
#[case(json!({"s3_max_retry_age_seconds": ""}), Some(3600))]
|
||||
fn retry_age_parses_as_python(#[case] params: Value, #[case] seconds: Option<u64>) {
|
||||
assert_eq!(
|
||||
settings(params).max_retry_age,
|
||||
seconds.map(Duration::from_secs)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(json!({"s3_sse_kms_key_id": "k"}), Some("aws:kms"), Some("k"))]
|
||||
#[case(json!({"s3_server_side_encryption": "AES256", "s3_sse_kms_key_id": "k"}), Some("AES256"), None)]
|
||||
#[case(json!({"s3_server_side_encryption": 3}), None, None)]
|
||||
fn encryption_combines_as_python(
|
||||
#[case] params: Value,
|
||||
#[case] algorithm: Option<&str>,
|
||||
#[case] key: Option<&str>,
|
||||
) {
|
||||
let settings = settings(params);
|
||||
assert_eq!(settings.server_side_encryption.as_deref(), algorithm);
|
||||
assert_eq!(settings.sse_kms_key_id.as_deref(), key);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(json!({"s3_partition_granularity": "hour"}), None, Granularity::Hour)]
|
||||
#[case(json!({}), Some("hour"), Granularity::Hour)]
|
||||
#[case(json!({"s3_partition_granularity": "day"}), Some("hour"), Granularity::Day)]
|
||||
#[case(json!({"s3_partition_granularity": "minute"}), None, Granularity::Day)]
|
||||
fn granularity_reads_the_param_then_the_environment(
|
||||
#[case] params: Value,
|
||||
#[case] env: Option<&'static str>,
|
||||
#[case] expected: Granularity,
|
||||
) {
|
||||
let environment = move |name: &str| {
|
||||
(name == PARTITION_GRANULARITY_ENV_VAR)
|
||||
.then_some(env)
|
||||
.flatten()
|
||||
.map(str::to_owned)
|
||||
};
|
||||
assert_eq!(
|
||||
settings(params).partition_granularity(&environment),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default(json!({}), Ok(None), Ok("us-west-2"))]
|
||||
#[case::param(json!({"s3_region_name": "eu-west-1"}), Ok(Some("ap-south-1")), Ok("eu-west-1"))]
|
||||
#[case::bad_param(json!({"s3_region_name": "EU"}), Ok(None), Err(()))]
|
||||
#[case::environment(json!({}), Ok(Some("ap-south-1")), Ok("ap-south-1"))]
|
||||
#[case::bad_environment(json!({}), Ok(Some("Ap_South")), Err(()))]
|
||||
#[case::empty_environment(json!({}), Ok(Some("")), Err(()))]
|
||||
#[case::secret_failure(json!({}), Err(()), Err(()))]
|
||||
fn the_signing_region_falls_back_like_python(
|
||||
#[case] params: Value,
|
||||
#[case] environment: Result<Option<&str>, ()>,
|
||||
#[case] expected: Result<&str, ()>,
|
||||
) {
|
||||
let from_environment = || {
|
||||
environment
|
||||
.map(|region| region.map(str::to_owned))
|
||||
.map_err(|()| "secret manager down".to_owned())
|
||||
};
|
||||
assert_eq!(
|
||||
settings(params)
|
||||
.signing_region(from_environment)
|
||||
.map_err(|_| ()),
|
||||
expected.map(str::to_owned)
|
||||
);
|
||||
}
|
||||
}
|
||||
670
litellm/rust/crates/callbacks-s3-log/src/sink.rs
Normal file
670
litellm/rust/crates/callbacks-s3-log/src/sink.rs
Normal file
|
|
@ -0,0 +1,670 @@
|
|||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc, Mutex, MutexGuard, PoisonError,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::{
|
||||
FutureExt,
|
||||
future::{BoxFuture, join_all},
|
||||
};
|
||||
use litellm_auth_aws::AwsAuthService;
|
||||
use litellm_callbacks::{
|
||||
Activation, Callback, Field, FieldError, FieldSet, FieldValues, Hook, HookInput, HookOutput,
|
||||
LoggedKeyIdentity, Manifest, MetadataKind, OptOutBy, PayloadRef, RenderOptions, Scalar,
|
||||
Selection, Subscription, typed,
|
||||
};
|
||||
use litellm_tracing::Logger;
|
||||
use serde_json::Value;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::{
|
||||
key::{Stamp, download_name, logging_id, object_key, parent},
|
||||
limiter::Limiter,
|
||||
settings::{Environment, Granularity, Settings, resolve_partition_granularity},
|
||||
upload::{Backoff, Body, Clock, Held, Object, Outcome, Uploader, panic_message, render_bodies},
|
||||
};
|
||||
|
||||
pub const NAME: &str = "s3_v2";
|
||||
const JSON: &str = "application/json";
|
||||
const NDJSON: &str = "application/x-ndjson";
|
||||
const PROMPTS_ONLY_NULL_KEYS: [&str; 1] = ["response"];
|
||||
|
||||
pub trait Host: Send + Sync {
|
||||
fn callback_failed(&self);
|
||||
fn web_identity_token(&self, reference: &str) -> Result<String, String>;
|
||||
}
|
||||
|
||||
pub struct S3LogParts {
|
||||
pub settings: Settings,
|
||||
pub client: litellm_http::Client,
|
||||
pub download_client: litellm_http::Client,
|
||||
pub environment: Arc<Environment>,
|
||||
pub clock: Clock,
|
||||
pub backoff: Backoff,
|
||||
pub host: Arc<dyn Host>,
|
||||
pub cold_storage_is_s3_v2: bool,
|
||||
pub region_from_environment: Option<String>,
|
||||
pub logger: Logger,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct BatchUploadFailure {
|
||||
pub queued: usize,
|
||||
pub failed: usize,
|
||||
pub total: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct Element {
|
||||
object: Object,
|
||||
retrying_since: Option<Instant>,
|
||||
is_file: bool,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Queue {
|
||||
items: Vec<Element>,
|
||||
requeued: usize,
|
||||
}
|
||||
|
||||
pub struct S3Log {
|
||||
manifest: Manifest,
|
||||
uploader: Uploader,
|
||||
limiter: Limiter,
|
||||
host: Arc<dyn Host>,
|
||||
cold_storage_is_s3_v2: bool,
|
||||
logger: Logger,
|
||||
granularity: Mutex<Option<(Option<Value>, Granularity)>>,
|
||||
queue: Mutex<Queue>,
|
||||
flush_lock: tokio::sync::Mutex<()>,
|
||||
held: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
enum Alias {
|
||||
Absent,
|
||||
Text(String),
|
||||
NotText,
|
||||
}
|
||||
|
||||
impl Alias {
|
||||
fn of(identity: &LoggedKeyIdentity, name: &str, value: Option<&Scalar>) -> Self {
|
||||
let rejected = identity
|
||||
.rejected
|
||||
.as_deref()
|
||||
.is_some_and(|rejected| rejected.iter().any(|member| member == name));
|
||||
match value {
|
||||
_ if rejected => Self::NotText,
|
||||
Some(value) if !value.truthy() => Self::Absent,
|
||||
None => Self::Absent,
|
||||
Some(Scalar::Text(text)) => Self::Text(text.clone()),
|
||||
Some(_) => Self::NotText,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Event {
|
||||
start: Option<Stamp>,
|
||||
id: Option<String>,
|
||||
metadata: MetadataKind,
|
||||
team_alias: Alias,
|
||||
key_alias: Alias,
|
||||
cold_storage_object_key: Option<String>,
|
||||
cold_storage_s3_v2: bool,
|
||||
}
|
||||
|
||||
fn manifest() -> Manifest {
|
||||
Manifest {
|
||||
selection: Selection {
|
||||
activation: Activation::DefaultOn,
|
||||
opt_out_by: vec![OptOutBy::Admin],
|
||||
honors_no_log: true,
|
||||
..Selection::named(NAME)
|
||||
},
|
||||
subscriptions: vec![Subscription {
|
||||
reads: FieldSet::of(&[
|
||||
Field::DeprecatedKwargs,
|
||||
Field::StandardLoggingPayloadRef,
|
||||
Field::LogId,
|
||||
Field::ColdStorageObjectKey,
|
||||
Field::LoggedKeyIdentity,
|
||||
Field::AttemptTiming,
|
||||
]),
|
||||
awaited: true,
|
||||
..Subscription::on(Hook::AttemptEnded)
|
||||
}],
|
||||
has_request_state: false,
|
||||
}
|
||||
}
|
||||
|
||||
impl S3Log {
|
||||
pub fn new(parts: S3LogParts) -> Arc<Self> {
|
||||
let settings = Arc::new(parts.settings);
|
||||
let limiter = Limiter::new(
|
||||
settings.max_concurrent_uploads,
|
||||
settings.adaptive_concurrency,
|
||||
settings.max_adaptive_concurrency,
|
||||
);
|
||||
let resolver_host = Arc::clone(&parts.host);
|
||||
let session_policy = settings.web_identity_session_policy.clone();
|
||||
Arc::new(Self {
|
||||
manifest: manifest(),
|
||||
uploader: Uploader {
|
||||
settings,
|
||||
client: parts.client,
|
||||
download_client: parts.download_client,
|
||||
environment_region: parts.region_from_environment,
|
||||
auth: AwsAuthService::for_web_identity(
|
||||
Arc::new(move |reference: &str| resolver_host.web_identity_token(reference)),
|
||||
session_policy,
|
||||
),
|
||||
environment: parts.environment,
|
||||
clock: parts.clock,
|
||||
backoff: parts.backoff,
|
||||
retries: Default::default(),
|
||||
},
|
||||
limiter,
|
||||
host: parts.host,
|
||||
cold_storage_is_s3_v2: parts.cold_storage_is_s3_v2,
|
||||
logger: parts.logger,
|
||||
granularity: Mutex::default(),
|
||||
queue: Mutex::default(),
|
||||
flush_lock: tokio::sync::Mutex::new(()),
|
||||
held: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn settings(&self) -> &Settings {
|
||||
&self.uploader.settings
|
||||
}
|
||||
|
||||
pub fn partition_granularity(&self) -> Granularity {
|
||||
let raw = self
|
||||
.settings()
|
||||
.partition_granularity_raw(self.uploader.environment.as_ref());
|
||||
let mut cached = self
|
||||
.granularity
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner);
|
||||
if let Some((seen, resolved)) = cached.as_ref()
|
||||
&& *seen == raw
|
||||
{
|
||||
return *resolved;
|
||||
}
|
||||
let resolved = resolve_partition_granularity(raw.as_ref());
|
||||
*cached = Some((raw, resolved));
|
||||
resolved
|
||||
}
|
||||
|
||||
pub fn log_prompts_only(&self) -> bool {
|
||||
self.settings()
|
||||
.log_prompts_only(self.uploader.environment.as_ref())
|
||||
}
|
||||
|
||||
fn queue(&self) -> MutexGuard<'_, Queue> {
|
||||
self.queue.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
pub fn queued_keys(&self) -> Vec<String> {
|
||||
self.queue()
|
||||
.items
|
||||
.iter()
|
||||
.map(|element| element.object.key.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn push(&self, element: Element) -> usize {
|
||||
let mut queue = self.queue();
|
||||
queue.items.push(element);
|
||||
queue.items.len()
|
||||
}
|
||||
|
||||
pub fn held_payloads(&self) -> usize {
|
||||
self.held.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn hold(&self, payload: Arc<PayloadRef>, options: RenderOptions) -> Body {
|
||||
Body::Payload(Arc::new(Held::new(
|
||||
payload,
|
||||
options,
|
||||
Arc::clone(&self.held),
|
||||
)))
|
||||
}
|
||||
|
||||
fn element(&self, event: Event, body: Body) -> Result<Element, String> {
|
||||
let settings = self.settings();
|
||||
let start = event
|
||||
.start
|
||||
.ok_or_else(|| "the payload carries no start time".to_owned())?;
|
||||
let mut prefixes = Vec::new();
|
||||
for (enabled, alias) in [
|
||||
(settings.use_team_prefix, &event.team_alias),
|
||||
(settings.use_key_prefix, &event.key_alias),
|
||||
] {
|
||||
if !enabled {
|
||||
continue;
|
||||
}
|
||||
if matches!(event.metadata, MetadataKind::Null | MetadataKind::Other) {
|
||||
return Err("metadata has no get".to_owned());
|
||||
}
|
||||
match alias {
|
||||
Alias::Absent => {}
|
||||
Alias::Text(alias) => prefixes.push(alias.clone()),
|
||||
Alias::NotText => return Err("sequence item: expected str instance".to_owned()),
|
||||
}
|
||||
}
|
||||
let mut prefix = prefixes.join("/");
|
||||
if !prefix.is_empty() {
|
||||
prefix.push('/');
|
||||
}
|
||||
let file_name = event
|
||||
.id
|
||||
.as_deref()
|
||||
.map(|id| logging_id(&start, id))
|
||||
.unwrap_or_default();
|
||||
let path = settings.path.clone().unwrap_or_default();
|
||||
let key_for = |granularity| object_key(&path, &prefix, &start, &file_name, granularity);
|
||||
let cold_storage_object_key = match (event.cold_storage_s3_v2, &event.metadata) {
|
||||
(true, MetadataKind::Other) => return Err("metadata has no get".to_owned()),
|
||||
(true, MetadataKind::Mapping) => event.cold_storage_object_key,
|
||||
_ => None,
|
||||
};
|
||||
let key = match cold_storage_object_key {
|
||||
Some(cold)
|
||||
if cold == key_for(Granularity::Day) || cold == key_for(Granularity::Hour) =>
|
||||
{
|
||||
cold
|
||||
}
|
||||
_ => key_for(self.partition_granularity()),
|
||||
};
|
||||
let id = event
|
||||
.id
|
||||
.ok_or_else(|| "the payload id is missing or not a string".to_owned())?;
|
||||
Ok(Element {
|
||||
object: Object {
|
||||
key,
|
||||
body,
|
||||
content_type: JSON,
|
||||
download_name: download_name(&start, &id),
|
||||
},
|
||||
retrying_since: None,
|
||||
is_file: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn event(&self, fields: &FieldValues) -> Result<Event, FieldError> {
|
||||
let identity = fields
|
||||
.get::<typed::LoggedKeyIdentity>()?
|
||||
.unwrap_or_default();
|
||||
let metadata = identity.metadata_kind.unwrap_or(MetadataKind::Missing);
|
||||
let cold_storage_object_key = fields.get::<typed::ColdStorageObjectKey>()?;
|
||||
let reads_cold_storage = match metadata {
|
||||
MetadataKind::Other => true,
|
||||
MetadataKind::Mapping => cold_storage_object_key.is_some(),
|
||||
MetadataKind::Missing | MetadataKind::Null => false,
|
||||
};
|
||||
let cold_storage_s3_v2 = reads_cold_storage && self.cold_storage_is_s3_v2;
|
||||
Ok(Event {
|
||||
start: fields
|
||||
.get::<typed::AttemptTiming>()?
|
||||
.and_then(|timing| timing.start)
|
||||
.as_ref()
|
||||
.map(Stamp::from),
|
||||
id: fields.get::<typed::LogId>()?,
|
||||
metadata,
|
||||
team_alias: Alias::of(&identity, "team_alias", identity.team_alias.as_ref()),
|
||||
key_alias: Alias::of(&identity, "key_alias", identity.key_alias.as_ref()),
|
||||
cold_storage_object_key,
|
||||
cold_storage_s3_v2,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn submit(&self, fields: &FieldValues) -> Result<(), FieldError> {
|
||||
let Some(payload) = fields.payload()? else {
|
||||
return Ok(());
|
||||
};
|
||||
let event = self.event(fields)?;
|
||||
let options = if self.log_prompts_only() {
|
||||
RenderOptions {
|
||||
null_keys: PROMPTS_ONLY_NULL_KEYS.map(str::to_owned).to_vec(),
|
||||
}
|
||||
} else {
|
||||
RenderOptions::default()
|
||||
};
|
||||
match self.element(event, self.hold(payload, options)) {
|
||||
Ok(element) => {
|
||||
self.push(element);
|
||||
}
|
||||
Err(error) => self.failed(&error),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn failed(&self, error: &str) {
|
||||
tracing::error!("s3 Layer Error - {error}");
|
||||
self.host.callback_failed();
|
||||
}
|
||||
|
||||
pub async fn submit_audit(&self, payload: PayloadRef, id: &str) -> Option<BatchUploadFailure> {
|
||||
let body = self.hold(Arc::new(payload), RenderOptions::default());
|
||||
let now = Stamp::utc((self.uploader.clock)());
|
||||
let key = object_key(
|
||||
self.settings().path.as_deref().unwrap_or_default(),
|
||||
"audit_logs/",
|
||||
&now,
|
||||
&format!("{}_{id}", now.clock()),
|
||||
self.partition_granularity(),
|
||||
);
|
||||
let queued = self.push(Element {
|
||||
object: Object {
|
||||
key,
|
||||
body,
|
||||
content_type: JSON,
|
||||
download_name: format!("audit-{id}.json"),
|
||||
},
|
||||
retrying_since: None,
|
||||
is_file: false,
|
||||
});
|
||||
if queued >= self.settings().batch_size {
|
||||
return self.flush().await;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn upload_now(&self, key: String, body: String, download_name: String) -> bool {
|
||||
let body = Bytes::from(body);
|
||||
let object = Object {
|
||||
key,
|
||||
body: Body::Ready(body.clone()),
|
||||
content_type: NDJSON,
|
||||
download_name,
|
||||
};
|
||||
let outcome = self
|
||||
.uploader
|
||||
.upload(&object, &body, &self.limiter, false)
|
||||
.await;
|
||||
if outcome != Outcome::Delivered {
|
||||
self.host.callback_failed();
|
||||
}
|
||||
outcome == Outcome::Delivered
|
||||
}
|
||||
|
||||
pub async fn download(&self, key: &str) -> Option<Bytes> {
|
||||
self.uploader.download(key).await
|
||||
}
|
||||
|
||||
pub async fn flush(&self) -> Option<BatchUploadFailure> {
|
||||
let _flushing = self.flush_lock.lock().await;
|
||||
let (batch, requeued) = {
|
||||
let queue = self.queue();
|
||||
if queue.items.is_empty() {
|
||||
return None;
|
||||
}
|
||||
(queue.items.clone(), queue.requeued)
|
||||
};
|
||||
let length = batch.len();
|
||||
let failure = match std::panic::AssertUnwindSafe(self.send_batch(batch, requeued))
|
||||
.catch_unwind()
|
||||
.await
|
||||
{
|
||||
Ok(Ok(())) => {
|
||||
self.queue().items.drain(..length);
|
||||
return None;
|
||||
}
|
||||
Ok(Err(failure)) => failure,
|
||||
Err(panic) => {
|
||||
tracing::error!(
|
||||
"CustomLogger: async_send_batch raised; preserving {length} events in queue for retry: {}",
|
||||
panic_message(panic.as_ref())
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
let max_queue_size = self.settings().max_queue_size;
|
||||
let mut queue = self.queue();
|
||||
let overflow = queue.items.len().saturating_sub(max_queue_size);
|
||||
if overflow > 0 {
|
||||
queue.items.drain(..overflow);
|
||||
tracing::warn!(
|
||||
"CustomLogger: log queue exceeded max_queue_size={max_queue_size}; dropped {overflow} oldest events."
|
||||
);
|
||||
}
|
||||
failure
|
||||
}
|
||||
|
||||
fn batch_file_mode(&self) -> bool {
|
||||
if !self.settings().batch_file_upload {
|
||||
return false;
|
||||
}
|
||||
if self.cold_storage_is_s3_v2 {
|
||||
tracing::warn!(
|
||||
"s3 logging: s3_batch_file_upload is ignored because s3_v2 is the cold storage logger; per-request objects are required for spend log lookups"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn batch_files(&self, batch: &[Element]) -> Result<Vec<Element>, String> {
|
||||
let now = Stamp::utc((self.uploader.clock)());
|
||||
let mut groups: BTreeMap<&str, Vec<&Element>> = BTreeMap::new();
|
||||
for element in batch.iter().filter(|element| !element.is_file) {
|
||||
groups
|
||||
.entry(parent(&element.object.key))
|
||||
.or_default()
|
||||
.push(element);
|
||||
}
|
||||
let bodies: Vec<&Body> = groups
|
||||
.values()
|
||||
.flatten()
|
||||
.map(|element| &element.object.body)
|
||||
.collect();
|
||||
let mut rendered = render_bodies(&bodies).await?.into_iter();
|
||||
let mut files = Vec::with_capacity(groups.len());
|
||||
for (folder, elements) in groups {
|
||||
let name = format!("batch_{}_{}", now.clock(), uuid::Uuid::new_v4().simple());
|
||||
let mut body = Vec::new();
|
||||
for index in 0..elements.len() {
|
||||
if index > 0 {
|
||||
body.push(b'\n');
|
||||
}
|
||||
body.extend_from_slice(&rendered.next().unwrap_or_default());
|
||||
}
|
||||
files.push(Element {
|
||||
object: Object {
|
||||
key: if folder.is_empty() {
|
||||
format!("{name}.jsonl")
|
||||
} else {
|
||||
format!("{folder}/{name}.jsonl")
|
||||
},
|
||||
body: Body::Ready(Bytes::from(body)),
|
||||
content_type: NDJSON,
|
||||
download_name: format!("{name}.jsonl"),
|
||||
},
|
||||
retrying_since: elements
|
||||
.iter()
|
||||
.filter_map(|element| element.retrying_since)
|
||||
.min(),
|
||||
is_file: true,
|
||||
});
|
||||
}
|
||||
Ok(batch
|
||||
.iter()
|
||||
.filter(|element| element.is_file)
|
||||
.cloned()
|
||||
.chain(files)
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn upload_bounded(&self, element: &Element) -> Outcome {
|
||||
let _permit = self.limiter.acquire().await;
|
||||
let outcome = match element.object.body.render().await {
|
||||
Ok(body) => {
|
||||
self.uploader
|
||||
.upload(&element.object, &body, &self.limiter, true)
|
||||
.await
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::error!("Error uploading to s3: {error}");
|
||||
Outcome::Retry
|
||||
}
|
||||
};
|
||||
if outcome != Outcome::Delivered {
|
||||
self.host.callback_failed();
|
||||
}
|
||||
outcome
|
||||
}
|
||||
|
||||
async fn send_batch(
|
||||
&self,
|
||||
batch: Vec<Element>,
|
||||
requeued_count: usize,
|
||||
) -> Result<(), Option<BatchUploadFailure>> {
|
||||
let uploads = if self.batch_file_mode() {
|
||||
match self.batch_files(&batch).await {
|
||||
Ok(files) => files,
|
||||
Err(error) => {
|
||||
tracing::error!(
|
||||
"CustomLogger: async_send_batch raised; preserving {} events in queue for retry: {error}",
|
||||
batch.len()
|
||||
);
|
||||
return Err(None);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
batch.clone()
|
||||
};
|
||||
self.uploader.retries.store(0, Ordering::Relaxed);
|
||||
let stale = if uploads.len() == batch.len() {
|
||||
requeued_count.min(uploads.len())
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let order: Vec<usize> = (stale..uploads.len()).chain(0..stale).collect();
|
||||
let finished = join_all(
|
||||
order
|
||||
.iter()
|
||||
.map(|index| self.upload_bounded(&uploads[*index])),
|
||||
)
|
||||
.await;
|
||||
let mut outcomes = vec![Outcome::Retry; uploads.len()];
|
||||
for (index, outcome) in order.into_iter().zip(finished) {
|
||||
outcomes[index] = outcome;
|
||||
}
|
||||
let retries = self.uploader.retries.load(Ordering::Relaxed);
|
||||
if retries > 0 {
|
||||
tracing::warn!(
|
||||
"s3 logging: {retries} in-call retries across {} uploads this flush",
|
||||
uploads.len()
|
||||
);
|
||||
}
|
||||
let delivered = outcomes
|
||||
.iter()
|
||||
.filter(|outcome| **outcome == Outcome::Delivered)
|
||||
.count();
|
||||
let failed: Vec<(&Element, Outcome)> = uploads
|
||||
.iter()
|
||||
.zip(outcomes)
|
||||
.filter(|(_, outcome)| *outcome != Outcome::Delivered)
|
||||
.collect();
|
||||
let now = Instant::now();
|
||||
let max_age = self.settings().max_retry_age;
|
||||
let requeued: Vec<Element> = if delivered == 0 {
|
||||
failed
|
||||
.iter()
|
||||
.map(|(element, _)| (*element).clone())
|
||||
.collect()
|
||||
} else {
|
||||
failed
|
||||
.iter()
|
||||
.filter(|(element, outcome)| {
|
||||
*outcome != Outcome::Dropped
|
||||
&& !matches!(
|
||||
(max_age, element.retrying_since),
|
||||
(Some(max_age), Some(since)) if now.duration_since(since) > max_age
|
||||
)
|
||||
})
|
||||
.map(|(element, _)| {
|
||||
let mut element = (*element).clone();
|
||||
if element.retrying_since.is_none() && max_age.is_some() {
|
||||
element.retrying_since = Some(now);
|
||||
}
|
||||
element
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let dropped = failed.len() - requeued.len();
|
||||
if dropped > 0 {
|
||||
let terminal = failed
|
||||
.iter()
|
||||
.filter(|(_, outcome)| *outcome == Outcome::Dropped)
|
||||
.count();
|
||||
tracing::warn!(
|
||||
"s3 logging: {dropped} uploads dropped this flush: {terminal} rejected as terminal by S3, {} retrying longer than s3_max_retry_age_seconds={}",
|
||||
dropped - terminal,
|
||||
max_age.map_or_else(|| "None".to_owned(), |age| age.as_secs().to_string())
|
||||
);
|
||||
}
|
||||
let mut queue = self.queue();
|
||||
if requeued.is_empty() {
|
||||
queue.requeued = 0;
|
||||
return Ok(());
|
||||
}
|
||||
let max_queue_size = self.settings().max_queue_size;
|
||||
let split = batch.len().min(queue.items.len());
|
||||
let arrivals = queue.items.split_off(split);
|
||||
let overflow = (requeued.len() + arrivals.len()).saturating_sub(max_queue_size);
|
||||
if overflow > 0 {
|
||||
tracing::warn!(
|
||||
"s3 logging: queue exceeded max_queue_size={max_queue_size} after a failed flush, dropped {overflow} oldest events"
|
||||
);
|
||||
}
|
||||
let requeued_length = requeued.len();
|
||||
queue.items = requeued
|
||||
.into_iter()
|
||||
.chain(arrivals)
|
||||
.skip(overflow)
|
||||
.collect();
|
||||
queue.requeued = requeued_length.saturating_sub(overflow);
|
||||
Err(Some(BatchUploadFailure {
|
||||
queued: batch.len(),
|
||||
failed: failed.len(),
|
||||
total: uploads.len(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
impl Callback for S3Log {
|
||||
fn manifest(&self) -> &Manifest {
|
||||
&self.manifest
|
||||
}
|
||||
|
||||
type State = ();
|
||||
|
||||
fn handle<'a>(
|
||||
&'a self,
|
||||
input: &'a HookInput,
|
||||
_state: Option<&'a ()>,
|
||||
) -> BoxFuture<'a, HookOutput> {
|
||||
Box::pin(async move {
|
||||
if input.hook == Hook::AttemptEnded {
|
||||
self.logger.instrument(self.submit(&input.fields)).await?;
|
||||
}
|
||||
Ok(None)
|
||||
})
|
||||
}
|
||||
|
||||
fn flush(&self) -> BoxFuture<'_, ()> {
|
||||
Box::pin(self.logger.instrument(S3Log::flush(self)).map(drop))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn system_clock() -> SystemTime {
|
||||
SystemTime::now()
|
||||
}
|
||||
470
litellm/rust/crates/callbacks-s3-log/src/upload.rs
Normal file
470
litellm/rust/crates/callbacks-s3-log/src/upload.rs
Normal file
|
|
@ -0,0 +1,470 @@
|
|||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use base64::Engine;
|
||||
use bytes::Bytes;
|
||||
use futures_util::FutureExt;
|
||||
use litellm_auth_aws::{AwsAuthService, Credentials};
|
||||
use litellm_callbacks::{PayloadRef, RenderOptions, render_all};
|
||||
use litellm_s3::signed::{Reply, SendError, SignedRequest, send};
|
||||
use reqwest::Method;
|
||||
|
||||
use crate::{
|
||||
key::{hex_sha256, object_url},
|
||||
limiter::Limiter,
|
||||
settings::{Environment, Settings},
|
||||
};
|
||||
|
||||
const ATTEMPTS: u32 = 3;
|
||||
const RETRYABLE: [u16; 4] = [403, 429, 500, 503];
|
||||
const TERMINAL_CODES: [&str; 7] = [
|
||||
"EntityTooLarge",
|
||||
"InvalidArgument",
|
||||
"MalformedXML",
|
||||
"InvalidDigest",
|
||||
"KeyTooLongError",
|
||||
"BadDigest",
|
||||
"InvalidRequest",
|
||||
];
|
||||
const CACHE_CONTROL: &str = "private, immutable, max-age=31536000, s-maxage=0";
|
||||
const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Outcome {
|
||||
Delivered,
|
||||
Retry,
|
||||
Dropped,
|
||||
}
|
||||
|
||||
pub struct Held {
|
||||
payload: Arc<PayloadRef>,
|
||||
options: RenderOptions,
|
||||
live: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Held {
|
||||
pub fn new(payload: Arc<PayloadRef>, options: RenderOptions, live: Arc<AtomicUsize>) -> Self {
|
||||
live.fetch_add(1, Ordering::Relaxed);
|
||||
Self {
|
||||
payload,
|
||||
options,
|
||||
live,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Held {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("Held")
|
||||
.field("payload", &self.payload)
|
||||
.field("options", &self.options)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Held {
|
||||
fn drop(&mut self) {
|
||||
self.live.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Body {
|
||||
Ready(Bytes),
|
||||
Payload(Arc<Held>),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Object {
|
||||
pub key: String,
|
||||
pub body: Body,
|
||||
pub content_type: &'static str,
|
||||
pub download_name: String,
|
||||
}
|
||||
|
||||
pub fn panic_message(panic: &(dyn std::any::Any + Send)) -> String {
|
||||
panic
|
||||
.downcast_ref::<&str>()
|
||||
.map(|message| (*message).to_owned())
|
||||
.or_else(|| panic.downcast_ref::<String>().cloned())
|
||||
.unwrap_or_else(|| "a panic without a message".to_owned())
|
||||
}
|
||||
|
||||
pub async fn render_bodies(bodies: &[&Body]) -> Result<Vec<Bytes>, String> {
|
||||
let held: Vec<&Held> = bodies
|
||||
.iter()
|
||||
.filter_map(|body| match body {
|
||||
Body::Payload(held) => Some(held.as_ref()),
|
||||
Body::Ready(_) => None,
|
||||
})
|
||||
.collect();
|
||||
let mut groups: Vec<(&RenderOptions, Vec<usize>)> = Vec::new();
|
||||
for (index, item) in held.iter().enumerate() {
|
||||
match groups
|
||||
.iter_mut()
|
||||
.find(|(options, _)| **options == item.options)
|
||||
{
|
||||
Some((_, indexes)) => indexes.push(index),
|
||||
None => groups.push((&item.options, vec![index])),
|
||||
}
|
||||
}
|
||||
let mut rendered: Vec<Option<Bytes>> = vec![None; held.len()];
|
||||
for (options, indexes) in groups {
|
||||
let payloads: Vec<&PayloadRef> = indexes
|
||||
.iter()
|
||||
.map(|index| held[*index].payload.as_ref())
|
||||
.collect();
|
||||
let encoded = std::panic::AssertUnwindSafe(render_all(&payloads, options.clone()))
|
||||
.catch_unwind()
|
||||
.await
|
||||
.map_err(|panic| panic_message(panic.as_ref()))??;
|
||||
for (index, bytes) in indexes.into_iter().zip(encoded) {
|
||||
rendered[index] = Some(Bytes::from(bytes));
|
||||
}
|
||||
}
|
||||
let mut rendered = rendered.into_iter();
|
||||
bodies
|
||||
.iter()
|
||||
.map(|body| match body {
|
||||
Body::Ready(bytes) => Ok(bytes.clone()),
|
||||
Body::Payload(_) => rendered
|
||||
.next()
|
||||
.flatten()
|
||||
.ok_or_else(|| "the host rendered too few payloads".to_owned()),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Body {
|
||||
pub async fn render(&self) -> Result<Bytes, String> {
|
||||
render_bodies(&[self])
|
||||
.await?
|
||||
.pop()
|
||||
.ok_or_else(|| "the host rendered no payload".to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
pub type Clock = fn() -> SystemTime;
|
||||
pub type Backoff = Arc<dyn Fn(u32) -> Duration + Send + Sync>;
|
||||
|
||||
pub struct Uploader {
|
||||
pub settings: Arc<Settings>,
|
||||
pub client: litellm_http::Client,
|
||||
pub download_client: litellm_http::Client,
|
||||
pub environment_region: Option<String>,
|
||||
pub auth: AwsAuthService,
|
||||
pub environment: Arc<Environment>,
|
||||
pub clock: Clock,
|
||||
pub backoff: Backoff,
|
||||
pub retries: AtomicUsize,
|
||||
}
|
||||
|
||||
enum Attempt {
|
||||
Reply(Reply),
|
||||
Failed(SendError),
|
||||
}
|
||||
|
||||
pub fn error_code(body: &str) -> Option<&str> {
|
||||
let mut rest = body;
|
||||
while let Some(start) = rest.find("<Code>") {
|
||||
let after = &rest[start + "<Code>".len()..];
|
||||
let code_length = after.find('<').unwrap_or(after.len());
|
||||
if code_length > 0 && after[code_length..].starts_with("</Code>") {
|
||||
return Some(&after[..code_length]);
|
||||
}
|
||||
rest = after;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn is_terminal(status: u16, body: &str) -> bool {
|
||||
matches!(status, 400 | 403)
|
||||
&& error_code(body).is_some_and(|code| TERMINAL_CODES.contains(&code))
|
||||
}
|
||||
|
||||
pub fn python_backoff(attempt: u32) -> Duration {
|
||||
Duration::from_secs_f64(f64::from(1u32 << attempt) * (0.5 + rand::random::<f64>()))
|
||||
}
|
||||
|
||||
fn throttled(attempt: &Attempt) -> bool {
|
||||
match attempt {
|
||||
Attempt::Failed(_) => true,
|
||||
Attempt::Reply(reply) => {
|
||||
matches!(reply.status, 429 | 503) || error_code(&reply.text()) == Some("SlowDown")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Uploader {
|
||||
pub fn headers(&self, object: &Object, body: &Bytes) -> Vec<(String, String)> {
|
||||
let md5 = base64::engine::general_purpose::STANDARD
|
||||
.encode(<md5::Md5 as md5::Digest>::digest(body));
|
||||
let mut headers = vec![
|
||||
("Content-Type".to_owned(), object.content_type.to_owned()),
|
||||
("Content-MD5".to_owned(), md5),
|
||||
("x-amz-content-sha256".to_owned(), hex_sha256(body)),
|
||||
("Content-Language".to_owned(), "en".to_owned()),
|
||||
(
|
||||
"Content-Disposition".to_owned(),
|
||||
format!("inline; filename=\"{}\"", object.download_name),
|
||||
),
|
||||
("Cache-Control".to_owned(), CACHE_CONTROL.to_owned()),
|
||||
];
|
||||
if let Some(algorithm) = &self.settings.server_side_encryption {
|
||||
headers.push(("x-amz-server-side-encryption".to_owned(), algorithm.clone()));
|
||||
}
|
||||
if let Some(key_id) = &self.settings.sse_kms_key_id {
|
||||
headers.push((
|
||||
"x-amz-server-side-encryption-aws-kms-key-id".to_owned(),
|
||||
key_id.clone(),
|
||||
));
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
async fn credentials(&self) -> Result<Credentials, String> {
|
||||
let environment = Arc::clone(&self.environment);
|
||||
let lookup = move |name: &str| environment(name);
|
||||
self.auth
|
||||
.resolve_credentials(self.settings.auth.clone(), &lookup)
|
||||
.await
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
async fn forget_credentials(&self) {
|
||||
let environment = Arc::clone(&self.environment);
|
||||
let lookup = move |name: &str| environment(name);
|
||||
self.auth
|
||||
.invalidate_credentials(self.settings.auth.clone(), &lookup)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn attempt(
|
||||
&self,
|
||||
url: &str,
|
||||
headers: &[(String, String)],
|
||||
body: &Bytes,
|
||||
) -> Result<Attempt, String> {
|
||||
let credentials = self.credentials().await?;
|
||||
let region = self
|
||||
.settings
|
||||
.signing_region(|| Ok(self.environment_region.clone()))?;
|
||||
let put = || {
|
||||
send(
|
||||
&self.client,
|
||||
SignedRequest {
|
||||
method: Method::PUT,
|
||||
url,
|
||||
headers,
|
||||
body: body.clone(),
|
||||
region: ®ion,
|
||||
credentials: &credentials,
|
||||
time: (self.clock)(),
|
||||
read_timeout: Some(self.settings.attempt_timeout),
|
||||
},
|
||||
)
|
||||
};
|
||||
let sent = match put().await {
|
||||
Err(error) if error.is_reconnectable() => put().await,
|
||||
sent => sent,
|
||||
};
|
||||
match sent {
|
||||
Ok(reply) => Ok(Attempt::Reply(reply)),
|
||||
Err(error) if error.is_transient() => Ok(Attempt::Failed(error)),
|
||||
Err(error) => Err(error.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn retry_reason(&self, attempt: &Attempt) -> Option<String> {
|
||||
match attempt {
|
||||
Attempt::Failed(error) => Some(error.to_string()),
|
||||
Attempt::Reply(reply) => {
|
||||
if !RETRYABLE.contains(&reply.status) {
|
||||
return None;
|
||||
}
|
||||
if self.settings.drop_on_terminal_error && is_terminal(reply.status, &reply.text())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(reply.status.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn upload(
|
||||
&self,
|
||||
object: &Object,
|
||||
body: &Bytes,
|
||||
limiter: &Limiter,
|
||||
in_flush: bool,
|
||||
) -> Outcome {
|
||||
let url = object_url(&self.settings, &object.key);
|
||||
let headers = self.headers(object, body);
|
||||
let mut attempt_number = 0;
|
||||
let last = loop {
|
||||
let attempt = match self.attempt(&url, &headers, body).await {
|
||||
Ok(attempt) => attempt,
|
||||
Err(error) => {
|
||||
tracing::error!("Error uploading to s3: {error}");
|
||||
return Outcome::Retry;
|
||||
}
|
||||
};
|
||||
limiter.record(throttled(&attempt));
|
||||
if attempt_number + 1 >= ATTEMPTS {
|
||||
break attempt;
|
||||
}
|
||||
let Some(reason) = self.retry_reason(&attempt) else {
|
||||
break attempt;
|
||||
};
|
||||
if matches!(&attempt, Attempt::Reply(reply) if reply.status == 403) {
|
||||
self.forget_credentials().await;
|
||||
}
|
||||
let wait = (self.backoff)(attempt_number);
|
||||
if in_flush {
|
||||
tracing::debug!(
|
||||
"S3 upload attempt {}/{ATTEMPTS} failed with {reason}, retrying in {:.2}s key={}",
|
||||
attempt_number + 1,
|
||||
wait.as_secs_f64(),
|
||||
object.key
|
||||
);
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"S3 upload attempt {}/{ATTEMPTS} failed with {reason}, retrying in {:.2}s key={}",
|
||||
attempt_number + 1,
|
||||
wait.as_secs_f64(),
|
||||
object.key
|
||||
);
|
||||
}
|
||||
self.retries.fetch_add(1, Ordering::Relaxed);
|
||||
tokio::time::sleep(wait).await;
|
||||
attempt_number += 1;
|
||||
};
|
||||
match last {
|
||||
Attempt::Reply(reply) if (200..300).contains(&reply.status) => Outcome::Delivered,
|
||||
Attempt::Reply(reply) => {
|
||||
let body = reply.text();
|
||||
tracing::error!(
|
||||
"Error uploading to s3: status {} for {}: {}",
|
||||
reply.status,
|
||||
object.key,
|
||||
body.chars().take(512).collect::<String>()
|
||||
);
|
||||
if self.settings.drop_on_terminal_error && is_terminal(reply.status, &body) {
|
||||
tracing::warn!(
|
||||
"s3 logging: dropping object {} after terminal status {}",
|
||||
object.key,
|
||||
reply.status
|
||||
);
|
||||
return Outcome::Dropped;
|
||||
}
|
||||
Outcome::Retry
|
||||
}
|
||||
Attempt::Failed(error) => {
|
||||
tracing::error!("Error uploading to s3: {error}");
|
||||
Outcome::Retry
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn download(&self, key: &str) -> Option<Bytes> {
|
||||
let credentials = match self.credentials().await {
|
||||
Ok(credentials) => credentials,
|
||||
Err(error) => {
|
||||
tracing::error!("Error downloading from S3: {error}");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let Some(region) = self
|
||||
.settings
|
||||
.region_name
|
||||
.as_ref()
|
||||
.map(crate::pyjson::py_str)
|
||||
else {
|
||||
tracing::error!("Error downloading from S3: no s3_region_name to sign with");
|
||||
return None;
|
||||
};
|
||||
let headers = [("x-amz-content-sha256".to_owned(), EMPTY_SHA256.to_owned())];
|
||||
let reply = send(
|
||||
&self.download_client,
|
||||
SignedRequest {
|
||||
method: Method::GET,
|
||||
url: &object_url(&self.settings, key),
|
||||
headers: &headers,
|
||||
body: Bytes::new(),
|
||||
region: ®ion,
|
||||
credentials: &credentials,
|
||||
time: (self.clock)(),
|
||||
read_timeout: Some(self.settings.download_read_timeout),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
match reply {
|
||||
Ok(reply) if reply.status == 200 => Some(reply.body),
|
||||
Ok(reply) => {
|
||||
tracing::error!("S3 object not found, saw response={}", reply.text());
|
||||
None
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::error!("Error downloading from S3: {error}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn md5_base64(body: &[u8]) -> String {
|
||||
base64::engine::general_purpose::STANDARD.encode(<md5::Md5 as md5::Digest>::digest(body))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn sha256_hex(body: &[u8]) -> String {
|
||||
crate::key::hex(&<sha2::Sha256 as sha2::Digest>::digest(body))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case(400, "<Error><Code>EntityTooLarge</Code></Error>", true)]
|
||||
#[case(403, "<Error><Code>BadDigest</Code></Error>", true)]
|
||||
#[case(403, "<Error><Code>AccessDenied</Code></Error>", false)]
|
||||
#[case(503, "<Error><Code>InvalidRequest</Code></Error>", false)]
|
||||
#[case(400, "bad request", false)]
|
||||
#[case(400, "<Code></Code><Code>InvalidDigest</Code>", true)]
|
||||
fn only_object_rejections_are_terminal(
|
||||
#[case] status: u16,
|
||||
#[case] body: &str,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
assert_eq!(is_terminal(status, body), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(0, 0.5, 1.5)]
|
||||
#[case(1, 1.0, 3.0)]
|
||||
fn backoff_doubles_with_jitter(#[case] attempt: u32, #[case] low: f64, #[case] high: f64) {
|
||||
for _ in 0..50 {
|
||||
let wait = python_backoff(attempt).as_secs_f64();
|
||||
assert!((low..high).contains(&wait), "{wait}");
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn digests_match_python() {
|
||||
assert_eq!(md5_base64(b"{}"), "mZFLkyvTelC5g8XnyQrpOw==");
|
||||
assert_eq!(
|
||||
sha256_hex(b""),
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
);
|
||||
}
|
||||
}
|
||||
693
litellm/rust/crates/callbacks-s3-log/tests/sink.rs
Normal file
693
litellm/rust/crates/callbacks-s3-log/tests/sink.rs
Normal file
|
|
@ -0,0 +1,693 @@
|
|||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicU64, AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use futures_util::future::BoxFuture;
|
||||
use litellm_callbacks::{
|
||||
Callback, Field, FieldSet, FieldValues, Hook, HookInput, LoggedKeyIdentity, MetadataKind,
|
||||
PayloadHost, PayloadRef, RenderOptions, Rendered, Scalar,
|
||||
};
|
||||
use litellm_callbacks_s3_log::{BatchUploadFailure, Constants, Host, S3Log, S3LogParts, Settings};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
use wiremock::{
|
||||
Mock, MockServer, Request, ResponseTemplate,
|
||||
matchers::{method, path_regex},
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
struct Recorder {
|
||||
failures: AtomicUsize,
|
||||
next: AtomicU64,
|
||||
payloads: Mutex<HashMap<u64, Option<String>>>,
|
||||
}
|
||||
|
||||
const PANICS: &str = "panics while rendering";
|
||||
|
||||
impl Recorder {
|
||||
fn stash(self: &Arc<Self>, body: Option<&str>) -> PayloadRef {
|
||||
let id = self.next.fetch_add(1, Ordering::Relaxed);
|
||||
self.payloads
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(id, body.map(str::to_owned));
|
||||
PayloadRef::new(id, Arc::clone(self) as Arc<dyn PayloadHost>)
|
||||
}
|
||||
|
||||
fn rewrite(&self, id: u64, body: &str) {
|
||||
self.payloads
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(id, Some(body.to_owned()));
|
||||
}
|
||||
|
||||
fn held(&self) -> usize {
|
||||
self.payloads.lock().unwrap().len()
|
||||
}
|
||||
|
||||
fn failures(&self) -> usize {
|
||||
self.failures.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
impl PayloadHost for Recorder {
|
||||
fn render(&self, payloads: Vec<u64>, options: RenderOptions) -> BoxFuture<'static, Rendered> {
|
||||
let bodies: Vec<Option<String>> = payloads
|
||||
.iter()
|
||||
.map(|payload| {
|
||||
self.payloads
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(payload)
|
||||
.cloned()
|
||||
.flatten()
|
||||
})
|
||||
.collect();
|
||||
let rendered = bodies
|
||||
.into_iter()
|
||||
.map(|body| match body {
|
||||
Some(body) if body == PANICS => panic!("dictionary changed size during iteration"),
|
||||
Some(body) if options.null_keys.is_empty() => Ok(body.into_bytes()),
|
||||
Some(body) => {
|
||||
Ok(format!("{body} without {}", options.null_keys.join(",")).into_bytes())
|
||||
}
|
||||
None => Err("the payload cannot be encoded".to_owned()),
|
||||
})
|
||||
.collect();
|
||||
Box::pin(std::future::ready(rendered))
|
||||
}
|
||||
|
||||
fn release(&self, payload: u64) {
|
||||
self.payloads.lock().unwrap().remove(&payload);
|
||||
}
|
||||
}
|
||||
|
||||
impl Host for Recorder {
|
||||
fn callback_failed(&self) {
|
||||
self.failures.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn web_identity_token(&self, reference: &str) -> Result<String, String> {
|
||||
Ok(reference.to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_clock() -> SystemTime {
|
||||
UNIX_EPOCH + Duration::from_secs(1_763_024_707)
|
||||
}
|
||||
|
||||
fn no_env(_: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
fn sink(server: &MockServer, extra: Value) -> (Arc<S3Log>, Arc<Recorder>) {
|
||||
sink_with(server, extra, Constants::default())
|
||||
}
|
||||
|
||||
fn sink_with(
|
||||
server: &MockServer,
|
||||
extra: Value,
|
||||
constants: Constants,
|
||||
) -> (Arc<S3Log>, Arc<Recorder>) {
|
||||
sink_from(server, extra, constants, false, None)
|
||||
}
|
||||
|
||||
fn sink_from(
|
||||
server: &MockServer,
|
||||
extra: Value,
|
||||
constants: Constants,
|
||||
cold_storage_is_s3_v2: bool,
|
||||
region_from_environment: Option<&str>,
|
||||
) -> (Arc<S3Log>, Arc<Recorder>) {
|
||||
let mut params = json!({
|
||||
"s3_bucket_name": "logs",
|
||||
"s3_region_name": "us-east-1",
|
||||
"s3_endpoint_url": server.uri(),
|
||||
"s3_aws_access_key_id": "AKIDEXAMPLE",
|
||||
"s3_aws_secret_access_key": "secret",
|
||||
"s3_path": "base",
|
||||
});
|
||||
params
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.extend(extra.as_object().unwrap().clone());
|
||||
params
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.retain(|_, value| !value.is_null());
|
||||
let host = Arc::new(Recorder::default());
|
||||
let sink = S3Log::new(S3LogParts {
|
||||
settings: Settings::from_params(params.as_object().unwrap(), &constants),
|
||||
client: litellm_http::Client::for_test(),
|
||||
download_client: litellm_http::Client::for_test(),
|
||||
environment: Arc::new(no_env),
|
||||
clock: fixed_clock,
|
||||
backoff: Arc::new(|_| Duration::ZERO),
|
||||
host: Arc::clone(&host) as Arc<dyn Host>,
|
||||
cold_storage_is_s3_v2,
|
||||
region_from_environment: region_from_environment.map(str::to_owned),
|
||||
logger: Default::default(),
|
||||
});
|
||||
(sink, host)
|
||||
}
|
||||
|
||||
const READS: [Field; 6] = [
|
||||
Field::DeprecatedKwargs,
|
||||
Field::StandardLoggingPayloadRef,
|
||||
Field::LogId,
|
||||
Field::ColdStorageObjectKey,
|
||||
Field::LoggedKeyIdentity,
|
||||
Field::AttemptTiming,
|
||||
];
|
||||
|
||||
struct Event {
|
||||
payload: Option<PayloadRef>,
|
||||
id: &'static str,
|
||||
identity: Value,
|
||||
cold_storage_object_key: Option<&'static str>,
|
||||
}
|
||||
|
||||
impl Event {
|
||||
fn input(self) -> HookInput {
|
||||
let mut fields = FieldValues::new(FieldSet::of(&READS))
|
||||
.with(Field::DeprecatedKwargs, Value::Null)
|
||||
.with(Field::LogId, json!(self.id))
|
||||
.with(Field::LoggedKeyIdentity, self.identity)
|
||||
.with(
|
||||
Field::AttemptTiming,
|
||||
json!({"start": {"year": 2026, "month": 11, "day": 13, "hour": 9, "minute": 5, "second": 7, "microsecond": 42}}),
|
||||
);
|
||||
if let Some(key) = self.cold_storage_object_key {
|
||||
fields = fields.with(Field::ColdStorageObjectKey, json!(key));
|
||||
}
|
||||
if let Some(payload) = self.payload {
|
||||
fields.set_payload(payload);
|
||||
}
|
||||
HookInput {
|
||||
hook: Hook::AttemptEnded,
|
||||
fields,
|
||||
call: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn event(host: &Arc<Recorder>, id: &'static str, body: &str) -> Event {
|
||||
event_with(host.stash(Some(body)), id)
|
||||
}
|
||||
|
||||
fn event_with(payload: PayloadRef, id: &'static str) -> Event {
|
||||
Event {
|
||||
payload: Some(payload),
|
||||
id,
|
||||
identity: json!({"metadata_kind": "mapping", "team_alias": "team-a"}),
|
||||
cold_storage_object_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn submit(sink: &S3Log, event: Event) {
|
||||
sink.handle(&event.input(), None).await.unwrap();
|
||||
}
|
||||
|
||||
fn header(request: &Request, name: &str) -> Option<String> {
|
||||
request
|
||||
.headers
|
||||
.get(name)
|
||||
.map(|value| value.to_str().unwrap().to_owned())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_flush_puts_one_object_per_event_with_the_python_headers() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink(
|
||||
&server,
|
||||
json!({"s3_use_team_prefix": true, "s3_sse_kms_key_id": "kms-1"}),
|
||||
);
|
||||
submit(
|
||||
&sink,
|
||||
event(&host, "chatcmpl-1", "{\"id\": \"chatcmpl-1\"}"),
|
||||
)
|
||||
.await;
|
||||
let failure = sink.flush().await;
|
||||
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
let put = &requests[0];
|
||||
assert_eq!(
|
||||
put.url.path(),
|
||||
"/logs/base/team-a/2026-11-13/time-09-05-07-000042_chatcmpl-1.json"
|
||||
);
|
||||
assert_eq!(put.body, b"{\"id\": \"chatcmpl-1\"}");
|
||||
assert_eq!(
|
||||
header(put, "content-type").as_deref(),
|
||||
Some("application/json")
|
||||
);
|
||||
assert_eq!(
|
||||
header(put, "content-md5").as_deref(),
|
||||
Some("Ma6yKrRUR7fU10HikmP0wA==")
|
||||
);
|
||||
assert_eq!(
|
||||
header(put, "content-disposition").as_deref(),
|
||||
Some("inline; filename=\"time-2026-11-13T09-05-07-000042_chatcmpl-1.json\"")
|
||||
);
|
||||
assert_eq!(
|
||||
header(put, "x-amz-server-side-encryption").as_deref(),
|
||||
Some("aws:kms")
|
||||
);
|
||||
assert_eq!(
|
||||
header(put, "x-amz-date").as_deref(),
|
||||
Some("20251113T090507Z")
|
||||
);
|
||||
assert!(
|
||||
header(put, "authorization").unwrap().starts_with(
|
||||
"AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20251113/us-east-1/s3/aws4_request"
|
||||
)
|
||||
);
|
||||
assert!(sink.queued_keys().is_empty());
|
||||
assert_eq!(host.failures.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(failure, None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_bucket_that_fails_every_upload_keeps_the_queue() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(503))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink(&server, json!({}));
|
||||
submit(&sink, event(&host, "a", "{}")).await;
|
||||
submit(&sink, event(&host, "b", "{}")).await;
|
||||
let failure = sink.flush().await;
|
||||
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 6);
|
||||
assert_eq!(sink.queued_keys().len(), 2);
|
||||
assert_eq!(host.failures.load(Ordering::Relaxed), 2);
|
||||
assert_eq!(
|
||||
failure,
|
||||
Some(BatchUploadFailure {
|
||||
queued: 2,
|
||||
failed: 2,
|
||||
total: 2
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_terminal_rejection_is_dropped_when_another_upload_lands() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.and(path_regex("_bad\\.json$"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(400)
|
||||
.set_body_string("<Error><Code>EntityTooLarge</Code></Error>"),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("PUT"))
|
||||
.and(path_regex("_slow\\.json$"))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink(&server, json!({}));
|
||||
for id in ["ok", "bad", "slow"] {
|
||||
submit(&sink, event(&host, id, "{}")).await;
|
||||
}
|
||||
sink.flush().await;
|
||||
|
||||
assert_eq!(
|
||||
sink.queued_keys(),
|
||||
["base/2026-11-13/time-09-05-07-000042_slow.json"]
|
||||
);
|
||||
assert_eq!(host.held(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn batch_files_group_one_flush_by_folder() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink(&server, json!({"s3_batch_file_upload": "yes"}));
|
||||
submit(&sink, event(&host, "a", "{\"n\": 1}")).await;
|
||||
submit(&sink, event(&host, "b", "{\"n\": 2}")).await;
|
||||
sink.flush().await;
|
||||
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(
|
||||
requests[0]
|
||||
.url
|
||||
.path()
|
||||
.starts_with("/logs/base/2026-11-13/batch_09-05-07_")
|
||||
);
|
||||
assert_eq!(requests[0].body, b"{\"n\": 1}\n{\"n\": 2}");
|
||||
assert_eq!(
|
||||
header(&requests[0], "content-type").as_deref(),
|
||||
Some("application/x-ndjson")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_event_python_would_reject_is_counted_and_skipped() {
|
||||
let server = MockServer::start().await;
|
||||
let (sink, host) = sink(&server, json!({"s3_use_key_prefix": true}));
|
||||
let mut bad = event(&host, "a", "{}");
|
||||
bad.identity["key_alias"] = json!(true);
|
||||
submit(&sink, bad).await;
|
||||
submit(
|
||||
&sink,
|
||||
Event {
|
||||
payload: None,
|
||||
id: "b",
|
||||
identity: json!({"metadata_kind": "mapping"}),
|
||||
cold_storage_object_key: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(sink.queued_keys().is_empty());
|
||||
assert_eq!(host.failures(), 1);
|
||||
assert_eq!(host.held(), 0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn the_body_is_encoded_at_flush_and_released_once_delivered() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink(&server, json!({}));
|
||||
let payload = host.stash(Some("{\"n\": 1}"));
|
||||
let handle = payload.handle();
|
||||
submit(&sink, event_with(payload, "a")).await;
|
||||
host.rewrite(handle, "{\"n\": 2}");
|
||||
sink.flush().await;
|
||||
|
||||
assert_eq!(
|
||||
server.received_requests().await.unwrap()[0].body,
|
||||
b"{\"n\": 2}"
|
||||
);
|
||||
assert_eq!(host.held(), 0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_body_that_cannot_be_encoded_stays_queued_and_counts_a_failure() {
|
||||
let server = MockServer::start().await;
|
||||
let (sink, host) = sink(&server, json!({}));
|
||||
submit(&sink, event_with(host.stash(None), "a")).await;
|
||||
sink.flush().await;
|
||||
|
||||
assert!(server.received_requests().await.unwrap().is_empty());
|
||||
assert_eq!(sink.queued_keys().len(), 1);
|
||||
assert_eq!(host.failures(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::environment(Some("ap-south-1"), Some("/ap-south-1/"))]
|
||||
#[case::default(None, Some("/us-west-2/"))]
|
||||
#[case::invalid(Some("Ap_South"), None)]
|
||||
#[tokio::test]
|
||||
async fn without_a_region_param_the_constructor_region_signs(
|
||||
#[case] region: Option<&str>,
|
||||
#[case] scope: Option<&str>,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink_from(
|
||||
&server,
|
||||
json!({"s3_region_name": null, "s3_aws_session_token": "session"}),
|
||||
Constants::default(),
|
||||
false,
|
||||
region,
|
||||
);
|
||||
submit(&sink, event(&host, "a", "{}")).await;
|
||||
sink.flush().await;
|
||||
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
let signed = requests
|
||||
.first()
|
||||
.and_then(|put| header(put, "authorization"))
|
||||
.map(|authorization| scope.is_some_and(|scope| authorization.contains(scope)));
|
||||
assert_eq!(signed, scope.map(|_| true));
|
||||
assert_eq!(sink.queued_keys().len(), usize::from(scope.is_none()));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_reply_slower_than_the_attempt_timeout_is_retried_then_kept() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200).set_delay(Duration::from_millis(600)))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink_with(
|
||||
&server,
|
||||
json!({}),
|
||||
Constants {
|
||||
upload_attempt_timeout_seconds: 0.1,
|
||||
..Constants::default()
|
||||
},
|
||||
);
|
||||
submit(&sink, event(&host, "a", "{}")).await;
|
||||
sink.flush().await;
|
||||
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 3);
|
||||
assert_eq!(sink.queued_keys().len(), 1);
|
||||
assert_eq!(host.failures(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_panic_while_rendering_keeps_that_object_and_the_flush_goes_on() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink(&server, json!({}));
|
||||
let racing = host.stash(Some(PANICS));
|
||||
let handle = racing.handle();
|
||||
submit(&sink, event_with(racing, "a")).await;
|
||||
submit(&sink, event(&host, "b", "{}")).await;
|
||||
sink.flush().await;
|
||||
|
||||
let paths: Vec<String> = server
|
||||
.received_requests()
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|put| put.url.path().to_owned())
|
||||
.collect();
|
||||
assert_eq!(paths, ["/logs/base/2026-11-13/time-09-05-07-000042_b.json"]);
|
||||
assert_eq!(
|
||||
sink.queued_keys(),
|
||||
["base/2026-11-13/time-09-05-07-000042_a.json"]
|
||||
);
|
||||
assert_eq!(host.failures(), 1);
|
||||
|
||||
host.rewrite(handle, "{}");
|
||||
sink.flush().await;
|
||||
assert!(sink.queued_keys().is_empty());
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_panic_while_rendering_a_batch_file_keeps_the_whole_queue() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink(&server, json!({"s3_batch_file_upload": true}));
|
||||
let racing = host.stash(Some(PANICS));
|
||||
let handle = racing.handle();
|
||||
submit(&sink, event_with(racing, "a")).await;
|
||||
submit(&sink, event(&host, "b", "{}")).await;
|
||||
sink.flush().await;
|
||||
|
||||
assert!(server.received_requests().await.unwrap().is_empty());
|
||||
assert_eq!(sink.queued_keys().len(), 2);
|
||||
|
||||
host.rewrite(handle, "{}");
|
||||
sink.flush().await;
|
||||
assert!(sink.queued_keys().is_empty());
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bool_alias(json!({"metadata_kind": "mapping", "team_alias": true}))]
|
||||
#[case::int_alias(json!({"metadata_kind": "mapping", "team_alias": 7}))]
|
||||
#[case::nan_alias(serde_json::to_value(LoggedKeyIdentity {
|
||||
metadata_kind: Some(MetadataKind::Mapping),
|
||||
team_alias: Some(Scalar::Float(f64::NAN)),
|
||||
..LoggedKeyIdentity::default()
|
||||
}).unwrap())]
|
||||
#[case::list_alias(json!({"metadata_kind": "mapping", "team_alias": {"text": "['team-a']", "truthy": true, "hashable": false, "integer": false, "type_name": "list", "hash_error": "unhashable type: 'list'"}}))]
|
||||
#[case::alias_whose_str_raises(json!({"metadata_kind": "mapping", "rejected": ["team_alias"]}))]
|
||||
#[case::null_metadata(json!({"metadata_kind": "null"}))]
|
||||
#[case::metadata_without_get(json!({"metadata_kind": "other"}))]
|
||||
#[tokio::test]
|
||||
async fn a_team_prefix_python_cannot_build_is_counted_and_released(#[case] identity: Value) {
|
||||
let server = MockServer::start().await;
|
||||
let (sink, host) = sink(&server, json!({"s3_use_team_prefix": true}));
|
||||
let mut bad = event(&host, "a", "{}");
|
||||
bad.identity = identity;
|
||||
submit(&sink, bad).await;
|
||||
|
||||
assert!(sink.queued_keys().is_empty());
|
||||
assert_eq!(host.failures(), 1);
|
||||
assert_eq!(host.held(), 0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty_text(json!(""))]
|
||||
#[case::zero(json!(0))]
|
||||
#[case::zero_float(json!(0.0))]
|
||||
#[case::false_flag(json!(false))]
|
||||
#[case::null(Value::Null)]
|
||||
#[case::empty_list(json!({"text": "[]", "truthy": false, "hashable": false, "integer": false, "type_name": "list", "hash_error": "unhashable type: 'list'"}))]
|
||||
#[tokio::test]
|
||||
async fn a_falsy_team_alias_adds_no_prefix(#[case] alias: Value) {
|
||||
let server = MockServer::start().await;
|
||||
let (sink, host) = sink(&server, json!({"s3_use_team_prefix": true}));
|
||||
let mut falsy = event(&host, "a", "{}");
|
||||
falsy.identity = json!({"metadata_kind": "mapping", "team_alias": alias});
|
||||
submit(&sink, falsy).await;
|
||||
|
||||
assert_eq!(
|
||||
sink.queued_keys(),
|
||||
["base/2026-11-13/time-09-05-07-000042_a.json"]
|
||||
);
|
||||
assert_eq!(host.failures(), 0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::prompts_only(json!({"s3_log_prompts_only": true}), "{} without response")]
|
||||
#[case::whole_payload(json!({}), "{}")]
|
||||
#[tokio::test]
|
||||
async fn prompts_only_renders_the_response_as_null(#[case] extra: Value, #[case] body: &str) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink(&server, extra);
|
||||
submit(&sink, event(&host, "a", "{}")).await;
|
||||
sink.flush().await;
|
||||
|
||||
assert_eq!(
|
||||
server.received_requests().await.unwrap()[0].body,
|
||||
body.as_bytes()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_region_param_wins_over_the_constructor_region() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink_from(
|
||||
&server,
|
||||
json!({}),
|
||||
Constants::default(),
|
||||
false,
|
||||
Some("ap-south-1"),
|
||||
);
|
||||
submit(&sink, event(&host, "a", "{}")).await;
|
||||
sink.flush().await;
|
||||
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert!(
|
||||
header(&requests[0], "authorization")
|
||||
.unwrap()
|
||||
.contains("/us-east-1/")
|
||||
);
|
||||
}
|
||||
|
||||
const HOUR_KEY: &str = "base/2026-11-13/09/time-09-05-07-000042_a.json";
|
||||
const DAY_KEY: &str = "base/2026-11-13/time-09-05-07-000042_a.json";
|
||||
|
||||
#[rstest]
|
||||
#[case::s3_v2_reuses_its_hour_key(true, json!({"metadata_kind": "mapping"}), Some(HOUR_KEY))]
|
||||
#[case::another_owner_keeps_the_day_key(false, json!({"metadata_kind": "mapping"}), Some(DAY_KEY))]
|
||||
#[case::s3_v2_rejects_metadata_without_get(true, json!({"metadata_kind": "other"}), None)]
|
||||
#[case::another_owner_ignores_metadata_without_get(false, json!({"metadata_kind": "other"}), Some(DAY_KEY))]
|
||||
#[tokio::test]
|
||||
async fn the_constructor_cold_storage_owner_decides_the_cold_key(
|
||||
#[case] cold_storage_is_s3_v2: bool,
|
||||
#[case] identity: Value,
|
||||
#[case] key: Option<&str>,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
let (sink, host) = sink_from(
|
||||
&server,
|
||||
json!({}),
|
||||
Constants::default(),
|
||||
cold_storage_is_s3_v2,
|
||||
None,
|
||||
);
|
||||
let mut cold = event(&host, "a", "{}");
|
||||
cold.identity = identity;
|
||||
cold.cold_storage_object_key = Some(HOUR_KEY);
|
||||
submit(&sink, cold).await;
|
||||
|
||||
assert_eq!(sink.queued_keys(), key.into_iter().collect::<Vec<_>>());
|
||||
assert_eq!(host.failures(), usize::from(key.is_none()));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::s3_v2_owner(true, "/logs/base/2026-11-13/time-09-05-07-000042_a.json")]
|
||||
#[case::another_owner(false, "/logs/base/2026-11-13/batch_09-05-07_")]
|
||||
#[tokio::test]
|
||||
async fn the_constructor_cold_storage_owner_turns_batch_files_off(
|
||||
#[case] cold_storage_is_s3_v2: bool,
|
||||
#[case] path: &str,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("PUT"))
|
||||
.respond_with(ResponseTemplate::new(200))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (sink, host) = sink_from(
|
||||
&server,
|
||||
json!({"s3_batch_file_upload": true}),
|
||||
Constants::default(),
|
||||
cold_storage_is_s3_v2,
|
||||
None,
|
||||
);
|
||||
submit(&sink, event(&host, "a", "{}")).await;
|
||||
sink.flush().await;
|
||||
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(requests[0].url.path().starts_with(path));
|
||||
}
|
||||
|
|
@ -46,6 +46,9 @@ litellm-auth-aws.workspace = true
|
|||
litellm-auth-azure.workspace = true
|
||||
azure_core = "1.0.0"
|
||||
litellm-callbacks-legacy-python.workspace = true
|
||||
litellm-callbacks.workspace = true
|
||||
litellm-callbacks-python.workspace = true
|
||||
litellm-callbacks-s3-log.workspace = true
|
||||
litellm-inference.workspace = true
|
||||
litellm-inference-transcription.workspace = true
|
||||
litellm-inference-responses.workspace = true
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ mod python_settings;
|
|||
mod redis;
|
||||
mod router;
|
||||
mod routes;
|
||||
mod s3_log;
|
||||
mod secrets;
|
||||
mod tokenizer;
|
||||
|
||||
|
|
@ -97,6 +98,8 @@ mod _native {
|
|||
use crate::routes::token_counter::TokenCounter;
|
||||
#[pymodule_export]
|
||||
use crate::routes::traces::trace_encode_error;
|
||||
#[pymodule_export]
|
||||
use crate::s3_log::NativeS3Log;
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pymodule_export]
|
||||
use crate::tokenizer::HuggingFaceEncoding;
|
||||
|
|
@ -202,6 +205,7 @@ mod tests {
|
|||
"process_state_started",
|
||||
"reserve_process_for_forking",
|
||||
"unsupported_tls_settings",
|
||||
"NativeS3Log",
|
||||
];
|
||||
#[cfg(feature = "huggingface")]
|
||||
expected.push("HuggingFaceEncoding");
|
||||
|
|
|
|||
225
litellm/rust/crates/python-bridge/src/s3_log.rs
Normal file
225
litellm/rust/crates/python-bridge/src/s3_log.rs
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use litellm_callbacks::ErasedCallback;
|
||||
use litellm_callbacks_python::{RustCallback, payload};
|
||||
use litellm_callbacks_s3_log::{
|
||||
BatchUploadFailure, Constants, Host, S3Log, S3LogParts, Settings, python_backoff, system_clock,
|
||||
};
|
||||
use litellm_host_python::{from_py, json_from_py};
|
||||
use litellm_http::ClientVariant;
|
||||
use pyo3::{
|
||||
prelude::*,
|
||||
types::{PyBool, PyDict, PyInt, PyString},
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
struct PythonHost {
|
||||
host: Arc<Py<PyAny>>,
|
||||
}
|
||||
|
||||
impl PythonHost {
|
||||
fn ask<T>(
|
||||
&self,
|
||||
method: &str,
|
||||
argument: Option<&str>,
|
||||
read: impl for<'py> FnOnce(&Bound<'py, PyAny>) -> PyResult<T>,
|
||||
) -> Result<T, String> {
|
||||
Python::attach(|py| {
|
||||
let answer = match argument {
|
||||
Some(argument) => self.host.call_method1(py, method, (argument,)),
|
||||
None => self.host.call_method0(py, method),
|
||||
};
|
||||
answer
|
||||
.and_then(|answer| read(answer.bind(py)))
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Host for PythonHost {
|
||||
fn callback_failed(&self) {
|
||||
let host = Arc::clone(&self.host);
|
||||
let report = move || {
|
||||
Python::attach(|py| {
|
||||
if let Err(error) = host.call_method0(py, "callback_failed") {
|
||||
error.print(py);
|
||||
}
|
||||
});
|
||||
};
|
||||
match tokio::runtime::Handle::try_current() {
|
||||
Ok(runtime) => drop(runtime.spawn_blocking(report)),
|
||||
Err(_) => report(),
|
||||
}
|
||||
}
|
||||
|
||||
fn web_identity_token(&self, reference: &str) -> Result<String, String> {
|
||||
self.ask("web_identity_token", Some(reference), |answer| {
|
||||
answer.extract::<String>()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type Failure = Option<(usize, usize, usize)>;
|
||||
|
||||
fn failure(failure: Option<BatchUploadFailure>) -> Failure {
|
||||
failure.map(|failure| (failure.queued, failure.failed, failure.total))
|
||||
}
|
||||
|
||||
#[pyclass(frozen, name = "NativeS3Log")]
|
||||
pub struct NativeS3Log {
|
||||
sink: Arc<S3Log>,
|
||||
}
|
||||
|
||||
fn client(
|
||||
py: Python<'_>,
|
||||
params: &Bound<'_, PyDict>,
|
||||
connect_timeout_seconds: f64,
|
||||
) -> PyResult<litellm_http::Client> {
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("ssl_verify", params.get_item("s3_verify")?)?;
|
||||
let mut config = crate::http::call_config(py, &kwargs, true)?;
|
||||
config.connect_timeout = Duration::from_secs_f64(connect_timeout_seconds.max(0.0));
|
||||
crate::http::pool()
|
||||
.client(&config, ClientVariant::Provider)
|
||||
.map_err(crate::http::client_error)
|
||||
}
|
||||
|
||||
fn settings_param(value: &Bound<'_, PyAny>) -> Value {
|
||||
if let Ok(integer) = value.cast::<PyInt>()
|
||||
&& !value.is_instance_of::<PyBool>()
|
||||
&& integer.extract::<i64>().is_err()
|
||||
&& integer.extract::<u64>().is_err()
|
||||
{
|
||||
return value
|
||||
.str()
|
||||
.map_or(Value::Null, |text| Value::String(text.to_string()));
|
||||
}
|
||||
json_from_py(value).unwrap_or(Value::Null)
|
||||
}
|
||||
|
||||
fn settings_params(params: &Bound<'_, PyDict>) -> Map<String, Value> {
|
||||
params
|
||||
.iter()
|
||||
.filter_map(|(key, value)| {
|
||||
let key = key.cast_into::<PyString>().ok()?;
|
||||
Some((key.to_str().ok()?.to_owned(), settings_param(&value)))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl NativeS3Log {
|
||||
#[new]
|
||||
#[pyo3(signature = (params, constants, host, cold_storage_is_s3_v2, region_from_environment))]
|
||||
fn new(
|
||||
py: Python<'_>,
|
||||
params: &Bound<'_, PyDict>,
|
||||
constants: &Bound<'_, PyAny>,
|
||||
host: Py<PyAny>,
|
||||
cold_storage_is_s3_v2: bool,
|
||||
region_from_environment: Option<String>,
|
||||
) -> PyResult<Self> {
|
||||
let resolved = settings_params(params);
|
||||
let constants: Constants = from_py(constants)?;
|
||||
let logger = crate::logger::capture(py);
|
||||
let settings = logger.scope(|| Settings::from_params(&resolved, &constants));
|
||||
let host = Arc::new(PythonHost {
|
||||
host: Arc::new(host),
|
||||
});
|
||||
let sink = S3Log::new(S3LogParts {
|
||||
settings,
|
||||
client: client(py, params, constants.upload_attempt_timeout_seconds)?,
|
||||
download_client: client(py, params, constants.download_connect_timeout_seconds)?,
|
||||
environment: Arc::new(|name: &str| std::env::var(name).ok()),
|
||||
clock: system_clock,
|
||||
backoff: Arc::new(python_backoff),
|
||||
host: host as Arc<dyn Host>,
|
||||
cold_storage_is_s3_v2,
|
||||
region_from_environment,
|
||||
logger,
|
||||
});
|
||||
Ok(Self { sink })
|
||||
}
|
||||
|
||||
fn callback(&self) -> PyResult<RustCallback> {
|
||||
Ok(RustCallback::new(
|
||||
Arc::clone(&self.sink) as Arc<dyn ErasedCallback>
|
||||
)?)
|
||||
}
|
||||
|
||||
fn stashed(&self) -> usize {
|
||||
self.sink.held_payloads()
|
||||
}
|
||||
|
||||
fn partition_granularity(&self) -> &'static str {
|
||||
self.sink.partition_granularity().as_str()
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn s3_path(&self) -> Option<String> {
|
||||
self.sink.settings().path.clone()
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn s3_bucket_name(&self) -> Option<String> {
|
||||
self.sink.settings().bucket_name.clone()
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn strip_base64_files(&self) -> bool {
|
||||
self.sink.settings().strip_base64_files
|
||||
}
|
||||
|
||||
fn queued_keys(&self) -> Vec<String> {
|
||||
self.sink.queued_keys()
|
||||
}
|
||||
|
||||
fn submit_audit<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
audit_log: Py<PyAny>,
|
||||
id: String,
|
||||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let sink = Arc::clone(&self.sink);
|
||||
let stashed = payload::stash(audit_log);
|
||||
let event_loop = payload::running_loop(py);
|
||||
crate::execution::run_async_value(
|
||||
py,
|
||||
payload::on_render_loop(event_loop, async move {
|
||||
Ok(failure(sink.submit_audit(stashed, &id).await))
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
fn upload_line_items<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
key: String,
|
||||
body: String,
|
||||
download_name: String,
|
||||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let sink = Arc::clone(&self.sink);
|
||||
crate::execution::run_async_value(py, async move {
|
||||
Ok(sink.upload_now(key, body, download_name).await)
|
||||
})
|
||||
}
|
||||
|
||||
fn download<'py>(&self, py: Python<'py>, key: String) -> PyResult<Bound<'py, PyAny>> {
|
||||
let sink = Arc::clone(&self.sink);
|
||||
crate::execution::run_async_value(py, async move {
|
||||
Ok(sink
|
||||
.download(&key)
|
||||
.await
|
||||
.map(|body| String::from_utf8_lossy(&body).into_owned()))
|
||||
})
|
||||
}
|
||||
|
||||
fn flush<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let sink = Arc::clone(&self.sink);
|
||||
let event_loop = payload::running_loop(py);
|
||||
crate::execution::run_async_value(
|
||||
py,
|
||||
payload::on_render_loop(event_loop, async move { Ok(failure(sink.flush().await)) }),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"python": "3.14.7",
|
||||
"python": "3.14.6",
|
||||
"rows": [
|
||||
{
|
||||
"name": "None",
|
||||
|
|
@ -495,6 +495,25 @@
|
|||
},
|
||||
"view": "1e+22"
|
||||
},
|
||||
{
|
||||
"name": "0.07872390747070312",
|
||||
"source": "0.07872390747070312",
|
||||
"literal": true,
|
||||
"plain": true,
|
||||
"repr": "0.07872390747070312",
|
||||
"str": "0.07872390747070312",
|
||||
"truthy": true,
|
||||
"json": "0.07872390747070312",
|
||||
"pickle": {
|
||||
"0": "46302e30373837323339303734373037303331320a2e",
|
||||
"1": "473fb42740000000002e",
|
||||
"2": "8002473fb42740000000002e",
|
||||
"3": "8003473fb42740000000002e",
|
||||
"4": "8004950a00000000000000473fb42740000000002e",
|
||||
"5": "8005950a00000000000000473fb42740000000002e"
|
||||
},
|
||||
"view": "0.07872390747070312"
|
||||
},
|
||||
{
|
||||
"name": "float('inf')",
|
||||
"source": "float('inf')",
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ CORPUS = [
|
|||
"5e-324",
|
||||
"1.7976931348623157e308",
|
||||
"1e22",
|
||||
"0.07872390747070312",
|
||||
"float('inf')",
|
||||
"float('-inf')",
|
||||
"float('nan')",
|
||||
|
|
|
|||
|
|
@ -113,8 +113,14 @@ fn format_float(value: f64, add_dot_0: bool) -> String {
|
|||
if value.is_infinite() {
|
||||
return if value > 0.0 { "inf" } else { "-inf" }.to_owned();
|
||||
}
|
||||
// Rust's `{:e}` prints the shortest round-trip digits, like CPython's 'r' mode.
|
||||
let scientific = format!("{value:e}");
|
||||
let shortest = format!("{value:e}");
|
||||
let length = shortest
|
||||
.split_once('e')
|
||||
.map_or(shortest.as_str(), |(mantissa, _)| mantissa)
|
||||
.chars()
|
||||
.filter(char::is_ascii_digit)
|
||||
.count();
|
||||
let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1));
|
||||
let (mantissa, exponent) = scientific
|
||||
.split_once('e')
|
||||
.expect("`{:e}` always prints an exponent");
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-auth-aws.workspace = true
|
||||
litellm-http.workspace = true
|
||||
hyper = "1.12.0"
|
||||
bytes.workspace = true
|
||||
aws-sdk-s3.workspace = true
|
||||
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] }
|
||||
aws-types = "1.4.0"
|
||||
|
|
@ -16,6 +19,7 @@ aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] }
|
|||
http.workspace = true
|
||||
reqwest.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["time"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
mod error;
|
||||
pub mod signed;
|
||||
mod transport;
|
||||
mod uri;
|
||||
|
||||
|
|
|
|||
137
litellm/rust/crates/s3/src/signed.rs
Normal file
137
litellm/rust/crates/s3/src/signed.rs
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use bytes::Bytes;
|
||||
use litellm_auth_aws::{Credentials, SigningScope, sign_s3_headers};
|
||||
use reqwest::Method;
|
||||
|
||||
pub struct SignedRequest<'a> {
|
||||
pub method: Method,
|
||||
pub url: &'a str,
|
||||
pub headers: &'a [(String, String)],
|
||||
pub body: Bytes,
|
||||
pub region: &'a str,
|
||||
pub credentials: &'a Credentials,
|
||||
pub time: SystemTime,
|
||||
pub read_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Reply {
|
||||
pub status: u16,
|
||||
pub body: Bytes,
|
||||
}
|
||||
|
||||
impl Reply {
|
||||
pub fn text(&self) -> String {
|
||||
String::from_utf8_lossy(&self.body).into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum SendError {
|
||||
#[error("the request cannot be signed: {0}")]
|
||||
Signing(#[from] litellm_auth_aws::Error),
|
||||
#[error("the request was not sent: {0}")]
|
||||
Request(String),
|
||||
#[error("{kind}: {message}")]
|
||||
Transport { kind: &'static str, message: String },
|
||||
}
|
||||
|
||||
impl SendError {
|
||||
pub fn is_transient(&self) -> bool {
|
||||
matches!(self, Self::Transport { .. })
|
||||
}
|
||||
|
||||
pub fn is_reconnectable(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::Transport {
|
||||
kind: "ConnectError" | "RemoteProtocolError",
|
||||
..
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn signature_headers(request: &SignedRequest<'_>) -> Result<Vec<(String, String)>, SendError> {
|
||||
Ok(sign_s3_headers(
|
||||
request.method.as_str(),
|
||||
request.url,
|
||||
&request.body,
|
||||
request.headers,
|
||||
SigningScope {
|
||||
region: request.region,
|
||||
service: "s3",
|
||||
credentials: request.credentials,
|
||||
time: request.time,
|
||||
},
|
||||
)?)
|
||||
}
|
||||
|
||||
pub async fn send(
|
||||
client: &litellm_http::Client,
|
||||
request: SignedRequest<'_>,
|
||||
) -> Result<Reply, SendError> {
|
||||
let signature = signature_headers(&request)?;
|
||||
let mut builder = client.request(request.method.clone(), request.url);
|
||||
for (name, value) in request.headers.iter().chain(&signature) {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
if !request.body.is_empty() || request.method == Method::PUT {
|
||||
builder = builder.body(request.body.clone());
|
||||
}
|
||||
let mut response = timed(request.read_timeout, builder.send()).await?;
|
||||
let status = response.status().as_u16();
|
||||
let mut body = Vec::new();
|
||||
while let Some(chunk) = timed(request.read_timeout, response.chunk()).await? {
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(Reply {
|
||||
status,
|
||||
body: Bytes::from(body),
|
||||
})
|
||||
}
|
||||
|
||||
async fn timed<T>(
|
||||
limit: Option<Duration>,
|
||||
pending: impl Future<Output = Result<T, reqwest::Error>>,
|
||||
) -> Result<T, SendError> {
|
||||
let finished = match limit {
|
||||
Some(limit) => tokio::time::timeout(limit, pending)
|
||||
.await
|
||||
.map_err(|elapsed| SendError::Transport {
|
||||
kind: "ReadTimeout",
|
||||
message: elapsed.to_string(),
|
||||
})?,
|
||||
None => pending.await,
|
||||
};
|
||||
finished.map_err(transport)
|
||||
}
|
||||
|
||||
fn transport(error: reqwest::Error) -> SendError {
|
||||
let message = error.to_string();
|
||||
if error.is_builder() || error.is_redirect() {
|
||||
return SendError::Request(message);
|
||||
}
|
||||
let kind = if error.is_timeout() {
|
||||
"Timeout"
|
||||
} else if error.is_connect() {
|
||||
"ConnectError"
|
||||
} else if remote_protocol(&error) {
|
||||
"RemoteProtocolError"
|
||||
} else {
|
||||
"TransportError"
|
||||
};
|
||||
SendError::Transport { kind, message }
|
||||
}
|
||||
|
||||
fn remote_protocol(error: &reqwest::Error) -> bool {
|
||||
let mut cause = std::error::Error::source(error);
|
||||
while let Some(current) = cause {
|
||||
if let Some(hyper) = current.downcast_ref::<hyper::Error>() {
|
||||
return hyper.is_incomplete_message() || hyper.is_parse();
|
||||
}
|
||||
cause = current.source();
|
||||
}
|
||||
false
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import httpx
|
|||
from pydantic import JsonValue
|
||||
from typing_extensions import disjoint_base
|
||||
|
||||
from litellm.integrations.custom_logger_v2.rust import RustHandle
|
||||
from litellm.redis.store_settings import RedisBreakerSettings, RedisStoreSettings
|
||||
from litellm.redis.stores.router_session_pins.keys import PinKey
|
||||
from litellm.rust_bridge.execution import Execution
|
||||
|
|
@ -119,6 +120,35 @@ class NativeModelCostMap:
|
|||
@final
|
||||
class NativeModelCostMapState: ...
|
||||
|
||||
class _S3LogHost(Protocol):
|
||||
def callback_failed(self) -> None: ...
|
||||
def web_identity_token(self, reference: str) -> str: ...
|
||||
|
||||
@final
|
||||
class NativeS3Log:
|
||||
def __new__(
|
||||
cls,
|
||||
params: dict[str, object],
|
||||
constants: Mapping[str, object],
|
||||
host: _S3LogHost,
|
||||
cold_storage_is_s3_v2: bool,
|
||||
region_from_environment: str | None,
|
||||
) -> NativeS3Log: ...
|
||||
@property
|
||||
def s3_path(self) -> str | None: ...
|
||||
@property
|
||||
def s3_bucket_name(self) -> str | None: ...
|
||||
@property
|
||||
def strip_base64_files(self) -> bool: ...
|
||||
def callback(self) -> RustHandle: ...
|
||||
def stashed(self) -> int: ...
|
||||
def partition_granularity(self) -> Literal["day", "hour"]: ...
|
||||
def queued_keys(self) -> list[str]: ...
|
||||
def submit_audit(self, audit_log: dict[str, object], id: str) -> Future[tuple[int, int, int] | None]: ...
|
||||
def upload_line_items(self, key: str, body: str, download_name: str) -> Future[bool]: ...
|
||||
def download(self, key: str) -> Future[str | None]: ...
|
||||
def flush(self) -> Future[tuple[int, int, int] | None]: ...
|
||||
|
||||
@final
|
||||
class NativeClickHouseSpendConfig:
|
||||
def __new__(cls, database: str, url: str, retention_days: int) -> NativeClickHouseSpendConfig: ...
|
||||
|
|
@ -786,6 +816,7 @@ __all__ = [
|
|||
"NativeRouterCooldowns",
|
||||
"NativeRouterSessionPins",
|
||||
"NativeRouterUsage",
|
||||
"NativeS3Log",
|
||||
"NativeSend",
|
||||
"NativeSensitiveRoutePins",
|
||||
"NativeSpendCounters",
|
||||
|
|
|
|||
|
|
@ -1,18 +1,41 @@
|
|||
from typing import Literal
|
||||
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
S3PartitionGranularity = Literal["day", "hour"]
|
||||
|
||||
|
||||
class s3BatchLoggingElement(LiteLLMBaseModel):
|
||||
"""
|
||||
Type of element stored in self.log_queue in S3Logger
|
||||
"""
|
||||
|
||||
payload: dict
|
||||
s3_object_key: str
|
||||
s3_object_download_filename: str
|
||||
body: str | None = None
|
||||
content_type: str = "application/json"
|
||||
retrying_since: float | None = None
|
||||
class S3LoggerSettings(TypedDict, total=False):
|
||||
s3_bucket_name: ReadOnly[str | None]
|
||||
s3_path: ReadOnly[str | None]
|
||||
s3_region_name: ReadOnly[str | None]
|
||||
s3_api_version: ReadOnly[str | None]
|
||||
s3_use_ssl: ReadOnly[bool]
|
||||
s3_verify: ReadOnly[bool | str | None]
|
||||
s3_endpoint_url: ReadOnly[str | None]
|
||||
s3_aws_access_key_id: ReadOnly[str | None]
|
||||
s3_aws_secret_access_key: ReadOnly[str | None]
|
||||
s3_aws_session_token: ReadOnly[str | None]
|
||||
s3_aws_session_name: ReadOnly[str | None]
|
||||
s3_aws_profile_name: ReadOnly[str | None]
|
||||
s3_aws_role_name: ReadOnly[str | None]
|
||||
s3_aws_web_identity_token: ReadOnly[str | None]
|
||||
s3_aws_sts_endpoint: ReadOnly[str | None]
|
||||
s3_flush_interval: ReadOnly[int | None]
|
||||
s3_batch_size: ReadOnly[int | None]
|
||||
s3_config: ReadOnly[object]
|
||||
s3_use_team_prefix: ReadOnly[bool]
|
||||
s3_strip_base64_files: ReadOnly[bool]
|
||||
s3_use_key_prefix: ReadOnly[bool]
|
||||
s3_use_virtual_hosted_style: ReadOnly[bool]
|
||||
s3_server_side_encryption: ReadOnly[str | None]
|
||||
s3_sse_kms_key_id: ReadOnly[str | None]
|
||||
s3_log_prompts_only: ReadOnly[bool | None]
|
||||
s3_partition_granularity: ReadOnly[str | None]
|
||||
s3_max_concurrent_uploads: ReadOnly[int]
|
||||
s3_max_queue_size: ReadOnly[int | None]
|
||||
s3_max_retry_age_seconds: ReadOnly[int | None]
|
||||
s3_drop_on_terminal_error: ReadOnly[bool]
|
||||
s3_adaptive_concurrency: ReadOnly[bool]
|
||||
s3_max_adaptive_concurrency: ReadOnly[int | None]
|
||||
s3_batch_file_upload: ReadOnly[bool]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Logging integration delivery (behavior features). Grounded in litellm/integrations/.
|
||||
- {id: logging.s3.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/s3_v2.py", rationale: "Primary audit trail; batch flush no-drop"}
|
||||
- {id: logging.s3.failure.writes_object, module: logging, tier: P0, event: failure, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/s3_v2.py", rationale: "Failed calls persisted for compliance"}
|
||||
- {id: logging.s3.success.partition_layout, module: logging, tier: P1, event: success, assertions: [object_key_layout], exercised_on: [chat_completions], source: "integrations/s3_v2.py / LIT-8985", rationale: "s3_partition_granularity picks the date or date/hour folder every downstream query and lifecycle rule reads"}
|
||||
- {id: logging.s3.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/s3_v3.py", rationale: "Primary audit trail; batch flush no-drop"}
|
||||
- {id: logging.s3.failure.writes_object, module: logging, tier: P0, event: failure, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/s3_v3.py", rationale: "Failed calls persisted for compliance"}
|
||||
- {id: logging.s3.success.partition_layout, module: logging, tier: P1, event: success, assertions: [object_key_layout], exercised_on: [chat_completions], source: "integrations/s3_v3.py / LIT-8985", rationale: "s3_partition_granularity picks the date or date/hour folder every downstream query and lifecycle rule reads"}
|
||||
- {id: logging.gcs_bucket.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/gcs_bucket/gcs_bucket.py", rationale: "GCS parallel to S3"}
|
||||
- {id: logging.datadog.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/datadog/datadog.py", rationale: "Powers dashboards/alerts; cardinality regressions common"}
|
||||
- {id: logging.datadog.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/datadog/datadog.py", rationale: "Streaming aggregates usage after the last chunk; delivery and cost must survive that path"}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from integration._support.process import owned_proxy
|
|||
from integration._support.wire import wire_server
|
||||
from integration.caching.response_cache_case import entry_keys, redis_store
|
||||
|
||||
|
||||
pytestmark: Final = pytest.mark.shares_worker_state
|
||||
|
||||
PER_REQUEST_KEY: Final = re.compile(rf"^/{BUCKET}/{PREFIX}/\d{{4}}-\d{{2}}-\d{{2}}/.+\.json$")
|
||||
|
|
@ -29,7 +30,6 @@ BATCH_KEY: Final = re.compile(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.mixed_surface_burst_bounds_puts_one_object_per_response_id")
|
||||
def test_s3_v2_mixed_surface_burst_bounds_puts_one_object_per_response_id(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3mix" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
|
@ -54,7 +54,6 @@ def test_s3_v2_mixed_surface_burst_bounds_puts_one_object_per_response_id(gatewa
|
|||
assert matched_ids(payloads, answered)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.mixed_surface_batch_writes_ndjson_lines_per_response_id")
|
||||
def test_s3_v2_mixed_surface_batch_writes_ndjson_lines_per_response_id(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3mixb" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
|
@ -80,7 +79,6 @@ def test_s3_v2_mixed_surface_batch_writes_ndjson_lines_per_response_id(gateway:
|
|||
assert len(payloads) == 48
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.sink_outage_mid_mixed_burst_recovers_every_response_id")
|
||||
def test_s3_v2_sink_outage_mid_mixed_burst_recovers_every_response_id(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3mixo" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink(fail_attempts=30, fail_status=503, delay_seconds=0.2)
|
||||
|
|
@ -33,6 +33,7 @@ from integration._support.database_relay import database_relay
|
|||
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
|
||||
pytestmark: Final = pytest.mark.shares_worker_state
|
||||
|
||||
FLUSH: Final = {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "1"}
|
||||
|
|
@ -14,6 +14,7 @@ from integration._support.client import Gateway, JsonValue, Scenario, eventually
|
|||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
|
||||
Surface = Literal["chat", "messages", "responses"]
|
||||
SURFACES: Final[tuple[Surface, ...]] = ("chat", "messages", "responses")
|
||||
FAILURE_BODY: Final[bytes] = json.dumps(
|
||||
|
|
@ -77,9 +78,7 @@ def _responses_stream_id(response: httpx.Response) -> str:
|
|||
if line.startswith("data: {")
|
||||
)
|
||||
completed: Final = tuple(
|
||||
object_value(event["response"])
|
||||
for event in events
|
||||
if event.get("type") == "response.completed"
|
||||
object_value(event["response"]) for event in events if event.get("type") == "response.completed"
|
||||
)
|
||||
assert len(completed) == 1, events
|
||||
response_id: Final = completed[0]["id"]
|
||||
|
|
@ -137,13 +136,11 @@ def _assert_one_payload(
|
|||
return_last_on_timeout=True,
|
||||
)
|
||||
assert len(payloads) == 1, (
|
||||
f"expected one {status} payload after {upstream_posts} upstream POSTs, "
|
||||
f"observed {len(payloads)} payloads"
|
||||
f"expected one {status} payload after {upstream_posts} upstream POSTs, observed {len(payloads)} payloads"
|
||||
)
|
||||
assert (
|
||||
_request_key(payloads[0], response, marker) == response.headers.get("x-litellm-call-id")
|
||||
or marker in json.dumps(payloads[0])
|
||||
), f"payload did not match request id or marker: {payloads[0]!r}"
|
||||
assert _request_key(payloads[0], response, marker) == response.headers.get(
|
||||
"x-litellm-call-id"
|
||||
) or marker in json.dumps(payloads[0]), f"payload did not match request id or marker: {payloads[0]!r}"
|
||||
return payloads
|
||||
|
||||
|
||||
|
|
@ -337,9 +334,7 @@ def test_failure_burst_through_sink_outage_lands_each_request_once(
|
|||
with ThreadPoolExecutor(max_workers=len(jobs)) as pool:
|
||||
responses: Final = tuple(pool.map(call, jobs))
|
||||
|
||||
upstream_requests: Final = tuple(
|
||||
request for request in upstream.drain() if request.method == "POST"
|
||||
)
|
||||
upstream_requests: Final = tuple(request for request in upstream.drain() if request.method == "POST")
|
||||
route_counts: Final = Counter(request.target for request in upstream_requests)
|
||||
expected_route_counts: Final = {
|
||||
_surface_target(surface): 2 * requests_per_variant * 3 for surface in SURFACES
|
||||
|
|
@ -358,8 +353,7 @@ def test_failure_burst_through_sink_outage_lands_each_request_once(
|
|||
rejected: Final = sink.attempts - len(sink.store)
|
||||
assert rejected >= 1, f"expected at least one rejected sink upload, rejected={rejected}"
|
||||
assert len(sink.store) == 0, (
|
||||
f"expected all sink uploads to fail before recovery, rejected={rejected}, "
|
||||
f"attempts={sink.attempts}"
|
||||
f"expected all sink uploads to fail before recovery, rejected={rejected}, attempts={sink.attempts}"
|
||||
)
|
||||
sink.fail_until = 0.0
|
||||
payloads: Final = eventually(
|
||||
|
|
@ -20,6 +20,7 @@ from integration._support.process import owned_proxy, owned_proxy_process
|
|||
from integration._support.timing import assert_enforced_within
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
|
||||
BUCKET: Final = "integration-bucket"
|
||||
PREFIX: Final = "integration-logs"
|
||||
REQUESTS: Final = 64
|
||||
|
|
@ -121,7 +122,6 @@ BATCH_KEY: Final = re.compile(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.flush_bounds_concurrent_puts_to_default_ceiling_and_keeps_every_log")
|
||||
def test_s3_v2_flush_bounds_concurrent_puts_to_the_default_ceiling(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3fan" + uuid.uuid4().hex[:8]
|
||||
sink: Final = S3Sink()
|
||||
|
|
@ -144,7 +144,6 @@ def test_s3_v2_flush_bounds_concurrent_puts_to_the_default_ceiling(gateway: Gate
|
|||
assert len({put.target for put in puts}) == REQUESTS
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.configured_bound_and_env_backed_false_keeps_per_request_objects")
|
||||
def test_s3_v2_honors_configured_bound_and_env_backed_false_batch_flag(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3cap" + uuid.uuid4().hex[:8]
|
||||
sink: Final = S3Sink()
|
||||
|
|
@ -173,7 +172,6 @@ def test_s3_v2_honors_configured_bound_and_env_backed_false_batch_flag(gateway:
|
|||
assert frozenset(json.loads(put.body)["id"] for put in puts) == ids
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.batch_file_upload_writes_one_ndjson_object_per_flush")
|
||||
def test_s3_v2_batch_file_upload_writes_one_jsonl_object_per_flush(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3jsonl" + uuid.uuid4().hex[:8]
|
||||
sink: Final = S3Sink()
|
||||
|
|
@ -196,7 +194,6 @@ def test_s3_v2_batch_file_upload_writes_one_jsonl_object_per_flush(gateway: Gate
|
|||
assert len(lines) == REQUESTS
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.batch_file_upload_keeps_team_prefix_in_object_key")
|
||||
def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3team" + uuid.uuid4().hex[:8]
|
||||
team_alias: Final = f"alpha-{uuid.uuid4().hex[:8]}"
|
||||
|
|
@ -223,7 +220,6 @@ def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_p
|
|||
assert len(lines) == REQUESTS
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.upstream_failure_events_land_alongside_successes")
|
||||
def test_s3_v2_upstream_failure_events_land_alongside_successes(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3fail" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
|
@ -276,7 +272,6 @@ def test_s3_v2_upstream_failure_events_land_alongside_successes(gateway: Gateway
|
|||
assert all("synthetic upstream rejection" in json.dumps(payload["error_information"]) for payload in failures)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.invalid_or_empty_bound_falls_back_to_default_ceiling")
|
||||
@pytest.mark.parametrize(
|
||||
("bad", "warns"),
|
||||
[
|
||||
|
|
@ -313,7 +308,6 @@ def test_s3_v2_invalid_or_empty_bound_falls_back_to_default_ceiling(
|
|||
assert frozenset(payload["id"] for payload in payloads) == ids
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.sink_rejection_requeues_and_delivers_every_id_once")
|
||||
def test_s3_v2_sink_rejection_requeues_and_delivers_every_id_once(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3deny" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink(fail_status=503, delay_seconds=0.2)
|
||||
|
|
@ -632,7 +626,6 @@ def test_s3_v2_default_config_retries_access_denied_and_every_event_lands(gatewa
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.batch_retry_resends_identical_key_and_body")
|
||||
def test_s3_v2_batch_retry_resends_identical_key_and_body(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3retry" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink(fail_status=500, delay_seconds=0.2)
|
||||
|
|
@ -658,7 +651,6 @@ def test_s3_v2_batch_retry_resends_identical_key_and_body(gateway: Gateway, tmp_
|
|||
assert len(payloads) == REQUESTS
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.unknown_model_rejection_keeps_other_requests_logging")
|
||||
def test_s3_v2_unknown_model_rejection_keeps_other_requests_logging(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3ghost" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
|
@ -690,7 +682,6 @@ def test_s3_v2_unknown_model_rejection_keeps_other_requests_logging(gateway: Gat
|
|||
assert all(payload["status"] == "failure" for payload in extras), extras
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.batch_flag_ignored_when_s3_v2_is_cold_storage_logger")
|
||||
def test_s3_v2_batch_flag_ignored_when_s3_v2_is_cold_storage_logger(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3cold" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
|
@ -734,7 +725,6 @@ def test_s3_v2_batch_flag_ignored_when_s3_v2_is_cold_storage_logger(gateway: Gat
|
|||
assert payloads[0]["id"] == request_id
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.identical_requests_land_distinct_objects")
|
||||
def test_s3_v2_identical_requests_land_distinct_objects(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3same" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
|
@ -766,7 +756,6 @@ def test_s3_v2_identical_requests_land_distinct_objects(gateway: Gateway, tmp_pa
|
|||
assert all(payload["id"] == marker for payload in payloads)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.two_workers_bound_and_deliver_every_id")
|
||||
def test_s3_v2_two_workers_bound_and_deliver_every_id(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3work" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
|
@ -788,7 +777,6 @@ def test_s3_v2_two_workers_bound_and_deliver_every_id(gateway: Gateway, tmp_path
|
|||
assert frozenset(payload["id"] for payload in payloads) == ids
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.slow_sink_never_duplicates_or_stalls_readiness")
|
||||
def test_s3_v2_slow_sink_never_duplicates_or_stalls_readiness(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3slow" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink(delay_seconds=1.5)
|
||||
|
|
@ -817,7 +805,6 @@ def test_s3_v2_slow_sink_never_duplicates_or_stalls_readiness(gateway: Gateway,
|
|||
assert len(payloads) == REQUESTS
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.worker_kill_mid_burst_keeps_surviving_deliveries")
|
||||
def test_s3_v2_worker_kill_mid_burst_keeps_surviving_deliveries(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3kill" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
|
@ -874,7 +861,6 @@ def test_s3_v2_worker_kill_mid_burst_keeps_surviving_deliveries(gateway: Gateway
|
|||
assert landed <= frozenset(burst) | frozenset(after), "an id that was never sent landed"
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.sigterm_mid_burst_loses_only_inflight_without_duplicates")
|
||||
def test_s3_v2_sigterm_mid_burst_loses_only_inflight_without_duplicates(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3term" + uuid.uuid4().hex[:8]
|
||||
sink: Final = RecordingS3Sink()
|
||||
|
|
@ -37,14 +37,26 @@ def test_sigv4_verifier_matches_published_put_and_rejects_corruption() -> None:
|
|||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.s3.sync_async_reserved_keys_are_signed_and_accepted")
|
||||
async def test_s3_sync_and_async_uploads_pass_independent_wire_verification(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
async def test_s3_sync_and_async_uploads_pass_independent_wire_verification() -> None:
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
|
||||
from litellm.types.utils import StandardBatchLineItem, StandardBatchLineItemMetadata, StandardBatchLineItemsPayload
|
||||
|
||||
monkeypatch.setattr("botocore.auth.get_current_datetime", lambda: datetime(2026, 9, 14))
|
||||
payload: Final = {"id": "synthetic-event", "content": "synthetic snow 雪"}
|
||||
metadata: Final = StandardBatchLineItemMetadata(
|
||||
user_api_key_hash=None, user_api_key_alias=None, user_api_key_team_id=None,
|
||||
user_api_key_team_alias=None, user_api_key_user_id=None, user_api_key_org_id=None,
|
||||
)
|
||||
lines: Final = (
|
||||
StandardBatchLineItem(
|
||||
batch_id="synthetic-batch", custom_id="synthetic-event", status="success",
|
||||
request={"content": "synthetic snow 雪"}, response=None, usage=None, response_cost=0.0,
|
||||
model="synthetic-model", custom_llm_provider=None, metadata=metadata,
|
||||
),
|
||||
)
|
||||
expected_path = ""
|
||||
|
||||
def scope_of(request: Request) -> str:
|
||||
return f"{request.headers.get('x-amz-date', '')[:8]}/us-east-1/s3/aws4_request"
|
||||
|
||||
def verify(request: Request) -> Reply:
|
||||
if request.method != "PUT" or request.target != expected_path:
|
||||
return Reply(status=403)
|
||||
|
|
@ -53,8 +65,8 @@ async def test_s3_sync_and_async_uploads_pass_independent_wire_verification(monk
|
|||
assert authorization.startswith("AWS4-HMAC-SHA256 ")
|
||||
fields: Final = dict(part.split("=", 1) for part in authorization.removeprefix("AWS4-HMAC-SHA256 ").split(", "))
|
||||
access, scope = fields["Credential"].split("/", 1)
|
||||
assert access == ACCESS and scope == "20260914/us-east-1/s3/aws4_request"
|
||||
assert request.headers["x-amz-date"] == "20260914T000000Z"
|
||||
assert access == ACCESS and scope == scope_of(request)
|
||||
assert datetime.strptime(request.headers["x-amz-date"], "%Y%m%dT%H%M%SZ")
|
||||
signed: Final = fields["SignedHeaders"].split(";")
|
||||
assert signed == sorted(set(signed))
|
||||
assert {"host", "content-md5", "x-amz-date"}.issubset(signed)
|
||||
|
|
@ -69,40 +81,37 @@ async def test_s3_sync_and_async_uploads_pass_independent_wire_verification(monk
|
|||
with wire_server(verify) as wire:
|
||||
prior: Final = asyncio.all_tasks()
|
||||
logger: Final = S3Logger(s3_bucket_name="integration-bucket", s3_region_name="us-east-1", s3_endpoint_url=wire.url,
|
||||
s3_aws_access_key_id=ACCESS, s3_aws_secret_access_key=SECRET, s3_callback_params_override={})
|
||||
owned: Final = {task for task in asyncio.all_tasks() - prior if task.get_coro().__qualname__ == "CustomBatchLogger.periodic_flush"}
|
||||
assert len(owned) == 1
|
||||
s3_aws_access_key_id=ACCESS, s3_aws_secret_access_key=SECRET, s3_callback_params_override={},
|
||||
s3_path="a=b+c/d e/雪")
|
||||
owned: Final = asyncio.all_tasks() - prior
|
||||
assert owned == {logger.periodic_flush_task}
|
||||
try:
|
||||
for mode in ("sync", "async"):
|
||||
for key in ("plain.json", "a=b+c/d e/雪.json", "percent%2Fplus+.json"):
|
||||
expected_path = encoded_path(f"/integration-bucket/{key}")
|
||||
element: Final = s3BatchLoggingElement(payload=payload, s3_object_key=key, s3_object_download_filename="event.json")
|
||||
if mode == "sync":
|
||||
await asyncio.to_thread(logger.upload_data_to_s3, element)
|
||||
else:
|
||||
await logger.async_upload_data_to_s3(element)
|
||||
requests: Final = wire.drain()
|
||||
assert len(requests) == 1, "Upload must be accepted on its first actual PUT"
|
||||
request: Final = requests[0]
|
||||
assert request.target == expected_path
|
||||
assert json.loads(request.body) == payload
|
||||
assert verify(request).status == 200
|
||||
with httpx.Client(timeout=5, trust_env=False) as client:
|
||||
corrupt: Final = {**request.headers, "authorization": request.headers["authorization"][:-1] + ("0" if request.headers["authorization"][-1] != "0" else "1")}
|
||||
assert client.put(wire.url + expected_path, content=request.body, headers=corrupt).status_code == 403
|
||||
assert client.put(wire.url + expected_path + "-wrong", content=request.body, headers=request.headers).status_code == 403
|
||||
assert client.put(wire.url + expected_path, content=request.body + b" ", headers={name: value for name, value in request.headers.items() if name != "content-length"}).status_code == 403
|
||||
fields: Final = dict(part.split("=", 1) for part in request.headers["authorization"].removeprefix("AWS4-HMAC-SHA256 ").split(", "))
|
||||
for signed, scope, md5 in (
|
||||
(fields["SignedHeaders"].replace("host;", ""), "20260914/us-east-1/s3/aws4_request", request.headers["content-md5"]),
|
||||
(fields["SignedHeaders"], "20260914/us-west-2/s3/aws4_request", request.headers["content-md5"]),
|
||||
(fields["SignedHeaders"], "20260914/us-east-1/s3/aws4_request", "AAAAAAAAAAAAAAAAAAAAAA=="),
|
||||
):
|
||||
candidate_headers: Final = {**request.headers, "content-md5": md5}
|
||||
digest: Final = signature("PUT", request.target, candidate_headers, signed, request.body, SECRET, scope)[1]
|
||||
candidate_headers["authorization"] = f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, SignedHeaders={signed}, Signature={digest}"
|
||||
assert client.put(wire.url + expected_path, content=request.body, headers=candidate_headers).status_code == 403
|
||||
assert len(wire.drain()) == 6
|
||||
for batch_id in ("plain", "a=b+c d e", "percent%2Fplus+"):
|
||||
expected_path = encoded_path(f"/integration-bucket/a=b+c/d e/雪/batch_line_items/2026-09-14/{batch_id}/part-00001.jsonl")
|
||||
await logger.async_log_batch_line_items(StandardBatchLineItemsPayload(batch_id=batch_id, finished_at=1789344000, part=1, lines=lines))
|
||||
requests: Final = wire.drain()
|
||||
assert len(requests) == 1, "Upload must be accepted on its first actual PUT"
|
||||
request: Final = requests[0]
|
||||
assert request.target == expected_path
|
||||
assert tuple(json.loads(line) for line in request.body.decode().splitlines()) == lines
|
||||
assert verify(request).status == 200
|
||||
day: Final = scope_of(request).split("/", 1)[0]
|
||||
with httpx.Client(timeout=5, trust_env=False) as client:
|
||||
corrupt: Final = {**request.headers, "authorization": request.headers["authorization"][:-1] + ("0" if request.headers["authorization"][-1] != "0" else "1")}
|
||||
assert client.put(wire.url + expected_path, content=request.body, headers=corrupt).status_code == 403
|
||||
assert client.put(wire.url + expected_path + "-wrong", content=request.body, headers=request.headers).status_code == 403
|
||||
assert client.put(wire.url + expected_path, content=request.body + b" ", headers={name: value for name, value in request.headers.items() if name != "content-length"}).status_code == 403
|
||||
fields: Final = dict(part.split("=", 1) for part in request.headers["authorization"].removeprefix("AWS4-HMAC-SHA256 ").split(", "))
|
||||
for signed, scope, md5 in (
|
||||
(fields["SignedHeaders"].replace("host;", ""), f"{day}/us-east-1/s3/aws4_request", request.headers["content-md5"]),
|
||||
(fields["SignedHeaders"], f"{day}/us-west-2/s3/aws4_request", request.headers["content-md5"]),
|
||||
(fields["SignedHeaders"], f"{day}/us-east-1/s3/aws4_request", "AAAAAAAAAAAAAAAAAAAAAA=="),
|
||||
):
|
||||
candidate_headers: Final = {**request.headers, "content-md5": md5}
|
||||
digest: Final = signature("PUT", request.target, candidate_headers, signed, request.body, SECRET, scope)[1]
|
||||
candidate_headers["authorization"] = f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, SignedHeaders={signed}, Signature={digest}"
|
||||
assert client.put(wire.url + expected_path, content=request.body, headers=candidate_headers).status_code == 403
|
||||
assert len(wire.drain()) == 6
|
||||
|
||||
finally:
|
||||
for task in owned:
|
||||
|
|
|
|||
0
tests/unit/integrations/s3_v2_golden/__init__.py
Normal file
0
tests/unit/integrations/s3_v2_golden/__init__.py
Normal file
5945
tests/unit/integrations/s3_v2_golden/python_s3_v2.json
Normal file
5945
tests/unit/integrations/s3_v2_golden/python_s3_v2.json
Normal file
File diff suppressed because it is too large
Load diff
145
tests/unit/integrations/s3_v3_rust_support.py
Normal file
145
tests/unit/integrations/s3_v3_rust_support.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import threading
|
||||
import time
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.integrations.custom_logger_v2.call import CallContext
|
||||
from litellm.integrations.custom_logger_v2.derivations import DERIVED_ATTEMPT_FIELDS
|
||||
from litellm.integrations.custom_logger_v2.enums import Hook
|
||||
from litellm.integrations.custom_logger_v2.enforcement import FieldView
|
||||
from litellm.integrations.custom_logger_v2.field_sources import HookArgs, with_sources
|
||||
from litellm.integrations.custom_logger_v2.legacy import KwargsFields
|
||||
from litellm.integrations.custom_logger_v2.rust import WireManifest
|
||||
from litellm.integrations.s3_v3 import S3Logger
|
||||
from litellm.litellm_core_utils.litellm_logging import LOGGING_KEYS, V2_ASYNC_LOGGING_LINE
|
||||
from litellm.types.integrations.custom_logger_v2 import Outcome
|
||||
|
||||
DROP: Final = -1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Received:
|
||||
method: str
|
||||
path: str
|
||||
headers: Mapping[str, str]
|
||||
body: bytes
|
||||
at: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScriptedBucket:
|
||||
port: int = 0
|
||||
received: list[Received] = field(default_factory=list) # mutable-ok: appended by the server thread
|
||||
objects: dict[str, bytes] = field(default_factory=dict) # mutable-ok: GET serves what PUT stored
|
||||
script: list[tuple[str, list[tuple[int, str]]]] = field(default_factory=list) # mutable-ok: replies are consumed
|
||||
delay: float = 0.0
|
||||
in_flight: int = 0
|
||||
peak: int = 0
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
|
||||
def reply(self, fragment: str, *replies: tuple[int, str]) -> None:
|
||||
self.script.append((fragment, list(replies)))
|
||||
|
||||
def puts(self) -> list[Received]:
|
||||
return [request for request in self.received if request.method == "PUT"]
|
||||
|
||||
def next_reply(self, path: str) -> tuple[int, str]:
|
||||
with self.lock:
|
||||
for fragment, replies in self.script:
|
||||
if fragment in path and replies:
|
||||
return replies.pop(0) if len(replies) > 1 else replies[0]
|
||||
return 200, ""
|
||||
|
||||
|
||||
def _handler(bucket: ScriptedBucket) -> type[BaseHTTPRequestHandler]:
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def log_message(self, *_: object) -> None:
|
||||
return
|
||||
|
||||
def _send(self, status: int, body: bytes) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_PUT(self) -> None:
|
||||
body: Final = self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
||||
headers: Final = {name.lower(): value for name, value in self.headers.items()}
|
||||
with bucket.lock:
|
||||
bucket.received.append(Received("PUT", self.path, headers, body, time.monotonic()))
|
||||
bucket.in_flight += 1
|
||||
bucket.peak = max(bucket.peak, bucket.in_flight)
|
||||
if bucket.delay:
|
||||
time.sleep(bucket.delay)
|
||||
with bucket.lock:
|
||||
bucket.in_flight -= 1
|
||||
status, text = bucket.next_reply(self.path)
|
||||
if status == DROP:
|
||||
self.close_connection = True
|
||||
return
|
||||
if status == 200:
|
||||
with bucket.lock:
|
||||
bucket.objects[self.path] = body
|
||||
self._send(status, text.encode())
|
||||
|
||||
def do_GET(self) -> None:
|
||||
headers: Final = {name.lower(): value for name, value in self.headers.items()}
|
||||
with bucket.lock:
|
||||
bucket.received.append(Received("GET", self.path, headers, b"", time.monotonic()))
|
||||
stored: Final = bucket.objects.get(self.path)
|
||||
if stored is None:
|
||||
self._send(404, b"<Error><Code>NoSuchKey</Code></Error>")
|
||||
return
|
||||
self._send(200, stored)
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
class _BucketServer(ThreadingHTTPServer):
|
||||
request_queue_size = 256
|
||||
|
||||
|
||||
@contextmanager
|
||||
def scripted_bucket() -> Iterator[ScriptedBucket]:
|
||||
bucket: Final = ScriptedBucket()
|
||||
server: Final = _BucketServer(("127.0.0.1", 0), _handler(bucket))
|
||||
bucket.port = server.server_address[1]
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
yield bucket
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def rust_s3_logger(monkeypatch: pytest.MonkeyPatch, **params: object) -> S3Logger: # kwargs-ok: the S3Logger keywords
|
||||
return S3Logger(params) if params else S3Logger()
|
||||
|
||||
|
||||
async def log_attempt(
|
||||
logger: S3Logger, kwargs: Mapping[str, object], start_time: datetime, outcome: Outcome = Outcome.success
|
||||
) -> None:
|
||||
args: Final = HookArgs(request=kwargs, start_time=start_time, end_time=start_time, outcome=outcome)
|
||||
view: Final = FieldView(with_sources(KwargsFields(kwargs, LOGGING_KEYS), DERIVED_ATTEMPT_FIELDS, args))
|
||||
manifest: Final = WireManifest.model_validate_json(logger.rust.manifest())
|
||||
reads: Final = next(s.reads for s in manifest.subscriptions if s.hook is Hook.attempt_ended)
|
||||
line: Final = V2_ASYNC_LOGGING_LINE.set(True)
|
||||
try:
|
||||
_ = await logger.run_rust(Hook.attempt_ended, reads, view, CallContext(Hook.attempt_ended, None, None))
|
||||
finally:
|
||||
V2_ASYNC_LOGGING_LINE.reset(line)
|
||||
|
||||
|
||||
async def log_payload(logger: S3Logger, payload: Mapping[str, object], start_time: datetime) -> None:
|
||||
await log_attempt(logger, {"standard_logging_object": payload}, start_time)
|
||||
|
|
@ -1,179 +0,0 @@
|
|||
import asyncio
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.integrations.adaptive_concurrency import AdaptiveConcurrencyLimiter, PutSample
|
||||
|
||||
_real_sleep: Final = asyncio.sleep
|
||||
|
||||
|
||||
def _limiter(initial: int = 4, floor: int = 1, ceiling: int = 16) -> AdaptiveConcurrencyLimiter:
|
||||
return AdaptiveConcurrencyLimiter(initial=initial, floor=floor, ceiling=ceiling)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_limit_grows_after_limit_clean_samples() -> None:
|
||||
limiter: Final = _limiter(initial=4)
|
||||
for _ in range(4):
|
||||
limiter.record(PutSample(throttled=False))
|
||||
assert limiter.limit == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_limit_does_not_grow_before_the_streak_completes() -> None:
|
||||
limiter: Final = _limiter(initial=4)
|
||||
for _ in range(3):
|
||||
limiter.record(PutSample(throttled=False))
|
||||
assert limiter.limit == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_throttled_sample_halves_the_limit() -> None:
|
||||
limiter: Final = _limiter(initial=16)
|
||||
limiter.record(PutSample(throttled=True))
|
||||
assert limiter.limit == 8
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_limit_clamps_at_the_floor() -> None:
|
||||
limiter: Final = _limiter(initial=4, floor=4)
|
||||
limiter.record(PutSample(throttled=True))
|
||||
assert limiter.limit == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_limit_clamps_at_the_ceiling() -> None:
|
||||
limiter: Final = _limiter(initial=15, ceiling=16)
|
||||
for _ in range(1000):
|
||||
limiter.record(PutSample(throttled=False))
|
||||
assert limiter.limit == 16
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_throttled_sample_resets_the_clean_streak() -> None:
|
||||
limiter: Final = _limiter(initial=4, ceiling=32)
|
||||
for _ in range(3):
|
||||
limiter.record(PutSample(throttled=False))
|
||||
limiter.record(PutSample(throttled=True))
|
||||
limiter.record(PutSample(throttled=False))
|
||||
assert limiter.limit == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_growing_the_limit_wakes_a_waiting_acquirer() -> None:
|
||||
limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=4)
|
||||
acquired: Final[list[str]] = [] # mutable-ok: the waiter task appends to it across the await boundary
|
||||
released: Final = asyncio.Event()
|
||||
|
||||
async def hold() -> None:
|
||||
async with limiter:
|
||||
await released.wait()
|
||||
|
||||
holder: Final = asyncio.create_task(hold())
|
||||
|
||||
async def waiter() -> None:
|
||||
async with limiter:
|
||||
acquired.append("waiter")
|
||||
|
||||
pending: Final = asyncio.create_task(waiter())
|
||||
await _real_sleep(0)
|
||||
assert not acquired
|
||||
|
||||
limiter.record(PutSample(throttled=False))
|
||||
await asyncio.wait_for(asyncio.shield(pending), timeout=5)
|
||||
released.set()
|
||||
await asyncio.wait_for(holder, timeout=5)
|
||||
assert tuple(acquired) == ("waiter",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_releasing_a_slot_wakes_exactly_one_waiter() -> None:
|
||||
limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=4)
|
||||
acquired: Final[list[str]] = [] # mutable-ok: the waiter tasks append to it across the await boundary
|
||||
release: Final = asyncio.Event()
|
||||
|
||||
async def hold() -> None:
|
||||
async with limiter:
|
||||
await _real_sleep(0)
|
||||
|
||||
async def waiter(name: str) -> None:
|
||||
async with limiter:
|
||||
acquired.append(name)
|
||||
await release.wait()
|
||||
|
||||
holder: Final = asyncio.create_task(hold())
|
||||
waiters: Final = tuple(asyncio.create_task(waiter(f"w{i}")) for i in range(3))
|
||||
await _real_sleep(0)
|
||||
await asyncio.wait_for(holder, timeout=5)
|
||||
await _real_sleep(0)
|
||||
assert len(acquired) == 1
|
||||
|
||||
for _ in range(3):
|
||||
limiter.record(PutSample(throttled=False))
|
||||
await _real_sleep(0)
|
||||
assert len(acquired) == 3
|
||||
release.set()
|
||||
await asyncio.gather(*waiters)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_double_cancel_during_release_leaves_in_flight_at_zero() -> None:
|
||||
limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=1)
|
||||
entered: Final = asyncio.Event()
|
||||
release: Final = asyncio.Event()
|
||||
|
||||
async def hold() -> None:
|
||||
async with limiter:
|
||||
entered.set()
|
||||
await release.wait()
|
||||
|
||||
holder: Final = asyncio.create_task(hold())
|
||||
await entered.wait()
|
||||
|
||||
waiter: Final = asyncio.create_task(hold())
|
||||
await _real_sleep(0)
|
||||
waiter.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await waiter
|
||||
|
||||
release.set()
|
||||
await asyncio.wait_for(holder, timeout=5)
|
||||
holder.cancel()
|
||||
try:
|
||||
await holder
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
assert limiter._in_flight == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_cancelled_waiter_is_skipped_when_a_slot_frees() -> None:
|
||||
limiter: Final = AdaptiveConcurrencyLimiter(initial=1, floor=1, ceiling=1)
|
||||
acquired: Final[list[str]] = [] # mutable-ok: waiter tasks append across the await boundary
|
||||
first_entered: Final = asyncio.Event()
|
||||
release: Final = asyncio.Event()
|
||||
|
||||
async def hold(name: str, entered: asyncio.Event | None = None) -> None:
|
||||
async with limiter:
|
||||
acquired.append(name)
|
||||
if entered is not None:
|
||||
entered.set()
|
||||
await release.wait()
|
||||
|
||||
holder: Final = asyncio.create_task(hold("holder", first_entered))
|
||||
await first_entered.wait()
|
||||
doomed: Final = asyncio.create_task(hold("doomed"))
|
||||
next_waiter: Final = asyncio.create_task(hold("next"))
|
||||
await _real_sleep(0)
|
||||
doomed.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await doomed
|
||||
release.set()
|
||||
await asyncio.wait_for(holder, timeout=5)
|
||||
await asyncio.wait_for(next_waiter, timeout=5)
|
||||
|
||||
assert "doomed" not in acquired
|
||||
assert "next" in acquired
|
||||
assert limiter._in_flight == 0
|
||||
File diff suppressed because it is too large
Load diff
3105
tests/unit/integrations/test_s3_v2_rust.py
Normal file
3105
tests/unit/integrations/test_s3_v2_rust.py
Normal file
File diff suppressed because it is too large
Load diff
1525
tests/unit/integrations/test_s3_v3_native.py
Normal file
1525
tests/unit/integrations/test_s3_v3_native.py
Normal file
File diff suppressed because it is too large
Load diff
445
tests/unit/integrations/test_s3_v3_strip_writer.py
Normal file
445
tests/unit/integrations/test_s3_v3_strip_writer.py
Normal file
|
|
@ -0,0 +1,445 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_logger_v2.enums import Field, Hook
|
||||
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
|
||||
from litellm.integrations.custom_logger_v2.manifest import build_manifest
|
||||
from litellm.integrations.s3_v3 import STRIP_CALLBACK_NAME, S3Logger, S3StripBase64Files
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.rust_bridge import _native
|
||||
from litellm.types.integrations.custom_logger_v2 import Outcome
|
||||
from litellm.types.utils import ModelResponse
|
||||
from tests.unit.integrations.s3_v3_rust_support import ScriptedBucket, scripted_bucket
|
||||
|
||||
pytestmark = pytest.mark.skipif(not hasattr(_native, "NativeS3Log"), reason="the built _native has no NativeS3Log")
|
||||
|
||||
START: Final = datetime.datetime(2025, 9, 14, 9, 5, 7, 42)
|
||||
|
||||
RECORDED_AT_86B70FEF16: Final = {
|
||||
"stripped": (
|
||||
"/test-bucket/2025-09-14/time-09-05-07-000042_stripped.json",
|
||||
"31e903035e2185abea6cac867139417cb42a37a0da6b7d4efea47b9d1c24f1e1",
|
||||
),
|
||||
"excluded": (
|
||||
"/test-bucket/2025-09-14/time-09-05-07-000042_excluded.json",
|
||||
"5958206c8eee87b3221e6ee760b48a0ead92343768adbf8f6f80295bd4d51329",
|
||||
),
|
||||
"failed": (
|
||||
"/test-bucket/2025-09-14/time-09-05-07-000042_failed.json",
|
||||
"04dcb95630cb5b61fbf96560be30411b5fcacebbb4f4ed03aeaccc2e0687c95f",
|
||||
),
|
||||
}
|
||||
|
||||
STRIPPED_MESSAGES: Final = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "see [BASE64_REDACTED]"}]},
|
||||
{"role": "assistant", "content": "plain [BASE64_REDACTED]", "name": "[BASE64_REDACTED]"},
|
||||
]
|
||||
|
||||
|
||||
def _messages() -> list[object]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "see data:image/png;base64,AAAA== inline"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,BBBB"}},
|
||||
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBER"}},
|
||||
],
|
||||
},
|
||||
{"role": "assistant", "content": "plain data:audio/wav;base64,CCCC tail", "name": "data:image/gif;base64,DDDD"},
|
||||
"not a message",
|
||||
]
|
||||
|
||||
|
||||
def _payload(response_id: str, status: str = "success") -> dict[str, object]:
|
||||
return {
|
||||
"id": response_id,
|
||||
"status": status,
|
||||
"metadata": {},
|
||||
"response": {"text": "ok"},
|
||||
"messages": _messages(),
|
||||
}
|
||||
|
||||
|
||||
class _Reader(CustomLogger):
|
||||
def __init__(self, name: str) -> None:
|
||||
super().__init__()
|
||||
self.name: Final = name
|
||||
self.seen: list[object] = []
|
||||
|
||||
def saw(self, kwargs: object) -> None:
|
||||
payload: Final = kwargs.get("standard_logging_object") if isinstance(kwargs, Mapping) else None
|
||||
self.seen.append(copy.deepcopy(payload.get("messages")) if isinstance(payload, Mapping) else None)
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: object, response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.saw(kwargs)
|
||||
|
||||
|
||||
class _SyncReader(_Reader):
|
||||
def log_success_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
|
||||
self.saw(kwargs)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
for name in ("S3_PARTITION_GRANULARITY", "S3_LOG_PROMPTS_ONLY", "AWS_REGION", "AWS_REGION_NAME"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setattr(litellm, "cold_storage_custom_logger", None)
|
||||
monkeypatch.setattr(litellm, "s3_callback_params", None)
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", None)
|
||||
for name in (
|
||||
"callbacks",
|
||||
"input_callback",
|
||||
"success_callback",
|
||||
"failure_callback",
|
||||
"_async_success_callback",
|
||||
"_async_failure_callback",
|
||||
):
|
||||
monkeypatch.setattr(litellm, name, [])
|
||||
monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [])
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bucket() -> Iterator[ScriptedBucket]:
|
||||
with scripted_bucket() as scripted:
|
||||
yield scripted
|
||||
|
||||
|
||||
def _params(bucket: ScriptedBucket, strip: bool, region: str = "us-east-1") -> dict[str, object]:
|
||||
return {
|
||||
"s3_bucket_name": "test-bucket",
|
||||
"s3_endpoint_url": bucket.url,
|
||||
"s3_aws_access_key_id": "test-key",
|
||||
"s3_aws_secret_access_key": "test-secret",
|
||||
"s3_region_name": region,
|
||||
**({"s3_strip_base64_files": True} if strip else {}),
|
||||
}
|
||||
|
||||
|
||||
def _s3(bucket: ScriptedBucket, strip: bool = True, region: str = "us-east-1") -> S3Logger:
|
||||
logger: Final = S3Logger(_params(bucket, strip, region))
|
||||
_ = logger.periodic_flush_task.cancel()
|
||||
return logger
|
||||
|
||||
|
||||
def _registered(*callbacks: object) -> list[object]:
|
||||
for callback in callbacks:
|
||||
assert isinstance(callback, CustomLogger | S3Logger)
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
return list(litellm._async_success_callback)
|
||||
|
||||
|
||||
def _deliver(callback: object, kwargs: object) -> None:
|
||||
if isinstance(callback, _Reader):
|
||||
callback.saw(kwargs)
|
||||
|
||||
|
||||
def _prepared(_callback: object, _kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
async def _run_line(callbacks: Sequence[object], kwargs: dict[str, object], outcome: Outcome = Outcome.success) -> None:
|
||||
if outcome is Outcome.success:
|
||||
await litellm_logging._run_callbacks(
|
||||
Hook.log_prepare,
|
||||
callbacks,
|
||||
kwargs,
|
||||
_prepared,
|
||||
args=HookArgs(request=kwargs, start_time=START, end_time=START),
|
||||
)
|
||||
await litellm_logging._run_callbacks(
|
||||
Hook.attempt_ended,
|
||||
callbacks,
|
||||
kwargs,
|
||||
_deliver,
|
||||
args=HookArgs(request=kwargs, start_time=START, end_time=START, outcome=outcome),
|
||||
)
|
||||
|
||||
|
||||
async def _flushed(logger: S3Logger) -> None:
|
||||
for _ in range(100):
|
||||
if logger.sink.queued_keys():
|
||||
break
|
||||
await asyncio.sleep(0.02)
|
||||
await logger.flush_queue()
|
||||
|
||||
|
||||
def _recorded(bucket: ScriptedBucket, case: str) -> bytes:
|
||||
(put,) = bucket.puts()
|
||||
assert (put.path, hashlib.sha256(put.body).hexdigest()) == RECORDED_AT_86B70FEF16[case]
|
||||
return put.body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strip_on_success_patches_readers_before_and_after_s3_and_uploads_the_recorded_object(
|
||||
bucket: ScriptedBucket,
|
||||
) -> None:
|
||||
before: Final = _Reader("before")
|
||||
after: Final = _Reader("after")
|
||||
s3: Final = _s3(bucket)
|
||||
callbacks: Final = _registered(before, s3, after)
|
||||
assert callbacks == [before, s3.v2_companion, s3, after]
|
||||
kwargs: Final[dict[str, object]] = {"standard_logging_object": _payload("stripped"), "litellm_params": {}}
|
||||
|
||||
await _run_line(callbacks, kwargs)
|
||||
await _flushed(s3)
|
||||
|
||||
assert before.seen == after.seen == [STRIPPED_MESSAGES]
|
||||
body: Final = _recorded(bucket, "stripped")
|
||||
assert json.loads(body)["messages"] == STRIPPED_MESSAGES
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_object_matches_the_attempt_ended_strip_alone(bucket: ScriptedBucket) -> None:
|
||||
patched: Final = _s3(bucket)
|
||||
await _run_line(_registered(patched), {"standard_logging_object": _payload("same"), "litellm_params": {}})
|
||||
await _flushed(patched)
|
||||
unpatched: Final = _s3(bucket)
|
||||
await _run_line([unpatched], {"standard_logging_object": _payload("same"), "litellm_params": {}})
|
||||
await _flushed(unpatched)
|
||||
|
||||
first, second = bucket.puts()
|
||||
assert (first.path, first.body) == (second.path, second.body)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strip_on_with_excluded_fields_uploads_the_recorded_object(
|
||||
bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["response"])
|
||||
after: Final = _Reader("after")
|
||||
s3: Final = _s3(bucket)
|
||||
kwargs: Final[dict[str, object]] = {"standard_logging_object": _payload("excluded"), "litellm_params": {}}
|
||||
|
||||
await _run_line(_registered(s3, after), kwargs)
|
||||
await _flushed(s3)
|
||||
|
||||
assert after.seen == [STRIPPED_MESSAGES]
|
||||
body: Final = json.loads(_recorded(bucket, "excluded"))
|
||||
assert "response" not in body
|
||||
assert body["messages"] == STRIPPED_MESSAGES
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strip_on_failure_still_uploads_the_recorded_stripped_object(bucket: ScriptedBucket) -> None:
|
||||
s3: Final = _s3(bucket)
|
||||
kwargs: Final[dict[str, object]] = {
|
||||
"standard_logging_object": _payload("failed", status="failure"),
|
||||
"litellm_params": {},
|
||||
}
|
||||
|
||||
await _run_line(_registered(s3), kwargs, Outcome.failure)
|
||||
await _flushed(s3)
|
||||
|
||||
assert json.loads(_recorded(bucket, "failed"))["messages"] == STRIPPED_MESSAGES
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strip_off_registers_no_writer_and_readers_see_the_original_messages(bucket: ScriptedBucket) -> None:
|
||||
before: Final = _Reader("before")
|
||||
s3: Final = _s3(bucket, strip=False)
|
||||
callbacks: Final = _registered(before, s3)
|
||||
kwargs: Final[dict[str, object]] = {"standard_logging_object": _payload("kept"), "litellm_params": {}}
|
||||
|
||||
await _run_line(callbacks, kwargs)
|
||||
await _flushed(s3)
|
||||
|
||||
assert s3.v2_companion is None
|
||||
assert callbacks == [before, s3]
|
||||
assert before.seen == [_messages()]
|
||||
assert json.loads(bucket.puts()[0].body)["messages"] == _messages()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"litellm_params",
|
||||
[
|
||||
pytest.param({"no-log": True}, id="no-log"),
|
||||
pytest.param({"proxy_server_request": {"headers": {"x-litellm-disable-callbacks": "s3_v2"}}}, id="opt-out"),
|
||||
],
|
||||
)
|
||||
async def test_a_request_s3_skips_gets_no_patch_and_nothing_is_uploaded(
|
||||
bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, object]
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(litellm, "allow_dynamic_callback_disabling", True)
|
||||
monkeypatch.setattr(proxy_server, "premium_user", True)
|
||||
before: Final = _Reader("before")
|
||||
after: Final = _Reader("after")
|
||||
s3: Final = _s3(bucket)
|
||||
kwargs: Final[dict[str, object]] = {
|
||||
"standard_logging_object": _payload("skipped"),
|
||||
"litellm_params": litellm_params,
|
||||
}
|
||||
|
||||
await _run_line(_registered(before, s3, after), kwargs)
|
||||
await s3.flush_queue()
|
||||
|
||||
assert before.seen == after.seen == [_messages()]
|
||||
assert bucket.puts() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_writer_patches_nothing_on_the_sync_line(bucket: ScriptedBucket) -> None:
|
||||
s3: Final = _s3(bucket)
|
||||
writer: Final = s3.v2_companion
|
||||
assert isinstance(writer, S3StripBase64Files)
|
||||
payload: Final = _payload("sync")
|
||||
kwargs: Final = {"standard_logging_object": payload, "litellm_params": {}}
|
||||
|
||||
await asyncio.to_thread(
|
||||
litellm_logging._run_callbacks_sync,
|
||||
Hook.log_prepare,
|
||||
[writer, s3],
|
||||
kwargs,
|
||||
lambda _callback, _kwargs: None,
|
||||
args=HookArgs(request=kwargs, start_time=START, end_time=START),
|
||||
)
|
||||
|
||||
assert payload["messages"] == _messages()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_acompletion_shows_every_reader_the_messages_s3_stores(
|
||||
bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
before: Final = _Reader("before")
|
||||
after: Final = _Reader("after")
|
||||
s3: Final = _s3(bucket)
|
||||
monkeypatch.setattr(litellm, "callbacks", [before, s3, after])
|
||||
|
||||
await litellm.acompletion(
|
||||
model="gpt-4o",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "see data:image/png;base64,AAAA== inline"},
|
||||
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBER"}},
|
||||
],
|
||||
}
|
||||
],
|
||||
mock_response="hi there",
|
||||
)
|
||||
for _ in range(100):
|
||||
if after.seen:
|
||||
break
|
||||
await asyncio.sleep(0.02)
|
||||
await _flushed(s3)
|
||||
|
||||
assert litellm._async_success_callback == [before, s3.v2_companion, s3, after]
|
||||
expected: Final = [{"role": "user", "content": [{"type": "text", "text": "see [BASE64_REDACTED]"}]}]
|
||||
assert before.seen == after.seen == [expected]
|
||||
assert json.loads(bucket.puts()[0].body)["messages"] == expected
|
||||
|
||||
|
||||
def test_a_sync_success_leaves_the_messages_for_readers_on_the_sync_line(bucket: ScriptedBucket) -> None:
|
||||
async def built() -> S3Logger:
|
||||
return _s3(bucket)
|
||||
|
||||
s3: Final = asyncio.run(built())
|
||||
before: Final = _SyncReader("before")
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(before)
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(s3)
|
||||
logging_obj: Final = Logging(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "see data:image/png;base64,AAAA== inline"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=START,
|
||||
litellm_call_id="call-sync",
|
||||
function_id="fn-sync",
|
||||
)
|
||||
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
|
||||
|
||||
logging_obj.success_handler(result=ModelResponse(id="served"), start_time=START, end_time=START)
|
||||
|
||||
assert litellm.success_callback == [before, s3.v2_companion, s3]
|
||||
assert before.seen == [[{"role": "user", "content": "see data:image/png;base64,AAAA== inline"}]]
|
||||
assert bucket.puts() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registering_s3_again_adds_no_second_writer(bucket: ScriptedBucket) -> None:
|
||||
s3: Final = _s3(bucket)
|
||||
_ = _registered(s3, s3)
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(s3)
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(s3)
|
||||
|
||||
assert litellm._async_success_callback == litellm.success_callback == [s3.v2_companion, s3]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reinitialising_proxy_callbacks_keeps_one_writer_right_before_s3(
|
||||
bucket: ScriptedBucket, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
monkeypatch.setattr(ProxyLogging, "_add_proxy_hooks", lambda _self, _router=None: None)
|
||||
monkeypatch.setattr(litellm, "s3_callback_params", _params(bucket, strip=True))
|
||||
reader: Final = _Reader("reader")
|
||||
monkeypatch.setattr(litellm, "callbacks", [reader, "s3_v2"])
|
||||
proxy_logging: Final = ProxyLogging(user_api_key_cache=MagicMock())
|
||||
|
||||
proxy_logging._init_litellm_callbacks(llm_router=None)
|
||||
proxy_logging._init_litellm_callbacks(llm_router=None)
|
||||
|
||||
(s3,) = (entry for entry in litellm.callbacks if isinstance(entry, S3Logger))
|
||||
_ = s3.periodic_flush_task.cancel()
|
||||
lists: Final = (
|
||||
litellm.callbacks,
|
||||
litellm.success_callback,
|
||||
litellm.failure_callback,
|
||||
litellm._async_success_callback,
|
||||
litellm._async_failure_callback,
|
||||
)
|
||||
assert [[entry for entry in listed if entry is not proxy_logging.service_logging_obj] for listed in lists] == [
|
||||
[reader, s3.v2_companion, s3]
|
||||
] * len(lists)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_removing_s3_removes_its_writer(bucket: ScriptedBucket) -> None:
|
||||
dropped: Final = _s3(bucket)
|
||||
removed: Final = _s3(bucket, region="eu-west-1")
|
||||
reader: Final = _Reader("reader")
|
||||
registered: Final = _registered(dropped, reader, removed)
|
||||
assert registered == [dropped.v2_companion, dropped, reader, removed.v2_companion, removed]
|
||||
|
||||
_ = litellm.logging_callback_manager.drop_callbacks((dropped,))
|
||||
litellm.logging_callback_manager.remove_callback_from_all_lists(removed)
|
||||
|
||||
assert litellm._async_success_callback == [reader]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_writer_manifest_builds_as_a_blocking_log_prepare_writer(bucket: ScriptedBucket) -> None:
|
||||
s3: Final = _s3(bucket)
|
||||
writer: Final = s3.v2_companion
|
||||
assert isinstance(writer, S3StripBase64Files)
|
||||
|
||||
manifest: Final = build_manifest(writer)
|
||||
subscription: Final = manifest.subscriptions[Hook.log_prepare]
|
||||
|
||||
assert manifest.selection.name == STRIP_CALLBACK_NAME
|
||||
assert manifest.selection.honors_no_log is False
|
||||
assert set(manifest.subscriptions) == {Hook.log_prepare}
|
||||
assert subscription.writes == frozenset({Field.standard_logging_payload})
|
||||
assert subscription.reads == frozenset({Field.deprecated_kwargs, Field.standard_logging_payload})
|
||||
assert build_manifest(s3).selection.honors_no_log is True
|
||||
assert [entry["hook"] for entry in json.loads(s3.sink.callback().manifest())["subscriptions"]] == ["attempt_ended"]
|
||||
|
|
@ -7,7 +7,6 @@ from urllib.parse import urlparse
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
from litellm.litellm_core_utils.aws_partition import (
|
||||
AwsPartition,
|
||||
contains_aws_arn,
|
||||
|
|
@ -100,14 +99,6 @@ def _agentcore_model(region: str) -> str:
|
|||
return f"agentcore/{get_aws_arn_prefix(region)}bedrock-agentcore:{region}:111122223333:runtime/my-agent"
|
||||
|
||||
|
||||
def _s3_object_url(region: str) -> str:
|
||||
logger = S3Logger.__new__(S3Logger)
|
||||
logger.s3_endpoint_url = None
|
||||
logger.s3_bucket_name = "audit-bucket"
|
||||
logger.s3_region_name = region
|
||||
return logger._build_object_url("2025-01-01/key.json")
|
||||
|
||||
|
||||
ENDPOINT_BUILDERS: Final = {
|
||||
"bedrock_runtime_default": lambda region: BaseAWSLLM()._select_default_endpoint_url("runtime", region),
|
||||
"bedrock_agent_default": lambda region: BaseAWSLLM()._select_default_endpoint_url("agent", region),
|
||||
|
|
@ -158,7 +149,6 @@ ENDPOINT_BUILDERS: Final = {
|
|||
api_base=None,
|
||||
litellm_params={"aws_region_name": region},
|
||||
),
|
||||
"s3_object_url": _s3_object_url,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ import pytest
|
|||
from botocore.session import get_session
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
from litellm.integrations.sqs import SQSLogger
|
||||
from litellm.llms.aws_polly.text_to_speech.transformation import AWSPollyTextToSpeechConfig
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
|
@ -55,7 +54,11 @@ from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
|
|||
from litellm.llms.bedrock.model_listing import BedrockModelLister
|
||||
from litellm.llms.bedrock.passthrough.transformation import BedrockAgentRuntimeAWS
|
||||
from litellm.llms.bedrock.realtime.handler import BedrockRealtime
|
||||
from litellm.llms.bedrock.web_identity_session_policy import WebIdentitySessionPolicy, _SessionPolicyStatement
|
||||
from litellm.llms.bedrock.web_identity_session_policy import (
|
||||
S3_LOGGING_SESSION_POLICY,
|
||||
WebIdentitySessionPolicy,
|
||||
_SessionPolicyStatement,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.opensearch.vector_stores.transformation import OpenSearchVectorStoreConfig
|
||||
from litellm.llms.s3_vectors.vector_stores.transformation import S3VectorsVectorStoreConfig
|
||||
|
|
@ -433,9 +436,12 @@ class TestSessionPolicyGrantsEveryBedrockRoute:
|
|||
assert len(_captured_policy_document()) <= _STS_SESSION_POLICY_PLAINTEXT_LIMIT
|
||||
|
||||
|
||||
def _s3_logger() -> S3Logger:
|
||||
with patch("asyncio.create_task"):
|
||||
return S3Logger(s3_bucket_name="litellm-logs", s3_region_name="us-east-1")
|
||||
class _S3LoggingSession(BaseAWSLLM):
|
||||
_web_identity_session_policy = S3_LOGGING_SESSION_POLICY
|
||||
|
||||
|
||||
def _s3_logger() -> _S3LoggingSession:
|
||||
return _S3LoggingSession()
|
||||
|
||||
|
||||
def _sqs_logger() -> SQSLogger:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from litellm.proxy.management_helpers.audit_logs import (
|
|||
is_audit_logging_enabled,
|
||||
)
|
||||
from litellm.types.utils import StandardAuditLogPayload
|
||||
from tests.unit.integrations.s3_v3_rust_support import scripted_bucket
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -313,16 +314,25 @@ class TestAuditLogTaskDoneCallback:
|
|||
|
||||
|
||||
class TestS3LoggerAuditLogEvent:
|
||||
@pytest.mark.asyncio
|
||||
async def test_queues_audit_log_with_correct_s3_key(self):
|
||||
with patch("litellm.integrations.s3_v2.S3Logger.__init__", return_value=None):
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
@staticmethod
|
||||
def _logger(monkeypatch: pytest.MonkeyPatch, endpoint: str, s3_path: str | None):
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
|
||||
logger = S3Logger()
|
||||
logger.s3_path = "my-prefix"
|
||||
logger.log_queue = []
|
||||
logger.batch_size = 100
|
||||
logger.max_queue_size = 100
|
||||
monkeypatch.setattr(litellm, "s3_callback_params", None)
|
||||
return S3Logger(
|
||||
s3_bucket_name="audit-bucket",
|
||||
s3_region_name="us-east-1",
|
||||
s3_endpoint_url=endpoint,
|
||||
s3_aws_access_key_id="test-key",
|
||||
s3_aws_secret_access_key="test-secret",
|
||||
s3_path=s3_path,
|
||||
s3_batch_size=100,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queues_audit_log_with_correct_s3_key(self, monkeypatch: pytest.MonkeyPatch):
|
||||
with scripted_bucket() as bucket:
|
||||
logger = self._logger(monkeypatch, bucket.url, "my-prefix")
|
||||
|
||||
audit_log = StandardAuditLogPayload(
|
||||
id="audit-123",
|
||||
|
|
@ -338,25 +348,26 @@ class TestS3LoggerAuditLogEvent:
|
|||
|
||||
await logger.async_log_audit_log_event(audit_log)
|
||||
|
||||
assert len(logger.log_queue) == 1
|
||||
element = logger.log_queue[0]
|
||||
assert element.s3_object_key.startswith("my-prefix/audit_logs/")
|
||||
assert "audit-123" in element.s3_object_key
|
||||
assert element.s3_object_key.endswith(".json")
|
||||
assert element.s3_object_download_filename == "audit-audit-123.json"
|
||||
assert element.payload["id"] == "audit-123"
|
||||
assert element.payload["action"] == "created"
|
||||
queued = logger.sink.queued_keys()
|
||||
assert len(queued) == 1
|
||||
s3_object_key = queued[0]
|
||||
assert s3_object_key.startswith("my-prefix/audit_logs/")
|
||||
assert "audit-123" in s3_object_key
|
||||
assert s3_object_key.endswith(".json")
|
||||
|
||||
await logger.flush_queue()
|
||||
|
||||
put = bucket.puts()[0]
|
||||
assert put.path == f"/audit-bucket/{s3_object_key}"
|
||||
assert put.headers["content-disposition"] == 'inline; filename="audit-audit-123.json"'
|
||||
payload = json.loads(put.body)
|
||||
assert payload["id"] == "audit-123"
|
||||
assert payload["action"] == "created"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_s3_key_format_no_path(self):
|
||||
with patch("litellm.integrations.s3_v2.S3Logger.__init__", return_value=None):
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
|
||||
logger = S3Logger()
|
||||
logger.s3_path = None
|
||||
logger.log_queue = []
|
||||
logger.batch_size = 100
|
||||
logger.max_queue_size = 100
|
||||
async def test_s3_key_format_no_path(self, monkeypatch: pytest.MonkeyPatch):
|
||||
with scripted_bucket() as bucket:
|
||||
logger = self._logger(monkeypatch, bucket.url, None)
|
||||
|
||||
audit_log = StandardAuditLogPayload(
|
||||
id="audit-456",
|
||||
|
|
@ -372,10 +383,10 @@ class TestS3LoggerAuditLogEvent:
|
|||
|
||||
await logger.async_log_audit_log_event(audit_log)
|
||||
|
||||
assert len(logger.log_queue) == 1
|
||||
element = logger.log_queue[0]
|
||||
assert element.s3_object_key.startswith("audit_logs/")
|
||||
assert "audit-456" in element.s3_object_key
|
||||
queued = logger.sink.queued_keys()
|
||||
assert len(queued) == 1
|
||||
assert queued[0].startswith("audit_logs/")
|
||||
assert "audit-456" in queued[0]
|
||||
|
||||
|
||||
class TestS3AuditCallbackParamsDecoupling:
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ from unittest.mock import AsyncMock, patch
|
|||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
from litellm.proxy._types import SpendLogsMetadata
|
||||
from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
|
||||
|
|
@ -22,6 +24,7 @@ from litellm.responses.litellm_completion_transformation.session_handler import
|
|||
ResponsesSessionHandler,
|
||||
)
|
||||
from litellm.types.utils import StandardLoggingMetadata
|
||||
from tests.unit.integrations.s3_v3_rust_support import scripted_bucket
|
||||
|
||||
|
||||
class TestColdStorageObjectKeyIntegration:
|
||||
|
|
@ -172,24 +175,29 @@ class TestColdStorageObjectKeyIntegration:
|
|||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_task") # Mock asyncio.create_task to avoid event loop issues
|
||||
async def test_s3_logger_supports_object_key_retrieval(self, mock_create_task):
|
||||
async def test_s3_logger_supports_object_key_retrieval(self, monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Test: S3Logger supports retrieval using provided object key.
|
||||
|
||||
This test verifies that the S3Logger can retrieve objects using
|
||||
the object key directly without generating it from request_id and start_time.
|
||||
"""
|
||||
# Create S3Logger instance
|
||||
s3_logger = S3Logger(s3_bucket_name="test-bucket")
|
||||
monkeypatch.setattr(litellm, "s3_callback_params", None)
|
||||
with scripted_bucket() as bucket:
|
||||
bucket.objects["/test-bucket/test/path/to/object.json"] = b'{"test": "data"}'
|
||||
s3_logger = S3Logger(
|
||||
s3_bucket_name="test-bucket",
|
||||
s3_region_name="us-east-1",
|
||||
s3_endpoint_url=bucket.url,
|
||||
s3_aws_access_key_id="test-key",
|
||||
s3_aws_secret_access_key="test-secret",
|
||||
)
|
||||
|
||||
# Mock the _download_object_from_s3 method
|
||||
with patch.object(
|
||||
s3_logger, "_download_object_from_s3", return_value={"test": "data"}
|
||||
) as mock_download:
|
||||
result = await s3_logger.get_proxy_server_request_from_cold_storage_with_object_key(
|
||||
object_key="test/path/to/object.json"
|
||||
)
|
||||
|
||||
assert result == {"test": "data"}
|
||||
mock_download.assert_called_once_with("test/path/to/object.json")
|
||||
assert [(request.method, request.path) for request in bucket.received] == [
|
||||
("GET", "/test-bucket/test/path/to/object.json")
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue