diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py
index b0ba7470433..e21e8822b41 100644
--- a/litellm/llms/bedrock/base_aws_llm.py
+++ b/litellm/llms/bedrock/base_aws_llm.py
@@ -181,14 +181,41 @@ class BaseAWSLLM(SignsRequestsWithAWS):
)
_web_identity_session_policy: ClassVar[WebIdentitySessionPolicy] = BEDROCK_SESSION_POLICY
+ _configured_web_identity_session_policy_document: str | None = None
@classmethod
@cache
def _web_identity_session_policy_document(cls) -> str:
return session_policy_document(cls._web_identity_session_policy)
- def __init__(self) -> None:
- self.iam_cache = BaseAWSLLM._shared_iam_cache
+ @property
+ def _effective_web_identity_session_policy_document(self) -> str:
+ return self._configured_web_identity_session_policy_document or self._web_identity_session_policy_document()
+
+ @overload
+ def __init__(self) -> None: ...
+
+ @overload
+ def __init__(
+ self,
+ *,
+ web_identity_session_policy: WebIdentitySessionPolicy | None,
+ credential_cache: DualCache | None = None,
+ ) -> None: ...
+
+ @overload
+ def __init__(self, *, credential_cache: DualCache) -> None: ...
+
+ def __init__(
+ self,
+ *,
+ web_identity_session_policy: WebIdentitySessionPolicy | None = None,
+ credential_cache: DualCache | None = None,
+ ) -> None:
+ self._configured_web_identity_session_policy_document = (
+ None if web_identity_session_policy is None else session_policy_document(web_identity_session_policy)
+ )
+ self.iam_cache = credential_cache if credential_cache is not None else BaseAWSLLM._shared_iam_cache
super().__init__()
self.aws_authentication_params = [
"aws_access_key_id",
@@ -402,7 +429,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
aws_session_name,
):
return self._get_or_set_cached_credentials(
- {**args, "web_identity_session_policy": self._web_identity_session_policy_document()},
+ {**args, "web_identity_session_policy": self._effective_web_identity_session_policy_document},
lambda: self._auth_with_web_identity_token(
aws_web_identity_token=cast(str, aws_web_identity_token),
aws_role_name=cast(str, aws_role_name),
@@ -1030,7 +1057,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
"RoleSessionName": aws_session_name,
"WebIdentityToken": oidc_token,
"DurationSeconds": 3600,
- "Policy": self._web_identity_session_policy_document(),
+ "Policy": self._effective_web_identity_session_policy_document,
}
# Add ExternalId parameter if provided
diff --git a/litellm/llms/bedrock/web_identity_session_policy.py b/litellm/llms/bedrock/web_identity_session_policy.py
index 8f31cb23171..93e8acf06ec 100644
--- a/litellm/llms/bedrock/web_identity_session_policy.py
+++ b/litellm/llms/bedrock/web_identity_session_policy.py
@@ -27,20 +27,26 @@ _KmsViaService: TypeAlias = Literal["s3.*", "sqs.*", "secretsmanager.*"]
_ViaServiceLike: Final = TypedDict("_ViaServiceLike", {"kms:ViaService": ReadOnly[_KmsViaService]})
-_StatementResource: TypeAlias = Literal["*", "arn:*:iam::*:role/litellm-bedrock-kb-*"]
+_S3ViaServiceEquals: Final = TypedDict("_S3ViaServiceEquals", {"kms:ViaService": ReadOnly[str]})
+
+_S3EncryptionContextLike: Final = TypedDict(
+ "_S3EncryptionContextLike", {"kms:EncryptionContext:aws:s3:arn": ReadOnly[tuple[str, ...]]}
+)
class _SessionPolicyCondition(TypedDict):
Bool: ReadOnly[_SecureTransportBool]
- StringEquals: NotRequired[ReadOnly[_CalledViaLastEquals | _PassedToServiceEquals | _ServiceLinkedRoleEquals]]
- StringLike: NotRequired[ReadOnly[_ViaServiceLike]]
+ StringEquals: NotRequired[
+ ReadOnly[_CalledViaLastEquals | _PassedToServiceEquals | _ServiceLinkedRoleEquals | _S3ViaServiceEquals]
+ ]
+ StringLike: NotRequired[ReadOnly[_ViaServiceLike | _S3EncryptionContextLike]]
class _SessionPolicyStatement(TypedDict):
Sid: ReadOnly[str]
Effect: ReadOnly[Literal["Allow"]]
Action: ReadOnly[tuple[str, ...]]
- Resource: ReadOnly[_StatementResource]
+ Resource: ReadOnly[str]
Condition: NotRequired[ReadOnly[_SessionPolicyCondition]]
@@ -58,7 +64,7 @@ def _allow(
sid: str,
actions: tuple[str, ...],
condition: _SessionPolicyCondition = _SECURE_TRANSPORT_ONLY,
- resource: _StatementResource = "*",
+ resource: str = "*",
) -> _SessionPolicyStatement:
return _SessionPolicyStatement(Sid=sid, Effect="Allow", Action=actions, Resource=resource, Condition=condition)
diff --git a/litellm/proxy/control_plane/audit/s3.py b/litellm/proxy/control_plane/audit/s3.py
new file mode 100644
index 00000000000..9e04ed0edae
--- /dev/null
+++ b/litellm/proxy/control_plane/audit/s3.py
@@ -0,0 +1,218 @@
+import base64
+import hashlib
+import re
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
+from urllib.parse import quote
+from uuid import uuid4
+
+import httpx
+
+from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix, get_aws_partition
+from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
+from litellm.llms.bedrock.web_identity_session_policy import WebIdentitySessionPolicy
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.proxy.control_plane.audit.pipeline import Batch, Delivered, DeliveryResult, Rejected, Undelivered
+from litellm.proxy.control_plane.audit.sinks import ocsf_line
+from litellm.proxy.control_plane.settings import S3SinkSettings
+
+if TYPE_CHECKING:
+ from botocore.credentials import Credentials
+
+CONTENT_TYPE: Final = "application/x-ndjson"
+RETAIN_UNTIL_FORMAT: Final = "%Y-%m-%dT%H:%M:%SZ"
+_ERROR_CODE: Final = re.compile(r"([^<]+)")
+_EXPIRED_CREDENTIAL_CODES: Final = frozenset(
+ {"ExpiredToken", "ExpiredTokenException", "RequestExpired", "TokenRefreshRequired"}
+)
+_RETRYABLE_CODES: Final = frozenset(
+ {
+ "RequestTimeout",
+ "RequestTimeoutException",
+ "SlowDown",
+ "ConditionalRequestConflict",
+ "OperationAborted",
+ "ExpiredToken",
+ "ExpiredTokenException",
+ "RequestExpired",
+ "RequestTimeTooSkewed",
+ "TokenRefreshRequired",
+ }
+)
+Clock: TypeAlias = Callable[[], datetime]
+
+
+def _utc_now() -> datetime:
+ return datetime.now(UTC)
+
+
+def audit_session_policy(settings: S3SinkSettings, region: str) -> WebIdentitySessionPolicy:
+ partition: Final = get_aws_partition(region)
+ # KMS ViaService uses the commercial suffix in every AWS partition.
+ kms_dns_suffix: Final = get_aws_dns_suffix(None)
+ bucket: Final = f"arn:{partition.partition}:s3:::{settings.bucket}"
+ prefix: Final = settings.prefix.strip("/")
+ objects: Final = f"{bucket}/{prefix}/*" if prefix else f"{bucket}/*"
+ return {
+ "Version": "2012-10-17",
+ "Statement": (
+ {
+ "Sid": "S3AuditObjectsLiteLLM",
+ "Effect": "Allow",
+ "Action": ("s3:PutObject", "s3:PutObjectRetention"),
+ "Resource": objects,
+ "Condition": {"Bool": {"aws:SecureTransport": "true"}},
+ },
+ {
+ "Sid": "S3AuditEncryptionLiteLLM",
+ "Effect": "Allow",
+ "Action": ("kms:GenerateDataKey", "kms:Decrypt"),
+ "Resource": f"arn:{partition.partition}:kms:{region}:*:key/*",
+ "Condition": {
+ "Bool": {"aws:SecureTransport": "true"},
+ "StringEquals": {"kms:ViaService": f"s3.{region}.{kms_dns_suffix}"},
+ "StringLike": {"kms:EncryptionContext:aws:s3:arn": (objects, bucket)},
+ },
+ },
+ ),
+ }
+
+
+class ObjectStore(Protocol):
+ async def put(self, url: str, content: bytes, headers: Mapping[str, str], timeout: float) -> httpx.Response: ...
+
+
+class Signer(Protocol):
+ def __call__(self, url: str, content: bytes, headers: Mapping[str, str]) -> Mapping[str, str]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class HttpxObjectStore:
+ """Holds the handler, not its client: the shared cache may close and replace the client between batches."""
+
+ handler: AsyncHTTPHandler
+
+ async def put(self, url: str, content: bytes, headers: Mapping[str, str], timeout: float) -> httpx.Response:
+ return await self.handler.client.put(
+ url, content=content, headers=dict(headers), timeout=timeout, follow_redirects=False
+ )
+
+
+@dataclass(frozen=True, slots=True)
+class SigV4Signer:
+ """SigV4 over the usual credential chain (env, profile, IRSA, IMDS), the same way the proxy's s3 logger signs.
+ Runs in the signing executor because the credential chain may block on STS or IMDS."""
+
+ region: str
+ aws: BaseAWSLLM
+
+ def invalidate_credentials(self) -> None:
+ self.aws.iam_cache.flush_cache()
+
+ def __call__(self, url: str, content: bytes, headers: Mapping[str, str]) -> Mapping[str, str]:
+ from botocore.auth import S3SigV4Auth
+ from botocore.awsrequest import AWSRequest
+ from botocore.credentials import RefreshableCredentials
+
+ credentials: Final[Credentials] = self.aws.get_credentials(aws_region_name=self.region)
+ frozen: Final = (
+ credentials.get_frozen_credentials() if isinstance(credentials, RefreshableCredentials) else credentials
+ )
+ request: Final = AWSRequest(method="PUT", url=url, data=content, headers=dict(headers))
+ S3SigV4Auth(frozen, "s3", self.region).add_auth(request)
+ return dict(request.headers.items())
+
+
+def ndjson(batch: Batch) -> bytes:
+ return "".join(f"{ocsf_line(record)}\n" for record in batch).encode()
+
+
+def object_key(prefix: str, run_id: str, batch: Batch) -> str:
+ normalized_prefix: Final = prefix.strip("/")
+ first: Final = batch[0]
+ last: Final = batch[-1]
+ suffix: Final = (
+ f"{run_id}/{first.event.at:%Y/%m/%d/%H}/{first.sequence:012d}-{last.sequence:012d}-{last.hash[-12:]}.ndjson"
+ )
+ return f"{normalized_prefix}/{suffix}" if normalized_prefix else suffix
+
+
+def object_url(settings: S3SinkSettings, key: str) -> str:
+ if settings.endpoint_url is not None:
+ return f"{settings.endpoint_url.rstrip('/')}/{settings.bucket}/{quote(key)}"
+ suffix: Final = get_aws_dns_suffix(settings.region)
+ host: Final = f"s3.{settings.region}.{suffix}" if settings.region else f"s3.{suffix}"
+ if "." in settings.bucket:
+ return f"https://{host}/{quote(settings.bucket, safe='')}/{quote(key)}"
+ return f"https://{settings.bucket}.{host}/{quote(key)}"
+
+
+def put_headers(settings: S3SinkSettings, batch: Batch, content: bytes, now: datetime) -> Mapping[str, str]:
+ """Integrity headers, the never-overwrite condition and, when a lock mode is set, retention counted from the
+ newest event in the batch so the clock in the record is the clock of the lock."""
+ base: Final = {
+ "Content-Type": CONTENT_TYPE,
+ "Content-MD5": base64.b64encode(hashlib.md5(content, usedforsecurity=False).digest()).decode(),
+ "x-amz-content-sha256": hashlib.sha256(content).hexdigest(),
+ "If-None-Match": "*",
+ }
+ if settings.object_lock_mode is None:
+ return base
+ retain_until: Final = max(record.event.at for record in batch) + timedelta(days=settings.retention_days)
+ if retain_until <= now:
+ return base
+ return {
+ **base,
+ "x-amz-object-lock-mode": settings.object_lock_mode,
+ "x-amz-object-lock-retain-until-date": retain_until.strftime(RETAIN_UNTIL_FORMAT),
+ }
+
+
+def _error_code(response: httpx.Response) -> str:
+ found: Final = _ERROR_CODE.search(response.text)
+ return found.group(1) if found else "no error code"
+
+
+@dataclass(frozen=True, slots=True)
+class S3ObjectLockSink:
+ """A 412 means the key already exists from an earlier attempt whose acknowledgement was lost. The key carries the
+ last record's hash, which covers every record in the batch, so the stored object is this batch and it counts
+ as delivered."""
+
+ settings: S3SinkSettings
+ store: ObjectStore
+ sign: Signer
+ name: str = "s3"
+ run_id: str = field(default_factory=lambda: uuid4().hex)
+ clock: Clock = _utc_now
+
+ async def deliver(self, batch: Batch) -> DeliveryResult:
+ key: Final = object_key(self.settings.prefix, self.run_id, batch)
+ url: Final = object_url(self.settings, key)
+ content: Final = ndjson(batch)
+ try:
+ headers: Final = put_headers(self.settings, batch, content, self.clock())
+ signed: Final = await run_aws_signing(self.sign, url, content, headers)
+ response: Final = await self.store.put(url, content, signed, self.settings.timeout_seconds)
+ except httpx.HTTPError as error:
+ return Undelivered(f"{type(error).__name__} putting {key}")
+ except Exception as error: # noqa: BLE001 # a failing credential chain is a failed delivery, not a dead worker
+ return Undelivered(f"{type(error).__name__} signing {key}")
+ match response.status_code:
+ case 200 | 412:
+ return Delivered(len(batch))
+ case status:
+ code: Final = _error_code(response)
+ reason: Final = f"HTTP {status} {code} putting {key}"
+ if code in _EXPIRED_CREDENTIAL_CODES and isinstance(self.sign, SigV4Signer):
+ self.sign.invalidate_credentials()
+ if 300 <= status < 400:
+ return Rejected(reason, len(batch))
+ if 400 <= status < 500 and status not in (408, 429) and code not in _RETRYABLE_CODES:
+ return Rejected(reason, len(batch))
+ return Undelivered(reason)
+
+ async def close(self) -> None:
+ return None
diff --git a/litellm/proxy/control_plane/audit/sinks.py b/litellm/proxy/control_plane/audit/sinks.py
index 6f8a47c05da..b62faec4157 100644
--- a/litellm/proxy/control_plane/audit/sinks.py
+++ b/litellm/proxy/control_plane/audit/sinks.py
@@ -27,7 +27,7 @@ EVENT_NAME: Final = "litellm.control_plane.audit"
LOG_PREFIX: Final = "control_plane.audit"
-def _line(record: AuditRecord) -> str:
+def ocsf_line(record: AuditRecord) -> str:
return json.dumps(record.body, default=str, separators=(",", ":"))
@@ -38,7 +38,7 @@ class LogSink:
async def deliver(self, batch: Batch) -> DeliveryResult:
stream: Final = sys.stderr if self.stream is None else self.stream
- for index, line in enumerate(f"{LOG_PREFIX} {_line(record)}\n" for record in batch):
+ for index, line in enumerate(f"{LOG_PREFIX} {ocsf_line(record)}\n" for record in batch):
try:
if stream.write(line) != len(line):
return Undelivered("log stream short write", settled=index)
@@ -98,7 +98,7 @@ def _log_data(record: AuditRecord, resource: Resource) -> ReadableLogRecord:
),
severity_text=severity_text,
severity_number=severity_number,
- body=_line(record),
+ body=ocsf_line(record),
attributes=dict(_attributes(record)),
),
resource,
diff --git a/litellm/proxy/control_plane/container.py b/litellm/proxy/control_plane/container.py
index 921a8b051bb..8af81a59d1c 100644
--- a/litellm/proxy/control_plane/container.py
+++ b/litellm/proxy/control_plane/container.py
@@ -2,17 +2,21 @@
from __future__ import annotations
+import os
import time
from collections.abc import Mapping, Sequence
from contextlib import AbstractAsyncContextManager
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from datetime import timedelta
from importlib.metadata import PackageNotFoundError, version
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import httpx
+from pydantic import TypeAdapter
+from litellm.caching.dual_cache import DualCache
+from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # params is an untyped dict upstream
@@ -20,6 +24,12 @@ from litellm.llms.custom_httpx.http_handler import (
from litellm.proxy.control_plane.audit.log import AuditLog, build_audit_log
from litellm.proxy.control_plane.audit.ocsf import Product
from litellm.proxy.control_plane.audit.pipeline import AuditPipeline, AuditSink, build_pipeline
+from litellm.proxy.control_plane.audit.s3 import (
+ HttpxObjectStore,
+ S3ObjectLockSink,
+ SigV4Signer,
+ audit_session_policy,
+)
from litellm.proxy.control_plane.audit.sinks import LogSink, OtlpLogSink
from litellm.proxy.control_plane.auth.jwt import JwksKeyLookup, SessionTokenCodec, TokenVerifier, build_token_verifier
from litellm.proxy.control_plane.auth.login import LoginService, LoginThrottle
@@ -119,6 +129,31 @@ def _shared_http_client(settings: ControlPlaneSettings) -> HttpTransport:
return HandlerTransport(_shared_http_handler(settings))
+_REGION: Final = TypeAdapter(str)
+_OPTIONAL_REGION: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
+
+
+def _s3_sink(settings: ControlPlaneSettings) -> AuditSink:
+ s3: Final = settings.audit_s3
+ if s3 is None:
+ raise ValueError("CONTROL_PLANE_AUDIT_SINKS names s3 but CONTROL_PLANE_AUDIT_S3_BUCKET is not set.")
+ region: Final = s3.region or _environment_region(BaseAWSLLM())
+ aws: Final = BaseAWSLLM(web_identity_session_policy=audit_session_policy(s3, region), credential_cache=DualCache())
+ return S3ObjectLockSink(
+ replace(s3, region=region), HttpxObjectStore(_shared_http_handler(settings)), SigV4Signer(region, aws)
+ )
+
+
+def _environment_region(aws: BaseAWSLLM) -> str:
+ """The AWS SDK order: ``AWS_REGION``, then botocore's chain (``AWS_DEFAULT_REGION``, profile, config), then
+ litellm's resolver (``AWS_REGION_NAME`` and its default)."""
+ from botocore.session import Session
+
+ explicit: Final = os.environ.get("AWS_REGION")
+ configured: Final = explicit or _OPTIONAL_REGION.validate_python(Session().get_config_variable("region"))
+ return configured or _REGION.validate_python(aws.get_aws_region_name_for_non_llm_api_calls())
+
+
def _sink(kind: str, settings: ControlPlaneSettings) -> AuditSink:
match kind:
case "otlp":
@@ -132,6 +167,8 @@ def _sink(kind: str, settings: ControlPlaneSettings) -> AuditSink:
"Set OTEL_EXPORTER=otlp_http or configure OTEL_EXPORTER_OTLP_ENDPOINT."
)
return OtlpLogSink(exporter, build_resource(settings))
+ case "s3":
+ return _s3_sink(settings)
case _:
return LogSink()
diff --git a/litellm/proxy/control_plane/settings.py b/litellm/proxy/control_plane/settings.py
index f9e11f962f5..e9e3d91f454 100644
--- a/litellm/proxy/control_plane/settings.py
+++ b/litellm/proxy/control_plane/settings.py
@@ -4,6 +4,7 @@ import os
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, Literal, TypeAlias
+from urllib.parse import urlsplit
import yaml
from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, field_validator, model_validator
@@ -15,10 +16,22 @@ from litellm.proxy.control_plane.auth.rbac import ADMIN_ROLE
from litellm.types.proxy.control_plane.auth import PASSWORD_MAX_LENGTH, USERNAME_MAX_LENGTH
DEFAULT_SESSION_TTL_SECONDS: Final = 12 * 60 * 60
-AuditSinkKind: TypeAlias = Literal["log", "otlp"]
+AuditSinkKind: TypeAlias = Literal["log", "otlp", "s3"]
+ObjectLockMode: TypeAlias = Literal["COMPLIANCE", "GOVERNANCE"]
DEFAULT_DATAPLANE_TIMEOUT_SECONDS: Final = 10.0
+@dataclass(frozen=True, slots=True)
+class S3SinkSettings:
+ bucket: str
+ prefix: str
+ region: str | None
+ endpoint_url: str | None
+ object_lock_mode: ObjectLockMode | None
+ retention_days: int
+ timeout_seconds: float
+
+
@dataclass(frozen=True, slots=True)
class OidcSettings:
"""External IdP used to verify caller JWTs and, later, to exchange them for dataplane tokens."""
@@ -124,6 +137,51 @@ class ControlPlaneSettings(BaseSettings):
audit_batch_size: int = Field(default=100, ge=1)
audit_drain_seconds: float = Field(default=5.0, gt=0)
audit_strict: bool = False
+ audit_s3_bucket: str | None = None
+ audit_s3_prefix: str = "control-plane-audit"
+ audit_s3_region: str | None = None
+ audit_s3_endpoint_url: str | None = None
+ audit_s3_allow_insecure_http: bool = False
+ audit_s3_object_lock_mode: ObjectLockMode | None = None
+ audit_s3_retention_days: int = Field(default=365, ge=1)
+ audit_s3_timeout_seconds: float = 30.0
+
+ @field_validator("audit_s3_bucket")
+ @classmethod
+ def _s3_bucket_is_not_blank(cls, value: str | None) -> str | None:
+ if value is not None and not value.strip():
+ raise ValueError("CONTROL_PLANE_AUDIT_S3_BUCKET must not be empty or whitespace.")
+ return value
+
+ @model_validator(mode="after")
+ def _s3_sink_needs_a_bucket(self) -> "ControlPlaneSettings":
+ if "s3" in self.audit_sinks and self.audit_s3_bucket is None:
+ raise ValueError("CONTROL_PLANE_AUDIT_SINKS names s3 but CONTROL_PLANE_AUDIT_S3_BUCKET is not set.")
+ return self
+
+ @model_validator(mode="after")
+ def _s3_endpoint_is_https_unless_opted_out(self) -> "ControlPlaneSettings":
+ endpoint: Final = self.audit_s3_endpoint_url
+ if endpoint is None or self.audit_s3_allow_insecure_http or urlsplit(endpoint).scheme.lower() == "https":
+ return self
+ raise ValueError(
+ f"audit s3 endpoint {endpoint!r} is not https, audit records and the SigV4 authorization would travel "
+ "unencrypted. Use https or set CONTROL_PLANE_AUDIT_S3_ALLOW_INSECURE_HTTP=true for an in-cluster store."
+ )
+
+ @property
+ def audit_s3(self) -> S3SinkSettings | None:
+ if self.audit_s3_bucket is None:
+ return None
+ return S3SinkSettings(
+ bucket=self.audit_s3_bucket,
+ prefix=self.audit_s3_prefix.strip("/"),
+ region=self.audit_s3_region,
+ endpoint_url=self.audit_s3_endpoint_url,
+ object_lock_mode=self.audit_s3_object_lock_mode,
+ retention_days=self.audit_s3_retention_days,
+ timeout_seconds=self.audit_s3_timeout_seconds,
+ )
@model_validator(mode="after")
def _strict_audit_requires_a_sink(self) -> "ControlPlaneSettings":
diff --git a/tests/unit/llms/bedrock/test_base_aws_llm.py b/tests/unit/llms/bedrock/test_base_aws_llm.py
index c220e39f4d1..d137b9bbdef 100644
--- a/tests/unit/llms/bedrock/test_base_aws_llm.py
+++ b/tests/unit/llms/bedrock/test_base_aws_llm.py
@@ -34,6 +34,11 @@ from litellm.llms.bedrock.base_aws_llm import (
)
from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
from litellm.llms.bedrock.common_utils import BedrockModelInfo
+from litellm.llms.bedrock.web_identity_session_policy import (
+ BEDROCK_SESSION_POLICY,
+ S3_LOGGING_SESSION_POLICY,
+ session_policy_document,
+)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
@@ -312,6 +317,37 @@ def test_web_identity_token_oidc_reference_still_resolved():
assert exc.value.status_code == 401
+def test_configured_web_identity_policies_isolate_sts_permissions_and_cached_credentials(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv("UNIT_WEB_IDENTITY_TOKEN", "policy-test-token")
+ sts: Final = MagicMock()
+ sts.assume_role_with_web_identity.side_effect = tuple(
+ {
+ "Credentials": {"AccessKeyId": key, "SecretAccessKey": "test-secret", "SessionToken": "test-session"},
+ "PackedPolicySize": 1,
+ }
+ for key in ("s3-key", "bedrock-key")
+ )
+ s3: Final = BaseAWSLLM(web_identity_session_policy=S3_LOGGING_SESSION_POLICY)
+ bedrock: Final = BaseAWSLLM()
+ with patch("boto3.client", return_value=sts):
+ credentials: Final = tuple(
+ helper.get_credentials(
+ aws_web_identity_token="oidc/env/UNIT_WEB_IDENTITY_TOKEN",
+ aws_role_name="arn:aws:iam::123456789012:role/policy-test",
+ aws_session_name="policy-test",
+ aws_region_name="eu-west-1",
+ )
+ for helper in (s3, bedrock, s3)
+ )
+ assert tuple(item.access_key for item in credentials) == ("s3-key", "bedrock-key", "s3-key")
+ assert tuple(call.kwargs["Policy"] for call in sts.assume_role_with_web_identity.call_args_list) == (
+ session_policy_document(S3_LOGGING_SESSION_POLICY),
+ session_policy_document(BEDROCK_SESSION_POLICY),
+ )
+
+
def test_web_identity_credentials_cached_in_iam_cache():
"""
Web identity STS credentials are cached for their STS lifetime, so repeated
diff --git a/tests/unit/proxy/control_plane/audit/test_s3.py b/tests/unit/proxy/control_plane/audit/test_s3.py
new file mode 100644
index 00000000000..4fd4c6a99a5
--- /dev/null
+++ b/tests/unit/proxy/control_plane/audit/test_s3.py
@@ -0,0 +1,300 @@
+import base64
+import hashlib
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass, field, replace
+from datetime import UTC, datetime, timedelta
+from functools import reduce
+from typing import Final
+from uuid import UUID
+
+import httpx
+import pytest
+
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.proxy.control_plane.audit.event import AuditEvent
+from litellm.proxy.control_plane.audit.ocsf import AuditRecord, ChainState, Product, chain
+from litellm.proxy.control_plane.audit.pipeline import Accepted, Batch, Delivered, Rejected, Undelivered, build_pipeline
+from litellm.proxy.control_plane.audit.s3 import HttpxObjectStore, S3ObjectLockSink, object_key
+from litellm.proxy.control_plane.settings import S3SinkSettings
+
+T0: Final = datetime(2026, 10, 13, 12, 0, tzinfo=UTC)
+
+
+def test_empty_s3_prefix_does_not_add_a_leading_slash() -> None:
+ assert not object_key("", "run", _batch(1)).startswith("/")
+
+
+SETTINGS: Final = S3SinkSettings(
+ bucket="audit",
+ prefix="cp",
+ region="eu-west-1",
+ endpoint_url=None,
+ object_lock_mode="COMPLIANCE",
+ retention_days=30,
+ timeout_seconds=5.0,
+)
+
+
+@dataclass
+class FakeStore: # mutable-ok: scripts responses and records puts for assertions
+ outcomes: list[httpx.Response | Exception]
+ puts: list[tuple[str, bytes, Mapping[str, str], float]] = field(default_factory=list)
+
+ async def put(self, url: str, content: bytes, headers: Mapping[str, str], timeout: float) -> httpx.Response:
+ self.puts.append((url, content, headers, timeout))
+ outcome: Final = self.outcomes.pop(0)
+ if isinstance(outcome, Exception):
+ raise outcome
+ return outcome
+
+
+@pytest.mark.asyncio
+async def test_s3_put_does_not_follow_redirects_from_a_following_http_client() -> None:
+ transport: Final = httpx.MockTransport(
+ lambda request: httpx.Response(
+ 307 if request.url.host == "bucket.example" else 200,
+ headers={"Location": "https://redirected.example/object"} if request.url.host == "bucket.example" else {},
+ )
+ )
+ handler: Final = AsyncHTTPHandler(transport=transport, follow_redirects=True)
+
+ response: Final = await HttpxObjectStore(handler).put("https://bucket.example/object", b"body", {}, 1.0)
+ await handler.close()
+
+ assert (response.status_code, str(response.request.url)) == (307, "https://bucket.example/object")
+
+
+def _sign(url: str, content: bytes, headers: Mapping[str, str]) -> Mapping[str, str]:
+ return {**headers, "Authorization": f"AWS4-HMAC-SHA256 test {len(content)}"}
+
+
+def _append_record(state: tuple[ChainState, Batch], index: int) -> tuple[ChainState, Batch]:
+ previous, records = state
+ event: Final = AuditEvent(T0, "alice", None, "eu", "POST", f"/key/{index}", "allowed", 200, None, None)
+ next_state, record = chain(event, Product("t"), previous)
+ return next_state, (*records, record)
+
+
+def _batch(count: int) -> Batch:
+ return reduce(_append_record, range(count), (ChainState(), ()))[1]
+
+
+@pytest.mark.asyncio
+async def test_a_batch_is_one_locked_ndjson_object_under_a_dated_key_that_is_never_overwritten() -> None:
+ store: Final = FakeStore([httpx.Response(200)])
+ batch: Final = _batch(3)
+ sink: Final = S3ObjectLockSink(SETTINGS, store, _sign, clock=lambda: T0)
+ assert await sink.deliver(batch) == Delivered(3)
+ ((url, content, headers, timeout),) = store.puts
+ suffix: Final = batch[-1].hash[-12:]
+ assert (
+ url == f"https://audit.s3.eu-west-1.amazonaws.com/cp/{sink.run_id}/2026/10/13/12/"
+ f"000000000001-000000000003-{suffix}.ndjson"
+ )
+ assert [json.loads(line) for line in content.decode().splitlines()] == [record.body for record in batch]
+ assert headers["Authorization"] == f"AWS4-HMAC-SHA256 test {len(content)}" and timeout == 5.0
+ assert headers["If-None-Match"] == "*" and headers["Content-Type"] == "application/x-ndjson"
+ assert headers["Content-MD5"] == base64.b64encode(hashlib.md5(content, usedforsecurity=False).digest()).decode()
+ assert headers["x-amz-content-sha256"] == hashlib.sha256(content).hexdigest()
+ assert (headers["x-amz-object-lock-mode"], headers["x-amz-object-lock-retain-until-date"]) == (
+ "COMPLIANCE",
+ "2026-11-12T12:00:00Z",
+ )
+
+
+@pytest.mark.asyncio
+async def test_without_a_lock_mode_there_are_no_lock_headers_and_a_custom_endpoint_is_path_style() -> None:
+ store: Final = FakeStore([httpx.Response(200)])
+ settings: Final = replace(SETTINGS, object_lock_mode=None, endpoint_url="https://minio.internal:9000/")
+ sink: Final = S3ObjectLockSink(settings, store, _sign)
+ assert await sink.deliver(_batch(1)) == Delivered(1)
+ ((url, _, headers, _),) = store.puts
+ assert url.startswith(f"https://minio.internal:9000/audit/cp/{sink.run_id}/2026/10/13/12/000000000001-")
+ assert not [key for key in headers if key.startswith("x-amz-object-lock")]
+
+
+@pytest.mark.asyncio
+async def test_a_china_partition_region_gets_its_own_dns_suffix() -> None:
+ store: Final = FakeStore([httpx.Response(200)])
+ sink: Final = S3ObjectLockSink(replace(SETTINGS, region="cn-north-1"), store, _sign)
+ await sink.deliver(_batch(1))
+ ((url, _, _, _),) = store.puts
+ assert url.startswith(f"https://audit.s3.cn-north-1.amazonaws.com.cn/cp/{sink.run_id}/")
+
+
+@pytest.mark.asyncio
+async def test_sink_keys_are_unique_per_sink_and_stable_across_retries() -> None:
+ batch: Final = _batch(1)
+ first_store: Final = FakeStore([httpx.Response(200), httpx.Response(200)])
+ second_store: Final = FakeStore([httpx.Response(200)])
+ first: Final = S3ObjectLockSink(SETTINGS, first_store, _sign)
+ second: Final = S3ObjectLockSink(SETTINGS, second_store, _sign)
+
+ await first.deliver(batch)
+ await first.deliver(batch)
+ await second.deliver(batch)
+
+ first_urls: Final = tuple(url for url, _, _, _ in first_store.puts)
+ ((second_url, _, _, _),) = second_store.puts
+ assert UUID(first.run_id).version == UUID(second.run_id).version == 4
+ assert first_urls[0] == first_urls[1] and first_urls[0] != second_url
+
+
+@pytest.mark.asyncio
+async def test_expired_retention_omits_lock_headers() -> None:
+ store: Final = FakeStore([httpx.Response(200)])
+ sink: Final = S3ObjectLockSink(SETTINGS, store, _sign, clock=lambda: T0 + timedelta(days=30))
+
+ await sink.deliver(_batch(1))
+
+ ((_, _, headers, _),) = store.puts
+ assert not [key for key in headers if key.startswith("x-amz-object-lock")]
+
+
+@pytest.mark.asyncio
+async def test_dotted_bucket_uses_regional_path_style_url() -> None:
+ store: Final = FakeStore([httpx.Response(200)])
+ sink: Final = S3ObjectLockSink(replace(SETTINGS, bucket="audit.logs"), store, _sign)
+
+ await sink.deliver(_batch(1))
+
+ ((url, _, _, _),) = store.puts
+ assert url.startswith(f"https://s3.eu-west-1.amazonaws.com/audit.logs/cp/{sink.run_id}/")
+
+
+@pytest.mark.asyncio
+async def test_an_existing_key_is_delivered_while_s3_errors_and_network_errors_are_retried() -> None:
+ store: Final = FakeStore(
+ [
+ httpx.Response(412, text="PreconditionFailed"),
+ httpx.Response(503, text="SlowDown"),
+ httpx.ConnectError("boom"),
+ ]
+ )
+ sink: Final = S3ObjectLockSink(SETTINGS, store, _sign)
+ batch: Final = _batch(2)
+ assert await sink.deliver(batch) == Delivered(2), "the object is already there from an unacknowledged attempt"
+ slow: Final = await sink.deliver(batch)
+ assert isinstance(slow, Undelivered) and slow.reason.startswith(
+ f"HTTP 503 SlowDown putting cp/{sink.run_id}/2026/10/13/12/"
+ )
+ down: Final = await sink.deliver(batch)
+ assert isinstance(down, Undelivered) and down.reason.startswith(f"ConnectError putting cp/{sink.run_id}/")
+
+
+@pytest.mark.asyncio
+async def test_a_signer_that_cannot_get_credentials_is_a_failed_delivery_not_a_dead_worker() -> None:
+ def no_credentials(url: str, content: bytes, headers: Mapping[str, str]) -> Mapping[str, str]:
+ raise RuntimeError("no credential chain")
+
+ store: Final = FakeStore([])
+ sink: Final = S3ObjectLockSink(SETTINGS, store, no_credentials)
+ result: Final = await sink.deliver(_batch(1))
+ assert isinstance(result, Undelivered) and result.reason.startswith(f"RuntimeError signing cp/{sink.run_id}/")
+ assert store.puts == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("status", "code"),
+ [
+ (301, "PermanentRedirect"),
+ (301, "RequestTimeout"),
+ (307, "TemporaryRedirect"),
+ (400, "InvalidRequest"),
+ (403, "AccessDenied"),
+ (404, "NoSuchBucket"),
+ (409, "Conflict"),
+ (422, ""),
+ (400, ""),
+ ],
+)
+async def test_permanent_s3_failures_release_the_batch_and_count_loss(status: int, code: str) -> None:
+ store: Final = FakeStore([httpx.Response(status, text=f"{code}"), httpx.Response(200)])
+ sink: Final = S3ObjectLockSink(SETTINGS, store, _sign)
+ pipeline: Final = build_pipeline((sink,), queue_size=3, batch_size=2, strict=False)
+ first, second = _batch(2)
+ assert pipeline.publish(first) == pipeline.publish(second) == (Accepted(),)
+ (worker,) = pipeline.workers
+
+ assert await worker.flush_once() == 2
+ assert worker.rejected == 2 and worker.inflight is None and worker.queue.empty()
+ assert pipeline.publish(first) == (Accepted(),)
+ assert await worker.flush_once() == 1
+ assert worker.rejected == 2 and worker.inflight is None and worker.queue.empty()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("status", "code"),
+ [
+ (408, ""),
+ (429, ""),
+ (500, "InternalError"),
+ (503, "SlowDown"),
+ (400, "RequestTimeout"),
+ (400, "RequestTimeoutException"),
+ (400, "SlowDown"),
+ (409, "ConditionalRequestConflict"),
+ (409, "OperationAborted"),
+ ],
+)
+async def test_transient_s3_failures_remain_retryable(status: int, code: str) -> None:
+ store: Final = FakeStore([httpx.Response(status, text=f"{code}")])
+ batch: Final = _batch(2)
+ result: Final = await S3ObjectLockSink(SETTINGS, store, _sign).deliver(batch)
+ assert isinstance(result, Undelivered)
+ assert result.reason.startswith(f"HTTP {status} {code or 'no error code'} putting cp/")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("code", ["ExpiredToken", "ExpiredTokenException", "RequestExpired", "TokenRefreshRequired"])
+@pytest.mark.parametrize("status", [400, 403])
+async def test_expired_s3_credentials_retry_the_same_batch_with_a_new_signature(status: int, code: str) -> None:
+ signatures: Final = iter(("expired-local-credential", "refreshed-local-credential"))
+
+ def sign(url: str, content: bytes, headers: Mapping[str, str]) -> Mapping[str, str]:
+ return {**headers, "Authorization": next(signatures)}
+
+ store: Final = FakeStore([httpx.Response(status, text=f"{code}"), httpx.Response(200)])
+ sink: Final = S3ObjectLockSink(SETTINGS, store, sign)
+ pipeline: Final = build_pipeline((sink,), queue_size=2, batch_size=2, strict=False)
+ (worker,) = pipeline.workers
+ first, second = _batch(2)
+ assert pipeline.publish(first) == pipeline.publish(second) == (Accepted(),)
+ assert await worker.flush_once() == 2
+ assert worker.rejected == 0 and worker.inflight is None and worker.queue.empty()
+ before, after = store.puts
+ assert before[:2] == after[:2]
+ assert before[2]["Authorization"] == "expired-local-credential"
+ assert after[2]["Authorization"] == "refreshed-local-credential"
+
+
+@pytest.mark.asyncio
+async def test_a_permanent_s3_reply_reports_the_rejected_count() -> None:
+ sink: Final = S3ObjectLockSink(SETTINGS, FakeStore([httpx.Response(403)]), _sign)
+ result: Final = await sink.deliver(_batch(2))
+ assert isinstance(result, Rejected) and result.count == 2 and result.reason.startswith("HTTP 403 no error code")
+
+
+@pytest.mark.asyncio
+async def test_clock_skew_retries_the_strict_batch_without_permanent_audit_loss() -> None:
+ store: Final = FakeStore(
+ [httpx.Response(403, text="RequestTimeTooSkewed"), httpx.Response(200)]
+ )
+ sink: Final = S3ObjectLockSink(SETTINGS, store, _sign)
+ pipeline: Final = build_pipeline((sink,), queue_size=1, batch_size=1, strict=True)
+ (worker,) = pipeline.workers
+ batch: Final = _batch(1)
+
+ async def retry(delay: float) -> None:
+ assert worker.inflight == batch and worker.rejected == 0
+
+ worker.sleep = retry
+ assert pipeline.publish(batch[0]) == (Accepted(),)
+ assert await worker.flush_once() == 1
+ assert len(store.puts) == 2 and store.puts[0][:2] == store.puts[1][:2]
+ assert worker.rejected == 0 and worker.inflight is None and worker.queue.empty()
+ assert pipeline.accepting()
+ await pipeline.close()
diff --git a/tests/unit/proxy/control_plane/test_container.py b/tests/unit/proxy/control_plane/test_container.py
index 317618d71d7..cdc0e454077 100644
--- a/tests/unit/proxy/control_plane/test_container.py
+++ b/tests/unit/proxy/control_plane/test_container.py
@@ -1,18 +1,179 @@
import os
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from typing import Final, cast
+from unittest.mock import MagicMock, patch
import httpx
import pytest
from pydantic import TypeAdapter
+from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
-from litellm.proxy.control_plane.container import HandlerTransport, _shared_http_handler, build_audit_pipeline
+from litellm.proxy.control_plane.audit.log import build_audit_log
+from litellm.proxy.control_plane.audit.pipeline import Delivered, Undelivered
+from litellm.proxy.control_plane.audit.s3 import HttpxObjectStore, S3ObjectLockSink
+from litellm.proxy.control_plane.container import (
+ HandlerTransport,
+ _environment_region,
+ _shared_http_handler,
+ build_audit_pipeline,
+)
from litellm.proxy.control_plane.settings import ControlPlaneSettings
_JSON_COUNT: Final = TypeAdapter(dict[str, int])
+@pytest.mark.asyncio
+@pytest.mark.parametrize("code", ["ExpiredToken", "ExpiredTokenException", "RequestExpired", "TokenRefreshRequired"])
+async def test_expired_s3_web_identity_credentials_are_refetched_without_flushing_other_sinks(
+ monkeypatch: pytest.MonkeyPatch, code: str
+) -> None:
+ for name in tuple(os.environ):
+ if name.startswith("AWS_"):
+ monkeypatch.delenv(name)
+ monkeypatch.setenv("AWS_WEB_IDENTITY_TOKEN", "oidc/env/CONTROL_PLANE_REFRESH_TEST_TOKEN")
+ monkeypatch.setenv("CONTROL_PLANE_REFRESH_TEST_TOKEN", "test-token")
+ monkeypatch.setenv("AWS_ROLE_NAME", "arn:aws:iam::123456789012:role/audit-refresh-test")
+ monkeypatch.setenv("AWS_SESSION_NAME", code)
+ sts: Final = MagicMock()
+ sts.assume_role_with_web_identity.side_effect = (
+ {
+ "Credentials": {"AccessKeyId": key, "SecretAccessKey": "test-secret", "SessionToken": key},
+ "PackedPolicySize": 1,
+ }
+ for key in ("expired-key", "other-sink-key", "fresh-key")
+ )
+ settings: Final = ControlPlaneSettings.model_validate(
+ {
+ "LITELLM_MASTER_KEY": "01234567890123456789012345678901",
+ "audit_sinks": ("s3",),
+ "audit_s3_bucket": "audit",
+ }
+ )
+ sink: Final = build_audit_pipeline(settings).workers[0].sink
+ other: Final = build_audit_pipeline(settings).workers[0].sink
+ assert isinstance(sink, S3ObjectLockSink) and isinstance(other, S3ObjectLockSink)
+ sent: Final[list[httpx.Request]] = []
+
+ def reply(request: httpx.Request) -> httpx.Response:
+ sent.append(request)
+ if len(sent) == 1:
+ return httpx.Response(403, text=f"{code}")
+ return httpx.Response(200)
+
+ handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(reply))
+ audit: Final = build_audit_log()
+ audit.record("operator", None, "control_plane", "PUT", "/roles/new", "allowed", 200)
+ batch: Final = audit.recent_records(1)
+ configured: Final = replace(sink, store=HttpxObjectStore(handler))
+ try:
+ with patch("boto3.client", return_value=sts):
+ expired: Final = configured.sign("https://audit.s3.test/event", b"{}", {})
+ isolated: Final = other.sign("https://audit.s3.test/event", b"{}", {})
+ assert "expired-key/" in expired["Authorization"]
+ assert "other-sink-key/" in isolated["Authorization"]
+ assert isinstance(await configured.deliver(batch), Undelivered)
+ assert await configured.deliver(batch) == Delivered(1)
+ assert "other-sink-key/" in other.sign("https://audit.s3.test/event", b"{}", {})["Authorization"]
+ assert len(sent) == 2 and sent[0].url == sent[1].url and sent[0].content == sent[1].content
+ assert "expired-key/" in sent[0].headers["Authorization"]
+ assert "fresh-key/" in sent[1].headers["Authorization"]
+ assert sts.assume_role_with_web_identity.call_count == 3
+ finally:
+ await handler.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("lock_mode", [None, "COMPLIANCE", "GOVERNANCE"])
+@pytest.mark.parametrize(
+ ("bucket", "prefix", "region", "resolved_region", "partition", "dns_suffix", "scoped_prefix"),
+ [
+ ("audit", "/cp/", "eu-west-1", "eu-west-1", "aws", "amazonaws.com", "cp/"),
+ ("other-audit", "/separate/", "eu-west-1", "eu-west-1", "aws", "amazonaws.com", "separate/"),
+ ("audit", "", None, "cn-north-1", "aws-cn", "amazonaws.com.cn", ""),
+ ("audit", "/cp/sub/", "us-gov-west-1", "us-gov-west-1", "aws-us-gov", "amazonaws.com", "cp/sub/"),
+ ],
+)
+async def test_s3_web_identity_signing_allows_uploads_and_object_retention(
+ monkeypatch: pytest.MonkeyPatch,
+ lock_mode: str | None,
+ bucket: str,
+ prefix: str,
+ region: str | None,
+ resolved_region: str,
+ partition: str,
+ dns_suffix: str,
+ scoped_prefix: str,
+) -> None:
+ for name in tuple(os.environ):
+ if name.startswith("AWS_"):
+ monkeypatch.delenv(name)
+ monkeypatch.setenv("AWS_WEB_IDENTITY_TOKEN", "oidc/env/CONTROL_PLANE_AUDIT_TEST_WEB_IDENTITY")
+ monkeypatch.setenv("CONTROL_PLANE_AUDIT_TEST_WEB_IDENTITY", "audit-test-token")
+ monkeypatch.setenv("AWS_ROLE_NAME", f"arn:{partition}:iam::123456789012:role/control-plane-audit")
+ monkeypatch.setenv("AWS_SESSION_NAME", f"audit-{lock_mode}")
+ monkeypatch.setenv("AWS_REGION", resolved_region)
+ sts: Final = MagicMock()
+ sts.assume_role_with_web_identity.return_value = {
+ "Credentials": {
+ "AccessKeyId": "audit-access-key",
+ "SecretAccessKey": "audit-secret-key",
+ "SessionToken": "audit-session-token",
+ },
+ "PackedPolicySize": 1,
+ }
+ settings: Final = ControlPlaneSettings.model_validate(
+ {
+ "LITELLM_MASTER_KEY": "01234567890123456789012345678901",
+ "audit_sinks": ("s3",),
+ "audit_s3_bucket": bucket,
+ "audit_s3_prefix": prefix,
+ "audit_s3_region": region,
+ "audit_s3_object_lock_mode": lock_mode,
+ }
+ )
+ pipeline: Final = build_audit_pipeline(settings)
+ sink: Final = pipeline.workers[0].sink
+ assert isinstance(sink, S3ObjectLockSink)
+ with patch("boto3.client", return_value=sts) as client:
+ signed: Final = sink.sign(f"https://{bucket}.s3.{resolved_region}.{dns_suffix}/event", b"{}", {})
+ assert "audit-access-key/" in signed["Authorization"]
+ assert f"/{resolved_region}/s3/aws4_request" in signed["Authorization"]
+ assert signed["X-Amz-Security-Token"] == "audit-session-token"
+ assert client.call_args.args == ("sts",)
+ assert client.call_args.kwargs["region_name"] == resolved_region
+ request: Final = sts.assume_role_with_web_identity.call_args.kwargs
+ assert request["WebIdentityToken"] == "audit-test-token"
+ assert request["RoleArn"] == os.environ["AWS_ROLE_NAME"]
+ assert request["RoleSessionName"] == os.environ["AWS_SESSION_NAME"]
+ policy: Final = TypeAdapter(dict[str, object]).validate_json(request["Policy"])
+ statements: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(policy["Statement"])
+ actions: Final = tuple(TypeAdapter(tuple[str, ...]).validate_python(item["Action"]) for item in statements)
+ assert frozenset().union(*actions) == {
+ "s3:PutObject",
+ "s3:PutObjectRetention",
+ "kms:GenerateDataKey",
+ "kms:Decrypt",
+ }
+ bucket_arn: Final = f"arn:{partition}:s3:::{bucket}"
+ objects: Final = f"{bucket_arn}/{scoped_prefix}*"
+ assert tuple(item["Resource"] for item in statements) == (
+ objects,
+ f"arn:{partition}:kms:{resolved_region}:*:key/*",
+ )
+ # AWS KMS ViaService uses amazonaws.com in every partition, verified 2026-11-15.
+ # https://docs.aws.amazon.com/kms/latest/developerguide/conditions-kms.html#conditions-kms-via-service
+ assert tuple(item["Condition"] for item in statements) == (
+ {"Bool": {"aws:SecureTransport": "true"}},
+ {
+ "Bool": {"aws:SecureTransport": "true"},
+ "StringEquals": {"kms:ViaService": f"s3.{resolved_region}.amazonaws.com"},
+ "StringLike": {"kms:EncryptionContext:aws:s3:arn": [objects, bucket_arn]},
+ },
+ )
+ await pipeline.close()
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("strict", [False, True])
@pytest.mark.parametrize(
@@ -104,3 +265,14 @@ async def test_transport_never_follows_a_dataplane_redirect_even_on_a_following_
async with transport.stream("GET", "https://dp.test/v1/mcp/server/oauth/x/authorize", timeout=1.0) as response:
status: Final = response.status_code
assert status == 302 and handler.hops == ["https://dp.test/v1/mcp/server/oauth/x/authorize"]
+
+
+def test_the_s3_region_follows_the_aws_sdk_order_aws_region_then_default_region(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv("AWS_REGION", "us-east-1")
+ monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-west-1")
+ monkeypatch.delenv("AWS_REGION_NAME", raising=False)
+ assert _environment_region(BaseAWSLLM()) == "us-east-1"
+ monkeypatch.delenv("AWS_REGION")
+ assert _environment_region(BaseAWSLLM()) == "eu-west-1"
diff --git a/tests/unit/proxy/control_plane/test_settings.py b/tests/unit/proxy/control_plane/test_settings.py
index e4318dce632..3304b66d118 100644
--- a/tests/unit/proxy/control_plane/test_settings.py
+++ b/tests/unit/proxy/control_plane/test_settings.py
@@ -4,7 +4,7 @@ import pytest
from pydantic import ValidationError
from litellm.proxy.control_plane.audit.event import CONTROL_PLANE_ID
-from litellm.proxy.control_plane.settings import ControlPlaneSettings, load_settings
+from litellm.proxy.control_plane.settings import ControlPlaneSettings, S3SinkSettings, load_settings
from litellm.types.proxy.control_plane.auth import PASSWORD_MAX_LENGTH, USERNAME_MAX_LENGTH
MASTER_KEY: Final = "01234567890123456789012345678901"
@@ -43,6 +43,45 @@ def test_password_login_without_default_roles_requires_a_custom_admin_role() ->
assert not settings.include_default_roles and "admin" in settings.roles
+def test_the_s3_sink_refuses_to_start_without_a_bucket_and_reads_its_block_with_one() -> None:
+ with pytest.raises(ValidationError, match="CONTROL_PLANE_AUDIT_S3_BUCKET"):
+ ControlPlaneSettings.model_validate({"LITELLM_MASTER_KEY": MASTER_KEY, "audit_sinks": ["log", "s3"]})
+ settings: Final = ControlPlaneSettings.model_validate(
+ {
+ "LITELLM_MASTER_KEY": MASTER_KEY,
+ "audit_sinks": ["s3"],
+ "audit_s3_bucket": "audit",
+ "audit_s3_prefix": "/cp/",
+ "audit_s3_object_lock_mode": "COMPLIANCE",
+ "audit_s3_retention_days": 7,
+ }
+ )
+ assert settings.audit_s3 == S3SinkSettings("audit", "cp", None, None, "COMPLIANCE", 7, 30.0)
+ assert ControlPlaneSettings.model_validate({"LITELLM_MASTER_KEY": MASTER_KEY}).audit_s3 is None
+
+
+@pytest.mark.parametrize("bucket", ["", " \t "])
+def test_the_s3_sink_rejects_a_blank_bucket(bucket: str) -> None:
+ with pytest.raises(ValidationError, match="must not be empty or whitespace"):
+ ControlPlaneSettings.model_validate(
+ {"LITELLM_MASTER_KEY": MASTER_KEY, "audit_sinks": ["s3"], "audit_s3_bucket": bucket}
+ )
+
+
+def test_a_plain_http_s3_endpoint_is_refused_unless_explicitly_allowed() -> None:
+ base: Final = {
+ "LITELLM_MASTER_KEY": MASTER_KEY,
+ "audit_s3_bucket": "audit",
+ "audit_s3_endpoint_url": "http://minio:9000",
+ }
+ with pytest.raises(ValidationError, match="not https"):
+ ControlPlaneSettings.model_validate(base)
+ with pytest.raises(ValidationError, match="not https"):
+ ControlPlaneSettings.model_validate({**base, "audit_s3_endpoint_url": "HTTP://minio:9000"})
+ allowed: Final = ControlPlaneSettings.model_validate({**base, "audit_s3_allow_insecure_http": True}).audit_s3
+ assert allowed is not None and allowed.endpoint_url == "http://minio:9000"
+
+
def test_credentials_at_the_bound_are_accepted() -> None:
settings: Final = ControlPlaneSettings.model_validate(
{