fix(s3_v2): keep tenant prefixes, stable retries and cold storage safety in batch file mode

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-23 08:34:11 +00:00
parent b16e2f42f8
commit 5272cde7d6
4 changed files with 227 additions and 37 deletions

View file

@ -40,12 +40,7 @@ def resolve_s3_max_concurrent_uploads(configured: object, fallback: int) -> int:
if configured is None or configured == "":
return fallback
try:
bound: Final = cast( # cast-ok: TypeAdapter.validate_python is untyped in the pydantic stubs
int,
_UPLOAD_BOUND.validate_python( # pyright: ignore[reportUnknownMemberType] # TypeAdapter members are untyped in pydantic stubs
configured.strip() if isinstance(configured, str) else configured
),
)
bound: Final = _UPLOAD_BOUND.validate_python(configured.strip() if isinstance(configured, str) else configured)
except ValidationError:
verbose_logger.warning(
"s3 logging: s3_max_concurrent_uploads=%r is not an integer, using %s", configured, fallback
@ -63,13 +58,7 @@ def resolve_s3_batch_file_upload(configured: object) -> bool:
if configured is None or configured == "":
return False
try:
enabled: Final = cast( # cast-ok: TypeAdapter.validate_python is untyped in the pydantic stubs
bool,
_S3_BOOL.validate_python( # pyright: ignore[reportUnknownMemberType] # TypeAdapter members are untyped in pydantic stubs
configured.strip() if isinstance(configured, str) else configured
),
)
return enabled
return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
except ValidationError:
verbose_logger.warning(
"s3 logging: s3_batch_file_upload=%r is not a boolean, keeping per-request objects", configured
@ -318,26 +307,25 @@ def get_s3_object_key(
prefix: str,
start_time: datetime,
s3_file_name: str,
extension: str = ".json",
) -> str:
sanitized_s3_file_name: Final = s3_file_name.replace("/", "_").replace(":", "_")
configured_prefix: Final = (s3_path.rstrip("/") + "/" if s3_path else "") + prefix
date_segment: Final = start_time.strftime("%Y-%m-%d") + "/"
# we need the s3 key to include the time, so we log cache hits too
s3_object_key: Final = configured_prefix + date_segment + sanitized_s3_file_name + extension
s3_object_key: Final = configured_prefix + date_segment + sanitized_s3_file_name + ".json"
if len(s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES:
return s3_object_key
# shorten the response id first and only trim the configured prefix if that is what does not
# fit, so prefix scoped IAM policies and lifecycle rules keep matching
budget: Final = MAX_S3_OBJECT_KEY_BYTES - len(date_segment.encode("utf-8")) - len(extension.encode("utf-8"))
budget: Final = MAX_S3_OBJECT_KEY_BYTES - len(date_segment.encode("utf-8")) - len(b".json")
prefix_bytes: Final = len(configured_prefix.encode("utf-8"))
if prefix_bytes + S3_MIN_BOUNDED_FILE_NAME_BYTES <= budget:
bounded_file_name: Final = _bounded_s3_file_name(s3_file_name, sanitized_s3_file_name, budget - prefix_bytes)
return configured_prefix + date_segment + bounded_file_name + extension
return configured_prefix + date_segment + bounded_file_name + ".json"
shortest_file_name: Final = _bounded_s3_file_name(
s3_file_name, sanitized_s3_file_name, S3_MIN_BOUNDED_FILE_NAME_BYTES
)
bounded_prefix: Final = _bounded_s3_prefix(configured_prefix, budget - len(shortest_file_name.encode("utf-8")))
return bounded_prefix + date_segment + shortest_file_name + extension
return bounded_prefix + date_segment + shortest_file_name + ".json"

View file

@ -50,6 +50,10 @@ if TYPE_CHECKING:
from botocore.credentials import Credentials
def _s3_key_parent(s3_object_key: str) -> str:
return s3_object_key.rsplit("/", 1)[0] if "/" in s3_object_key else ""
class S3BatchUploadError(Exception):
def __init__(self, failed: int, total: int) -> None:
self.failed = failed
@ -478,36 +482,50 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
# the log queue can be bounded by DEFAULT_S3_BATCH_SIZE
# see custom_batch_logger.py which triggers the flush
#########################################################
if self.s3_batch_file_upload:
if not await self.async_upload_data_to_s3(self._build_batch_file_element(batch)):
raise S3BatchUploadError(failed=len(batch), total=len(batch))
return
results: Final = await asyncio.gather(*(self._upload_bounded(element) for element in batch))
failed: Final = tuple(element for element, ok in zip(batch, results, strict=True) if not ok)
uploads: Final = self._batch_file_elements(batch) if self._batch_file_mode_active() else batch
results: Final = await asyncio.gather(*(self._upload_bounded(element) for element in uploads))
failed: Final = tuple(element for element, ok in zip(uploads, results, strict=True) if not ok)
if not failed:
return
self.log_queue[:] = [*failed, *self.log_queue[len(batch) :]]
raise S3BatchUploadError(failed=len(failed), total=len(batch))
raise S3BatchUploadError(failed=len(failed), total=len(uploads))
def _batch_file_mode_active(self) -> bool:
if not self.s3_batch_file_upload:
return False
if litellm.cold_storage_custom_logger == "s3_v2":
verbose_logger.warning(
"s3 logging: s3_batch_file_upload is ignored because s3_v2 is the cold storage logger; "
"per-request objects are required for spend log lookups"
)
return False
return True
async def _upload_bounded(self, element: s3BatchLoggingElement) -> bool:
async with self._upload_semaphore:
return await self.async_upload_data_to_s3(element)
def _build_batch_file_element(self, batch: tuple[s3BatchLoggingElement, ...]) -> s3BatchLoggingElement:
def _batch_file_elements(self, batch: tuple[s3BatchLoggingElement, ...]) -> tuple[s3BatchLoggingElement, ...]:
now: Final = datetime.now(timezone.utc)
groups: Final = {
parent: tuple(
element for element in batch if element.body is None and _s3_key_parent(element.s3_object_key) == parent
)
for parent in sorted({_s3_key_parent(element.s3_object_key) for element in batch if element.body is None})
}
return tuple(element for element in batch if element.body is not None) + tuple(
self._build_batch_file_element(elements, parent, now) for parent, elements in groups.items()
)
def _build_batch_file_element(
self, elements: tuple[s3BatchLoggingElement, ...], parent: str, now: datetime
) -> s3BatchLoggingElement:
batch_name: Final = f"batch_{now.strftime('%H-%M-%S')}_{uuid4().hex}"
return s3BatchLoggingElement(
payload={},
body="\n".join(safe_dumps(element.payload) for element in batch),
body="\n".join(safe_dumps(element.payload) for element in elements),
content_type="application/x-ndjson",
s3_object_key=get_s3_object_key(
s3_path=cast(str | None, self.s3_path) # cast-ok: self.s3_path comes from untyped s3_callback_params
or "",
prefix="",
start_time=now,
s3_file_name=batch_name,
extension=".jsonl",
),
s3_object_key=f"{parent}/{batch_name}.jsonl" if parent else f"{batch_name}.jsonl",
s3_object_download_filename=f"{batch_name}.jsonl",
)

View file

@ -185,3 +185,30 @@ def test_s3_v2_batch_file_upload_writes_one_jsonl_object_per_flush(gateway: Gate
lines: Final = tuple(line for put in puts for line in put.body.decode().splitlines())
assert frozenset(json.loads(line)["id"] for line in lines) == ids
assert len(lines) == REQUESTS
@pytest.mark.covers("other.observability.s3_v2.batch_file_upload_keeps_team_prefix_in_object_key")
def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = "s3team" + uuid.uuid4().hex[:8]
team_alias: Final = f"alpha-{uuid.uuid4().hex[:8]}"
team_batch_key: Final = re.compile(
rf"^/{BUCKET}/{PREFIX}/{team_alias}/\d{{4}}-\d{{2}}-\d{{2}}/batch_\d{{2}}-\d{{2}}-\d{{2}}_[0-9a-f]{{32}}\.jsonl$"
)
sink: Final = S3Sink()
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
config: Final = _s3_config(tmp_path, bucket.url, {"s3_batch_file_upload": True, "s3_use_team_prefix": True})
with (
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate,
candidate.scenario() as scenario,
):
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
team: Final = scenario.team(team_alias=team_alias, models=[model])
key: Final = scenario.key(team_id=team, models=[model])
ids: Final = _burst(candidate, model, key, marker)
puts: Final = _collect(bucket, count_lines=True, expected=REQUESTS)
assert len(provider.drain()) == REQUESTS
assert len(puts) >= 1
assert all(team_batch_key.match(put.target) for put in puts), [put.target for put in puts]
lines: Final = tuple(line for put in puts for line in put.body.decode().splitlines())
assert frozenset(json.loads(line)["id"] for line in lines) == ids
assert len(lines) == REQUESTS

View file

@ -2702,7 +2702,8 @@ async def test_batch_file_upload_failure_keeps_whole_batch() -> None:
await logger.flush_queue()
assert put_calls == 1
assert logger.log_queue == elements
assert len(logger.log_queue) == 1
assert logger.log_queue[0].body == "\n".join(json.dumps(element.payload) for element in elements)
@pytest.mark.asyncio
@ -2760,5 +2761,161 @@ async def test_batch_file_key_shape() -> None:
await logger.async_send_batch()
((url, headers),) = calls
assert re.search(r".*/logs/\d{4}-\d{2}-\d{2}/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", url)
assert re.search(r".*/2025-09-14/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", url)
assert headers["Content-Disposition"].endswith('.jsonl"')
@pytest.mark.asyncio
async def test_batch_file_groups_raw_elements_by_key_parent() -> None:
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
s3_batch_file_upload=True,
)
calls = []
async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock:
calls.append((url, data))
return _ok_response()
logger.async_httpx_client = AsyncMock()
logger.async_httpx_client.put = fake_put
alpha = s3BatchLoggingElement(
s3_object_key="logs/alpha/2026-01-01/a.json", payload={"id": "a"}, s3_object_download_filename="a.json"
)
beta = s3BatchLoggingElement(
s3_object_key="logs/beta/2026-01-01/b.json", payload={"id": "b"}, s3_object_download_filename="b.json"
)
plain = s3BatchLoggingElement(
s3_object_key="logs/2026-01-01/c.json", payload={"id": "c"}, s3_object_download_filename="c.json"
)
logger.log_queue = [alpha, beta, plain]
await logger.async_send_batch()
assert len(calls) == 3
by_parent = {
re.sub(r"/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", "", url.split(".com/", 1)[-1]): (url, data)
for url, data in calls
}
assert sorted(by_parent) == ["logs/2026-01-01", "logs/alpha/2026-01-01", "logs/beta/2026-01-01"]
assert [line for line in by_parent["logs/alpha/2026-01-01"][1].splitlines()] == [json.dumps({"id": "a"})]
assert [line for line in by_parent["logs/beta/2026-01-01"][1].splitlines()] == [json.dumps({"id": "b"})]
assert [line for line in by_parent["logs/2026-01-01"][1].splitlines()] == [json.dumps({"id": "c"})]
@pytest.mark.asyncio
async def test_failed_batch_file_is_requeued_and_resent_unchanged() -> None:
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
s3_batch_file_upload=True,
)
calls = []
fail_next = True
async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock:
calls.append((url, data))
if fail_next:
return _failure_response()
return _ok_response()
logger.async_httpx_client = AsyncMock()
logger.async_httpx_client.put = fake_put
logger.log_queue = [_element({"i": i}, f"{i}") for i in range(3)]
await logger.flush_queue()
assert len(logger.log_queue) == 1
assert logger.log_queue[0].body is not None
assert logger.log_queue[0].s3_object_key.endswith(".jsonl")
fail_next = False
await logger.flush_queue()
assert logger.log_queue == []
assert len(calls) == 2
assert calls[0] == calls[1]
@pytest.mark.asyncio
async def test_elements_appended_after_failed_batch_file_get_their_own_file() -> None:
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
s3_batch_file_upload=True,
)
calls = []
fail_next = True
async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock:
calls.append((url, data))
if fail_next:
return _failure_response()
return _ok_response()
logger.async_httpx_client = AsyncMock()
logger.async_httpx_client.put = fake_put
logger.log_queue = [_element({"id": "first"}, "first")]
await logger.flush_queue()
late = _element({"id": "late"}, "late")
logger.log_queue.append(late)
fail_next = False
await logger.flush_queue()
assert logger.log_queue == []
assert len(calls) == 3
assert calls[0] == calls[1]
assert calls[2][0] != calls[0][0]
assert calls[2][1] == json.dumps({"id": "late"})
@pytest.mark.asyncio
async def test_batch_file_mode_disabled_when_s3_v2_is_cold_storage_logger(monkeypatch: pytest.MonkeyPatch) -> None:
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
s3_batch_file_upload=True,
)
calls = []
async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock:
calls.append((url, headers))
return _ok_response()
logger.async_httpx_client = AsyncMock()
logger.async_httpx_client.put = fake_put
import litellm
monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2")
logger.log_queue = [_element({"id": "req-1"}, "0")]
await logger.async_send_batch()
assert len(calls) == 1
assert calls[0][0].endswith("test-0.json")
monkeypatch.setattr(litellm, "cold_storage_custom_logger", None)
logger.log_queue = [_element({"id": "req-2"}, "1")]
await logger.async_send_batch()
assert len(calls) == 2
assert calls[1][0].endswith(".jsonl")