diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index 6d451bb59ca..3c8619e82b2 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -40,12 +40,7 @@ def resolve_s3_max_concurrent_uploads(configured: object, fallback: int) -> int: if configured is None or configured == "": return fallback try: - bound: Final = cast( # cast-ok: TypeAdapter.validate_python is untyped in the pydantic stubs - int, - _UPLOAD_BOUND.validate_python( # pyright: ignore[reportUnknownMemberType] # TypeAdapter members are untyped in pydantic stubs - configured.strip() if isinstance(configured, str) else configured - ), - ) + bound: Final = _UPLOAD_BOUND.validate_python(configured.strip() if isinstance(configured, str) else configured) except ValidationError: verbose_logger.warning( "s3 logging: s3_max_concurrent_uploads=%r is not an integer, using %s", configured, fallback @@ -63,13 +58,7 @@ def resolve_s3_batch_file_upload(configured: object) -> bool: if configured is None or configured == "": return False try: - enabled: Final = cast( # cast-ok: TypeAdapter.validate_python is untyped in the pydantic stubs - bool, - _S3_BOOL.validate_python( # pyright: ignore[reportUnknownMemberType] # TypeAdapter members are untyped in pydantic stubs - configured.strip() if isinstance(configured, str) else configured - ), - ) - return enabled + return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured) except ValidationError: verbose_logger.warning( "s3 logging: s3_batch_file_upload=%r is not a boolean, keeping per-request objects", configured @@ -318,26 +307,25 @@ def get_s3_object_key( prefix: str, start_time: datetime, s3_file_name: str, - extension: str = ".json", ) -> str: sanitized_s3_file_name: Final = s3_file_name.replace("/", "_").replace(":", "_") configured_prefix: Final = (s3_path.rstrip("/") + "/" if s3_path else "") + prefix date_segment: Final = start_time.strftime("%Y-%m-%d") + "/" # we need the s3 key to include the time, so we log cache hits too - s3_object_key: Final = configured_prefix + date_segment + sanitized_s3_file_name + extension + s3_object_key: Final = configured_prefix + date_segment + sanitized_s3_file_name + ".json" if len(s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES: return s3_object_key # shorten the response id first and only trim the configured prefix if that is what does not # fit, so prefix scoped IAM policies and lifecycle rules keep matching - budget: Final = MAX_S3_OBJECT_KEY_BYTES - len(date_segment.encode("utf-8")) - len(extension.encode("utf-8")) + budget: Final = MAX_S3_OBJECT_KEY_BYTES - len(date_segment.encode("utf-8")) - len(b".json") prefix_bytes: Final = len(configured_prefix.encode("utf-8")) if prefix_bytes + S3_MIN_BOUNDED_FILE_NAME_BYTES <= budget: bounded_file_name: Final = _bounded_s3_file_name(s3_file_name, sanitized_s3_file_name, budget - prefix_bytes) - return configured_prefix + date_segment + bounded_file_name + extension + return configured_prefix + date_segment + bounded_file_name + ".json" shortest_file_name: Final = _bounded_s3_file_name( s3_file_name, sanitized_s3_file_name, S3_MIN_BOUNDED_FILE_NAME_BYTES ) bounded_prefix: Final = _bounded_s3_prefix(configured_prefix, budget - len(shortest_file_name.encode("utf-8"))) - return bounded_prefix + date_segment + shortest_file_name + extension + return bounded_prefix + date_segment + shortest_file_name + ".json" diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 1653e6c18d0..569f84ab2f2 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -50,6 +50,10 @@ if TYPE_CHECKING: from botocore.credentials import Credentials +def _s3_key_parent(s3_object_key: str) -> str: + return s3_object_key.rsplit("/", 1)[0] if "/" in s3_object_key else "" + + class S3BatchUploadError(Exception): def __init__(self, failed: int, total: int) -> None: self.failed = failed @@ -478,36 +482,50 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): # the log queue can be bounded by DEFAULT_S3_BATCH_SIZE # see custom_batch_logger.py which triggers the flush ######################################################### - if self.s3_batch_file_upload: - if not await self.async_upload_data_to_s3(self._build_batch_file_element(batch)): - raise S3BatchUploadError(failed=len(batch), total=len(batch)) - return - results: Final = await asyncio.gather(*(self._upload_bounded(element) for element in batch)) - failed: Final = tuple(element for element, ok in zip(batch, results, strict=True) if not ok) + uploads: Final = self._batch_file_elements(batch) if self._batch_file_mode_active() else batch + results: Final = await asyncio.gather(*(self._upload_bounded(element) for element in uploads)) + failed: Final = tuple(element for element, ok in zip(uploads, results, strict=True) if not ok) if not failed: return self.log_queue[:] = [*failed, *self.log_queue[len(batch) :]] - raise S3BatchUploadError(failed=len(failed), total=len(batch)) + raise S3BatchUploadError(failed=len(failed), total=len(uploads)) + + def _batch_file_mode_active(self) -> bool: + if not self.s3_batch_file_upload: + return False + if litellm.cold_storage_custom_logger == "s3_v2": + verbose_logger.warning( + "s3 logging: s3_batch_file_upload is ignored because s3_v2 is the cold storage logger; " + "per-request objects are required for spend log lookups" + ) + return False + return True async def _upload_bounded(self, element: s3BatchLoggingElement) -> bool: async with self._upload_semaphore: return await self.async_upload_data_to_s3(element) - def _build_batch_file_element(self, batch: tuple[s3BatchLoggingElement, ...]) -> s3BatchLoggingElement: + def _batch_file_elements(self, batch: tuple[s3BatchLoggingElement, ...]) -> tuple[s3BatchLoggingElement, ...]: now: Final = datetime.now(timezone.utc) + groups: Final = { + parent: tuple( + element for element in batch if element.body is None and _s3_key_parent(element.s3_object_key) == parent + ) + for parent in sorted({_s3_key_parent(element.s3_object_key) for element in batch if element.body is None}) + } + return tuple(element for element in batch if element.body is not None) + tuple( + self._build_batch_file_element(elements, parent, now) for parent, elements in groups.items() + ) + + def _build_batch_file_element( + self, elements: tuple[s3BatchLoggingElement, ...], parent: str, now: datetime + ) -> s3BatchLoggingElement: batch_name: Final = f"batch_{now.strftime('%H-%M-%S')}_{uuid4().hex}" return s3BatchLoggingElement( payload={}, - body="\n".join(safe_dumps(element.payload) for element in batch), + body="\n".join(safe_dumps(element.payload) for element in elements), content_type="application/x-ndjson", - s3_object_key=get_s3_object_key( - s3_path=cast(str | None, self.s3_path) # cast-ok: self.s3_path comes from untyped s3_callback_params - or "", - prefix="", - start_time=now, - s3_file_name=batch_name, - extension=".jsonl", - ), + s3_object_key=f"{parent}/{batch_name}.jsonl" if parent else f"{batch_name}.jsonl", s3_object_download_filename=f"{batch_name}.jsonl", ) diff --git a/tests/integration/observability/test_s3_v2_upload_fanout.py b/tests/integration/observability/test_s3_v2_upload_fanout.py index f0e35faab70..4481ab0bf88 100644 --- a/tests/integration/observability/test_s3_v2_upload_fanout.py +++ b/tests/integration/observability/test_s3_v2_upload_fanout.py @@ -185,3 +185,30 @@ def test_s3_v2_batch_file_upload_writes_one_jsonl_object_per_flush(gateway: Gate lines: Final = tuple(line for put in puts for line in put.body.decode().splitlines()) assert frozenset(json.loads(line)["id"] for line in lines) == ids assert len(lines) == REQUESTS + + +@pytest.mark.covers("other.observability.s3_v2.batch_file_upload_keeps_team_prefix_in_object_key") +def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "s3team" + uuid.uuid4().hex[:8] + team_alias: Final = f"alpha-{uuid.uuid4().hex[:8]}" + team_batch_key: Final = re.compile( + rf"^/{BUCKET}/{PREFIX}/{team_alias}/\d{{4}}-\d{{2}}-\d{{2}}/batch_\d{{2}}-\d{{2}}-\d{{2}}_[0-9a-f]{{32}}\.jsonl$" + ) + sink: Final = S3Sink() + with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket: + config: Final = _s3_config(tmp_path, bucket.url, {"s3_batch_file_upload": True, "s3_use_team_prefix": True}) + with ( + owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key") + team: Final = scenario.team(team_alias=team_alias, models=[model]) + key: Final = scenario.key(team_id=team, models=[model]) + ids: Final = _burst(candidate, model, key, marker) + puts: Final = _collect(bucket, count_lines=True, expected=REQUESTS) + assert len(provider.drain()) == REQUESTS + assert len(puts) >= 1 + assert all(team_batch_key.match(put.target) for put in puts), [put.target for put in puts] + lines: Final = tuple(line for put in puts for line in put.body.decode().splitlines()) + assert frozenset(json.loads(line)["id"] for line in lines) == ids + assert len(lines) == REQUESTS diff --git a/tests/test_litellm/integrations/test_s3_v2.py b/tests/test_litellm/integrations/test_s3_v2.py index 3e82adb21e3..ba977cc8650 100644 --- a/tests/test_litellm/integrations/test_s3_v2.py +++ b/tests/test_litellm/integrations/test_s3_v2.py @@ -2702,7 +2702,8 @@ async def test_batch_file_upload_failure_keeps_whole_batch() -> None: await logger.flush_queue() assert put_calls == 1 - assert logger.log_queue == elements + assert len(logger.log_queue) == 1 + assert logger.log_queue[0].body == "\n".join(json.dumps(element.payload) for element in elements) @pytest.mark.asyncio @@ -2760,5 +2761,161 @@ async def test_batch_file_key_shape() -> None: await logger.async_send_batch() ((url, headers),) = calls - assert re.search(r".*/logs/\d{4}-\d{2}-\d{2}/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", url) + assert re.search(r".*/2025-09-14/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", url) assert headers["Content-Disposition"].endswith('.jsonl"') + + +@pytest.mark.asyncio +async def test_batch_file_groups_raw_elements_by_key_parent() -> None: + logger = S3Logger( + s3_bucket_name="test-bucket", + s3_aws_access_key_id="test-key", + s3_aws_secret_access_key="test-secret", + s3_region_name="us-east-1", + s3_batch_file_upload=True, + ) + + calls = [] + + async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock: + calls.append((url, data)) + return _ok_response() + + logger.async_httpx_client = AsyncMock() + logger.async_httpx_client.put = fake_put + + alpha = s3BatchLoggingElement( + s3_object_key="logs/alpha/2026-01-01/a.json", payload={"id": "a"}, s3_object_download_filename="a.json" + ) + beta = s3BatchLoggingElement( + s3_object_key="logs/beta/2026-01-01/b.json", payload={"id": "b"}, s3_object_download_filename="b.json" + ) + plain = s3BatchLoggingElement( + s3_object_key="logs/2026-01-01/c.json", payload={"id": "c"}, s3_object_download_filename="c.json" + ) + logger.log_queue = [alpha, beta, plain] + + await logger.async_send_batch() + + assert len(calls) == 3 + by_parent = { + re.sub(r"/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", "", url.split(".com/", 1)[-1]): (url, data) + for url, data in calls + } + assert sorted(by_parent) == ["logs/2026-01-01", "logs/alpha/2026-01-01", "logs/beta/2026-01-01"] + assert [line for line in by_parent["logs/alpha/2026-01-01"][1].splitlines()] == [json.dumps({"id": "a"})] + assert [line for line in by_parent["logs/beta/2026-01-01"][1].splitlines()] == [json.dumps({"id": "b"})] + assert [line for line in by_parent["logs/2026-01-01"][1].splitlines()] == [json.dumps({"id": "c"})] + + +@pytest.mark.asyncio +async def test_failed_batch_file_is_requeued_and_resent_unchanged() -> None: + logger = S3Logger( + s3_bucket_name="test-bucket", + s3_aws_access_key_id="test-key", + s3_aws_secret_access_key="test-secret", + s3_region_name="us-east-1", + s3_batch_file_upload=True, + ) + + calls = [] + fail_next = True + + async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock: + calls.append((url, data)) + if fail_next: + return _failure_response() + return _ok_response() + + logger.async_httpx_client = AsyncMock() + logger.async_httpx_client.put = fake_put + logger.log_queue = [_element({"i": i}, f"{i}") for i in range(3)] + + await logger.flush_queue() + + assert len(logger.log_queue) == 1 + assert logger.log_queue[0].body is not None + assert logger.log_queue[0].s3_object_key.endswith(".jsonl") + + fail_next = False + await logger.flush_queue() + + assert logger.log_queue == [] + assert len(calls) == 2 + assert calls[0] == calls[1] + + +@pytest.mark.asyncio +async def test_elements_appended_after_failed_batch_file_get_their_own_file() -> None: + logger = S3Logger( + s3_bucket_name="test-bucket", + s3_aws_access_key_id="test-key", + s3_aws_secret_access_key="test-secret", + s3_region_name="us-east-1", + s3_batch_file_upload=True, + ) + + calls = [] + fail_next = True + + async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock: + calls.append((url, data)) + if fail_next: + return _failure_response() + return _ok_response() + + logger.async_httpx_client = AsyncMock() + logger.async_httpx_client.put = fake_put + logger.log_queue = [_element({"id": "first"}, "first")] + + await logger.flush_queue() + + late = _element({"id": "late"}, "late") + logger.log_queue.append(late) + + fail_next = False + await logger.flush_queue() + + assert logger.log_queue == [] + assert len(calls) == 3 + assert calls[0] == calls[1] + assert calls[2][0] != calls[0][0] + assert calls[2][1] == json.dumps({"id": "late"}) + + +@pytest.mark.asyncio +async def test_batch_file_mode_disabled_when_s3_v2_is_cold_storage_logger(monkeypatch: pytest.MonkeyPatch) -> None: + logger = S3Logger( + s3_bucket_name="test-bucket", + s3_aws_access_key_id="test-key", + s3_aws_secret_access_key="test-secret", + s3_region_name="us-east-1", + s3_batch_file_upload=True, + ) + + calls = [] + + async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock: + calls.append((url, headers)) + return _ok_response() + + logger.async_httpx_client = AsyncMock() + logger.async_httpx_client.put = fake_put + + import litellm + + monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2") + logger.log_queue = [_element({"id": "req-1"}, "0")] + + await logger.async_send_batch() + + assert len(calls) == 1 + assert calls[0][0].endswith("test-0.json") + + monkeypatch.setattr(litellm, "cold_storage_custom_logger", None) + logger.log_queue = [_element({"id": "req-2"}, "1")] + + await logger.async_send_batch() + + assert len(calls) == 2 + assert calls[1][0].endswith(".jsonl")