mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(s3_v2): keep tenant prefixes, stable retries and cold storage safety in batch file mode
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
b16e2f42f8
commit
5272cde7d6
4 changed files with 227 additions and 37 deletions
|
|
@ -40,12 +40,7 @@ def resolve_s3_max_concurrent_uploads(configured: object, fallback: int) -> int:
|
|||
if configured is None or configured == "":
|
||||
return fallback
|
||||
try:
|
||||
bound: Final = cast( # cast-ok: TypeAdapter.validate_python is untyped in the pydantic stubs
|
||||
int,
|
||||
_UPLOAD_BOUND.validate_python( # pyright: ignore[reportUnknownMemberType] # TypeAdapter members are untyped in pydantic stubs
|
||||
configured.strip() if isinstance(configured, str) else configured
|
||||
),
|
||||
)
|
||||
bound: Final = _UPLOAD_BOUND.validate_python(configured.strip() if isinstance(configured, str) else configured)
|
||||
except ValidationError:
|
||||
verbose_logger.warning(
|
||||
"s3 logging: s3_max_concurrent_uploads=%r is not an integer, using %s", configured, fallback
|
||||
|
|
@ -63,13 +58,7 @@ def resolve_s3_batch_file_upload(configured: object) -> bool:
|
|||
if configured is None or configured == "":
|
||||
return False
|
||||
try:
|
||||
enabled: Final = cast( # cast-ok: TypeAdapter.validate_python is untyped in the pydantic stubs
|
||||
bool,
|
||||
_S3_BOOL.validate_python( # pyright: ignore[reportUnknownMemberType] # TypeAdapter members are untyped in pydantic stubs
|
||||
configured.strip() if isinstance(configured, str) else configured
|
||||
),
|
||||
)
|
||||
return enabled
|
||||
return _S3_BOOL.validate_python(configured.strip() if isinstance(configured, str) else configured)
|
||||
except ValidationError:
|
||||
verbose_logger.warning(
|
||||
"s3 logging: s3_batch_file_upload=%r is not a boolean, keeping per-request objects", configured
|
||||
|
|
@ -318,26 +307,25 @@ def get_s3_object_key(
|
|||
prefix: str,
|
||||
start_time: datetime,
|
||||
s3_file_name: str,
|
||||
extension: str = ".json",
|
||||
) -> str:
|
||||
sanitized_s3_file_name: Final = s3_file_name.replace("/", "_").replace(":", "_")
|
||||
configured_prefix: Final = (s3_path.rstrip("/") + "/" if s3_path else "") + prefix
|
||||
date_segment: Final = start_time.strftime("%Y-%m-%d") + "/"
|
||||
# we need the s3 key to include the time, so we log cache hits too
|
||||
s3_object_key: Final = configured_prefix + date_segment + sanitized_s3_file_name + extension
|
||||
s3_object_key: Final = configured_prefix + date_segment + sanitized_s3_file_name + ".json"
|
||||
if len(s3_object_key.encode("utf-8")) <= MAX_S3_OBJECT_KEY_BYTES:
|
||||
return s3_object_key
|
||||
|
||||
# shorten the response id first and only trim the configured prefix if that is what does not
|
||||
# fit, so prefix scoped IAM policies and lifecycle rules keep matching
|
||||
budget: Final = MAX_S3_OBJECT_KEY_BYTES - len(date_segment.encode("utf-8")) - len(extension.encode("utf-8"))
|
||||
budget: Final = MAX_S3_OBJECT_KEY_BYTES - len(date_segment.encode("utf-8")) - len(b".json")
|
||||
prefix_bytes: Final = len(configured_prefix.encode("utf-8"))
|
||||
if prefix_bytes + S3_MIN_BOUNDED_FILE_NAME_BYTES <= budget:
|
||||
bounded_file_name: Final = _bounded_s3_file_name(s3_file_name, sanitized_s3_file_name, budget - prefix_bytes)
|
||||
return configured_prefix + date_segment + bounded_file_name + extension
|
||||
return configured_prefix + date_segment + bounded_file_name + ".json"
|
||||
|
||||
shortest_file_name: Final = _bounded_s3_file_name(
|
||||
s3_file_name, sanitized_s3_file_name, S3_MIN_BOUNDED_FILE_NAME_BYTES
|
||||
)
|
||||
bounded_prefix: Final = _bounded_s3_prefix(configured_prefix, budget - len(shortest_file_name.encode("utf-8")))
|
||||
return bounded_prefix + date_segment + shortest_file_name + extension
|
||||
return bounded_prefix + date_segment + shortest_file_name + ".json"
|
||||
|
|
|
|||
|
|
@ -50,6 +50,10 @@ if TYPE_CHECKING:
|
|||
from botocore.credentials import Credentials
|
||||
|
||||
|
||||
def _s3_key_parent(s3_object_key: str) -> str:
|
||||
return s3_object_key.rsplit("/", 1)[0] if "/" in s3_object_key else ""
|
||||
|
||||
|
||||
class S3BatchUploadError(Exception):
|
||||
def __init__(self, failed: int, total: int) -> None:
|
||||
self.failed = failed
|
||||
|
|
@ -478,36 +482,50 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
# the log queue can be bounded by DEFAULT_S3_BATCH_SIZE
|
||||
# see custom_batch_logger.py which triggers the flush
|
||||
#########################################################
|
||||
if self.s3_batch_file_upload:
|
||||
if not await self.async_upload_data_to_s3(self._build_batch_file_element(batch)):
|
||||
raise S3BatchUploadError(failed=len(batch), total=len(batch))
|
||||
return
|
||||
results: Final = await asyncio.gather(*(self._upload_bounded(element) for element in batch))
|
||||
failed: Final = tuple(element for element, ok in zip(batch, results, strict=True) if not ok)
|
||||
uploads: Final = self._batch_file_elements(batch) if self._batch_file_mode_active() else batch
|
||||
results: Final = await asyncio.gather(*(self._upload_bounded(element) for element in uploads))
|
||||
failed: Final = tuple(element for element, ok in zip(uploads, results, strict=True) if not ok)
|
||||
if not failed:
|
||||
return
|
||||
self.log_queue[:] = [*failed, *self.log_queue[len(batch) :]]
|
||||
raise S3BatchUploadError(failed=len(failed), total=len(batch))
|
||||
raise S3BatchUploadError(failed=len(failed), total=len(uploads))
|
||||
|
||||
def _batch_file_mode_active(self) -> bool:
|
||||
if not self.s3_batch_file_upload:
|
||||
return False
|
||||
if litellm.cold_storage_custom_logger == "s3_v2":
|
||||
verbose_logger.warning(
|
||||
"s3 logging: s3_batch_file_upload is ignored because s3_v2 is the cold storage logger; "
|
||||
"per-request objects are required for spend log lookups"
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _upload_bounded(self, element: s3BatchLoggingElement) -> bool:
|
||||
async with self._upload_semaphore:
|
||||
return await self.async_upload_data_to_s3(element)
|
||||
|
||||
def _build_batch_file_element(self, batch: tuple[s3BatchLoggingElement, ...]) -> s3BatchLoggingElement:
|
||||
def _batch_file_elements(self, batch: tuple[s3BatchLoggingElement, ...]) -> tuple[s3BatchLoggingElement, ...]:
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
groups: Final = {
|
||||
parent: tuple(
|
||||
element for element in batch if element.body is None and _s3_key_parent(element.s3_object_key) == parent
|
||||
)
|
||||
for parent in sorted({_s3_key_parent(element.s3_object_key) for element in batch if element.body is None})
|
||||
}
|
||||
return tuple(element for element in batch if element.body is not None) + tuple(
|
||||
self._build_batch_file_element(elements, parent, now) for parent, elements in groups.items()
|
||||
)
|
||||
|
||||
def _build_batch_file_element(
|
||||
self, elements: tuple[s3BatchLoggingElement, ...], parent: str, now: datetime
|
||||
) -> s3BatchLoggingElement:
|
||||
batch_name: Final = f"batch_{now.strftime('%H-%M-%S')}_{uuid4().hex}"
|
||||
return s3BatchLoggingElement(
|
||||
payload={},
|
||||
body="\n".join(safe_dumps(element.payload) for element in batch),
|
||||
body="\n".join(safe_dumps(element.payload) for element in elements),
|
||||
content_type="application/x-ndjson",
|
||||
s3_object_key=get_s3_object_key(
|
||||
s3_path=cast(str | None, self.s3_path) # cast-ok: self.s3_path comes from untyped s3_callback_params
|
||||
or "",
|
||||
prefix="",
|
||||
start_time=now,
|
||||
s3_file_name=batch_name,
|
||||
extension=".jsonl",
|
||||
),
|
||||
s3_object_key=f"{parent}/{batch_name}.jsonl" if parent else f"{batch_name}.jsonl",
|
||||
s3_object_download_filename=f"{batch_name}.jsonl",
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -185,3 +185,30 @@ def test_s3_v2_batch_file_upload_writes_one_jsonl_object_per_flush(gateway: Gate
|
|||
lines: Final = tuple(line for put in puts for line in put.body.decode().splitlines())
|
||||
assert frozenset(json.loads(line)["id"] for line in lines) == ids
|
||||
assert len(lines) == REQUESTS
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.s3_v2.batch_file_upload_keeps_team_prefix_in_object_key")
|
||||
def test_s3_v2_batch_file_upload_keeps_team_alias_prefix(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "s3team" + uuid.uuid4().hex[:8]
|
||||
team_alias: Final = f"alpha-{uuid.uuid4().hex[:8]}"
|
||||
team_batch_key: Final = re.compile(
|
||||
rf"^/{BUCKET}/{PREFIX}/{team_alias}/\d{{4}}-\d{{2}}-\d{{2}}/batch_\d{{2}}-\d{{2}}-\d{{2}}_[0-9a-f]{{32}}\.jsonl$"
|
||||
)
|
||||
sink: Final = S3Sink()
|
||||
with wire_server(_chat_reply) as provider, wire_server(sink.respond) as bucket:
|
||||
config: Final = _s3_config(tmp_path, bucket.url, {"s3_batch_file_upload": True, "s3_use_team_prefix": True})
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {"DEFAULT_S3_FLUSH_INTERVAL_SECONDS": "3"}, config=config) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key")
|
||||
team: Final = scenario.team(team_alias=team_alias, models=[model])
|
||||
key: Final = scenario.key(team_id=team, models=[model])
|
||||
ids: Final = _burst(candidate, model, key, marker)
|
||||
puts: Final = _collect(bucket, count_lines=True, expected=REQUESTS)
|
||||
assert len(provider.drain()) == REQUESTS
|
||||
assert len(puts) >= 1
|
||||
assert all(team_batch_key.match(put.target) for put in puts), [put.target for put in puts]
|
||||
lines: Final = tuple(line for put in puts for line in put.body.decode().splitlines())
|
||||
assert frozenset(json.loads(line)["id"] for line in lines) == ids
|
||||
assert len(lines) == REQUESTS
|
||||
|
|
|
|||
|
|
@ -2702,7 +2702,8 @@ async def test_batch_file_upload_failure_keeps_whole_batch() -> None:
|
|||
await logger.flush_queue()
|
||||
|
||||
assert put_calls == 1
|
||||
assert logger.log_queue == elements
|
||||
assert len(logger.log_queue) == 1
|
||||
assert logger.log_queue[0].body == "\n".join(json.dumps(element.payload) for element in elements)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -2760,5 +2761,161 @@ async def test_batch_file_key_shape() -> None:
|
|||
await logger.async_send_batch()
|
||||
|
||||
((url, headers),) = calls
|
||||
assert re.search(r".*/logs/\d{4}-\d{2}-\d{2}/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", url)
|
||||
assert re.search(r".*/2025-09-14/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", url)
|
||||
assert headers["Content-Disposition"].endswith('.jsonl"')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_file_groups_raw_elements_by_key_parent() -> None:
|
||||
logger = S3Logger(
|
||||
s3_bucket_name="test-bucket",
|
||||
s3_aws_access_key_id="test-key",
|
||||
s3_aws_secret_access_key="test-secret",
|
||||
s3_region_name="us-east-1",
|
||||
s3_batch_file_upload=True,
|
||||
)
|
||||
|
||||
calls = []
|
||||
|
||||
async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock:
|
||||
calls.append((url, data))
|
||||
return _ok_response()
|
||||
|
||||
logger.async_httpx_client = AsyncMock()
|
||||
logger.async_httpx_client.put = fake_put
|
||||
|
||||
alpha = s3BatchLoggingElement(
|
||||
s3_object_key="logs/alpha/2026-01-01/a.json", payload={"id": "a"}, s3_object_download_filename="a.json"
|
||||
)
|
||||
beta = s3BatchLoggingElement(
|
||||
s3_object_key="logs/beta/2026-01-01/b.json", payload={"id": "b"}, s3_object_download_filename="b.json"
|
||||
)
|
||||
plain = s3BatchLoggingElement(
|
||||
s3_object_key="logs/2026-01-01/c.json", payload={"id": "c"}, s3_object_download_filename="c.json"
|
||||
)
|
||||
logger.log_queue = [alpha, beta, plain]
|
||||
|
||||
await logger.async_send_batch()
|
||||
|
||||
assert len(calls) == 3
|
||||
by_parent = {
|
||||
re.sub(r"/batch_\d{2}-\d{2}-\d{2}_[0-9a-f]{32}\.jsonl$", "", url.split(".com/", 1)[-1]): (url, data)
|
||||
for url, data in calls
|
||||
}
|
||||
assert sorted(by_parent) == ["logs/2026-01-01", "logs/alpha/2026-01-01", "logs/beta/2026-01-01"]
|
||||
assert [line for line in by_parent["logs/alpha/2026-01-01"][1].splitlines()] == [json.dumps({"id": "a"})]
|
||||
assert [line for line in by_parent["logs/beta/2026-01-01"][1].splitlines()] == [json.dumps({"id": "b"})]
|
||||
assert [line for line in by_parent["logs/2026-01-01"][1].splitlines()] == [json.dumps({"id": "c"})]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_batch_file_is_requeued_and_resent_unchanged() -> None:
|
||||
logger = S3Logger(
|
||||
s3_bucket_name="test-bucket",
|
||||
s3_aws_access_key_id="test-key",
|
||||
s3_aws_secret_access_key="test-secret",
|
||||
s3_region_name="us-east-1",
|
||||
s3_batch_file_upload=True,
|
||||
)
|
||||
|
||||
calls = []
|
||||
fail_next = True
|
||||
|
||||
async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock:
|
||||
calls.append((url, data))
|
||||
if fail_next:
|
||||
return _failure_response()
|
||||
return _ok_response()
|
||||
|
||||
logger.async_httpx_client = AsyncMock()
|
||||
logger.async_httpx_client.put = fake_put
|
||||
logger.log_queue = [_element({"i": i}, f"{i}") for i in range(3)]
|
||||
|
||||
await logger.flush_queue()
|
||||
|
||||
assert len(logger.log_queue) == 1
|
||||
assert logger.log_queue[0].body is not None
|
||||
assert logger.log_queue[0].s3_object_key.endswith(".jsonl")
|
||||
|
||||
fail_next = False
|
||||
await logger.flush_queue()
|
||||
|
||||
assert logger.log_queue == []
|
||||
assert len(calls) == 2
|
||||
assert calls[0] == calls[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_elements_appended_after_failed_batch_file_get_their_own_file() -> None:
|
||||
logger = S3Logger(
|
||||
s3_bucket_name="test-bucket",
|
||||
s3_aws_access_key_id="test-key",
|
||||
s3_aws_secret_access_key="test-secret",
|
||||
s3_region_name="us-east-1",
|
||||
s3_batch_file_upload=True,
|
||||
)
|
||||
|
||||
calls = []
|
||||
fail_next = True
|
||||
|
||||
async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock:
|
||||
calls.append((url, data))
|
||||
if fail_next:
|
||||
return _failure_response()
|
||||
return _ok_response()
|
||||
|
||||
logger.async_httpx_client = AsyncMock()
|
||||
logger.async_httpx_client.put = fake_put
|
||||
logger.log_queue = [_element({"id": "first"}, "first")]
|
||||
|
||||
await logger.flush_queue()
|
||||
|
||||
late = _element({"id": "late"}, "late")
|
||||
logger.log_queue.append(late)
|
||||
|
||||
fail_next = False
|
||||
await logger.flush_queue()
|
||||
|
||||
assert logger.log_queue == []
|
||||
assert len(calls) == 3
|
||||
assert calls[0] == calls[1]
|
||||
assert calls[2][0] != calls[0][0]
|
||||
assert calls[2][1] == json.dumps({"id": "late"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_file_mode_disabled_when_s3_v2_is_cold_storage_logger(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
logger = S3Logger(
|
||||
s3_bucket_name="test-bucket",
|
||||
s3_aws_access_key_id="test-key",
|
||||
s3_aws_secret_access_key="test-secret",
|
||||
s3_region_name="us-east-1",
|
||||
s3_batch_file_upload=True,
|
||||
)
|
||||
|
||||
calls = []
|
||||
|
||||
async def fake_put(url: str, data: str | None = None, headers: dict[str, str] | None = None) -> MagicMock:
|
||||
calls.append((url, headers))
|
||||
return _ok_response()
|
||||
|
||||
logger.async_httpx_client = AsyncMock()
|
||||
logger.async_httpx_client.put = fake_put
|
||||
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2")
|
||||
logger.log_queue = [_element({"id": "req-1"}, "0")]
|
||||
|
||||
await logger.async_send_batch()
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0].endswith("test-0.json")
|
||||
|
||||
monkeypatch.setattr(litellm, "cold_storage_custom_logger", None)
|
||||
logger.log_queue = [_element({"id": "req-2"}, "1")]
|
||||
|
||||
await logger.async_send_batch()
|
||||
|
||||
assert len(calls) == 2
|
||||
assert calls[1][0].endswith(".jsonl")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue