fix(s3): avoid cryptographic long-name suffixes

This commit is contained in:
Raashish Aggarwal 2026-03-27 23:56:19 +05:30
parent 12bbd109ce
commit c9ecb07547
2 changed files with 31 additions and 4 deletions

View file

@ -1,7 +1,7 @@
#### What this does ####
# On success + failure, log events to Supabase
import hashlib
import zlib
from datetime import datetime
from typing import Optional, cast
@ -12,6 +12,15 @@ from litellm.types.utils import StandardLoggingPayload
MAX_S3_FILENAME_STEM_LENGTH = 250
def _long_filename_suffix(s3_file_name: str) -> str:
encoded_file_name = s3_file_name.encode("utf-8")
# Keep the suffix deterministic for collision avoidance without using a cryptographic hash.
return (
f"{zlib.crc32(encoded_file_name) & 0xFFFFFFFF:08x}"
f"{zlib.adler32(encoded_file_name) & 0xFFFFFFFF:08x}"
)
class S3Logger:
# Class variables or attributes
def __init__(
@ -189,9 +198,9 @@ def get_s3_object_key(
s3_file_name: str,
) -> str:
if len(s3_file_name) > MAX_S3_FILENAME_STEM_LENGTH:
digest = hashlib.sha256(s3_file_name.encode("utf-8")).hexdigest()[:16]
prefix_length = MAX_S3_FILENAME_STEM_LENGTH - len(digest) - 1
s3_file_name = f"{s3_file_name[:prefix_length]}_{digest}"
suffix = _long_filename_suffix(s3_file_name)
prefix_length = MAX_S3_FILENAME_STEM_LENGTH - len(suffix) - 1
s3_file_name = f"{s3_file_name[:prefix_length]}_{suffix}"
s3_object_key = (
(s3_path.rstrip("/") + "/" if s3_path else "")

View file

@ -904,6 +904,24 @@ def test_get_s3_object_key_truncates_long_filename_components():
assert filename.startswith(f"time-{start_time.strftime('%H-%M-%S-%f')}_resp_")
def test_get_s3_object_key_uses_deterministic_suffixes_for_long_filename_components():
start_time = datetime(2026, 3, 26, 12, 51, 27, 995047)
shared_prefix = "resp_" + ("a" * 400)
first_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{shared_prefix}first"
second_file_name = (
f"time-{start_time.strftime('%H-%M-%S-%f')}_{shared_prefix}second"
)
first_key = get_s3_object_key("", "", start_time, first_file_name)
first_key_again = get_s3_object_key("", "", start_time, first_file_name)
second_key = get_s3_object_key("", "", start_time, second_file_name)
assert first_key == first_key_again
assert first_key != second_key
assert len(first_key.split("/")[-1]) <= 255
assert len(second_key.split("/")[-1]) <= 255
def test_create_s3_batch_logging_element_limits_long_ids():
start_time = datetime(2026, 3, 26, 12, 51, 27, 995047)
long_response_id = "resp_" + ("a" * 400)