mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-17 23:51:30 +00:00
fix(s3): avoid cryptographic long-name suffixes
This commit is contained in:
parent
12bbd109ce
commit
c9ecb07547
2 changed files with 31 additions and 4 deletions
|
|
@ -1,7 +1,7 @@
|
|||
#### What this does ####
|
||||
# On success + failure, log events to Supabase
|
||||
|
||||
import hashlib
|
||||
import zlib
|
||||
from datetime import datetime
|
||||
from typing import Optional, cast
|
||||
|
||||
|
|
@ -12,6 +12,15 @@ from litellm.types.utils import StandardLoggingPayload
|
|||
MAX_S3_FILENAME_STEM_LENGTH = 250
|
||||
|
||||
|
||||
def _long_filename_suffix(s3_file_name: str) -> str:
|
||||
encoded_file_name = s3_file_name.encode("utf-8")
|
||||
# Keep the suffix deterministic for collision avoidance without using a cryptographic hash.
|
||||
return (
|
||||
f"{zlib.crc32(encoded_file_name) & 0xFFFFFFFF:08x}"
|
||||
f"{zlib.adler32(encoded_file_name) & 0xFFFFFFFF:08x}"
|
||||
)
|
||||
|
||||
|
||||
class S3Logger:
|
||||
# Class variables or attributes
|
||||
def __init__(
|
||||
|
|
@ -189,9 +198,9 @@ def get_s3_object_key(
|
|||
s3_file_name: str,
|
||||
) -> str:
|
||||
if len(s3_file_name) > MAX_S3_FILENAME_STEM_LENGTH:
|
||||
digest = hashlib.sha256(s3_file_name.encode("utf-8")).hexdigest()[:16]
|
||||
prefix_length = MAX_S3_FILENAME_STEM_LENGTH - len(digest) - 1
|
||||
s3_file_name = f"{s3_file_name[:prefix_length]}_{digest}"
|
||||
suffix = _long_filename_suffix(s3_file_name)
|
||||
prefix_length = MAX_S3_FILENAME_STEM_LENGTH - len(suffix) - 1
|
||||
s3_file_name = f"{s3_file_name[:prefix_length]}_{suffix}"
|
||||
|
||||
s3_object_key = (
|
||||
(s3_path.rstrip("/") + "/" if s3_path else "")
|
||||
|
|
|
|||
|
|
@ -904,6 +904,24 @@ def test_get_s3_object_key_truncates_long_filename_components():
|
|||
assert filename.startswith(f"time-{start_time.strftime('%H-%M-%S-%f')}_resp_")
|
||||
|
||||
|
||||
def test_get_s3_object_key_uses_deterministic_suffixes_for_long_filename_components():
|
||||
start_time = datetime(2026, 3, 26, 12, 51, 27, 995047)
|
||||
shared_prefix = "resp_" + ("a" * 400)
|
||||
first_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{shared_prefix}first"
|
||||
second_file_name = (
|
||||
f"time-{start_time.strftime('%H-%M-%S-%f')}_{shared_prefix}second"
|
||||
)
|
||||
|
||||
first_key = get_s3_object_key("", "", start_time, first_file_name)
|
||||
first_key_again = get_s3_object_key("", "", start_time, first_file_name)
|
||||
second_key = get_s3_object_key("", "", start_time, second_file_name)
|
||||
|
||||
assert first_key == first_key_again
|
||||
assert first_key != second_key
|
||||
assert len(first_key.split("/")[-1]) <= 255
|
||||
assert len(second_key.split("/")[-1]) <= 255
|
||||
|
||||
|
||||
def test_create_s3_batch_logging_element_limits_long_ids():
|
||||
start_time = datetime(2026, 3, 26, 12, 51, 27, 995047)
|
||||
long_response_id = "resp_" + ("a" * 400)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue