mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix: enforce S3 filename stem byte limit
This commit is contained in:
parent
f911a6d26a
commit
3af8b0845c
2 changed files with 45 additions and 8 deletions
|
|
@ -21,6 +21,24 @@ def _long_filename_suffix(s3_file_name: str) -> str:
|
|||
)
|
||||
|
||||
|
||||
def _truncate_to_utf8_bytes(value: str, max_bytes: int) -> str:
|
||||
if max_bytes <= 0:
|
||||
return ""
|
||||
|
||||
encoded_value = value.encode("utf-8")
|
||||
if len(encoded_value) <= max_bytes:
|
||||
return value
|
||||
|
||||
truncated_value = encoded_value[:max_bytes]
|
||||
while truncated_value:
|
||||
try:
|
||||
return truncated_value.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
truncated_value = truncated_value[:-1]
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
class S3Logger:
|
||||
# Class variables or attributes
|
||||
def __init__(
|
||||
|
|
@ -197,10 +215,14 @@ def get_s3_object_key(
|
|||
start_time: datetime,
|
||||
s3_file_name: str,
|
||||
) -> str:
|
||||
if len(s3_file_name) > MAX_S3_FILENAME_STEM_LENGTH:
|
||||
if len(s3_file_name.encode("utf-8")) > MAX_S3_FILENAME_STEM_LENGTH:
|
||||
suffix = _long_filename_suffix(s3_file_name)
|
||||
prefix_length = MAX_S3_FILENAME_STEM_LENGTH - len(suffix) - 1
|
||||
s3_file_name = f"{s3_file_name[:prefix_length]}_{suffix}"
|
||||
prefix_byte_length = max(
|
||||
MAX_S3_FILENAME_STEM_LENGTH - len(suffix.encode("utf-8")) - 1,
|
||||
0,
|
||||
)
|
||||
truncated_prefix = _truncate_to_utf8_bytes(s3_file_name, prefix_byte_length)
|
||||
s3_file_name = f"{truncated_prefix}_{suffix}"
|
||||
|
||||
s3_object_key = (
|
||||
(s3_path.rstrip("/") + "/" if s3_path else "")
|
||||
|
|
|
|||
|
|
@ -898,8 +898,8 @@ def test_get_s3_object_key_truncates_long_filename_components():
|
|||
)
|
||||
filename = key.split("/")[-1]
|
||||
|
||||
assert len(f"{original_file_name}.json") > 255
|
||||
assert len(filename) <= 255
|
||||
assert len(f"{original_file_name}.json".encode("utf-8")) > 255
|
||||
assert len(filename.encode("utf-8")) <= 255
|
||||
assert filename.endswith(".json")
|
||||
assert filename.startswith(f"time-{start_time.strftime('%H-%M-%S-%f')}_resp_")
|
||||
|
||||
|
|
@ -918,8 +918,23 @@ def test_get_s3_object_key_uses_deterministic_suffixes_for_long_filename_compone
|
|||
|
||||
assert first_key == first_key_again
|
||||
assert first_key != second_key
|
||||
assert len(first_key.split("/")[-1]) <= 255
|
||||
assert len(second_key.split("/")[-1]) <= 255
|
||||
assert len(first_key.split("/")[-1].encode("utf-8")) <= 255
|
||||
assert len(second_key.split("/")[-1].encode("utf-8")) <= 255
|
||||
|
||||
|
||||
def test_get_s3_object_key_truncates_non_ascii_filename_components_by_utf8_bytes():
|
||||
start_time = datetime(2026, 3, 26, 12, 51, 27, 995047)
|
||||
long_response_id = "resp_" + ("漢" * 120)
|
||||
original_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{long_response_id}"
|
||||
|
||||
key = get_s3_object_key("", "", start_time, original_file_name)
|
||||
filename = key.split("/")[-1]
|
||||
|
||||
assert len(original_file_name) < 250
|
||||
assert len(f"{original_file_name}.json".encode("utf-8")) > 255
|
||||
assert len(filename.encode("utf-8")) <= 255
|
||||
assert filename.endswith(".json")
|
||||
assert filename.startswith(f"time-{start_time.strftime('%H-%M-%S-%f')}_resp_")
|
||||
|
||||
|
||||
def test_create_s3_batch_logging_element_limits_long_ids():
|
||||
|
|
@ -939,5 +954,5 @@ def test_create_s3_batch_logging_element_limits_long_ids():
|
|||
result = logger.create_s3_batch_logging_element(start_time, payload)
|
||||
|
||||
assert result is not None
|
||||
assert len(result.s3_object_key.split("/")[-1]) <= 255
|
||||
assert len(result.s3_object_key.split("/")[-1].encode("utf-8")) <= 255
|
||||
assert result.s3_object_key.endswith(".json")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue