diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index a608c54e73d..63ac1054058 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -21,6 +21,24 @@ def _long_filename_suffix(s3_file_name: str) -> str: ) +def _truncate_to_utf8_bytes(value: str, max_bytes: int) -> str: + if max_bytes <= 0: + return "" + + encoded_value = value.encode("utf-8") + if len(encoded_value) <= max_bytes: + return value + + truncated_value = encoded_value[:max_bytes] + while truncated_value: + try: + return truncated_value.decode("utf-8") + except UnicodeDecodeError: + truncated_value = truncated_value[:-1] + + return "" + + class S3Logger: # Class variables or attributes def __init__( @@ -197,10 +215,14 @@ def get_s3_object_key( start_time: datetime, s3_file_name: str, ) -> str: - if len(s3_file_name) > MAX_S3_FILENAME_STEM_LENGTH: + if len(s3_file_name.encode("utf-8")) > MAX_S3_FILENAME_STEM_LENGTH: suffix = _long_filename_suffix(s3_file_name) - prefix_length = MAX_S3_FILENAME_STEM_LENGTH - len(suffix) - 1 - s3_file_name = f"{s3_file_name[:prefix_length]}_{suffix}" + prefix_byte_length = max( + MAX_S3_FILENAME_STEM_LENGTH - len(suffix.encode("utf-8")) - 1, + 0, + ) + truncated_prefix = _truncate_to_utf8_bytes(s3_file_name, prefix_byte_length) + s3_file_name = f"{truncated_prefix}_{suffix}" s3_object_key = ( (s3_path.rstrip("/") + "/" if s3_path else "") diff --git a/tests/test_litellm/integrations/test_s3_v2.py b/tests/test_litellm/integrations/test_s3_v2.py index 968023fbc57..7b4398fc934 100644 --- a/tests/test_litellm/integrations/test_s3_v2.py +++ b/tests/test_litellm/integrations/test_s3_v2.py @@ -898,8 +898,8 @@ def test_get_s3_object_key_truncates_long_filename_components(): ) filename = key.split("/")[-1] - assert len(f"{original_file_name}.json") > 255 - assert len(filename) <= 255 + assert len(f"{original_file_name}.json".encode("utf-8")) > 255 + assert len(filename.encode("utf-8")) <= 255 assert filename.endswith(".json") assert filename.startswith(f"time-{start_time.strftime('%H-%M-%S-%f')}_resp_") @@ -918,8 +918,23 @@ def test_get_s3_object_key_uses_deterministic_suffixes_for_long_filename_compone assert first_key == first_key_again assert first_key != second_key - assert len(first_key.split("/")[-1]) <= 255 - assert len(second_key.split("/")[-1]) <= 255 + assert len(first_key.split("/")[-1].encode("utf-8")) <= 255 + assert len(second_key.split("/")[-1].encode("utf-8")) <= 255 + + +def test_get_s3_object_key_truncates_non_ascii_filename_components_by_utf8_bytes(): + start_time = datetime(2026, 3, 26, 12, 51, 27, 995047) + long_response_id = "resp_" + ("漢" * 120) + original_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{long_response_id}" + + key = get_s3_object_key("", "", start_time, original_file_name) + filename = key.split("/")[-1] + + assert len(original_file_name) < 250 + assert len(f"{original_file_name}.json".encode("utf-8")) > 255 + assert len(filename.encode("utf-8")) <= 255 + assert filename.endswith(".json") + assert filename.startswith(f"time-{start_time.strftime('%H-%M-%S-%f')}_resp_") def test_create_s3_batch_logging_element_limits_long_ids(): @@ -939,5 +954,5 @@ def test_create_s3_batch_logging_element_limits_long_ids(): result = logger.create_s3_batch_logging_element(start_time, payload) assert result is not None - assert len(result.s3_object_key.split("/")[-1]) <= 255 + assert len(result.s3_object_key.split("/")[-1].encode("utf-8")) <= 255 assert result.s3_object_key.endswith(".json")