mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
fix(bedrock): count UTF-8 bytes, not string length (#36388)
This commit is contained in:
parent
2d2b32c3f8
commit
56dbaefd15
2 changed files with 39 additions and 1 deletions
|
|
@ -863,7 +863,13 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
litellm_params["upload_url"] = api_base
|
||||
# Store file size for response transformation (S3 PUT response
|
||||
# doesn't include Content-Length, so we pass it through).
|
||||
litellm_params["_file_content_size"] = len(file_content) if isinstance(file_content, (str, bytes)) else 0
|
||||
# Use UTF-8 byte length, not decoded string length, so multi-byte
|
||||
# characters are counted correctly.
|
||||
if isinstance(file_content, bytes):
|
||||
_file_content_bytes = len(file_content)
|
||||
else:
|
||||
_file_content_bytes = len(file_content.encode("utf-8"))
|
||||
litellm_params["_file_content_size"] = _file_content_bytes
|
||||
|
||||
# Return a dict that tells the HTTP handler exactly what to do
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -2093,6 +2093,38 @@ class TestBedrockFilesS3SignatureEncoding:
|
|||
|
||||
assert result.bytes == 2048
|
||||
|
||||
def test_file_content_size_counts_utf8_bytes_not_characters(self):
|
||||
"""Multi-byte characters must count as their encoded byte length.
|
||||
|
||||
Regression test for Greptile review on #36388: len() on a decoded
|
||||
string undercounts multi-byte (e.g. CJK) content.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
|
||||
|
||||
raw_response = httpx.Response(
|
||||
status_code=200,
|
||||
headers={}, # S3 PUT omits Content-Length
|
||||
content=b"",
|
||||
request=httpx.Request(
|
||||
"PUT",
|
||||
"https://s3.us-east-1.amazonaws.com/my-bucket/litellm-bedrock-files-abc.jsonl",
|
||||
),
|
||||
)
|
||||
|
||||
# 3 CJK characters = 9 UTF-8 bytes, not 3 characters.
|
||||
content_size = len("日本語".encode("utf-8"))
|
||||
|
||||
result = BedrockFilesConfig().transform_create_file_response(
|
||||
model=None,
|
||||
raw_response=raw_response,
|
||||
logging_obj=MagicMock(),
|
||||
litellm_params={"_file_content_size": content_size},
|
||||
)
|
||||
|
||||
assert result.bytes == 9
|
||||
|
||||
def test_file_content_signs_spaced_object_key_the_way_s3_does(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue