fix(s3_v2): sign S3 uploads with S3SigV4Auth so keys with spaces are not double-encoded

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-07-31 00:30:31 +00:00
parent 71b825a7f0
commit 3586cff5eb
2 changed files with 71 additions and 6 deletions

View file

@ -282,7 +282,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
import hashlib
import requests
from botocore.auth import SigV4Auth
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
@ -352,7 +352,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
headers=prepped.headers,
)
aws_region_name = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers = dict(aws_request.headers.items())
@ -466,7 +466,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
import hashlib
import requests
from botocore.auth import SigV4Auth
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
except ImportError:
@ -527,7 +527,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
headers=prepped.headers,
)
aws_region_name = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers = dict(aws_request.headers.items())
@ -571,7 +571,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
import hashlib
import requests
from botocore.auth import SigV4Auth
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
raise ImportError("Missing boto3 to call S3. Run 'pip install boto3'.")
@ -623,7 +623,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
url=prepped.url,
headers=prepped.headers,
)
SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request)
S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers = dict(aws_request.headers.items())

View file

@ -1388,3 +1388,68 @@ def test_s3_server_side_encryption_read_from_callback_params():
assert logger.s3_server_side_encryption == "aws:kms"
finally:
litellm.s3_callback_params = original
def _s3_signature_for(url: str, body: str, headers: dict) -> str:
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
signed_header_names = headers["Authorization"].split("SignedHeaders=")[1].split(",")[0].split(";")
request = AWSRequest(
method="PUT",
url=url,
data=body,
headers={k: v for k, v in headers.items() if k.lower() in signed_header_names},
)
request.context["timestamp"] = headers["X-Amz-Date"]
S3SigV4Auth(Credentials("test-key", "test-secret"), "s3", "us-east-1").add_auth(request)
return request.headers["Authorization"].split("Signature=")[1]
@pytest.mark.parametrize("upload_path", ["async", "sync"])
def test_upload_signature_matches_s3_canonicalization_for_keys_with_spaces(upload_path):
"""
S3 signs the object key single-encoded; generic SigV4Auth re-quotes the already
encoded path, so a team alias containing a space (key '.../AI Post_Chat/...' sent as
'%20') produced a signature over '%2520' and S3 answered 403 SignatureDoesNotMatch.
"""
from unittest.mock import AsyncMock, MagicMock
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
)
test_element = s3BatchLoggingElement(
s3_object_key="LOGS/AI Post_Chat/2026-07-23/test.json",
payload={"test": "spaces"},
s3_object_download_filename="test.json",
)
response = MagicMock()
response.status_code = 200
response.raise_for_status = MagicMock()
if upload_path == "async":
logger.async_httpx_client = AsyncMock()
logger.async_httpx_client.put.return_value = response
asyncio.run(logger.async_upload_data_to_s3(test_element))
call = logger.async_httpx_client.put.call_args
else:
mock_sync_client = MagicMock()
mock_sync_client.put.return_value = response
with patch("litellm.integrations.s3_v2._get_httpx_client", return_value=mock_sync_client):
logger.upload_data_to_s3(test_element)
call = mock_sync_client.put.call_args
url = call.args[0]
headers = call.kwargs["headers"]
assert "AI%20Post_Chat" in url
assert headers["Authorization"].split("Signature=")[1] == _s3_signature_for(
url, call.kwargs["data"], headers
)