fix(bedrock): move a Nova invoke cache point behind an image or tool result to the last text block

litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py

@@ -67,9 +67,10 @@ def _inline_block_cache_points(
         cache_point: Final = block.get("cachePoint")
         if cache_point is None or len(block) != 1:
             return (*inlined, block)
-        if not inlined:
+        anchor: Final = next((index for index in reversed(range(len(inlined))) if "text" in inlined[index]), None)
+        if anchor is None:
             return inlined
-        return (*inlined[:-1], with_cache_point(inlined[-1], cache_point))
+        return (*inlined[:anchor], with_cache_point(inlined[anchor], cache_point), *inlined[anchor + 1 :])
 
     return list(reduce(attach, blocks, ()))
 
@@ -148,8 +149,9 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
 
     @staticmethod
     def _inline_cache_points(request: BedrockInvokeNovaRequest) -> BedrockInvokeNovaRequest:
-        """InvokeModel takes ``cachePoint`` as a key of the block it caches and rejects the
-        standalone ``{"cachePoint": ...}`` blocks Converse accepts.
+        """InvokeModel takes ``cachePoint`` as a key of the text block it caches: it rejects the
+        standalone ``{"cachePoint": ...}`` blocks Converse accepts and the key on image, toolUse,
+        and toolResult blocks, so a point behind one of those moves back to the last text block.
         """
         return {
             **request,

tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_nova_transformation.py

@@ -9,6 +9,8 @@ MODEL = "us.amazon.nova-pro-v1:0"
 EPHEMERAL = {"type": "ephemeral"}
 DEFAULT_CACHE_POINT = {"type": "default"}
 TOOLS = [{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}]
+TOOL_CALL = {"id": "call_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}
+PNG_DATA_URL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
 
 
 def _transform_request(messages, optional_params, litellm_params=None):
@@ -42,6 +44,32 @@ def test_cache_points_are_inlined_into_the_block_they_cache(local_model_cost_map
     ]
 
 
+def test_cache_point_behind_a_non_text_block_moves_back_to_the_last_text_block(local_model_cost_map):
+    """InvokeModel rejects ``cachePoint`` on image, toolUse, and toolResult blocks
+    (``extraneous key [cachePoint] is not permitted``), so the point a user put on an image or a
+    tool result lands on the closest text block before it, and a message with no text block at
+    all sends no point rather than a request AWS refuses.
+    """
+    request = _transform_request(
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "what is in this picture?"},
+                    {"type": "image_url", "image_url": {"url": PNG_DATA_URL}, "cache_control": EPHEMERAL},
+                ],
+            },
+            {"role": "assistant", "content": None, "tool_calls": [TOOL_CALL]},
+            {"role": "tool", "tool_call_id": "call_1", "content": "sunny", "cache_control": EPHEMERAL},
+        ],
+        optional_params={"tools": TOOLS},
+    )
+    picture, image = request["messages"][0]["content"]
+    assert picture == {"text": "what is in this picture?", "cachePoint": DEFAULT_CACHE_POINT}
+    assert set(image) == {"image"}
+    assert [set(block) for block in request["messages"][2]["content"]] == [{"toolResult"}]
+
+
 def test_cache_point_with_nothing_before_it_is_dropped():
     request = AmazonInvokeNovaConfig._inline_cache_points(
         {