Merge pull request #41275 from BerriAI/litellm_azure_strip_file_format

fix(azure): strip litellm format field from file and image content parts

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -30,8 +30,10 @@ from litellm.types.llms.openai import (
     ChatCompletionAssistantMessage,
     ChatCompletionAssistantToolCall,
     ChatCompletionFileObject,
+    ChatCompletionFileObjectFile,
     ChatCompletionFunctionMessage,
     ChatCompletionImageObject,
+    ChatCompletionImageUrlObject,
     ChatCompletionTextObject,
     ChatCompletionToolCallFunctionChunk,
     ChatCompletionToolMessage,
@@ -1067,6 +1069,18 @@ def _azure_tool_call_invoke_helper(
 def _azure_image_url_helper(content: ChatCompletionImageObject):
     if isinstance(content["image_url"], str):
         content["image_url"] = {"url": content["image_url"]}
+    else:
+        content["image_url"] = cast(
+            ChatCompletionImageUrlObject,
+            {k: v for k, v in content["image_url"].items() if k != "format"},
+        )
+
+
+def _azure_file_helper(content: ChatCompletionFileObject) -> None:
+    content["file"] = cast(
+        ChatCompletionFileObjectFile,
+        {k: v for k, v in content.get("file", {}).items() if k != "format"},
+    )
 
 
 def convert_to_azure_openai_messages(
@@ -1081,7 +1095,9 @@ def convert_to_azure_openai_messages(
         if m["role"] == "user" and isinstance(m.get("content"), list):
             for content in m.get("content", []):
                 if isinstance(content, dict) and content.get("type") == "image_url":
-                    _azure_image_url_helper(content)
+                    _azure_image_url_helper(cast(ChatCompletionImageObject, content))
+                elif isinstance(content, dict) and content.get("type") == "file":
+                    _azure_file_helper(cast(ChatCompletionFileObject, content))
     return messages
 
 

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py

@@ -297,6 +297,35 @@ def test_convert_to_azure_openai_messages():
     assert content == expected_content
 
 
+def test_convert_to_azure_openai_messages_strips_litellm_format_from_file_and_image():
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        convert_to_azure_openai_messages,
+    )
+    from litellm.types.llms.openai import AllMessageValues
+
+    input: list[AllMessageValues] = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "file",
+                    "file": {"file_id": "assistant-xyz", "format": "application/pdf"},
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://x/y.png", "format": "image/png"},
+                },
+            ],
+        }
+    ]
+
+    output = convert_to_azure_openai_messages(input)
+
+    content = output[0].get("content")
+    assert content[0]["file"] == {"file_id": "assistant-xyz"}
+    assert content[1]["image_url"] == {"url": "https://x/y.png"}
+
+
 def test_bedrock_validate_format_image_or_video():
     """Test the _validate_format method for images, videos, and documents"""
 

tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py

@@ -333,3 +333,37 @@ class TestAzureToolSchemaCombinatorFlattening:
         )
         assert "tools" not in request
         assert request["temperature"] == 0.2
+
+
+def test_transform_request_strips_litellm_format_from_managed_file_id():
+    import base64
+
+    from litellm.litellm_core_utils.prompt_templates.common_utils import (
+        update_messages_with_model_file_ids,
+    )
+
+    managed_file_id: Final = base64.b64encode(
+        b"litellm_proxy:application/pdf;unified_id,abc123;llm_output_file_id,assistant-xyz;target_model_names,azure-gpt"
+    ).decode()
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Summarize this file"},
+                {"type": "file", "file": {"file_id": managed_file_id}},
+            ],
+        }
+    ]
+    updated_messages = update_messages_with_model_file_ids(messages, None, {})
+
+    request = AzureOpenAIConfig().transform_request(
+        model="gpt-5.4",
+        messages=updated_messages,
+        optional_params={},
+        litellm_params={},
+        headers={},
+    )
+
+    file_part = request["messages"][0]["content"][1]["file"]
+    assert "format" not in file_part
+    assert file_part["file_id"] == "assistant-xyz"