From a22860e589069daf3816cfa15dd69275d032b1fc Mon Sep 17 00:00:00 2001
From: Gerard-Devlin <alwenxuyouwen@gmail.com>
Date: Thu, 11 Jun 2026 00:47:29 +0800
Subject: [PATCH] fix: guard unsupported proxy image payloads

Send Anthropic image blocks as OpenAI image_url parts with high detail.

Detect endpoints known to reject OpenAI-style image parts and fail early with a clear provider error.
---
 apps/desktop/src-tauri/src/anthropic_proxy.rs | 40 +++++++++++++++++++
 .../src-tauri/src/anthropic_proxy/messages.rs | 11 +++++
 2 files changed, 51 insertions(+)

diff --git a/apps/desktop/src-tauri/src/anthropic_proxy.rs b/apps/desktop/src-tauri/src/anthropic_proxy.rs
index 1ecd0dd..8210e52 100644
--- a/apps/desktop/src-tauri/src/anthropic_proxy.rs
+++ b/apps/desktop/src-tauri/src/anthropic_proxy.rs
@@ -225,6 +225,14 @@ async fn handle_messages_to_stream(
         credential,
         wants_stream,
     );
+    if request_contains_openai_image_parts(&openai_request)
+        && provider_rejects_openai_image_parts(credential)
+    {
+        return Err(format!(
+            "{} does not accept OpenAI-style image_url message parts. Switch to Claude Code or a vision-capable OpenAI-compatible endpoint for image questions.",
+            credential.model
+        ));
+    }
 
     let client = reqwest::Client::builder()
         .timeout(std::time::Duration::from_secs(300))
@@ -313,6 +321,22 @@ fn with_optional_bearer_auth(
     }
 }
 
+fn request_contains_openai_image_parts(value: &Value) -> bool {
+    match value {
+        Value::Array(values) => values.iter().any(request_contains_openai_image_parts),
+        Value::Object(object) => {
+            object.get("type").and_then(Value::as_str) == Some("image_url")
+                || object.values().any(request_contains_openai_image_parts)
+        }
+        _ => false,
+    }
+}
+
+fn provider_rejects_openai_image_parts(credential: &OpenAiProxyCredential) -> bool {
+    let base_url = credential.base_url.to_ascii_lowercase();
+    base_url.contains("api.deepseek.com")
+}
+
 fn openai_compatible_base_url_has_chat_root(base_url: &str) -> bool {
     let lower = base_url.to_ascii_lowercase();
     if lower == "https://api.deepseek.com" || lower == "http://api.deepseek.com" {
@@ -385,6 +409,22 @@ mod tests {
         assert!(is_count_tokens_path("/v1/messages/count_tokens"));
     }
 
+    #[test]
+    fn detects_openai_image_parts_for_provider_guard() {
+        assert!(request_contains_openai_image_parts(&json!({
+            "messages": [{
+                "role": "user",
+                "content": [
+                    { "type": "text", "text": "what is this?" },
+                    { "type": "image_url", "image_url": { "url": "data:image/png;base64,abc" } }
+                ]
+            }]
+        })));
+        assert!(!request_contains_openai_image_parts(&json!({
+            "messages": [{ "role": "user", "content": "text only" }]
+        })));
+    }
+
     #[test]
     fn converts_tool_use_and_tool_result_messages() {
         let credential = OpenAiProxyCredential {
diff --git a/apps/desktop/src-tauri/src/anthropic_proxy/messages.rs b/apps/desktop/src-tauri/src/anthropic_proxy/messages.rs
index 5636c63..46f9204 100644
--- a/apps/desktop/src-tauri/src/anthropic_proxy/messages.rs
+++ b/apps/desktop/src-tauri/src/anthropic_proxy/messages.rs
@@ -271,6 +271,7 @@ fn anthropic_image_block_to_openai_part(block: &Value) -> Option<Value> {
         "type": "image_url",
         "image_url": {
             "url": url,
+            "detail": "high",
         },
     }))
 }
@@ -694,10 +695,15 @@ mod tests {
         let converted = anthropic_to_openai_request(&request, &credential()).unwrap();
 
         assert_eq!(converted["messages"][0]["content"][0]["type"], "text");
+        assert_eq!(converted["messages"][0]["content"][1]["type"], "image_url");
         assert_eq!(
             converted["messages"][0]["content"][1]["image_url"]["url"],
             "data:image/png;base64,abcd"
         );
+        assert_eq!(
+            converted["messages"][0]["content"][1]["image_url"]["detail"],
+            "high"
+        );
     }
 
     #[test]
@@ -745,10 +751,15 @@ mod tests {
         );
         assert_eq!(converted["messages"][2]["role"], "user");
         assert_eq!(converted["messages"][2]["content"][0]["type"], "text");
+        assert_eq!(converted["messages"][2]["content"][1]["type"], "image_url");
         assert_eq!(
             converted["messages"][2]["content"][1]["image_url"]["url"],
             "data:image/png;base64,abcd"
         );
+        assert_eq!(
+            converted["messages"][2]["content"][1]["image_url"]["detail"],
+            "high"
+        );
     }
 
     #[test]