fix: guard unsupported proxy image payloads

Send Anthropic image blocks as OpenAI image_url parts with high detail.

Detect endpoints known to reject OpenAI-style image parts and fail early with a clear provider error.
This commit is contained in:
Gerard-Devlin 2026-06-11 00:47:29 +08:00
parent 3eac6be0a8
commit a22860e589
2 changed files with 51 additions and 0 deletions

View file

@ -225,6 +225,14 @@ async fn handle_messages_to_stream(
credential,
wants_stream,
);
if request_contains_openai_image_parts(&openai_request)
&& provider_rejects_openai_image_parts(credential)
{
return Err(format!(
"{} does not accept OpenAI-style image_url message parts. Switch to Claude Code or a vision-capable OpenAI-compatible endpoint for image questions.",
credential.model
));
}
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(300))
@ -313,6 +321,22 @@ fn with_optional_bearer_auth(
}
}
fn request_contains_openai_image_parts(value: &Value) -> bool {
match value {
Value::Array(values) => values.iter().any(request_contains_openai_image_parts),
Value::Object(object) => {
object.get("type").and_then(Value::as_str) == Some("image_url")
|| object.values().any(request_contains_openai_image_parts)
}
_ => false,
}
}
fn provider_rejects_openai_image_parts(credential: &OpenAiProxyCredential) -> bool {
let base_url = credential.base_url.to_ascii_lowercase();
base_url.contains("api.deepseek.com")
}
fn openai_compatible_base_url_has_chat_root(base_url: &str) -> bool {
let lower = base_url.to_ascii_lowercase();
if lower == "https://api.deepseek.com" || lower == "http://api.deepseek.com" {
@ -385,6 +409,22 @@ mod tests {
assert!(is_count_tokens_path("/v1/messages/count_tokens"));
}
#[test]
fn detects_openai_image_parts_for_provider_guard() {
assert!(request_contains_openai_image_parts(&json!({
"messages": [{
"role": "user",
"content": [
{ "type": "text", "text": "what is this?" },
{ "type": "image_url", "image_url": { "url": "data:image/png;base64,abc" } }
]
}]
})));
assert!(!request_contains_openai_image_parts(&json!({
"messages": [{ "role": "user", "content": "text only" }]
})));
}
#[test]
fn converts_tool_use_and_tool_result_messages() {
let credential = OpenAiProxyCredential {

View file

@ -271,6 +271,7 @@ fn anthropic_image_block_to_openai_part(block: &Value) -> Option<Value> {
"type": "image_url",
"image_url": {
"url": url,
"detail": "high",
},
}))
}
@ -694,10 +695,15 @@ mod tests {
let converted = anthropic_to_openai_request(&request, &credential()).unwrap();
assert_eq!(converted["messages"][0]["content"][0]["type"], "text");
assert_eq!(converted["messages"][0]["content"][1]["type"], "image_url");
assert_eq!(
converted["messages"][0]["content"][1]["image_url"]["url"],
"data:image/png;base64,abcd"
);
assert_eq!(
converted["messages"][0]["content"][1]["image_url"]["detail"],
"high"
);
}
#[test]
@ -745,10 +751,15 @@ mod tests {
);
assert_eq!(converted["messages"][2]["role"], "user");
assert_eq!(converted["messages"][2]["content"][0]["type"], "text");
assert_eq!(converted["messages"][2]["content"][1]["type"], "image_url");
assert_eq!(
converted["messages"][2]["content"][1]["image_url"]["url"],
"data:image/png;base64,abcd"
);
assert_eq!(
converted["messages"][2]["content"][1]["image_url"]["detail"],
"high"
);
}
#[test]