diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index b3e0519b772..c2b08eee0c0 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -19,9 +19,9 @@ pub enum CoreError { InvalidRequest(String), #[error("{0}")] Auth(String), - #[error("OCR request failed with status {status}: {body}")] + #[error("upstream request failed with status {status}: {body}")] Http { status: u16, body: String }, - #[error("OCR network error: {0}")] + #[error("upstream network error: {0}")] Network(String), #[error("routing error: {0}")] Routing(String), diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index fd0fa3b60ee..13e79b087c7 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -7,11 +7,14 @@ use crate::messages::types::{ use crate::providers::anthropic::messages::transformation::{ non_empty, AnthropicMessagesConfig, ANTHROPIC_MESSAGES_CONFIG, }; +use serde_json::{Map, Value}; const AZURE_API_KEY_ENV: &str = "AZURE_API_KEY"; const AZURE_API_BASE_ENV: &str = "AZURE_API_BASE"; const ANTHROPIC_PATH_SEGMENT: &str = "/anthropic"; const MESSAGES_PATH_SUFFIX: &str = "/v1/messages"; +const SYSTEM_ROLE: &str = "system"; +const TEXT_BLOCK_TYPE: &str = "text"; pub struct AzureAnthropicMessagesConfig { anthropic: AnthropicMessagesConfig, @@ -83,6 +86,61 @@ fn strip_scope_from_message(message: &mut AnthropicMessage) { } } +fn text_content_block(text: String) -> ContentBlock { + let extra = Map::from_iter([ + ( + "type".to_string(), + Value::String(TEXT_BLOCK_TYPE.to_string()), + ), + ("text".to_string(), Value::String(text)), + ]); + ContentBlock { + cache_control: None, + extra, + } +} + +fn content_into_blocks(content: MessageContent) -> Vec { + match content { + MessageContent::Text(text) => vec![text_content_block(text)], + MessageContent::Blocks(blocks) => blocks, + } +} + +fn system_into_blocks(system: Option) -> Vec { + match system { + None => Vec::new(), + Some(SystemPrompt::Text(text)) => vec![text_content_block(text)], + Some(SystemPrompt::Blocks(blocks)) => blocks, + } +} + +fn fold_system_role_messages(request: AnthropicMessagesRequest) -> AnthropicMessagesRequest { + if !request.messages.iter().any(|msg| msg.role == SYSTEM_ROLE) { + return request; + } + + let (system_messages, chat_messages): (Vec, Vec) = request + .messages + .into_iter() + .partition(|msg| msg.role == SYSTEM_ROLE); + + let folded_system: Vec = system_into_blocks(request.system) + .into_iter() + .chain( + system_messages + .into_iter() + .flat_map(|msg| content_into_blocks(msg.content)), + ) + .collect(); + + AnthropicMessagesRequest { + messages: chat_messages, + system: (!folded_system.is_empty()).then_some(SystemPrompt::Blocks(folded_system)), + ..request + } +} + impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { fn complete_url( &self, @@ -111,8 +169,9 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { fn transform_request( &self, - mut request: AnthropicMessagesRequest, + request: AnthropicMessagesRequest, ) -> CoreResult { + let mut request = fold_system_role_messages(request); if let Some(system) = request.system.as_mut() { strip_scope_from_system(system); } @@ -345,6 +404,83 @@ mod tests { assert_eq!(transformed, body); } + #[test] + fn transform_request_folds_system_role_message_into_top_level_system() { + let request = request_from(json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 256, + "system": [{"type": "text", "text": "base system"}], + "messages": [ + {"role": "user", "content": "fix the bug"}, + {"role": "system", "content": "Available agent types: claude"} + ] + })); + + let transformed = to_value( + AZURE_ANTHROPIC_MESSAGES_CONFIG + .transform_request(request) + .expect("request transforms"), + ); + + assert_eq!( + transformed["messages"], + json!([{"role": "user", "content": "fix the bug"}]) + ); + assert_eq!( + transformed["system"], + json!([ + {"type": "text", "text": "base system"}, + {"type": "text", "text": "Available agent types: claude"} + ]) + ); + } + + #[test] + fn transform_request_folds_system_role_when_no_top_level_system() { + let request = request_from(json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 256, + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + {"role": "system", "content": [{"type": "text", "text": "sys block"}]} + ] + })); + + let transformed = to_value( + AZURE_ANTHROPIC_MESSAGES_CONFIG + .transform_request(request) + .expect("request transforms"), + ); + + assert_eq!( + transformed["messages"], + json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]) + ); + assert_eq!( + transformed["system"], + json!([{"type": "text", "text": "sys block"}]) + ); + } + + #[test] + fn transform_request_leaves_requests_without_system_role_untouched() { + let body = json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 256, + "system": "be terse", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"} + ] + }); + let transformed = to_value( + AZURE_ANTHROPIC_MESSAGES_CONFIG + .transform_request(request_from(body.clone())) + .expect("request transforms"), + ); + assert_eq!(transformed, body); + } + #[test] fn transform_request_rejects_non_object_body() { let err = serde_json::from_value::(json!("bad")) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 133ce99818b..3e6f9ee08ee 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2269,15 +2269,22 @@ class BaseLLMHTTPHandler: from litellm.rust_bridge import messages as rust_messages_bridge upstream_body = {key: value for key, value in request_body.items() if key != "stream"} - rust_response = await rust_messages_bridge.amessages( - model=model, - body=upstream_body, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=headers, - timeout=timeout, - ) + try: + rust_response = await rust_messages_bridge.amessages( + model=model, + body=upstream_body, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=headers, + timeout=timeout, + ) + except Exception as rust_error: # noqa: BLE001 # rollout-safety fallback: any Rust bridge failure must fall back to the Python path + verbose_logger.debug( + "Rust Anthropic messages bridge raised %s; falling back to Python path", + type(rust_error).__name__, + ) + return None if rust_response is None: return None diff --git a/tests/test_litellm/rust_bridge/test_messages.py b/tests/test_litellm/rust_bridge/test_messages.py index f34b8c05819..26ca7d27210 100644 --- a/tests/test_litellm/rust_bridge/test_messages.py +++ b/tests/test_litellm/rust_bridge/test_messages.py @@ -98,6 +98,15 @@ class ExplodingAsyncMessages: raise AssertionError("bridge must not be called") +class RaisingAsyncMessages: + def __init__(self) -> None: + self.calls = 0 + + async def __call__(self, **kwargs: object) -> dict[str, object]: + self.calls += 1 + raise RuntimeError("upstream request failed with status 400: bad request") + + @pytest.fixture(autouse=True) def _reset_rust_flag(): litellm.use_litellm_rust(False, messages=None, amessages=None) @@ -242,6 +251,17 @@ async def test_gate_invokes_rust_and_marks_response_header(): assert call["timeout_seconds"] == 30.0 +@pytest.mark.asyncio +async def test_gate_falls_back_to_python_when_bridge_raises(): + bridge = RaisingAsyncMessages() + litellm.use_litellm_rust(True, amessages=bridge) + + response = await _gate() + + assert response is None + assert bridge.calls == 1 + + @pytest.mark.asyncio async def test_gate_skips_rust_when_flag_absent(): bridge = ExplodingAsyncMessages()