diff --git a/litellm-rust/crates/gateway/src/hooks.rs b/litellm-rust/crates/gateway/src/hooks.rs index 579d84737d0..e5a3c739577 100644 --- a/litellm-rust/crates/gateway/src/hooks.rs +++ b/litellm-rust/crates/gateway/src/hooks.rs @@ -205,8 +205,8 @@ fn is_azure(model: &Model) -> bool { } /// Whether the deployment's chat param rules read model capabilities from the cost map: -/// Azure, the Anthropic, Bedrock and Gemini chat translations, and the OpenAI-compatible -/// providers. +/// Azure, the Anthropic, Bedrock, Gemini and Claude on Vertex AI chat translations, and the +/// OpenAI-compatible providers. fn reads_model_info(model: &Model) -> bool { is_azure(model) || litellm_inference::provider::resolve_llm_provider( diff --git a/litellm-rust/crates/gateway/tests/vertex_claude_chat.rs b/litellm-rust/crates/gateway/tests/vertex_claude_chat.rs new file mode 100644 index 00000000000..d2e8c711a02 --- /dev/null +++ b/litellm-rust/crates/gateway/tests/vertex_claude_chat.rs @@ -0,0 +1,624 @@ +//! Claude chat completions on Vertex AI through the gateway (`vertex_ai/claude-*`, a service +//! account traded for an access token): the `rawPredict` request Vertex gets, the OpenAI answer +//! the client gets, and the usage and cost the request is recorded with. + +use std::{ + io::Write, + sync::{Arc, Mutex, Once}, +}; + +use axum::{ + Router, + body::{Body, to_bytes}, + http::{Request, StatusCode, header::CONTENT_TYPE}, + response::IntoResponse, +}; +use futures_util::future::BoxFuture; +use litellm_config::Config; +use litellm_gateway_hooks::{Facts, Observer, Record}; +use rstest::rstest; +use serde_json::{Value, json}; +use tokio::net::TcpListener; +use tower::ServiceExt; + +/// Input and output prices of every deployment. +const RATES: (f64, f64) = (1.0, 2.0); + +const ACCESS_TOKEN: &str = "ya29.test-token"; +const PROJECT: &str = "test-project"; +const LOCATION: &str = "us-east5"; +const MODEL: &str = "claude-sonnet-4-5@20250929"; + +/// A throwaway RSA key the fake service account signs its token request with. +const TEST_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC95ihJQlfJuaQe +ZCwF+EFy/Y5vk7Lx6Nm74He9Oes+6ZbgA2LVdcoFJrNEYHd8S0CCv7cqEp4dpIoR +B4HAmQAQF92ZC/S2lcgLAezWQU21TSjDLngdVbn+Jr4syzOwVw/NUigydtQ66Iqv +1FpXs/4gQXMgs8FW5XgOhgfKhyMPf71A2qiRaA0vwsrfsmKOY0QVVooXPRfpLIvZ +VEOV89ZMQErnYan+m8OADoZ5NJ/Ij6QKb2HkQJNhQC2Zq2T1Y9VLyxY6pXURdVqc +XJgFT6YPGjLABLX/oId4g+Fs2YI5mM+g72pYEntnIaE5yicGKBdjyqeVfKZ6/7jg +g8evHi6vAgMBAAECggEAGdJjlP6b8Fa5bdaCM/ebcrbuuNZVJVbb0JPHxGfNSLs7 +pE9hj5QaOdQW2Uviw3h6F61ZCzQH4xD+Iy2po5ZKb2XHYKnDB1bboj+LRGER337T +9aJqe9at2VTMVEv3Rdm40NsEk0QcPLxlK16NQFK90gYEUSSQPDAswJDSG2R/zHn+ +vADI907mW/goEJHeLn8PWGlNlSiR6x+5JJtq+GXCzUzVvJYQSCLGxCSl2x2H+0g7 +NhFI0zPpdzNmO/h+yhzaFb6Rp5U8+ZsnZ3qYjQ/03gw1myTDKJt1YaO9JvArnNYX +hcJQQ8Rt0bHhcrZA16bBOpqZlo5pKCicwI/netgN8QKBgQDcFz7AzdJ26sMSV32V +rwrMgIoggt8qDjO1ARwqW35A1TIge0FoW4M4KpsXQGGfT341uU1esXEcyZ/1L/5X +3ql2gX4DbOYLZLWYzZGR2hq33oi8HkhN98QrEwL9emSH8NqYX3Xxja3PrmCrSYJe +Zbnd9TIm2XkxyMoyXJu6M/QvnwKBgQDc4dzqTbxoGEGa5MuJoGmMwPnqgdG9UM5J +eExVnh7osxc2sOdsiPeRjjQTxs9v2kJwctC359OJoo9yGaaJeSghU4LEWJo1sqnA +fzSCLammYvtVAtniyNv5Mxk/6Uimi4NNDKaAKB+m4K2uSn3U9AmY7KPYMGaSbS9W +XSnobjxm8QKBgC8bPpAvvWs8ZhIn7bY659nLbUT2HeO3dHO6UBf0yzn/J6JyHxbB +93zvCZDZc8uQTRgcmCW7XtVlhjoJUqvl+Wlm39zF0xr/LCsPXKfWAb/2/lcdOCaP +8Emz4QD10EyUTYUtcWYJB/mafhBLRH8F0Nlj4J8WDu2L51MOJTqeYhZLAoGAWffN +icocAbJPlo22sdoa4+/+W5yBF8GAJMDRJtZ+9H1t6SLpQHYRkMIBSETkXUTjZvX9 +Ocs9iIQkNW9pO/mTdO+VBfCo71JUfknR02xR+6m5gYjlws/ZeYlssXGN2/hbhNiw +QOcW7Vv6olFJK6Iy/oz0t6wPO3kpnN3Zogi0paECgYEAwo44M1DdYCtV0snhmYM9 +5u0mPfYt5P2SVLXyUbr+vFTfrTL/WKnXIJgbsnj3Gvf+GIZv9tKcXhSNmEHQCYX4 +X3w9iTPddCHuvZ1fpufi2TyArJh0OkoNtLXJHTKrHjf2N+61AQzFiv5WieJrdE+H +qr32PTUuVGPyO9LyTY4/RL0= +-----END PRIVATE KEY----- +"; + +/// The gateway reads its cost map from this variable; every test sets it before building a +/// gateway, so no test reads the environment while it changes. +fn use_local_cost_map() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + let map = json!({ + format!("vertex_ai/{MODEL}"): { + "mode": "chat", + "litellm_provider": "vertex_ai-anthropic_models", + "max_output_tokens": 64000, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + }, + }); + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.write_all(map.to_string().as_bytes()).unwrap(); + let (_, path) = file.keep().unwrap(); + // SAFETY: set once, before any test builds a gateway or spawns a thread that reads the + // environment. + unsafe { std::env::set_var("LITELLM_MODEL_COST_MAP_PATH", path) }; + }); +} + +fn message(content: Value, stop_reason: &str) -> Value { + json!({ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": content, + "stop_reason": stop_reason, + "stop_sequence": null, + "usage": {"input_tokens": 10, "output_tokens": 4}, + }) +} + +fn text_answer(text: &str) -> Value { + message(json!([{"type": "text", "text": text}]), "end_turn") +} + +fn tool_answer() -> Value { + message( + json!([{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}]), + "tool_use", + ) +} + +fn thinking_answer() -> Value { + message( + json!([ + {"type": "thinking", "thinking": "The user says hi.", "signature": "sig-1"}, + {"type": "text", "text": "Hello"} + ]), + "end_turn", + ) +} + +fn sse_answer() -> String { + [ + json!({"type": "message_start", "message": {"id": "msg_2", "type": "message", "role": "assistant", "model": "claude-sonnet-4-5", "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 10, "output_tokens": 1}}}), + json!({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + json!({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello"}}), + json!({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " world"}}), + json!({"type": "content_block_stop", "index": 0}), + json!({"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": null}, "usage": {"output_tokens": 12}}), + json!({"type": "message_stop"}), + ] + .iter() + .map(|event| format!("event: {}\ndata: {event}\n\n", event["type"].as_str().unwrap())) + .collect() +} + +#[derive(Clone, Debug)] +struct Sent { + path: String, + query: Option, + authorization: Option, + api_key: Option, + anthropic_version: Option, + anthropic_beta: Option, + body: Value, +} + +type Calls = Arc>>; + +fn has_tool_result(body: &Value) -> bool { + body["messages"].as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message["content"] + .as_array() + .is_some_and(|blocks| blocks.iter().any(|block| block["type"] == "tool_result")) + }) + }) +} + +/// Google's token endpoint and Vertex's `rawPredict` and `streamRawPredict`, answering by what +/// the body asks. +async fn upstream(calls: Calls) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = Router::new().fallback(move |request: Request| { + let calls = calls.clone(); + async move { + let (parts, body) = request.into_parts(); + let header = |name: &str| { + parts + .headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + let bytes = to_bytes(body, 1 << 20).await.unwrap(); + let path = parts.uri.path().to_owned(); + if path == "/token" { + return ( + [(CONTENT_TYPE, "application/json")], + json!({"access_token": ACCESS_TOKEN, "expires_in": 3600, "token_type": "Bearer"}) + .to_string(), + ) + .into_response(); + } + let body: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null); + calls.lock().unwrap().push(Sent { + path: path.clone(), + query: parts.uri.query().map(str::to_owned), + authorization: header("authorization"), + api_key: header("x-api-key"), + anthropic_version: header("anthropic-version"), + anthropic_beta: header("anthropic-beta"), + body: body.clone(), + }); + match path.ends_with(":streamRawPredict") { + true => ([(CONTENT_TYPE, "text/event-stream")], sse_answer()).into_response(), + false => { + let answer = match (has_tool_result(&body), body.get("tools"), body.get("thinking")) { + (true, _, _) => text_answer("It is sunny in Paris"), + (false, Some(_), _) => tool_answer(), + (false, None, Some(_)) => thinking_answer(), + (false, None, None) => text_answer("Hello world"), + }; + ( + StatusCode::OK, + [(CONTENT_TYPE, "application/json")], + answer.to_string(), + ) + .into_response() + } + } + } + }); + tokio::spawn(async move { axum::serve(listener, app).await }); + format!("http://{address}") +} + +#[derive(Debug)] +struct Seen { + deployment: Option, + usage: Option, + cost: Option, +} + +struct Collect(Mutex>); + +impl Observer for Collect { + fn name(&self) -> &'static str { + "collect" + } + + fn reads(&self) -> Facts { + Facts::DEPLOYMENT | Facts::USAGE | Facts::COST + } + + fn observe<'a>(&'a self, record: &'a Record) -> BoxFuture<'a, ()> { + Box::pin(async move { + self.0.lock().unwrap().push(Seen { + deployment: record + .outcome + .deployment + .as_ref() + .map(|deployment| deployment.id.to_string()), + usage: record.outcome.usage, + cost: record.outcome.cost, + }); + }) + } +} + +struct Harness { + app: Router, + inference: Arc, + calls: Calls, + seen: Arc, +} + +fn service_account(upstream: &str) -> String { + json!({ + "type": "service_account", + "project_id": "credentials-project", + "private_key_id": "test-key-id", + "private_key": TEST_PRIVATE_KEY, + "client_email": "gateway-test@credentials-project.iam.gserviceaccount.com", + "client_id": "1", + "token_uri": format!("{upstream}/token"), + }) + .to_string() +} + +async fn harness() -> Harness { + use_local_cost_map(); + let calls = Calls::default(); + let upstream = upstream(calls.clone()).await; + let (input, output) = RATES; + let prices = + format!(" input_cost_per_token: {input}\n output_cost_per_token: {output}\n"); + let credentials = service_account(&upstream); + let config = Config::from_yaml(&format!( + "model_list: + - model_name: claude + litellm_params: + model: vertex_ai/{MODEL} + api_base: {upstream} + vertex_project: {PROJECT} + vertex_location: {LOCATION} + vertex_credentials: '{credentials}' +{prices} model_info: + id: claude-1 + - model_name: claude-global + litellm_params: + model: vertex_ai/{MODEL} + api_base: {upstream} + vertex_project: {PROJECT} + vertex_location: global + vertex_credentials: '{credentials}' +{prices} model_info: + id: claude-global-1 +general_settings: + master_key: sk-master +" + )) + .unwrap(); + let inference = litellm_gateway::build_inference(&config).unwrap(); + let seen = Arc::new(Collect(Mutex::new(Vec::new()))); + litellm_gateway::hooks::install(&inference, &config, Vec::new(), vec![seen.clone()]) + .await + .unwrap(); + let app = litellm_gateway::router(inference.clone(), &config, None, None); + Harness { + app, + inference, + calls, + seen, + } +} + +async fn send(app: &Router, body: Value) -> (u16, String) { + let request = Request::post("/v1/chat/completions") + .header("content-type", "application/json") + .header("authorization", "Bearer sk-master") + .body(Body::from(body.to_string())) + .unwrap(); + let response = app.clone().oneshot(request).await.unwrap(); + let status = response.status().as_u16(); + let body = to_bytes(response.into_body(), 1 << 20).await.unwrap(); + (status, String::from_utf8(body.to_vec()).unwrap()) +} + +async fn seen(harness: &Harness) -> Vec { + litellm_gateway::hooks::close(&harness.inference).await; + std::mem::take(&mut *harness.seen.0.lock().unwrap()) +} + +fn cost(prompt: u64, completion: u64) -> f64 { + let (input, output) = RATES; + prompt as f64 * input + completion as f64 * output +} + +fn hi(model: &str) -> Value { + json!({ + "model": model, + "messages": [ + {"role": "system", "content": "Be brief"}, + {"role": "user", "content": "hi"} + ], + "temperature": 0.2, + "max_tokens": 64, + }) +} + +fn sent_body() -> Value { + json!({ + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + "temperature": 0.2, + "max_tokens": 64, + "anthropic_version": "vertex-2023-10-16", + "system": [{"type": "text", "text": "Be brief"}], + }) +} + +fn raw_predict_path(location: &str, method: &str) -> String { + format!( + "/v1/projects/{PROJECT}/locations/{location}/publishers/anthropic/models/{MODEL}:{method}" + ) +} + +#[rstest] +#[case::regional("claude", "claude-1", LOCATION)] +#[case::global("claude-global", "claude-global-1", "global")] +#[tokio::test] +async fn a_chat_completion_calls_raw_predict_and_records_its_cost( + #[case] model: &str, + #[case] id: &str, + #[case] location: &str, +) { + let harness = harness().await; + let (status, body) = send(&harness.app, hi(model)).await; + assert_eq!(status, 200, "{body}"); + let answer: Value = serde_json::from_str(&body).unwrap(); + // `VertexAIAnthropicConfig.transform_response` names the requested model. + assert_eq!(answer["model"], MODEL); + assert_eq!(answer["choices"][0]["message"]["role"], "assistant"); + assert_eq!(answer["choices"][0]["message"]["content"], "Hello world"); + assert_eq!(answer["choices"][0]["finish_reason"], "stop"); + assert_eq!(answer["usage"]["prompt_tokens"], 10); + assert_eq!(answer["usage"]["completion_tokens"], 4); + assert_eq!(answer["usage"]["total_tokens"], 14); + + let sent = harness.calls.lock().unwrap()[0].clone(); + assert_eq!(sent.path, raw_predict_path(location, "rawPredict")); + assert_eq!(sent.query, None); + assert_eq!( + sent.authorization.as_deref(), + Some("Bearer ya29.test-token") + ); + assert_eq!(sent.api_key, None); + assert_eq!(sent.anthropic_version.as_deref(), Some("2023-06-01")); + assert_eq!(sent.anthropic_beta, None); + assert_eq!(sent.body, sent_body()); + + let seen = seen(&harness).await; + assert_eq!(seen.len(), 1, "{seen:?}"); + assert_eq!(seen[0].deployment.as_deref(), Some(id)); + let usage = seen[0].usage.unwrap(); + assert_eq!((usage.prompt_tokens(), usage.output_tokens), (10, 4)); + assert_eq!(seen[0].cost, Some(cost(10, 4))); +} + +/// The JSON of each `data:` frame, and whether the stream closed with `[DONE]`. +fn frames(body: &str) -> (Vec, bool) { + let mut done = false; + let frames = body + .split("\n\n") + .filter(|frame| !frame.is_empty()) + .filter_map(|frame| { + let data = frame.strip_prefix("data: ").expect("bare data frames"); + match data { + "[DONE]" => { + done = true; + None + } + data => Some(serde_json::from_str(data).unwrap()), + } + }) + .collect(); + (frames, done) +} + +#[rstest] +#[case::regional("claude", "claude-1", LOCATION)] +#[case::global("claude-global", "claude-global-1", "global")] +#[tokio::test] +async fn a_stream_calls_stream_raw_predict_and_records_its_usage( + #[case] model: &str, + #[case] id: &str, + #[case] location: &str, +) { + let harness = harness().await; + let mut request = hi(model); + request["stream"] = json!(true); + request["stream_options"] = json!({"include_usage": true}); + let (status, body) = send(&harness.app, request).await; + assert_eq!(status, 200, "{body}"); + + let (frames, done) = frames(&body); + assert!(done, "{body}"); + assert!( + frames + .iter() + .all(|frame| frame["object"] == "chat.completion.chunk" && frame["model"] == model), + "{body}" + ); + let content: String = frames + .iter() + .filter_map(|frame| frame["choices"][0]["delta"]["content"].as_str()) + .collect(); + assert_eq!(content, "Hello world", "{body}"); + let finishes: Vec<&Value> = frames + .iter() + .filter_map(|frame| frame["choices"][0].get("finish_reason")) + .filter(|reason| !reason.is_null()) + .collect(); + assert_eq!(finishes, [&json!("stop")], "{body}"); + let usages: Vec<&Value> = frames + .iter() + .filter_map(|frame| frame.get("usage")) + .filter(|usage| !usage.is_null()) + .collect(); + assert_eq!(usages.len(), 1, "{body}"); + assert_eq!(usages[0]["prompt_tokens"], 10); + assert_eq!(usages[0]["completion_tokens"], 12); + + let sent = harness.calls.lock().unwrap()[0].clone(); + assert_eq!(sent.path, raw_predict_path(location, "streamRawPredict")); + // A custom `api_base` asks for SSE, as `_check_custom_proxy` does. + assert_eq!(sent.query.as_deref(), Some("alt=sse")); + assert_eq!( + sent.authorization.as_deref(), + Some("Bearer ya29.test-token") + ); + let mut expected = sent_body(); + expected["stream"] = json!(true); + assert_eq!(sent.body, expected); + + let seen = seen(&harness).await; + assert_eq!(seen.len(), 1, "{seen:?}"); + assert_eq!(seen[0].deployment.as_deref(), Some(id)); + let usage = seen[0].usage.unwrap(); + assert_eq!((usage.prompt_tokens(), usage.output_tokens), (10, 12)); + assert_eq!(seen[0].cost, Some(cost(10, 12))); +} + +#[tokio::test] +async fn a_stream_without_include_usage_still_reports_usage_to_the_hooks() { + let harness = harness().await; + let mut request = hi("claude"); + request["stream"] = json!(true); + let (status, body) = send(&harness.app, request).await; + assert_eq!(status, 200, "{body}"); + let (frames, done) = frames(&body); + assert!(done, "{body}"); + assert!( + frames + .iter() + .all(|frame| frame.get("usage").is_none_or(Value::is_null)), + "{body}" + ); + let seen = seen(&harness).await; + assert_eq!(seen.len(), 1, "{seen:?}"); + assert_eq!(seen[0].cost, Some(cost(10, 12))); +} + +#[tokio::test] +async fn a_tool_call_round_trips_through_tool_use_and_tool_result() { + let harness = harness().await; + let tools = json!([{"type": "function", "function": { + "name": "get_weather", + "description": "Weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]} + }}]); + let question = json!({"role": "user", "content": "Weather in Paris?"}); + let (status, body) = send( + &harness.app, + json!({"model": "claude", "messages": [question], "tools": tools, "tool_choice": "auto", "max_tokens": 64}), + ) + .await; + assert_eq!(status, 200, "{body}"); + let answer: Value = serde_json::from_str(&body).unwrap(); + assert_eq!(answer["choices"][0]["finish_reason"], "tool_calls"); + let call = &answer["choices"][0]["message"]["tool_calls"][0]; + assert_eq!(call["id"], "toolu_1"); + assert_eq!(call["type"], "function"); + assert_eq!(call["function"]["name"], "get_weather"); + assert_eq!(call["function"]["arguments"], "{\"city\": \"Paris\"}"); + + let first = harness.calls.lock().unwrap()[0].body.clone(); + assert_eq!( + first, + json!({ + "messages": [{"role": "user", "content": [{"type": "text", "text": "Weather in Paris?"}]}], + "tools": [{ + "name": "get_weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + "type": "custom", + "description": "Weather for a city" + }], + "tool_choice": {"type": "auto"}, + "max_tokens": 64, + "anthropic_version": "vertex-2023-10-16", + }) + ); + + let (status, body) = send( + &harness.app, + json!({"model": "claude", "tools": tools, "max_tokens": 64, "messages": [ + question, + {"role": "assistant", "content": null, "tool_calls": [call]}, + {"role": "tool", "tool_call_id": "toolu_1", "content": "sunny"} + ]}), + ) + .await; + assert_eq!(status, 200, "{body}"); + let answer: Value = serde_json::from_str(&body).unwrap(); + assert_eq!( + answer["choices"][0]["message"]["content"], + "It is sunny in Paris" + ); + let second = harness.calls.lock().unwrap()[1].body.clone(); + assert_eq!( + second["messages"], + json!([ + {"role": "user", "content": [{"type": "text", "text": "Weather in Paris?"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny"}]} + ]) + ); + + let seen = seen(&harness).await; + assert_eq!(seen.len(), 2, "{seen:?}"); + assert!(seen.iter().all(|seen| seen.cost == Some(cost(10, 4)))); +} + +#[tokio::test] +async fn reasoning_effort_becomes_a_thinking_budget_and_comes_back_as_reasoning() { + let harness = harness().await; + let (status, body) = send( + &harness.app, + json!({"model": "claude", "messages": [{"role": "user", "content": "hi"}], "reasoning_effort": "low"}), + ) + .await; + assert_eq!(status, 200, "{body}"); + let answer: Value = serde_json::from_str(&body).unwrap(); + let message = &answer["choices"][0]["message"]; + assert_eq!(message["content"], "Hello"); + assert_eq!(message["reasoning_content"], "The user says hi."); + assert_eq!( + message["thinking_blocks"], + json!([{"type": "thinking", "thinking": "The user says hi.", "signature": "sig-1"}]) + ); + + let sent = harness.calls.lock().unwrap()[0].body.clone(); + assert_eq!( + sent, + json!({ + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + "thinking": {"type": "enabled", "budget_tokens": 1024}, + "max_tokens": 1024 + 4096, + "anthropic_version": "vertex-2023-10-16", + }) + ); + let seen = seen(&harness).await; + assert_eq!(seen[0].cost, Some(cost(10, 4))); +} + +#[tokio::test] +async fn an_anthropic_api_only_param_is_refused() { + let harness = harness().await; + let mut request = hi("claude"); + request["service_tier"] = json!("auto"); + let (status, body) = send(&harness.app, request).await; + assert_eq!(status, 400, "{body}"); + assert!(harness.calls.lock().unwrap().is_empty()); +} diff --git a/litellm-rust/crates/inference-chat/src/common_utils.rs b/litellm-rust/crates/inference-chat/src/common_utils.rs index 6acd8cfa32a..89834504fcd 100644 --- a/litellm-rust/crates/inference-chat/src/common_utils.rs +++ b/litellm-rust/crates/inference-chat/src/common_utils.rs @@ -6,8 +6,13 @@ use litellm_llms::{ bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, gemini::chat::transformation::GOOGLE_AI_STUDIO_GEMINI_CHAT_COMPLETIONS_CONFIG, openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG, - vertex_ai::gemini::vertex_and_google_ai_studio_gemini::{ - VERTEX_GEMINI_CHAT_COMPLETIONS_CONFIG, is_vertex_gemini_model, + vertex_ai::{ + gemini::vertex_and_google_ai_studio_gemini::{ + VERTEX_GEMINI_CHAT_COMPLETIONS_CONFIG, is_vertex_gemini_model, + }, + vertex_ai_partner_models::anthropic::transformation::{ + VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG, is_vertex_claude_model, + }, }, }; use serde_json::{Map, Value}; @@ -21,6 +26,7 @@ pub(super) enum ChatProvider { Bedrock, Gemini, OpenaiLike, + VertexAnthropic, VertexGemini, } @@ -31,12 +37,13 @@ impl ChatProvider { Self::Bedrock => &BEDROCK_CHAT_COMPLETIONS_CONFIG, Self::Gemini => &GOOGLE_AI_STUDIO_GEMINI_CHAT_COMPLETIONS_CONFIG, Self::OpenaiLike => &OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG, + Self::VertexAnthropic => &VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG, Self::VertexGemini => &VERTEX_GEMINI_CHAT_COMPLETIONS_CONFIG, } } } -/// Vertex AI serves only its Gemini models here; Claude and partner models stay refused. +/// Vertex AI serves its Gemini and Claude models here; its other partner models stay refused. pub(super) fn chat_completions_provider( provider: LlmProviders, model: &str, @@ -47,6 +54,9 @@ pub(super) fn chat_completions_provider( LlmProviders::Gemini => Some(ChatProvider::Gemini), LlmProviders::OpenaiLike => Some(ChatProvider::OpenaiLike), LlmProviders::VertexAi if is_vertex_gemini_model(model) => Some(ChatProvider::VertexGemini), + LlmProviders::VertexAi if is_vertex_claude_model(model) => { + Some(ChatProvider::VertexAnthropic) + } _ => None, } } @@ -66,10 +76,15 @@ mod tests { #[rstest] #[case::studio(LlmProviders::Gemini, "gemini-2.5-flash", Some("gemini"))] #[case::vertex_gemini(LlmProviders::VertexAi, "gemini-2.5-pro", Some("vertex_gemini"))] - #[case::vertex_claude(LlmProviders::VertexAi, "claude-sonnet-4@20250514", None)] + #[case::vertex_claude( + LlmProviders::VertexAi, + "claude-sonnet-4@20250514", + Some("vertex_claude") + )] #[case::vertex_partner(LlmProviders::VertexAi, "meta/llama-3.1-405b-instruct-maas", None)] + #[case::vertex_mistral(LlmProviders::VertexAi, "mistral-large@2407", None)] #[case::vertex_gemma(LlmProviders::VertexAi, "gemma/gemma-3-27b-it", None)] - fn vertex_routes_only_its_gemini_models( + fn vertex_routes_its_gemini_and_claude_models( #[case] provider: LlmProviders, #[case] model: &str, #[case] expected: Option<&str>, @@ -77,6 +92,7 @@ mod tests { let routed = chat_completions_provider(provider, model).map(|provider| match provider { ChatProvider::Gemini => "gemini", ChatProvider::VertexGemini => "vertex_gemini", + ChatProvider::VertexAnthropic => "vertex_claude", ChatProvider::Anthropic | ChatProvider::Bedrock | ChatProvider::OpenaiLike => "other", }); assert_eq!(routed, expected); diff --git a/litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md b/litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md index dab8abcc5ad..1b0af4c5c8c 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md +++ b/litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md @@ -2,3 +2,4 @@ - Chat Completions translated onto Anthropic Messages. Both format specs live under `llms-types/src/formats/` - `top_k` is not forwarded, because Python gates it per model inside `transform_request` +- `build_messages_request` serves every host of the translation; `AnthropicHost` picks the host's cost-map reads, `service_tier`/`speed` support and whether the Anthropic API betas are added (Vertex AI sets its own in `vertex_ai/vertex_ai_partner_models/anthropic`) diff --git a/litellm-rust/crates/llms/src/anthropic/chat/params.rs b/litellm-rust/crates/llms/src/anthropic/chat/params.rs index be507b1c46c..cd377b3385a 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/params.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/params.rs @@ -166,35 +166,150 @@ pub fn model_map_candidates(model: &str) -> Vec { candidates } +/// The host an Anthropic Messages request built from a chat call goes to. The request is the +/// same; the cost-map reads and a few params depend on the host, as Python's +/// `custom_llm_provider` on the config decides them. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AnthropicHost { + /// The Anthropic API, `AnthropicConfig`. + #[default] + Anthropic, + /// Claude on Vertex AI, `VertexAIAnthropicConfig`. + VertexAi, +} + +impl AnthropicHost { + /// The provider an unsupported-params error names. + pub fn provider(self) -> &'static str { + match self { + Self::Anthropic => "anthropic", + Self::VertexAi => "vertex_ai", + } + } +} + +/// `_strip_stable_vertex_version`: a trailing `-` removed. +fn strip_stable_vertex_version(model: &str) -> String { + let trimmed = model.trim_end_matches(|c: char| c.is_ascii_digit()); + match trimmed.strip_suffix('-') { + Some(head) if trimmed.len() < model.len() => head.to_owned(), + _ => model.to_owned(), + } +} + +/// The keys `get_model_info_helper` tries for a Vertex AI model, most specific first: the +/// `vertex_ai/` entry, the bare id, then both with a stable version suffix removed. +pub fn vertex_model_info_candidates(model: &str) -> Vec { + let stripped = strip_stable_vertex_version(model); + let mut candidates: Vec = Vec::new(); + for candidate in [ + format!("vertex_ai/{model}"), + model.to_owned(), + format!("vertex_ai/{stripped}"), + stripped, + ] { + if !candidates.contains(&candidate) { + candidates.push(candidate); + } + } + candidates +} + /// The cost-map reads a model's request rules make, through its lookup candidates. pub struct ModelCapabilities<'a> { pub model: String, pub context: ChatRequestContext<'a>, pub candidates: Vec, + pub host: AnthropicHost, + /// The keys a provider-aware lookup (`_get_provider_resolved_capability`) resolves the + /// model through; the first entry found answers. Empty when the host has none. + pub resolved: Vec, } impl<'a> ModelCapabilities<'a> { pub fn new(model: &str, context: &ChatRequestContext<'a>) -> Self { + Self::on(AnthropicHost::Anthropic, model, context) + } + + pub fn on(host: AnthropicHost, model: &str, context: &ChatRequestContext<'a>) -> Self { Self { model: model.to_owned(), context: *context, candidates: model_map_candidates(model), + host, + resolved: match host { + AnthropicHost::Anthropic => Vec::new(), + AnthropicHost::VertexAi => vertex_model_info_candidates(model), + }, } } /// `_get_model_capability`: the flag the first candidate the map knows sets. - pub fn flag(&self, key: &str) -> Result, Error> { + pub fn walk_flag(&self, key: &str) -> Result, Error> { capability(&self.context, &self.candidates, key) } + /// `_get_provider_resolved_capability`: the flag of the entry the host's own lookup + /// resolves the model to, `None` when that entry does not set it. + pub fn resolved_flag(&self, key: &str) -> Result, Error> { + if self.resolved.is_empty() { + return Ok(None); + } + let Some(info) = self.context.model_info else { + return match self.context.drop_params { + None => Err(Error::Unsupported( + "a request whose translation reads the model cost map", + )), + Some(_) => Ok(None), + }; + }; + Ok(self + .resolved + .iter() + .find_map(|candidate| info.entry(candidate)) + .and_then(|entry| entry.get(key).and_then(Value::as_bool))) + } + + /// `_supports_model_capability`'s answer: the host's resolved flag when it sets one, else + /// the model-map walk. + pub fn flag(&self, key: &str) -> Result, Error> { + match self.resolved_flag(key)? { + Some(flag) => Ok(Some(flag)), + None => self.walk_flag(key), + } + } + /// `_supports_model_capability`: the flag is set to `true`. pub fn supports(&self, key: &str) -> Result { Ok(self.flag(key)? == Some(true)) } + /// `uses_native_structured_output`: Vertex AI reads only its own entry for it. + pub fn native_structured_output(&self) -> Result { + match self.host { + AnthropicHost::Anthropic => self.supports("supports_native_structured_output"), + AnthropicHost::VertexAi => { + Ok(self.resolved_flag("supports_native_structured_output")? == Some(true)) + } + } + } + + /// `_model_supports_effort_param`: `output_config` or any reasoning effort level. + pub fn supports_effort(&self) -> Result { + if self.supports("supports_output_config")? { + return Ok(true); + } + for level in ["low", "minimal", "medium", "high", "xhigh", "max"] { + if self.supports(&format!("supports_{level}_reasoning_effort"))? { + return Ok(true); + } + } + Ok(false) + } + /// `_supports_sampling_params`. pub fn supports_sampling(&self) -> Result { - if let Some(flag) = self.flag("supports_sampling_params")? { + if let Some(flag) = self.walk_flag("supports_sampling_params")? { return Ok(flag); } let lower = self.model.to_lowercase(); @@ -207,7 +322,7 @@ impl<'a> ModelCapabilities<'a> { } pub fn forced_tool_use_unsupported(&self) -> Result { - Ok(self.flag("supports_forced_tool_use")? == Some(false)) + Ok(self.walk_flag("supports_forced_tool_use")? == Some(false)) } pub fn is_adaptive(&self) -> Result { @@ -943,15 +1058,22 @@ pub fn supported_params(caps: &ModelCapabilities<'_>) -> Result { + let mut reasoning_keys = caps.candidates.clone(); + reasoning_keys.insert(1, format!("anthropic/{}", caps.model)); + capability(&caps.context, &reasoning_keys, "supports_reasoning")? == Some(true) + } + // `supports_reasoning(model, "vertex_ai")` reads the entry Vertex resolves to. + AnthropicHost::VertexAi => caps.resolved_flag("supports_reasoning")? == Some(true), + }; + if caps.model.contains("claude-3-7-sonnet") || caps.is_adaptive()? || supports_reasoning { params.extend(["thinking", "reasoning_effort"]); } - params.push("service_tier"); + // `SERVICE_TIER_PROVIDERS`: Vertex AI rejects the field. + if caps.host == AnthropicHost::Anthropic { + params.push("service_tier"); + } Ok(params) } @@ -1001,7 +1123,7 @@ pub fn map_openai_params( let Some(format) = value.as_object() else { continue; }; - if caps.supports("supports_native_structured_output")? { + if caps.native_structured_output()? { if let Some(output_format) = output_format(format) { out.insert("output_format".into(), output_format); } diff --git a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs index d45e1dfe6de..e281a07bcc4 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs @@ -26,8 +26,8 @@ use crate::{ split_leading_system_run, system_blocks, }, params::{ - MappedParams, ModelCapabilities, apply_sampling_param, json_tool_schema, - map_openai_params, map_tools, supported_params, valid_user_id, + AnthropicHost, MappedParams, ModelCapabilities, apply_sampling_param, + json_tool_schema, map_openai_params, map_tools, supported_params, valid_user_id, }, }, common_utils::{ @@ -369,18 +369,7 @@ fn apply_output_config( if output_config.is_empty() { return Ok(()); } - let supports_effort = || -> Result { - if caps.supports("supports_output_config")? { - return Ok(true); - } - for level in ["low", "minimal", "medium", "high", "xhigh", "max"] { - if caps.supports(&format!("supports_{level}_reasoning_effort"))? { - return Ok(true); - } - } - Ok(false) - }; - if drops && output_config.keys().any(|key| key != "format") && !supports_effort()? { + if drops && output_config.keys().any(|key| key != "format") && !caps.supports_effort()? { match output_config.get("format") { None | Some(Value::Null) => { body.remove("output_config"); @@ -447,61 +436,65 @@ fn default_max_tokens(model: &str, context: &ChatRequestContext<'_>) -> Result, - optional_params: Map, - context: &ChatRequestContext<'_>, - ) -> Result { - let caps = ModelCapabilities::new(model, context); - let request_metadata = optional_params.get("metadata").cloned(); - let sorted = sort_params( - optional_params, - &[], - context, - |_| supported_params(&caps), - "anthropic", - model, - )?; - let drop = sorted.drop; - let MappedParams { - mut params, - json_mode, - } = map_openai_params(&sorted.openai, &caps, drop)?; - for name in &sorted.allowed { - if let Some(value) = sorted.openai.get(name) - && !params.contains_key(name) - { - params.insert(name.clone(), value.clone()); - } - } - params.extend(sorted.provider); - - let mut messages: Vec = messages - .iter() - .map(serde_json::to_value) - .collect::>() - .map_err(|error| Error::InvalidRequest(ErrorDetail::invalid("messages", error)))?; - let original_messages = messages.clone(); - let prefix_prompt = prefix_prompt(&messages); - - if !params.contains_key("tools") - && messages.iter().any(|message| { - message - .get("tool_calls") - .is_some_and(|calls| !calls.is_null()) - }) +/// `AnthropicConfig.transform_request` after `map_openai_params`, for the host `caps` names: +/// the Messages body, `model` included, and the `anthropic-beta` values the Anthropic API +/// needs. A host with its own beta rules (Vertex AI) gets no header here and sets its own. +pub fn build_messages_request( + model: &str, + messages: Vec, + optional_params: Map, + context: &ChatRequestContext<'_>, + host: AnthropicHost, + config_params: &[&str], +) -> Result { + let caps = ModelCapabilities::on(host, model, context); + let request_metadata = optional_params.get("metadata").cloned(); + let sorted = sort_params( + optional_params, + config_params, + context, + |_| supported_params(&caps), + host.provider(), + model, + )?; + let drop = sorted.drop; + let MappedParams { + mut params, + json_mode, + } = map_openai_params(&sorted.openai, &caps, drop)?; + for name in &sorted.allowed { + if let Some(value) = sorted.openai.get(name) + && !params.contains_key(name) { - params.insert("tools".into(), Value::Array(map_tools(&dummy_tool())?)); + params.insert(name.clone(), value.clone()); } + } + params.extend(sorted.provider); - // `_maybe_drop_speed_param`: fast mode is the Anthropic API's own, per exact entry. - if let Some(speed) = params.get("speed").cloned() - && capability(context, &[model.to_owned()], "supports_speed")? != Some(true) - { - drop.unsupported(|| { + let mut messages: Vec = messages + .iter() + .map(serde_json::to_value) + .collect::>() + .map_err(|error| Error::InvalidRequest(ErrorDetail::invalid("messages", error)))?; + let original_messages = messages.clone(); + let prefix_prompt = prefix_prompt(&messages); + + if !params.contains_key("tools") + && messages.iter().any(|message| { + message + .get("tool_calls") + .is_some_and(|calls| !calls.is_null()) + }) + { + params.insert("tools".into(), Value::Array(map_tools(&dummy_tool())?)); + } + + // `_maybe_drop_speed_param`: fast mode is the Anthropic API's own, per exact entry. + if let Some(speed) = params.get("speed").cloned() + && (host != AnthropicHost::Anthropic + || capability(context, &[model.to_owned()], "supports_speed")? != Some(true)) + { + drop.unsupported(|| { let shown = match &speed { Value::String(text) => format!("'{text}'"), other => python_dumps(other), @@ -510,92 +503,109 @@ impl AnthropicConfig { "{model} does not support speed={shown}. To drop unsupported params, set `litellm.drop_params = True`." ))) })?; - params.remove("speed"); + params.remove("speed"); + } + + // `maybe_drop_disabled_thinking`. + if params + .get("thinking") + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + == Some("disabled") + && caps.supports("thinking_always_on")? + { + params.remove("thinking"); + } + + let tool_name_reverse_map = sanitize_tool_names(&mut params, &mut messages); + + let (leading, later) = split_leading_system_run(messages); + let system = system_blocks(&leading); + if !system.is_empty() { + params.insert("system".into(), Value::Array(system)); + } + let anthropic_messages = anthropic_messages_pt(place_mid_conversation_system(later))?; + + // `update_headers_with_optional_anthropic_beta` skips a Vertex request, and Vertex's + // `get_anthropic_headers` sends none of these. + let betas = match host { + AnthropicHost::Anthropic => { + request_betas(¶ms, &original_messages, &anthropic_messages, &caps)? } + AnthropicHost::VertexAi => Vec::new(), + }; - // `maybe_drop_disabled_thinking`. - if params - .get("thinking") - .and_then(|thinking| thinking.get("type")) - .and_then(Value::as_str) - == Some("disabled") - && caps.supports("thinking_always_on")? - { - params.remove("thinking"); - } + if !params.contains_key("max_tokens") { + params.insert("max_tokens".into(), default_max_tokens(model, context)?); + } - let tool_name_reverse_map = sanitize_tool_names(&mut params, &mut messages); - - let (leading, later) = split_leading_system_run(messages); - let system = system_blocks(&leading); - if !system.is_empty() { - params.insert("system".into(), Value::Array(system)); - } - let anthropic_messages = anthropic_messages_pt(place_mid_conversation_system(later))?; - - let betas = request_betas(¶ms, &original_messages, &anthropic_messages, &caps)?; - - if !params.contains_key("max_tokens") { - params.insert("max_tokens".into(), default_max_tokens(model, context)?); - } - - if let Some(user_id) = request_metadata - .as_ref() - .and_then(|metadata| metadata.get("user_id")) - .and_then(Value::as_str) - .filter(|user_id| valid_user_id(user_id)) - { - params.insert("metadata".into(), json!({"user_id": user_id})); - } - if let Some(Value::Object(metadata)) = params.get("metadata") { - match metadata.get("user_id").filter(|user_id| !user_id.is_null()) { - Some(user_id) => { - let reduced = json!({"user_id": user_id}); - params.insert("metadata".into(), reduced); - } - None => { - params.remove("metadata"); - } + if let Some(user_id) = request_metadata + .as_ref() + .and_then(|metadata| metadata.get("user_id")) + .and_then(Value::as_str) + .filter(|user_id| valid_user_id(user_id)) + { + params.insert("metadata".into(), json!({"user_id": user_id})); + } + if let Some(Value::Object(metadata)) = params.get("metadata") { + match metadata.get("user_id").filter(|user_id| !user_id.is_null()) { + Some(user_id) => { + let reduced = json!({"user_id": user_id}); + params.insert("metadata".into(), reduced); + } + None => { + params.remove("metadata"); } } - params.remove("is_vertex_request"); - params.remove("client_metadata"); - params.remove("json_mode"); - - if let Some(top_k) = params.remove("top_k").filter(|top_k| !top_k.is_null()) { - apply_sampling_param(&mut params, &caps, drop, "top_k", &top_k, "top_k")?; - } - - let speed = params - .get("speed") - .and_then(Value::as_str) - .map(str::to_owned); - let response_format_schema = json_mode - .then(|| json_tool_schema(params.get("tools"))) - .flatten(); - - let mut body = Map::from_iter([ - ("model".to_owned(), json!(model)), - ("messages".to_owned(), Value::Array(anthropic_messages)), - ]); - body.extend(params); - apply_output_config(&mut body, &caps, drop.drops())?; - - Ok(ProviderChatRequestData { - body: Value::Object(body), - stream_shape: StreamShape { - json_mode, - speed, - tool_name_reverse_map, - response_format_schema, - prefix_prompt, - }, - headers: match betas.is_empty() { - true => Vec::new(), - false => vec![(BETA_HEADER.to_owned(), betas.join(", "))], - }, - }) } + params.remove("is_vertex_request"); + params.remove("client_metadata"); + params.remove("json_mode"); + + if let Some(top_k) = params.remove("top_k").filter(|top_k| !top_k.is_null()) { + apply_sampling_param(&mut params, &caps, drop, "top_k", &top_k, "top_k")?; + } + + let speed = params + .get("speed") + .and_then(Value::as_str) + .map(str::to_owned); + let response_format_schema = json_mode + .then(|| json_tool_schema(params.get("tools"))) + .flatten(); + + let mut body = Map::from_iter([ + ("model".to_owned(), json!(model)), + ("messages".to_owned(), Value::Array(anthropic_messages)), + ]); + body.extend(params); + apply_output_config(&mut body, &caps, drop.drops())?; + + Ok(ProviderChatRequestData { + body: Value::Object(body), + stream_shape: StreamShape { + json_mode, + speed, + tool_name_reverse_map, + response_format_schema, + prefix_prompt, + }, + headers: match betas.is_empty() { + true => Vec::new(), + false => vec![(BETA_HEADER.to_owned(), betas.join(", "))], + }, + }) +} + +/// `transform_parsed_response`: a Messages API answer as a chat completion. +pub fn transform_messages_response( + body: Value, + shape: &StreamShape, +) -> Result { + let body: ResponseProjection = serde_json::from_value(body).map_err(|error| { + Error::InvalidResponse(ErrorDetail::invalid("messages response", error)) + })?; + anthropic_response(body, shape) } /// `convert_tool_use_to_openai_format`, the name put back the way the caller spelled it. @@ -819,7 +829,14 @@ impl BaseConfig for AnthropicConfig { optional_params: Map, context: &ChatRequestContext<'_>, ) -> Result { - self.build_request(model, messages, optional_params, context) + build_messages_request( + model, + messages, + optional_params, + context, + AnthropicHost::Anthropic, + &[], + ) } fn transform_response( @@ -828,10 +845,7 @@ impl BaseConfig for AnthropicConfig { response: ProviderChatResponseData, shape: &StreamShape, ) -> Result { - let body: ResponseProjection = serde_json::from_value(response.body).map_err(|error| { - Error::InvalidResponse(ErrorDetail::invalid("messages response", error)) - })?; - anthropic_response(body, shape) + transform_messages_response(response.body, shape) } /// `inlines_remote_media`: Anthropic fetches an `https` image itself, not an `http` one. diff --git a/litellm-rust/crates/llms/src/vertex_ai/mod.rs b/litellm-rust/crates/llms/src/vertex_ai/mod.rs index 9fb560df0bd..561fd3bcc79 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/mod.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/mod.rs @@ -2,3 +2,4 @@ pub mod common_utils; pub mod gemini; pub mod messages; pub mod ocr; +pub mod vertex_ai_partner_models; diff --git a/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/anthropic/AGENTS.md b/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/anthropic/AGENTS.md new file mode 100644 index 00000000000..ce3acb08b20 --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/anthropic/AGENTS.md @@ -0,0 +1,12 @@ +# rules + +- Claude on Vertex AI for Chat Completions: the `anthropic/chat` translation built with `AnthropicHost::VertexAi`, addressed to the publisher `rawPredict` URL and authenticated with a Google access token +- The body drops `model`, carries `anthropic_version: vertex-2023-10-16`, sanitizes `output_config` and names its betas in `anthropic_beta`, filtered to the ones Vertex accepts +- Capability reads resolve the `vertex_ai/` cost map entry first; fast mode and `service_tier` are the Anthropic API's own + +# references + +- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude +- litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +- litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +- litellm/llms/anthropic/chat/transformation.py diff --git a/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/anthropic/mod.rs b/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/anthropic/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/anthropic/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/anthropic/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/anthropic/transformation.rs new file mode 100644 index 00000000000..0fbc2a9203b --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/anthropic/transformation.rs @@ -0,0 +1,847 @@ +//! Python's `VertexAIAnthropicConfig` for `/chat/completions`: Claude on Vertex AI, reached +//! through `VertexAIPartnerModels.completion`. The request is `AnthropicConfig`'s, built for the +//! `vertex_ai` host, then addressed to the publisher URL: no `model` in the body, Vertex's +//! `anthropic_version`, a sanitized `output_config`, and the betas in `anthropic_beta`. + +use std::collections::BTreeMap; + +use litellm_auth::VertexParams; +use litellm_auth_gcp::{ + VertexConfig, get_vertex_ai_location, get_vertex_ai_project, + get_vertex_ai_project_from_credentials, +}; +use litellm_llms_types::{ + formats::chat_completions::{ChatCompletionsResponse, ChatMessage}, + providers::{ + anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet, DEFAULT_HEADERS}, + vertex_ai::{ + ANTHROPIC_VERSION, DEFAULT_LOCATION, GLOBAL_API_BASE, RAW_PREDICT, STREAM_RAW_PREDICT, + }, + }, +}; +use serde_json::{Map, Value, json}; + +use crate::{ + Error, + anthropic::{ + beta_headers::BetaPolicy, + chat::{ + handler::ModelResponseIterator, + params::{AnthropicHost, ModelCapabilities}, + transformation::{build_messages_request, transform_messages_response}, + }, + }, + base_llm::{ + auth::{AuthScheme, Headers, ValidatedEnvironment}, + chat::{ + media::{data_url, remote_urls}, + params::ChatRequestContext, + streaming::{ChatStream, StreamShape}, + transformation::{ + BaseConfig, ProviderChatRequestData, ProviderChatResponseData, STREAM_PARAM, + Unsupported, + }, + }, + messages::streaming::event_stream as messages_event_stream, + }, + vertex_ai::{ + common_utils::with_query, + gemini::vertex_and_google_ai_studio_gemini::{CONFIG_PARAMS, VERTEXAI_API_BASE_ENV}, + messages::transformation::complete_vertex_anthropic_url, + }, +}; + +/// `PartnerModelPrefixes`: the ids `is_vertex_partner_model` sends to the partner handler. +const PARTNER_MODEL_PREFIXES: &[&str] = &[ + "meta/", + "deepseek-ai", + "mistral", + "codestral", + "jamba", + "claude", + "qwen", + "openai/gpt-oss-", + "minimaxai/", + "moonshotai/", + "zai-org/", + "google/gemma-", +]; + +/// `should_use_openai_handler`: the partners served through Vertex's OpenAI-compatible API. +const OPENAI_LIKE_PARTNERS: &[&str] = &[ + "llama", + "deepseek-ai", + "qwen", + "openai/gpt-oss-", + "minimaxai/", + "moonshotai/", + "zai-org/", + "google/gemma-", +]; + +/// The tool type prefix `is_web_search_tool_used` looks for. +const WEB_SEARCH_TOOL_PREFIX: &str = "web_search"; + +/// `get_vertex_ai_model_route(model) == PARTNER_MODELS` and the partner handler picking its +/// Claude branch: a partner model that is neither OpenAI-compatible, Mistral, nor AI21. +pub fn is_vertex_claude_model(model: &str) -> bool { + !model.contains("agent_engine/") + && PARTNER_MODEL_PREFIXES + .iter() + .any(|prefix| model.starts_with(prefix)) + && !OPENAI_LIKE_PARTNERS + .iter() + .any(|partner| model.contains(partner)) + && !model.contains("mistral") + && !model.contains("codestral") + && !model.contains("jamba") + && model.contains("claude") +} + +/// `VertexAIAnthropicConfig`: Claude on Vertex AI, addressed to a project and location and +/// authenticated with a Google access token. +pub struct VertexAiAnthropicConfig; + +pub const VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG: VertexAiAnthropicConfig = + VertexAiAnthropicConfig; + +fn vertex_config(optional_params: &Map) -> Result { + Ok(VertexConfig::from_sourced_optional_params( + optional_params, + &BTreeMap::new(), + )?) +} + +/// Whether `url` is on a host `get_vertex_base_url` builds. Any other host is a custom +/// `api_base`, which `_check_custom_proxy` asks for `alt=sse` when streaming. +fn is_vertex_host(url: &str) -> bool { + let rest = url.split_once("://").map_or(url, |(_, rest)| rest); + let host = rest.split(['/', '?', '#']).next().unwrap_or(rest); + let global = GLOBAL_API_BASE + .split_once("://") + .map_or(GLOBAL_API_BASE, |(_, host)| host); + host == global + || host.ends_with("-aiplatform.googleapis.com") + || (host.starts_with("aiplatform.") && host.ends_with(".rep.googleapis.com")) +} + +/// The same call made as a stream: `create_vertex_url` with `stream=True`, which names +/// `streamRawPredict`, plus `alt=sse` on a custom `api_base`. +pub fn streaming_url(url: &str) -> String { + let method = format!(":{RAW_PREDICT}"); + let (head, query) = url + .split_once('?') + .map_or((url, None), |(head, query)| (head, Some(query))); + let head = match head.strip_suffix(&method) { + Some(stem) => format!("{stem}:{STREAM_RAW_PREDICT}"), + None => head.to_owned(), + }; + let swapped = match query { + Some(query) => format!("{head}?{query}"), + None => head, + }; + match is_vertex_host(&swapped) { + true => swapped, + false => with_query(&swapped, "alt=sse"), + } +} + +/// The betas a caller named in the request's `extra_headers`, as a comma separated +/// `anthropic-beta` string or a list. +fn extra_header_betas(optional_params: &Map) -> Vec { + let value = optional_params + .get("extra_headers") + .and_then(|headers| headers.get(BETA_HEADER)); + match value { + Some(Value::String(text)) => text + .split(',') + .map(str::trim) + .filter(|beta| !beta.is_empty()) + .map(str::to_owned) + .collect(), + Some(Value::Array(items)) => items + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + _ => Vec::new(), + } +} + +/// `_add_context_management_beta_headers`: `compact_20260112` edits need the compaction beta, +/// any other edit the context management one. +fn context_management_betas(context_management: Option<&Value>) -> Vec { + let Some(Value::Object(context_management)) = context_management else { + return Vec::new(); + }; + let edits = context_management + .get("edits") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + let compact = + |edit: &Value| edit.get("type").and_then(Value::as_str) == Some("compact_20260112"); + [ + edits + .iter() + .any(compact) + .then_some(AnthropicBeta::Compact20260112), + edits + .iter() + .any(|edit| !compact(edit)) + .then_some(AnthropicBeta::ContextManagement20250627), + ] + .into_iter() + .flatten() + .collect() +} + +/// `filter_and_transform_beta_headers` for Vertex AI: the betas it accepts, under its names. +fn accepted_on_vertex(betas: &BetaSet) -> BetaSet { + betas + .iter() + .filter_map(|beta| beta.on(BetaProvider::VertexAi)) + .collect() +} + +/// `sanitize_vertex_anthropic_output_params`: `output_format` folded into +/// `output_config.format`, `effort` kept only for models that accept it, and an emptied or +/// malformed `output_config` left out. +fn sanitize_output_params( + body: &mut Map, + caps: &ModelCapabilities<'_>, +) -> Result<(), Error> { + let output_format = body + .remove("output_format") + .filter(|value| !value.is_null()); + let output_config = match body.get("output_config") { + Some(Value::Object(config)) => Some(config.clone()), + _ => None, + }; + let output_config = match (output_config, output_format) { + (config, Some(format)) => { + let mut config = config.unwrap_or_default(); + config.insert("format".into(), format); + Some(config) + } + (config, None) => config, + }; + let Some(mut output_config) = output_config else { + body.remove("output_config"); + return Ok(()); + }; + if output_config.contains_key("effort") && !caps.supports_effort()? { + output_config.remove("effort"); + } + match output_config.is_empty() { + true => { + body.remove("output_config"); + } + false => { + body.insert("output_config".into(), Value::Object(output_config)); + } + } + Ok(()) +} + +impl VertexAiAnthropicConfig { + fn build_request( + &self, + model: &str, + messages: Vec, + mut optional_params: Map, + context: &ChatRequestContext<'_>, + ) -> Result { + let caller_betas = extra_header_betas(&optional_params); + // `VertexAIPartnerModels.completion` adds the version after the mapped params. + optional_params.insert("anthropic_version".into(), json!(ANTHROPIC_VERSION)); + let built = build_messages_request( + model, + messages, + optional_params, + context, + AnthropicHost::VertexAi, + CONFIG_PARAMS, + )?; + let Value::Object(mut body) = built.body else { + return Err(Error::InvalidRequest( + "the Messages request is not an object".into(), + )); + }; + body.remove("model"); + body.remove("extra_headers"); + let caps = ModelCapabilities::on(AnthropicHost::VertexAi, model, context); + sanitize_output_params(&mut body, &caps)?; + + let betas: BetaSet = context_management_betas(body.get("context_management")) + .into_iter() + .chain( + caller_betas + .iter() + .map(|beta| beta.parse().unwrap_or_else(|never| match never {})), + ) + .collect(); + let web_search = body + .get("tools") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|tool| { + tool.get("type") + .and_then(Value::as_str) + .is_some_and(|kind| kind.starts_with(WEB_SEARCH_TOOL_PREFIX)) + }); + // `get_anthropic_headers` sends Vertex only the web search beta; the request's own + // betas replace it when there are any. + let header_betas = match (betas.is_empty(), web_search) { + (false, _) => accepted_on_vertex(&betas), + (true, true) => { + accepted_on_vertex(&[AnthropicBeta::WebSearch20250305].into_iter().collect()) + } + (true, false) => BetaSet::default(), + }; + let body_betas = accepted_on_vertex(&betas); + if !body_betas.is_empty() { + body.insert( + "anthropic_beta".into(), + Value::Array(body_betas.iter().map(|beta| json!(beta.as_str())).collect()), + ); + } + Ok(ProviderChatRequestData { + body: Value::Object(body), + stream_shape: built.stream_shape, + headers: match header_betas.is_empty() { + true => Vec::new(), + false => vec![(BETA_HEADER.to_owned(), header_betas.to_string())], + }, + }) + } +} + +impl BaseConfig for VertexAiAnthropicConfig { + fn secret_names(&self) -> Vec<&'static str> { + litellm_auth_gcp::secret_names() + .iter() + .copied() + .chain([VERTEXAI_API_BASE_ENV]) + .collect() + } + + /// `get_complete_vertex_url` for the Claude partner, with `VERTEXAI_API_BASE` as the + /// fallback base `completion` reads. + fn get_complete_url( + &self, + api_base: Option<&str>, + model: &str, + optional_params: &Map, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let config = vertex_config(optional_params)?; + let project = get_vertex_ai_project(&config, env_lookup) + .or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup)) + .ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?; + let location = get_vertex_ai_location(&config, env_lookup) + .unwrap_or_else(|| DEFAULT_LOCATION.to_owned()); + let api_base = api_base + .map(str::to_owned) + .or_else(|| env_lookup(VERTEXAI_API_BASE_ENV)) + .filter(|base| !base.trim().is_empty()); + complete_vertex_anthropic_url(api_base.as_deref(), &project, &location, model, false) + } + + fn transform_request( + &self, + model: &str, + messages: Vec, + optional_params: Map, + context: &ChatRequestContext<'_>, + ) -> Result { + self.build_request(model, messages, optional_params, context) + } + + /// `transform_response` names the requested model, not the one Vertex reports. + fn transform_response( + &self, + model: &str, + response: ProviderChatResponseData, + shape: &StreamShape, + ) -> Result { + let mut body = response.body; + if let Value::Object(fields) = &mut body { + fields.insert("model".into(), json!(model)); + } + transform_messages_response(body, shape) + } + + /// `inline_remote_image_urls`: Vertex takes no image by URL, so every remote image is + /// downloaded. + fn remote_media(&self, messages: &[ChatMessage]) -> Vec { + remote_urls(messages, &["user"], false, |url| { + (url.starts_with("http://") || url.starts_with("https://")) && data_url(url).is_none() + }) + } + + /// Python's partner handler sets `Authorization: Bearer ` over whatever was + /// forwarded, so the Google token is always the credential. A forwarded `anthropic-beta` + /// keeps only what Vertex accepts. + fn validate_environment( + &self, + headers: Headers, + _api_key: Option<&str>, + _model: &str, + optional_params: &Map, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + Ok(ValidatedEnvironment { + headers: BetaPolicy::Filter(BetaProvider::VertexAi).apply(headers), + auth: AuthScheme::GcpAccessToken { + config: Box::new(vertex_config(optional_params)?), + }, + }) + } + + fn model_response_iterator(&self, shape: StreamShape) -> Option { + Some(ChatStream::new( + messages_event_stream, + ModelResponseIterator::new(shape), + )) + } + + /// `streamRawPredict` with `stream: true` in the body, after the other params. + fn streaming_request(&self, url: String, mut body: Value) -> Result<(String, Value), Error> { + if let Value::Object(fields) = &mut body { + fields.insert(STREAM_PARAM.into(), Value::Bool(true)); + } + Ok((streaming_url(&url), body)) + } + + fn default_headers(&self) -> &'static [(&'static str, &'static str)] { + DEFAULT_HEADERS + } + + fn config_params(&self) -> &'static [&'static str] { + CONFIG_PARAMS + } + + /// Every request is translated; a param or message Python refuses is refused here too. + fn unsupported_reason( + &self, + _messages: &[ChatMessage], + _optional_params: &Map, + ) -> Option { + None + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use rstest::rstest; + + use super::*; + use crate::base_llm::model_info::ModelInfo; + + struct CostMap(HashMap>); + + impl ModelInfo for CostMap { + fn entry(&self, model: &str) -> Option>> { + self.0.get(model).map(std::borrow::Cow::Borrowed) + } + + fn declares(&self, key: &str) -> bool { + self.0.values().any(|entry| entry.contains_key(key)) + } + } + + const SONNET: &str = "claude-sonnet-4-5@20250929"; + const OPUS: &str = "claude-opus-4-7"; + + fn cost_map() -> CostMap { + let entry = |value: Value| value.as_object().cloned().unwrap(); + CostMap(HashMap::from([ + ( + format!("vertex_ai/{SONNET}"), + entry(json!({"max_output_tokens": 64000, "supports_reasoning": true})), + ), + ( + "claude-sonnet-4-5".to_owned(), + entry( + json!({"max_output_tokens": 64000, "supports_native_structured_output": true}), + ), + ), + ( + format!("vertex_ai/{OPUS}"), + entry(json!({"supports_reasoning": true, "supports_adaptive_thinking": true})), + ), + ( + OPUS.to_owned(), + entry(json!({"supports_output_config": true, "supports_speed": true})), + ), + ( + "vertex_ai/claude-haiku-4-5@20251001".to_owned(), + entry(json!({"supports_reasoning": true})), + ), + ])) + } + + fn object(value: Value) -> Map { + value.as_object().cloned().expect("an object") + } + + fn request_with( + model: &str, + messages: Value, + params: Value, + drop_params: bool, + ) -> Result { + let info = cost_map(); + let context = ChatRequestContext { + drop_params: Some(drop_params), + model_info: Some(&info), + }; + VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG.transform_request( + model, + serde_json::from_value(messages).unwrap(), + object(params), + &context, + ) + } + + fn body(model: &str, params: Value) -> Value { + request_with(model, hi(), params, false) + .expect("request transforms") + .body + } + + fn hi() -> Value { + json!([{"role": "system", "content": "be terse"}, {"role": "user", "content": "hi"}]) + } + + fn no_env(_: &str) -> Option { + None + } + + #[rstest] + #[case::versioned("claude-sonnet-4-5@20250929", true)] + #[case::bare("claude-opus-4-7", true)] + #[case::gemini("gemini-2.5-pro", false)] + #[case::llama("meta/llama-3.1-405b-instruct-maas", false)] + #[case::mistral("mistral-large@2407", false)] + #[case::not_a_partner_prefix("my-claude", false)] + fn only_claude_partner_models_route_here(#[case] model: &str, #[case] expected: bool) { + assert_eq!(is_vertex_claude_model(model), expected); + } + + #[rstest] + fn builds_the_body_python_builds() { + let body = body( + SONNET, + json!({ + "max_tokens": 128, + "temperature": 0.2, + "stop": "END", + "vertex_project": "p", + "vertex_location": "us-east5", + "vertex_credentials": "{}" + }), + ); + assert_eq!( + body, + json!({ + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + "max_tokens": 128, + "temperature": 0.2, + "stop_sequences": ["END"], + "anthropic_version": "vertex-2023-10-16", + "system": [{"type": "text", "text": "be terse"}], + }) + ); + } + + #[rstest] + fn max_tokens_defaults_from_the_bare_entry_like_python() { + // `get_max_tokens` reads the bare id, which the cost map has no entry for. + assert_eq!(body(SONNET, json!({}))["max_tokens"], json!(4096)); + } + + #[rstest] + fn reasoning_effort_becomes_a_thinking_budget() { + let body = body(SONNET, json!({"reasoning_effort": "low"})); + assert_eq!( + body["thinking"], + json!({"type": "enabled", "budget_tokens": 1024}) + ); + assert_eq!(body["max_tokens"], json!(1024 + 4096)); + } + + #[rstest] + fn an_adaptive_model_gets_adaptive_thinking_and_its_effort() { + let body = body(OPUS, json!({"reasoning_effort": "high", "max_tokens": 64})); + assert_eq!( + body["thinking"], + json!({"type": "adaptive", "display": "summarized"}) + ); + assert_eq!(body["output_config"], json!({"effort": "high"})); + } + + #[rstest] + fn effort_is_dropped_for_a_model_that_takes_none() { + let body = body( + "claude-haiku-4-5@20251001", + json!({"output_config": {"effort": "low"}, "max_tokens": 64}), + ); + assert!(body.get("output_config").is_none(), "{body}"); + } + + #[rstest] + fn native_structured_output_is_read_from_the_vertex_entry_only() { + // The bare Anthropic entry supports it; the Vertex one does not say, so Python uses + // the forced JSON tool. + let request = request_with( + SONNET, + hi(), + json!({"response_format": {"type": "json_schema", "json_schema": {"name": "x", "schema": {"type": "object", "properties": {"a": {"type": "string"}}}}}, "max_tokens": 64}), + false, + ) + .unwrap(); + assert!(request.body.get("output_config").is_none()); + assert!(request.body.get("output_format").is_none()); + assert_eq!( + request.body["tool_choice"], + json!({"name": "json_tool_call", "type": "tool"}) + ); + assert!(request.stream_shape.json_mode); + } + + #[rstest] + #[case::service_tier(json!({"service_tier": "auto"}))] + #[case::speed(json!({"speed": "fast"}))] + fn anthropic_api_only_params_are_refused_or_dropped(#[case] params: Value) { + assert!(matches!( + request_with(OPUS, hi(), params.clone(), false), + Err(Error::InvalidRequest(_)) + )); + let body = request_with(OPUS, hi(), params, true).unwrap().body; + assert!(body.get("service_tier").is_none() && body.get("speed").is_none()); + } + + #[rstest] + fn context_management_names_its_betas_in_the_body_and_the_header() { + let request = request_with( + SONNET, + hi(), + json!({ + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "compact_20260112"}]}, + "extra_headers": {"anthropic-beta": "advanced-tool-use-2025-11-20, oauth-2025-04-20"}, + "max_tokens": 64 + }), + false, + ) + .unwrap(); + assert_eq!( + request.body["anthropic_beta"], + json!([ + "compact-2026-01-12", + "context-management-2025-06-27", + "tool-search-tool-2025-10-19" + ]) + ); + assert_eq!( + request.headers, + [( + "anthropic-beta".to_owned(), + "compact-2026-01-12,context-management-2025-06-27,tool-search-tool-2025-10-19" + .to_owned() + )] + ); + assert!(request.body.get("extra_headers").is_none()); + } + + #[rstest] + fn web_search_alone_sends_only_its_header() { + let request = request_with( + SONNET, + hi(), + json!({"web_search_options": {}, "max_tokens": 64}), + false, + ) + .unwrap(); + assert!(request.body.get("anthropic_beta").is_none()); + assert_eq!( + request.headers, + [( + "anthropic-beta".to_owned(), + "web-search-2025-03-05".to_owned() + )] + ); + } + + #[rstest] + fn without_betas_no_header_or_body_field() { + let request = request_with(SONNET, hi(), json!({"max_tokens": 8}), false).unwrap(); + assert!(request.headers.is_empty()); + assert!(request.body.get("anthropic_beta").is_none()); + } + + #[rstest] + fn tools_round_trip_into_anthropic_shape() { + let body = request_with( + SONNET, + json!([ + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": null, "tool_calls": [{"id": "toolu_1", "type": "function", "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"}}]}, + {"role": "tool", "tool_call_id": "toolu_1", "content": "sunny"} + ]), + json!({"tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}], "max_tokens": 64}), + false, + ) + .unwrap() + .body; + assert_eq!( + body["tools"], + json!([{"name": "get_weather", "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, "type": "custom"}]) + ); + assert_eq!( + body["messages"][1]["content"][0], + json!({"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}) + ); + assert_eq!(body["messages"][2]["content"][0]["type"], "tool_result"); + } + + fn url(api_base: Option<&str>, location: &str) -> String { + VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .get_complete_url( + api_base, + SONNET, + &object(json!({"vertex_project": "proj", "vertex_location": location})), + &no_env, + ) + .unwrap() + } + + const PATH: &str = "/v1/projects/proj/locations/{location}/publishers/anthropic/models/claude-sonnet-4-5@20250929:rawPredict"; + + #[rstest] + #[case::regional(None, "us-east5", "https://us-east5-aiplatform.googleapis.com")] + #[case::global(None, "global", "https://aiplatform.googleapis.com")] + #[case::custom_host(Some("http://127.0.0.1:9/"), "us-east5", "http://127.0.0.1:9")] + fn the_url_names_the_project_location_and_model( + #[case] api_base: Option<&str>, + #[case] location: &str, + #[case] host: &str, + ) { + assert_eq!( + url(api_base, location), + format!("{host}{}", PATH.replace("{location}", location)) + ); + } + + #[rstest] + fn the_location_defaults_to_us_central1() { + let url = VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .get_complete_url( + None, + SONNET, + &object(json!({"vertex_project": "proj"})), + &no_env, + ) + .unwrap(); + assert!( + url.starts_with("https://us-central1-aiplatform.googleapis.com/v1/projects/proj/locations/us-central1/"), + "{url}" + ); + } + + #[rstest] + #[case::regional( + "https://us-east5-aiplatform.googleapis.com/v1/x/models/m:rawPredict", + "https://us-east5-aiplatform.googleapis.com/v1/x/models/m:streamRawPredict" + )] + #[case::global( + "https://aiplatform.googleapis.com/v1/x/models/m:rawPredict", + "https://aiplatform.googleapis.com/v1/x/models/m:streamRawPredict" + )] + #[case::custom_base( + "http://127.0.0.1:9/v1/x/models/m:rawPredict", + "http://127.0.0.1:9/v1/x/models/m:streamRawPredict?alt=sse" + )] + #[case::custom_path( + "https://proxy.example/route:rawPredict", + "https://proxy.example/route:streamRawPredict?alt=sse" + )] + fn a_stream_calls_stream_raw_predict(#[case] url: &str, #[case] expected: &str) { + let (streamed, body) = VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .streaming_request(url.to_owned(), json!({"max_tokens": 1})) + .unwrap(); + assert_eq!(streamed, expected); + assert_eq!(body, json!({"max_tokens": 1, "stream": true})); + } + + #[rstest] + fn the_google_token_replaces_a_forwarded_bearer_and_betas_are_filtered() { + let validated = VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .validate_environment( + vec![ + ("Authorization".to_owned(), "Bearer caller".to_owned()), + ( + "anthropic-beta".to_owned(), + "oauth-2025-04-20,context-management-2025-06-27".to_owned(), + ), + ], + None, + SONNET, + &Map::new(), + &no_env, + ) + .unwrap(); + assert!(matches!(validated.auth, AuthScheme::GcpAccessToken { .. })); + assert!(validated.headers.contains(&( + "anthropic-beta".to_owned(), + "context-management-2025-06-27".to_owned() + ))); + } + + #[rstest] + fn the_response_names_the_requested_model() { + let response = VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .transform_response( + SONNET, + ProviderChatResponseData { + body: json!({ + "id": "msg_1", "type": "message", "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "hmm", "signature": "sig"}, + {"type": "text", "text": "hello"} + ], + "stop_reason": "end_turn", + "usage": {"input_tokens": 11, "output_tokens": 4, "cache_read_input_tokens": 2} + }), + }, + &StreamShape::default(), + ) + .unwrap(); + assert_eq!(response.model, SONNET); + let message = &response.choices[0].message; + assert_eq!(message.content.as_deref(), Some("hello")); + assert_eq!(message.reasoning_content.as_deref(), Some("hmm")); + assert_eq!(response.choices[0].finish_reason, "stop"); + let usage = &response.usage; + assert_eq!((usage.prompt_tokens, usage.completion_tokens), (13, 4)); + } + + #[rstest] + fn every_remote_image_is_downloaded() { + let messages: Vec = + serde_json::from_value(json!([{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + {"type": "image_url", "image_url": {"url": "http://example.com/b.png"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}} + ]}])) + .unwrap(); + assert_eq!( + VERTEX_ANTHROPIC_CHAT_COMPLETIONS_CONFIG.remote_media(&messages), + ["https://example.com/a.png", "http://example.com/b.png"] + ); + } +} diff --git a/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/mod.rs b/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/mod.rs new file mode 100644 index 00000000000..e529997219e --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/vertex_ai_partner_models/mod.rs @@ -0,0 +1 @@ +pub mod anthropic; diff --git a/litellm-rust/docs/customer-proxy/vertex-claude-chat.svg b/litellm-rust/docs/customer-proxy/vertex-claude-chat.svg new file mode 100644 index 00000000000..df44cc0062a --- /dev/null +++ b/litellm-rust/docs/customer-proxy/vertex-claude-chat.svg @@ -0,0 +1,98 @@ + + + + + + + + POST /v1/chat/completions to a vertex_ai/ Claude deployment + + + + Client + OpenAI SDK body + + + + gateway-inference chat_completions + auth, hooks admit, route deployment + os.environ/ refs resolved via the secret source, + deployment vertex_* params added to the call + + + + inference-chat chat_completions_provider + vertex_ai + Gemini model: VertexGeminiConfig + vertex_ai + Claude partner model: VertexAiAnthropicConfig + Llama, Mistral, AI21, Gemma on Vertex: still refused + + + + + llms anthropic/chat build_messages_request (AnthropicHost::VertexAi) + 1. anthropic_version: vertex-2023-10-16 added after the params + 2. sort_params + map_openai_params: the Anthropic port; cost map + read as vertex_ai/{m} first, then the model-map walk + 3. service_tier and speed are refused (400) or dropped + with drop_params: Anthropic API only + 4. messages, system, tools, thinking as for anthropic/ + VertexAIAnthropicConfig.transform_request + model popped; output_format folded into output_config, + effort kept only where the model takes it + context_management + extra_headers betas, filtered for + Vertex, in body anthropic_beta and the anthropic-beta header + + + + + URL and credential + https://{l}-aiplatform.googleapis.com + global: https://aiplatform.googleapis.com + /v1/projects/{p}/locations/{l}/publishers/ + anthropic/models/{m@version}:rawPredict + api_base or VERTEXAI_API_BASE: path grafted, + or {api_base}:rawPredict when it has its own + location default us-central1 + Bearer: a Google access token from + vertex_credentials, env or ADC; it replaces + a forwarded Authorization, as Python does + anthropic-version: 2023-06-01 header + + + + + Claude on Vertex AI + rawPredict: Messages API JSON + stream: :streamRawPredict, "stream": true + custom api_base also gets ?alt=sse + + + + + response and stream (anthropic/chat, shared) + model: the requested {m@version}, as Python sets it + text, tool_use to tool_calls with names put back, + thinking to reasoning_content and thinking_blocks + json_tool_call answer becomes the content + stream: Anthropic SSE through ModelResponseIterator and + StreamWrapper, usage handed to ReportedUsage + + + + + usage and cost (gateway-inference Finish) + prompt_tokens: input + cache read + cache write + completion_tokens: output_tokens + streams report it whether or not the client + asked for include_usage + config prices win, then the vertex_ai/{m} entry + + + gateway startup (hooks::install) + a vertex_ai deployment loads the cost map and + installs it as ModelInfo: supports_reasoning, + supports_adaptive_thinking, supports_output_config, + supports_native_structured_output (vertex entry only), + default max_tokens: the bare {m} entry, else 4096 + +