feat(gateway): native chat for openai-compatible providers (#47099)

Serve chat completions for hosted_vllm, nvidia_nim, groq, together_ai,
fireworks_ai, xai, mistral, deepseek, openrouter, cerebras and perplexity
through the OpenAI passthrough, with each provider's Python config ported
as a request transform in llms and its response and stream fixups applied
on the way back.

Streams ask for stream_options.include_usage on the client's behalf where
the provider takes it, record the usage through ReportedUsage for the spend
hooks, and keep it from the client unless the client asked for it.

litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs

@@ -21,31 +21,48 @@ pub enum LlmProviders {
     Bedrock,
     #[strum(serialize = "bedrock_mantle")]
     BedrockMantle,
+    #[strum(serialize = "cerebras")]
+    Cerebras,
     #[strum(serialize = "cohere")]
     Cohere,
     #[strum(serialize = "deepseek")]
     Deepseek,
     #[strum(serialize = "edenai")]
     Edenai,
+    #[strum(serialize = "fireworks_ai")]
+    FireworksAi,
     /// Google AI Studio (`gemini/<model>`), the Gemini API addressed with an API key.
     #[strum(serialize = "gemini")]
     Gemini,
+    #[strum(serialize = "groq")]
+    Groq,
+    /// A vLLM server (`hosted_vllm/<model>`) at the deployment's `api_base`.
+    #[strum(serialize = "hosted_vllm")]
+    HostedVllm,
     #[strum(serialize = "minimax")]
     Minimax,
     #[strum(serialize = "mistral")]
     Mistral,
+    #[strum(serialize = "nvidia_nim")]
+    NvidiaNim,
     #[strum(serialize = "openai")]
     Openai,
     #[strum(serialize = "openai_like")]
     OpenaiLike,
     #[strum(serialize = "openrouter")]
     Openrouter,
+    #[strum(serialize = "perplexity")]
+    Perplexity,
     #[strum(serialize = "reducto")]
     Reducto,
     #[strum(serialize = "tencent")]
     Tencent,
+    #[strum(serialize = "together_ai")]
+    TogetherAi,
     #[strum(serialize = "vertex_ai")]
     VertexAi,
+    #[strum(serialize = "xai")]
+    Xai,
 }
 
 pub fn get_custom_llm_provider<'a>(

litellm-rust/crates/gateway-inference/src/chat_completions.rs

@@ -274,6 +274,7 @@ async fn openai(
     body: Map<String, Value>,
     extra_headers: Option<Map<String, Value>>,
 ) -> Result<Response, Error> {
+    let usage = ReportedUsage::default();
     let machine = gateway.chat_completions.clone().openai_machine(
         OpenAiChatCall {
             model: deployment.model.clone(),
@@ -286,6 +287,7 @@ async fn openai(
             litellm_params: provider_params(deployment),
             drop_params: drop_params(gateway, deployment),
             model_info: gateway.model_info(),
+            usage: usage.clone(),
         },
         None,
     );
@@ -293,7 +295,9 @@ async fn openai(
         |body: Bytes| ([(CONTENT_TYPE, "application/json")], body),
         |error| Bytes::from(Error::from(error).openai_sse_frame()),
     );
-    Ok(litellm_host_http::serve(machine, (), (), encoder, None).await?)
+    let mut response = litellm_host_http::serve(machine, (), (), encoder, None).await?;
+    response.extensions_mut().insert(usage);
+    Ok(response)
 }
 
 /// A stream from a provider chat translates (Anthropic, Bedrock), sent as OpenAI chunks. The

litellm-rust/crates/gateway/src/hooks.rs

@@ -205,7 +205,8 @@ fn is_azure(model: &Model) -> bool {
 }
 
 /// Whether the deployment's chat param rules read model capabilities from the cost map:
-/// Azure, and the Anthropic, Bedrock and Gemini chat translations.
+/// Azure, the Anthropic, Bedrock and Gemini chat translations, and the OpenAI-compatible
+/// providers.
 fn reads_model_info(model: &Model) -> bool {
     is_azure(model)
         || litellm_inference::provider::resolve_llm_provider(
@@ -216,7 +217,21 @@ fn reads_model_info(model: &Model) -> bool {
         .is_ok_and(|resolved| {
             matches!(
                 <&str>::from(resolved.provider),
-                "anthropic" | "bedrock" | "gemini" | "vertex_ai"
+                "anthropic"
+                    | "bedrock"
+                    | "gemini"
+                    | "vertex_ai"
+                    | "cerebras"
+                    | "deepseek"
+                    | "fireworks_ai"
+                    | "groq"
+                    | "hosted_vllm"
+                    | "mistral"
+                    | "nvidia_nim"
+                    | "openrouter"
+                    | "perplexity"
+                    | "together_ai"
+                    | "xai"
             )
         })
 }

litellm-rust/crates/gateway/tests/openai_compat_chat.rs

@@ -0,0 +1,442 @@
+//! Chat completions for the OpenAI-compatible providers through the gateway: the URL, auth
+//! and exact body each provider's upstream sees, and the usage and cost a stream is recorded
+//! with whether or not the client asked to see the usage.
+
+use std::{
+    collections::BTreeMap,
+    io::Write,
+    sync::{Arc, Mutex, Once},
+};
+
+use axum::{
+    Router,
+    body::{Body, to_bytes},
+    http::{Request, header::CONTENT_TYPE},
+    response::IntoResponse,
+};
+use futures_util::future::BoxFuture;
+use litellm_config::Config;
+use litellm_gateway_hooks::{Facts, Observer, Record};
+use rstest::rstest;
+use serde_json::{Value, json};
+use tokio::net::TcpListener;
+use tower::ServiceExt;
+
+const COMPLETION: &str = r#"{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"m","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":6,"total_tokens":11}}"#;
+const CONTENT_CHUNK: &str = "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}]}\n\n";
+const USAGE_CHUNK: &str = "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[],\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":3,\"total_tokens\":10}}\n\n";
+/// xAI counts reasoning apart from `completion_tokens` but inside `total_tokens`.
+const XAI_USAGE_CHUNK: &str = "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[],\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":3,\"total_tokens\":14,\"completion_tokens_details\":{\"reasoning_tokens\":4}}}\n\n";
+/// Mistral ends its stream with the usage on the last content chunk, unasked.
+const MISTRAL_LAST_CHUNK: &str = "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":3,\"total_tokens\":10}}\n\n";
+const DONE: &str = "data: [DONE]\n\n";
+
+const RATES: (f64, f64) = (1.0, 2.0);
+
+fn use_local_cost_map() {
+    static ONCE: Once = Once::new();
+    ONCE.call_once(|| {
+        let price = json!({
+            "mode": "chat",
+            "input_cost_per_token": RATES.0,
+            "output_cost_per_token": RATES.1,
+        });
+        let map: serde_json::Map<String, Value> = [
+            ("groq/llama-3.1-8b-instant", "groq"),
+            ("mistral/mistral-small-latest", "mistral"),
+            ("xai/grok-3", "xai"),
+            (
+                "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct",
+                "hosted_vllm",
+            ),
+        ]
+        .into_iter()
+        .map(|(model, provider)| {
+            let mut entry = price.clone();
+            entry["litellm_provider"] = json!(provider);
+            (model.to_owned(), entry)
+        })
+        .collect();
+        let mut file = tempfile::NamedTempFile::new().unwrap();
+        file.write_all(Value::Object(map).to_string().as_bytes())
+            .unwrap();
+        let (_, path) = file.keep().unwrap();
+        // SAFETY: set once, before any test builds a gateway or spawns a thread that reads the
+        // environment.
+        unsafe { std::env::set_var("LITELLM_MODEL_COST_MAP_PATH", path) };
+    });
+}
+
+#[derive(Clone, Debug)]
+struct Sent {
+    path: String,
+    headers: BTreeMap<String, String>,
+    body: Value,
+}
+
+type Calls = Arc<Mutex<Vec<Sent>>>;
+
+fn stream_answer(path: &str) -> String {
+    match path.split('/').nth(1) {
+        Some("xai") => [CONTENT_CHUNK, XAI_USAGE_CHUNK, DONE].concat(),
+        Some("mistral") => [CONTENT_CHUNK, MISTRAL_LAST_CHUNK, DONE].concat(),
+        Some("vllm") => [
+            "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"hmm\"}}]}\n\n",
+            CONTENT_CHUNK,
+            USAGE_CHUNK,
+            DONE,
+        ]
+        .concat(),
+        _ => [CONTENT_CHUNK, USAGE_CHUNK, DONE].concat(),
+    }
+}
+
+async fn upstream(calls: Calls) -> String {
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    let app = Router::new().fallback(move |request: Request<Body>| {
+        let calls = calls.clone();
+        async move {
+            let (parts, body) = request.into_parts();
+            let headers = parts
+                .headers
+                .iter()
+                .filter_map(|(name, value)| {
+                    Some((name.as_str().to_owned(), value.to_str().ok()?.to_owned()))
+                })
+                .collect();
+            let path = parts.uri.path().to_owned();
+            let body: Value =
+                serde_json::from_slice(&to_bytes(body, 1 << 20).await.unwrap()).unwrap();
+            let stream = body["stream"].as_bool().unwrap_or(false);
+            calls.lock().unwrap().push(Sent {
+                path: path.clone(),
+                headers,
+                body,
+            });
+            match stream {
+                true => {
+                    ([(CONTENT_TYPE, "text/event-stream")], stream_answer(&path)).into_response()
+                }
+                false => ([(CONTENT_TYPE, "application/json")], COMPLETION).into_response(),
+            }
+        }
+    });
+    tokio::spawn(async move { axum::serve(listener, app).await });
+    format!("http://{address}")
+}
+
+#[derive(Debug)]
+struct Seen {
+    deployment: Option<String>,
+    usage: Option<(u64, u64)>,
+    cost: Option<f64>,
+}
+
+struct Collect(Mutex<Vec<Seen>>);
+
+impl Observer for Collect {
+    fn name(&self) -> &'static str {
+        "collect"
+    }
+
+    fn reads(&self) -> Facts {
+        Facts::DEPLOYMENT | Facts::USAGE | Facts::COST
+    }
+
+    fn observe<'a>(&'a self, record: &'a Record) -> BoxFuture<'a, ()> {
+        Box::pin(async move {
+            self.0.lock().unwrap().push(Seen {
+                deployment: record
+                    .outcome
+                    .deployment
+                    .as_ref()
+                    .map(|deployment| deployment.id.to_string()),
+                usage: record
+                    .outcome
+                    .usage
+                    .map(|usage| (usage.prompt_tokens(), usage.output_tokens)),
+                cost: record.outcome.cost,
+            });
+        })
+    }
+}
+
+struct Harness {
+    app: Router,
+    inference: Arc<litellm_gateway_inference::Gateway>,
+    calls: Calls,
+    seen: Arc<Collect>,
+}
+
+/// One deployment per provider, each at its own path on the mock upstream.
+const DEPLOYMENTS: &[(&str, &str, &str, Option<&str>)] = &[
+    (
+        "vllm",
+        "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct",
+        "/vllm/v1",
+        None,
+    ),
+    (
+        "nim",
+        "nvidia_nim/meta/llama3-70b-instruct",
+        "/nim/v1",
+        Some("sk-nim"),
+    ),
+    (
+        "groq",
+        "groq/llama-3.1-8b-instant",
+        "/groq/openai/v1",
+        Some("sk-groq"),
+    ),
+    (
+        "together",
+        "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo",
+        "/together/v1",
+        Some("sk-together"),
+    ),
+    (
+        "fireworks",
+        "fireworks_ai/llama-v3p1-8b-instruct",
+        "/fireworks/inference/v1",
+        Some("sk-fireworks"),
+    ),
+    ("xai", "xai/grok-3", "/xai/v1", Some("sk-xai")),
+    (
+        "mistral",
+        "mistral/mistral-small-latest",
+        "/mistral",
+        Some("sk-mistral"),
+    ),
+    (
+        "deepseek",
+        "deepseek/deepseek-chat",
+        "/deepseek/beta",
+        Some("sk-deepseek"),
+    ),
+    (
+        "openrouter",
+        "openrouter/openai/gpt-4o",
+        "/openrouter/api/v1",
+        Some("sk-openrouter"),
+    ),
+    (
+        "cerebras",
+        "cerebras/llama3.1-8b",
+        "/cerebras/v1",
+        Some("sk-cerebras"),
+    ),
+    (
+        "perplexity",
+        "perplexity/sonar",
+        "/perplexity",
+        Some("sk-perplexity"),
+    ),
+    (
+        "custom-openai",
+        "openai/my-served-model",
+        "/custom/v1",
+        Some("sk-custom"),
+    ),
+];
+
+async fn harness() -> Harness {
+    use_local_cost_map();
+    let calls = Calls::default();
+    let upstream = upstream(calls.clone()).await;
+    let models: String = DEPLOYMENTS
+        .iter()
+        .map(|(name, model, base, key)| {
+            let key = key.map_or_else(String::new, |key| format!("\n      api_key: {key}"));
+            format!(
+                "  - model_name: {name}
+    litellm_params:
+      model: {model}
+      api_base: {upstream}{base}{key}
+    model_info:
+      id: {name}-1
+"
+            )
+        })
+        .collect();
+    let config = Config::from_yaml(&format!(
+        "model_list:
+{models}router_settings:
+  num_retries: 0
+general_settings:
+  master_key: sk-master
+"
+    ))
+    .unwrap();
+    let inference = litellm_gateway::build_inference(&config).unwrap();
+    let seen = Arc::new(Collect(Mutex::new(Vec::new())));
+    litellm_gateway::hooks::install(&inference, &config, Vec::new(), vec![seen.clone()])
+        .await
+        .unwrap();
+    let app = litellm_gateway::router(inference.clone(), &config, None, None);
+    Harness {
+        app,
+        inference,
+        calls,
+        seen,
+    }
+}
+
+async fn send(app: &Router, body: Value) -> (u16, String) {
+    let request = Request::post("/v1/chat/completions")
+        .header("content-type", "application/json")
+        .header("authorization", "Bearer sk-master")
+        .body(Body::from(body.to_string()))
+        .unwrap();
+    let response = app.clone().oneshot(request).await.unwrap();
+    let status = response.status().as_u16();
+    let body = to_bytes(response.into_body(), 1 << 20).await.unwrap();
+    (status, String::from_utf8(body.to_vec()).unwrap())
+}
+
+async fn seen(harness: &Harness) -> Vec<Seen> {
+    litellm_gateway::hooks::close(&harness.inference).await;
+    std::mem::take(&mut *harness.seen.0.lock().unwrap())
+}
+
+fn hi(model: &str) -> Value {
+    json!({
+        "model": model,
+        "messages": [{"role": "user", "content": "hi"}],
+        "max_completion_tokens": 8,
+        "temperature": 0.5,
+        "metadata": {"trace": "t"},
+    })
+}
+
+fn messages() -> Value {
+    json!([{"role": "user", "content": "hi"}])
+}
+
+#[rstest]
+#[case::hosted_vllm("vllm", "/vllm/v1/chat/completions", "Bearer fake-api-key", None,
+    json!({"model": "meta-llama/Llama-3.1-8B-Instruct", "messages": messages(), "max_completion_tokens": 8, "temperature": 0.5}))]
+#[case::nvidia_nim("nim", "/nim/v1/chat/completions", "Bearer sk-nim", None,
+    json!({"model": "meta/llama3-70b-instruct", "messages": messages(), "max_tokens": 8, "temperature": 0.5}))]
+#[case::groq("groq", "/groq/openai/v1/chat/completions", "Bearer sk-groq", None,
+    json!({"model": "llama-3.1-8b-instant", "messages": messages(), "max_tokens": 8, "temperature": 0.5}))]
+#[case::together_ai("together", "/together/v1/chat/completions", "Bearer sk-together", None,
+    json!({"model": "meta-llama/Llama-3.3-70B-Instruct-Turbo", "messages": messages(), "max_completion_tokens": 8, "temperature": 0.5}))]
+#[case::fireworks_ai("fireworks", "/fireworks/inference/v1/chat/completions", "Bearer sk-fireworks", None,
+    json!({"model": "accounts/fireworks/models/llama-v3p1-8b-instruct", "messages": messages(), "max_tokens": 8, "temperature": 0.5}))]
+#[case::xai("xai", "/xai/v1/chat/completions", "Bearer sk-xai", None,
+    json!({"model": "grok-3", "messages": messages(), "max_tokens": 8, "temperature": 0.5}))]
+#[case::mistral("mistral", "/mistral/v1/chat/completions", "Bearer sk-mistral", None,
+    json!({"model": "mistral-small-latest", "messages": messages(), "max_tokens": 8, "temperature": 0.5}))]
+#[case::deepseek("deepseek", "/deepseek/beta/chat/completions", "Bearer sk-deepseek", None,
+    json!({"model": "deepseek-chat", "messages": messages(), "max_completion_tokens": 8, "temperature": 0.5}))]
+#[case::openrouter("openrouter", "/openrouter/api/v1/chat/completions", "Bearer sk-openrouter",
+    Some(("x-title", "liteLLM")),
+    json!({"model": "openai/gpt-4o", "messages": messages(), "max_completion_tokens": 8, "temperature": 0.5, "usage": {"include": true}}))]
+#[case::cerebras("cerebras", "/cerebras/v1/chat/completions", "Bearer sk-cerebras", None,
+    json!({"model": "llama3.1-8b", "messages": messages(), "max_tokens": 8, "temperature": 0.5}))]
+#[case::perplexity("perplexity", "/perplexity/chat/completions", "Bearer sk-perplexity",
+    Some(("x-pplx-integration", "litellm")),
+    json!({"model": "sonar", "messages": messages(), "max_completion_tokens": 8, "temperature": 0.5}))]
+#[case::openai_custom_base("custom-openai", "/custom/v1/chat/completions", "Bearer sk-custom", None,
+    json!({"model": "my-served-model", "messages": messages(), "max_completion_tokens": 8, "temperature": 0.5}))]
+#[tokio::test]
+async fn each_provider_gets_its_url_auth_and_body(
+    #[case] model: &str,
+    #[case] path: &str,
+    #[case] authorization: &str,
+    #[case] header: Option<(&str, &str)>,
+    #[case] body: Value,
+) {
+    let harness = harness().await;
+    let (status, answer) = send(&harness.app, hi(model)).await;
+    assert_eq!(status, 200, "{answer}");
+    assert_eq!(
+        serde_json::from_str::<Value>(&answer).unwrap()["choices"][0]["message"]["content"],
+        "hi"
+    );
+
+    let sent = harness.calls.lock().unwrap()[0].clone();
+    assert_eq!(sent.path, path);
+    assert_eq!(
+        sent.headers.get("authorization").map(String::as_str),
+        Some(authorization)
+    );
+    if let Some((name, value)) = header {
+        assert_eq!(sent.headers.get(name).map(String::as_str), Some(value));
+    }
+    assert_eq!(sent.body, body);
+
+    let seen = seen(&harness).await;
+    assert_eq!(seen.len(), 1, "{seen:?}");
+    assert_eq!(seen[0].deployment, Some(format!("{model}-1")));
+    assert_eq!(seen[0].usage, Some((5, 6)));
+}
+
+#[rstest]
+#[case::groq_asks_on_the_clients_behalf("groq", false, Some(json!({"include_usage": true})), (7, 3))]
+#[case::groq_client_asked("groq", true, Some(json!({"include_usage": true})), (7, 3))]
+#[case::mistral_reports_unasked("mistral", false, None, (7, 3))]
+#[case::xai_folds_reasoning_into_completion("xai", false, Some(json!({"include_usage": true})), (7, 7))]
+#[case::hosted_vllm("vllm", false, Some(json!({"include_usage": true})), (7, 3))]
+#[tokio::test]
+async fn a_stream_is_metered_and_priced(
+    #[case] model: &str,
+    #[case] client_usage: bool,
+    #[case] stream_options: Option<Value>,
+    #[case] usage: (u64, u64),
+) {
+    let harness = harness().await;
+    let mut request = hi(model);
+    request["stream"] = json!(true);
+    if client_usage {
+        request["stream_options"] = json!({"include_usage": true});
+    }
+    let (status, body) = send(&harness.app, request).await;
+    assert_eq!(status, 200, "{body}");
+    assert!(body.contains("\"content\":\"hi\""), "{body}");
+    assert!(body.ends_with(DONE), "{body}");
+    assert_eq!(body.contains("\"usage\":{"), client_usage, "{body}");
+
+    let sent = harness.calls.lock().unwrap()[0].clone();
+    assert_eq!(sent.body.get("stream_options").cloned(), stream_options);
+
+    let seen = seen(&harness).await;
+    assert_eq!(seen.len(), 1, "{seen:?}");
+    assert_eq!(seen[0].usage, Some(usage));
+    let expected = usage.0 as f64 * RATES.0 + usage.1 as f64 * RATES.1;
+    assert!((seen[0].cost.unwrap() - expected).abs() < 1e-9, "{seen:?}");
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_vllm_stream_names_its_reasoning_as_litellm_does() {
+    let harness = harness().await;
+    let mut request = hi("vllm");
+    request["stream"] = json!(true);
+    let (status, body) = send(&harness.app, request).await;
+    assert_eq!(status, 200, "{body}");
+    assert!(body.contains("\"reasoning_content\":\"hmm\""), "{body}");
+    assert!(!body.contains("\"reasoning\":"), "{body}");
+}
+
+#[rstest]
+#[case::groq_streamed_response_format("groq", json!({"stream": true, "response_format": {"type": "json_object"}}), 501)]
+#[case::param_the_model_lacks("cerebras", json!({"frequency_penalty": 1}), 400)]
+#[case::fireworks_file_part("fireworks", json!({"messages": [{"role": "user", "content": [{"type": "file", "file": {"file_id": "f"}}]}]}), 400)]
+#[tokio::test]
+async fn requests_a_provider_cannot_take_are_refused_before_the_call(
+    #[case] model: &str,
+    #[case] extra: Value,
+    #[case] status: u16,
+) {
+    let harness = harness().await;
+    let mut request = hi(model);
+    let (Value::Object(request_fields), Value::Object(extra)) = (&mut request, extra) else {
+        unreachable!()
+    };
+    request_fields.extend(extra);
+    let (sent_status, body) = send(&harness.app, request).await;
+    assert_eq!(sent_status, status, "{body}");
+    assert!(harness.calls.lock().unwrap().is_empty());
+}

litellm-rust/crates/inference-chat/Cargo.toml

@@ -12,6 +12,7 @@ futures-util.workspace = true
 litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }
 litellm-cache-response.workspace = true
 litellm-core-utils.workspace = true
+litellm-framer.workspace = true
 litellm-host.workspace = true
 litellm-http.workspace = true
 litellm-inference.workspace = true
@@ -21,17 +22,16 @@ litellm-secrets.workspace = true
 reqwest.workspace = true
 serde_json = { workspace = true, features = ["preserve_order"] }
 tokio = { workspace = true, features = ["time"] }
+tokio-util = { version = "0.7", features = ["codec"] }
 tracing.workspace = true
 uuid.workspace = true
 
 [dev-dependencies]
 litellm-cache-memory.workspace = true
-litellm-framer.workspace = true
 litellm-host-native.workspace = true
 litellm-http = { workspace = true, features = ["test-support"] }
 litellm-inference-testing.workspace = true
 litellm-tracing.workspace = true
 rstest.workspace = true
-tokio-util = { version = "0.7", features = ["codec"] }
 tokio = { workspace = true, features = ["io-util", "sync"] }
 wiremock.workspace = true

litellm-rust/crates/inference-chat/src/azure.rs

@@ -129,6 +129,7 @@ impl ChatCompletionsRoute {
             headers: authenticated.headers,
             body,
             api_key,
+            adapter: None,
         })
     }
 

litellm-rust/crates/inference-chat/src/lib.rs

@@ -10,6 +10,7 @@ pub mod constants;
 pub(crate) mod handler;
 mod media;
 pub mod openai;
+pub mod openai_compatible;
 mod prepare;
 pub mod stream;
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;

litellm-rust/crates/inference-chat/src/openai.rs

@@ -24,8 +24,11 @@ use litellm_llms::{
 use serde_json::{Map, Value};
 
 use crate::{
-    ChatCompletionsRoute, Error, common_utils::string_headers,
+    ChatCompletionsRoute, Error,
+    common_utils::string_headers,
     constants::CHAT_COMPLETIONS_TIMEOUT_SECS,
+    openai_compatible::{ResponseAdapter, compatible_chat_config},
+    stream::ReportedUsage,
 };
 
 pub struct OpenAiChatCompletions;
@@ -61,6 +64,9 @@ pub struct OpenAiChatCall {
     pub drop_params: bool,
     /// The model cost map, which Azure's param rules read.
     pub model_info: Option<Arc<dyn ModelInfo>>,
+    /// Where a stream's usage is recorded for the host when the client did not ask to see
+    /// it.
+    pub usage: ReportedUsage,
 }
 
 /// A provider request ready to go out: where, with which headers and body, and who serves it.
@@ -71,17 +77,18 @@ pub(crate) struct PreparedCall {
     pub(crate) headers: Headers,
     pub(crate) body: Map<String, Value>,
     pub(crate) api_key: Option<String>,
+    pub(crate) adapter: Option<ResponseAdapter>,
 }
 
 impl ChatCompletionsRoute {
-    /// Whether the deployment is served by the OpenAI wire passthrough: OpenAI itself, or
-    /// Azure OpenAI.
+    /// Whether the deployment is served by the OpenAI wire passthrough: OpenAI itself, Azure
+    /// OpenAI, or an OpenAI-compatible provider.
     pub fn serves_openai(model: &str, custom_llm_provider: Option<&str>) -> bool {
         resolve_llm_provider(model, custom_llm_provider, "chat completions").is_ok_and(|resolved| {
             matches!(
                 resolved.provider,
                 LlmProviders::Openai | LlmProviders::Azure
-            )
+            ) || compatible_chat_config(resolved.provider).is_some()
         })
     }
 
@@ -149,10 +156,14 @@ impl ChatCompletionsRoute {
                 "chat completions",
             )?;
             let model = resolved.model.to_string();
-            let prepared = match resolved.provider {
-                LlmProviders::Openai => self.prepare_openai(model, call).await?,
-                LlmProviders::Azure => self.prepare_azure(model, call).await?,
-                other => return Err(Error::InvalidProvider(<&str>::from(other).to_string())),
+            let usage = call.usage.clone();
+            let prepared = match (resolved.provider, compatible_chat_config(resolved.provider)) {
+                (LlmProviders::Openai, _) => self.prepare_openai(model, call).await?,
+                (LlmProviders::Azure, _) => self.prepare_azure(model, call).await?,
+                (_, Some(config)) => self.prepare_openai_compatible(config, model, call).await?,
+                (other, None) => {
+                    return Err(Error::InvalidProvider(<&str>::from(other).to_string()));
+                }
             };
             let PreparedCall {
                 model,
@@ -161,6 +172,7 @@ impl ChatCompletionsRoute {
                 headers,
                 body,
                 api_key,
+                adapter,
             } = prepared;
             litellm_inference::diagnostic::provider(&model, provider);
             let request_context = RequestContext {
@@ -192,12 +204,20 @@ impl ChatCompletionsRoute {
                 .unwrap_or(false);
             let response = send(&self.http, wire, stream, timeout).await?;
             let output = if stream {
+                let chunks = chunks(response, timeout);
                 CallOutput::Stream {
                     head: (),
-                    chunks: chunks(response, timeout),
+                    chunks: match adapter {
+                        Some(adapter) => adapter.stream(chunks, usage),
+                        None => chunks,
+                    },
                 }
             } else {
                 let body = response.bytes().await.map_err(network)?;
+                let body = match &adapter {
+                    Some(adapter) => adapter.complete(body),
+                    None => body,
+                };
                 context
                     .response_received(&String::from_utf8_lossy(&body))
                     .await?;
@@ -246,6 +266,7 @@ impl ChatCompletionsRoute {
             headers: authenticated.headers,
             body,
             api_key: call.api_key,
+            adapter: None,
         })
     }
 }
@@ -354,6 +375,9 @@ mod tests {
     #[case::azure("azure/gpt-4o", None, true)]
     #[case::azure_ai("azure_ai/gpt-4o", None, false)]
     #[case::openai_like("openai_like/gpt-6-astra", None, false)]
+    #[case::groq("groq/llama-3.1-8b-instant", None, true)]
+    #[case::hosted_vllm("meta-llama/Llama-3.1-8B", Some("hosted_vllm"), true)]
+    #[case::nvidia_nim("nvidia_nim/meta/llama3-70b-instruct", None, true)]
     #[case::anthropic("anthropic/claude", None, false)]
     #[case::unprefixed("gpt-6-astra", None, false)]
     fn only_openai_deployments_take_the_passthrough(

litellm-rust/crates/inference-chat/src/openai_compatible.rs

@@ -0,0 +1,416 @@
+//! The OpenAI-compatible providers (`litellm/llms/<provider>/chat/transformation.py`) through
+//! the OpenAI passthrough: each provider's config shapes the request, and the response goes
+//! back as the provider sent it but for the fixups its config makes. A stream is relayed
+//! event by event, with the usage the call asked for on the client's behalf recorded for the
+//! host and kept from the client.
+
+use bytes::{Bytes, BytesMut};
+use futures_util::{StreamExt, stream::BoxStream};
+use litellm_auth::{CredentialPlacement, SecretValue};
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use litellm_framer::sse::{SseBlock, SseBlockCodec};
+use litellm_llms::{
+    ErrorDetail,
+    base_llm::{
+        auth::{AuthScheme, ValidatedEnvironment, resolve_auth},
+        sse::DONE_FRAME,
+    },
+    cerebras::chat::transformation::CEREBRAS_CHAT_CONFIG,
+    deepseek::chat::transformation::DEEPSEEK_CHAT_CONFIG,
+    fireworks_ai::chat::transformation::FIREWORKS_AI_CHAT_CONFIG,
+    groq::chat::transformation::GROQ_CHAT_CONFIG,
+    hosted_vllm::chat::transformation::HOSTED_VLLM_CHAT_CONFIG,
+    mistral::chat::transformation::MISTRAL_CHAT_CONFIG,
+    nvidia_nim::chat::transformation::NVIDIA_NIM_CHAT_CONFIG,
+    openai::chat::compatible::{
+        CompatibleChatBody, CompatibleContext, OpenAiCompatibleChatConfig, ResponseShape,
+        fix_response, transform_request,
+    },
+    openrouter::chat::transformation::OPENROUTER_CHAT_CONFIG,
+    perplexity::chat::transformation::PERPLEXITY_CHAT_CONFIG,
+    together_ai::chat::transformation::TOGETHER_AI_CHAT_CONFIG,
+    xai::chat::transformation::XAI_CHAT_CONFIG,
+};
+use litellm_llms_types::formats::chat_completions::{
+    ChatCompletionsUsage, CompletionTokensDetails, PromptTokensDetails,
+};
+use serde_json::Value;
+use tokio_util::codec::Decoder;
+
+use crate::{
+    ChatCompletionsRoute, Error,
+    common_utils::string_headers,
+    openai::{OpenAiChatCall, PreparedCall},
+    stream::ReportedUsage,
+};
+
+const ENV_REFERENCE: &str = "os.environ/";
+
+/// The config of a provider served through the OpenAI passthrough with its own transforms.
+pub fn compatible_chat_config(
+    provider: LlmProviders,
+) -> Option<&'static dyn OpenAiCompatibleChatConfig> {
+    Some(match provider {
+        LlmProviders::Cerebras => &CEREBRAS_CHAT_CONFIG,
+        LlmProviders::Deepseek => &DEEPSEEK_CHAT_CONFIG,
+        LlmProviders::FireworksAi => &FIREWORKS_AI_CHAT_CONFIG,
+        LlmProviders::Groq => &GROQ_CHAT_CONFIG,
+        LlmProviders::HostedVllm => &HOSTED_VLLM_CHAT_CONFIG,
+        LlmProviders::Mistral => &MISTRAL_CHAT_CONFIG,
+        LlmProviders::NvidiaNim => &NVIDIA_NIM_CHAT_CONFIG,
+        LlmProviders::Openrouter => &OPENROUTER_CHAT_CONFIG,
+        LlmProviders::Perplexity => &PERPLEXITY_CHAT_CONFIG,
+        LlmProviders::TogetherAi => &TOGETHER_AI_CHAT_CONFIG,
+        LlmProviders::Xai => &XAI_CHAT_CONFIG,
+        _ => return None,
+    })
+}
+
+/// What the response of a call to an OpenAI-compatible provider goes through on its way back.
+#[derive(Clone)]
+pub(crate) struct ResponseAdapter {
+    config: &'static dyn OpenAiCompatibleChatConfig,
+    shape: ResponseShape,
+}
+
+impl ChatCompletionsRoute {
+    pub(crate) async fn prepare_openai_compatible(
+        &self,
+        config: &'static dyn OpenAiCompatibleChatConfig,
+        model: String,
+        call: OpenAiChatCall,
+    ) -> Result<PreparedCall, Error> {
+        let OpenAiChatCall {
+            body,
+            api_key,
+            api_base,
+            extra_headers,
+            drop_params,
+            model_info,
+            ..
+        } = call;
+        let references: Vec<&str> = [api_key.as_deref(), api_base.as_deref()]
+            .into_iter()
+            .flatten()
+            .filter_map(|value| value.strip_prefix(ENV_REFERENCE))
+            .collect();
+        let names: Vec<&str> = config
+            .secret_names()
+            .into_iter()
+            .chain(references)
+            .collect();
+        let secrets = self.secrets.resolve(&names).await?;
+        let lookup = |name: &str| secrets.get(name);
+        // Python's router reads every `os.environ/` setting from the environment when it
+        // loads the deployment. A reference to an unset variable counts as unset.
+        let resolve = |value: &str| match value.strip_prefix(ENV_REFERENCE) {
+            Some(name) => lookup(name),
+            None => Some(value.to_owned()),
+        };
+        let api_key = api_key.as_deref().and_then(resolve);
+        let api_base = api_base.as_deref().and_then(resolve);
+        let url = config.complete_url(&config.api_base(api_base.as_deref(), &lookup)?);
+        let key = config.api_key(api_key.as_deref(), &lookup)?;
+        let provider_headers = config.headers(&body, &lookup);
+        let context = CompatibleContext {
+            model: &model,
+            drop_params,
+            model_info: model_info.as_deref(),
+        };
+        let CompatibleChatBody { body, shape } = transform_request(config, &context, body)?;
+        let defaults: Vec<(&str, &str)> = provider_headers
+            .iter()
+            .map(|(name, value)| (name.as_str(), value.as_str()))
+            .chain([("content-type", "application/json")])
+            .collect();
+        let environment = ValidatedEnvironment {
+            headers: litellm_http::request::with_default_headers(
+                string_headers(extra_headers)?,
+                &defaults,
+            ),
+            auth: AuthScheme::Credential {
+                placement: CredentialPlacement::Bearer,
+                secret: SecretValue::new(key),
+            },
+        };
+        let authenticated = resolve_auth(&self.auth, environment, &lookup).await?;
+        Ok(PreparedCall {
+            model,
+            provider: config.name(),
+            url,
+            headers: authenticated.headers,
+            body,
+            api_key,
+            adapter: Some(ResponseAdapter { config, shape }),
+        })
+    }
+}
+
+impl ResponseAdapter {
+    /// A complete response, re-encoded only when a fixup changed it.
+    pub(crate) fn complete(&self, body: Bytes) -> Bytes {
+        let Ok(Value::Object(mut response)) = serde_json::from_slice::<Value>(&body) else {
+            return body;
+        };
+        match fix_response(self.config, &mut response, &self.shape) {
+            true => serde_json::to_vec(&response).map_or(body, Bytes::from),
+            false => body,
+        }
+    }
+
+    pub(crate) fn stream(
+        self,
+        upstream: BoxStream<'static, Result<Bytes, Error>>,
+        usage: ReportedUsage,
+    ) -> BoxStream<'static, Result<Bytes, Error>> {
+        let relay = Relay {
+            upstream,
+            buffer: BytesMut::new(),
+            codec: SseBlockCodec::default(),
+            adapter: self,
+            usage,
+            ended: false,
+        };
+        futures_util::stream::try_unfold(relay, |mut relay| async move {
+            relay
+                .next_frame()
+                .await
+                .map(|frame| frame.map(|frame| (frame, relay)))
+        })
+        .boxed()
+    }
+
+    /// One event's frame for the client, or `None` when the client is not to see it.
+    fn frame(&self, block: SseBlock, usage: &ReportedUsage) -> Option<Bytes> {
+        if block.event.data.trim() == "[DONE]" {
+            return Some(Bytes::from_static(DONE_FRAME));
+        }
+        let Ok(Value::Object(mut chunk)) = serde_json::from_str::<Value>(&block.event.data) else {
+            return Some(block.bytes);
+        };
+        let mut changed = self.config.fix_chunk(&mut chunk, &self.shape);
+        if let Some(reported) = chunk.get("usage").and_then(usage_from_json) {
+            usage.set(reported);
+        }
+        if !self.shape.client_usage {
+            if chunk.get("usage").is_some_and(|usage| !usage.is_null()) {
+                chunk.remove("usage");
+                changed = true;
+            }
+            if chunk
+                .get("choices")
+                .and_then(Value::as_array)
+                .is_some_and(Vec::is_empty)
+            {
+                return None;
+            }
+        }
+        if !changed {
+            return Some(block.bytes);
+        }
+        let mut frame = b"data: ".to_vec();
+        serde_json::to_writer(&mut frame, &chunk).ok()?;
+        frame.extend_from_slice(b"\n\n");
+        Some(Bytes::from(frame))
+    }
+}
+
+struct Relay {
+    upstream: BoxStream<'static, Result<Bytes, Error>>,
+    buffer: BytesMut,
+    codec: SseBlockCodec,
+    adapter: ResponseAdapter,
+    usage: ReportedUsage,
+    ended: bool,
+}
+
+impl Relay {
+    async fn next_frame(&mut self) -> Result<Option<Bytes>, Error> {
+        loop {
+            let block = match self.ended {
+                false => self.codec.decode(&mut self.buffer),
+                true => self.codec.decode_eof(&mut self.buffer),
+            }
+            .map_err(|error| {
+                Error::InvalidResponse(ErrorDetail::from(format!(
+                    "the provider sent a malformed stream: {error}"
+                )))
+            })?;
+            match block {
+                Some(block) => {
+                    if let Some(frame) = self.adapter.frame(block, &self.usage) {
+                        return Ok(Some(frame));
+                    }
+                }
+                None if self.ended => return Ok(None),
+                None => match self.upstream.next().await {
+                    Some(bytes) => self.buffer.extend_from_slice(&bytes?),
+                    None => self.ended = true,
+                },
+            }
+        }
+    }
+}
+
+/// The usage a chunk or response reports, read as leniently as Python's `Usage` reads it.
+fn usage_from_json(usage: &Value) -> Option<ChatCompletionsUsage> {
+    let usage = usage.as_object()?;
+    let count = |value: Option<&Value>| value.and_then(Value::as_u64);
+    let prompt = usage.get("prompt_tokens_details");
+    let completion = usage.get("completion_tokens_details");
+    Some(ChatCompletionsUsage {
+        prompt_tokens: count(usage.get("prompt_tokens")).unwrap_or(0),
+        completion_tokens: count(usage.get("completion_tokens")).unwrap_or(0),
+        total_tokens: count(usage.get("total_tokens")).unwrap_or(0),
+        prompt_tokens_details: PromptTokensDetails {
+            cached_tokens: count(prompt.and_then(|d| d.get("cached_tokens"))).unwrap_or(0),
+            cache_creation_tokens: count(prompt.and_then(|d| d.get("cache_creation_tokens")))
+                .unwrap_or(0),
+            text_tokens: count(prompt.and_then(|d| d.get("text_tokens"))).unwrap_or(0),
+            cache_write_tokens: None,
+            cache_creation_token_details: prompt
+                .and_then(|d| d.get("cache_creation_token_details"))
+                .filter(|details| details.is_object())
+                .cloned(),
+        },
+        completion_tokens_details: completion.filter(|details| details.is_object()).map(
+            |details| CompletionTokensDetails {
+                reasoning_tokens: count(details.get("reasoning_tokens")),
+                text_tokens: count(details.get("text_tokens")),
+            },
+        ),
+        server_tool_use: usage
+            .get("server_tool_use")
+            .filter(|tools| tools.is_object())
+            .cloned(),
+        cache_creation_input_tokens: None,
+        cache_read_input_tokens: None,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    fn adapter(
+        config: &'static dyn OpenAiCompatibleChatConfig,
+        client_usage: bool,
+    ) -> ResponseAdapter {
+        ResponseAdapter {
+            config,
+            shape: ResponseShape {
+                stream: true,
+                client_usage,
+                ..ResponseShape::default()
+            },
+        }
+    }
+
+    async fn relay(adapter: ResponseAdapter, parts: &[&'static str]) -> (String, ReportedUsage) {
+        let usage = ReportedUsage::default();
+        let upstream = futures_util::stream::iter(
+            parts
+                .iter()
+                .map(|part| Ok::<_, Error>(Bytes::from_static(part.as_bytes())))
+                .collect::<Vec<_>>(),
+        )
+        .boxed();
+        let frames: Vec<Bytes> = adapter
+            .stream(upstream, usage.clone())
+            .map(Result::unwrap)
+            .collect()
+            .await;
+        (String::from_utf8(frames.concat()).unwrap(), usage)
+    }
+
+    const CONTENT: &str =
+        "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}]}\n\n";
+    const USAGE: &str = "data: {\"id\":\"c\",\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":2,\"total_tokens\":5}}\n\n";
+    const DONE: &str = "data: [DONE]\n\n";
+
+    #[rstest]
+    #[case::kept_from_the_client(false, format!("{CONTENT}{DONE}"))]
+    #[case::shown_when_asked(true, format!("{CONTENT}{USAGE}{DONE}"))]
+    #[tokio::test]
+    async fn usage_is_recorded_and_shown_only_when_asked(
+        #[case] client_usage: bool,
+        #[case] expected: String,
+    ) {
+        let (sent, usage) = relay(
+            adapter(&GROQ_CHAT_CONFIG, client_usage),
+            &[
+                "data: {\"id\":\"c\",\"choices\":[{\"index\":0,",
+                "\"delta\":{\"content\":\"hi\"}}]}\n\n",
+                USAGE,
+                DONE,
+            ],
+        )
+        .await;
+        assert_eq!(sent, expected);
+        let usage = usage.get().unwrap();
+        assert_eq!(
+            (
+                usage.prompt_tokens,
+                usage.completion_tokens,
+                usage.total_tokens
+            ),
+            (3, 2, 5)
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn chunk_fixups_reencode_only_the_chunks_they_change() {
+        let (sent, _) = relay(
+            adapter(&HOSTED_VLLM_CHAT_CONFIG, true),
+            &[
+                CONTENT,
+                "data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"r\"}}]}\n\n",
+                DONE,
+            ],
+        )
+        .await;
+        assert_eq!(
+            sent,
+            format!(
+                "{CONTENT}data: {{\"choices\":[{{\"index\":0,\"delta\":{{\"reasoning_content\":\"r\"}}}}]}}\n\n{DONE}"
+            )
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn a_usage_chunk_with_choices_loses_only_its_usage() {
+        let (sent, usage) = relay(
+            adapter(&MISTRAL_CHAT_CONFIG, false),
+            &["data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\n"],
+        )
+        .await;
+        assert_eq!(
+            sent,
+            "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n"
+        );
+        assert_eq!(usage.get().unwrap().total_tokens, 2);
+    }
+
+    #[rstest]
+    fn a_complete_response_is_reencoded_only_when_fixed() {
+        let raw = Bytes::from_static(
+            br#"{"choices": [{"message": {"role": "assistant", "content": "x"}}]}"#,
+        );
+        assert_eq!(
+            adapter(&CEREBRAS_CHAT_CONFIG, false).complete(raw.clone()),
+            raw
+        );
+        let fixed = adapter(&HOSTED_VLLM_CHAT_CONFIG, false).complete(Bytes::from_static(
+            br#"{"choices":[{"message":{"role":"assistant","content":"x","reasoning":"r"}}]}"#,
+        ));
+        assert_eq!(
+            serde_json::from_slice::<Value>(&fixed).unwrap(),
+            json!({"choices": [{"message": {"role": "assistant", "content": "x", "reasoning_content": "r"}}]})
+        );
+    }
+}

litellm-rust/crates/inference-chat/src/stream.rs

@@ -71,7 +71,7 @@ impl ReportedUsage {
         self.0.lock().unwrap_or_else(|e| e.into_inner()).clone()
     }
 
-    fn set(&self, usage: ChatCompletionsUsage) {
+    pub(crate) fn set(&self, usage: ChatCompletionsUsage) {
         *self.0.lock().unwrap_or_else(|e| e.into_inner()) = Some(usage);
     }
 }

litellm-rust/crates/llms/src/cerebras/chat/AGENTS.md

@@ -0,0 +1,10 @@
+# rules
+
+- Cerebras through the OpenAI SDK URL rule at `https://api.cerebras.ai/v1`
+- `reasoning_effort` only for models the cost map marks as reasoning; `max_completion_tokens` goes as `max_tokens`
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://inference-docs.cerebras.ai/api-reference/chat-completions
+- litellm/llms/cerebras/chat.py

litellm-rust/crates/llms/src/cerebras/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/cerebras/chat/transformation.rs

@@ -0,0 +1,163 @@
+//! `litellm/llms/cerebras/chat.py`: Cerebras through the OpenAI SDK at
+//! `https://api.cerebras.ai/v1`, with `reasoning_effort` for the models the cost map marks as
+//! reasoning ones and `max_completion_tokens` sent as `max_tokens`.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value};
+
+use crate::{
+    Error,
+    openai::chat::compatible::{CompatibleContext, OpenAiCompatibleChatConfig, sdk_complete_url},
+};
+
+const SUPPORTED_PARAMS: &[&str] = &[
+    "max_tokens",
+    "max_completion_tokens",
+    "response_format",
+    "seed",
+    "stop",
+    "stream",
+    "temperature",
+    "top_p",
+    "tool_choice",
+    "tools",
+    "user",
+    "max_retries",
+    "extra_headers",
+];
+
+pub struct CerebrasChatConfig;
+
+pub const CEREBRAS_CHAT_CONFIG: CerebrasChatConfig = CerebrasChatConfig;
+
+impl OpenAiCompatibleChatConfig for CerebrasChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::Cerebras
+    }
+
+    fn label(&self) -> &'static str {
+        "Cerebras"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["CEREBRAS_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://api.cerebras.ai/v1")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &["CEREBRAS_API_KEY"]
+    }
+
+    fn complete_url(&self, api_base: &str) -> String {
+        sdk_complete_url(api_base)
+    }
+
+    fn supported_params(&self, context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        let mut params = SUPPORTED_PARAMS.to_vec();
+        if context.supports(self.name(), "supports_reasoning") {
+            params.push("reasoning_effort");
+        }
+        params
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        let supported = self.supported_params(context);
+        for (name, value) in non_default {
+            if name == "max_completion_tokens" {
+                optional.insert("max_tokens".into(), value.clone());
+            } else if supported.contains(&name.as_str()) {
+                optional.insert(name.clone(), value.clone());
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::borrow::Cow;
+
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+    use crate::{
+        ErrorDetail, base_llm::model_info::ModelInfo, openai::chat::compatible::transform_request,
+    };
+
+    struct Reasoning;
+
+    impl ModelInfo for Reasoning {
+        fn entry(&self, model: &str) -> Option<Cow<'_, Map<String, Value>>> {
+            (model == "cerebras/gpt-oss-120b").then(|| {
+                Cow::Owned(Map::from_iter([(
+                    "supports_reasoning".to_owned(),
+                    json!(true),
+                )]))
+            })
+        }
+
+        fn declares(&self, _key: &str) -> bool {
+            true
+        }
+    }
+
+    fn send(model: &str, params: Value) -> Result<Map<String, Value>, Error> {
+        let Value::Object(body) = params else {
+            panic!("an object")
+        };
+        transform_request(
+            &CEREBRAS_CHAT_CONFIG,
+            &CompatibleContext {
+                model,
+                drop_params: false,
+                model_info: Some(&Reasoning),
+            },
+            body,
+        )
+        .map(|sent| sent.body)
+    }
+
+    #[rstest]
+    fn reasoning_effort_goes_to_reasoning_models() {
+        let sent = send(
+            "gpt-oss-120b",
+            json!({"messages": [], "reasoning_effort": "low", "max_completion_tokens": 5}),
+        )
+        .unwrap();
+        assert_eq!(
+            Value::Object(sent),
+            json!({"model": "gpt-oss-120b", "messages": [], "reasoning_effort": "low", "max_tokens": 5})
+        );
+    }
+
+    #[rstest]
+    fn reasoning_effort_is_refused_elsewhere() {
+        assert!(matches!(
+            send(
+                "llama3.1-8b",
+                json!({"messages": [], "reasoning_effort": "low"})
+            ),
+            Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams { .. }))
+        ));
+    }
+
+    #[rstest]
+    fn frequency_penalty_is_refused() {
+        assert!(
+            send(
+                "llama3.1-8b",
+                json!({"messages": [], "frequency_penalty": 1})
+            )
+            .is_err()
+        );
+    }
+}

litellm-rust/crates/llms/src/cerebras/mod.rs

@@ -0,0 +1 @@
+pub mod chat;

litellm-rust/crates/llms/src/deepseek/chat/AGENTS.md

@@ -0,0 +1,11 @@
+# rules
+
+- DeepSeek at `https://api.deepseek.com/beta`; `/chat/completions` is appended unless the base ends with it
+- `thinking` and `reasoning_effort` become the `thinking` toggle, only function tools are sent
+- A thinking turn replays `reasoning_content` on every assistant message
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://api-docs.deepseek.com/api/create-chat-completion
+- litellm/llms/deepseek/chat/transformation.py

litellm-rust/crates/llms/src/deepseek/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/deepseek/chat/transformation.rs

@@ -0,0 +1,352 @@
+//! `litellm/llms/deepseek/chat/transformation.py`: DeepSeek at `https://api.deepseek.com/beta`.
+//! `thinking` and `reasoning_effort` become DeepSeek's thinking toggle, only function tools go,
+//! content lists collapse to strings unless a vision model can take them, and a thinking turn
+//! replays `reasoning_content` on every assistant message.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error,
+    base_llm::chat::params::truthy,
+    openai::chat::{
+        compatible::{
+            CompatibleContext, GPT_BASE_PARAMS, OpenAiCompatibleChatConfig, content_list_to_str,
+            copy_supported, gpt_messages, role, search_results_text,
+        },
+        transformation::OPENAI_CHAT_COMPLETIONS_PATH,
+    },
+};
+
+pub struct DeepSeekChatConfig;
+
+pub const DEEPSEEK_CHAT_CONFIG: DeepSeekChatConfig = DeepSeekChatConfig;
+
+impl OpenAiCompatibleChatConfig for DeepSeekChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::Deepseek
+    }
+
+    fn label(&self) -> &'static str {
+        "DeepSeek"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["DEEPSEEK_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://api.deepseek.com/beta")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &["DEEPSEEK_API_KEY"]
+    }
+
+    fn complete_url(&self, api_base: &str) -> String {
+        let base = api_base.trim_end_matches('/');
+        match base.ends_with(OPENAI_CHAT_COMPLETIONS_PATH) {
+            true => base.to_owned(),
+            false => format!("{base}{OPENAI_CHAT_COMPLETIONS_PATH}"),
+        }
+    }
+
+    fn supported_params(&self, _context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        GPT_BASE_PARAMS
+            .iter()
+            .copied()
+            .chain(["response_format", "thinking", "reasoning_effort"])
+            .collect()
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        copy_supported(non_default, optional, &self.supported_params(context));
+        let thinking = optional.remove("thinking");
+        let effort = optional.remove("reasoning_effort");
+        let thinking_type = thinking
+            .as_ref()
+            .and_then(|thinking| thinking.get("type"))
+            .and_then(Value::as_str)
+            .filter(|kind| ["enabled", "disabled"].contains(kind));
+        if let Some(kind) = thinking_type {
+            optional.insert("thinking".into(), json!({"type": kind}));
+        } else if let Some(effort) = effort.filter(|effort| !effort.is_null()) {
+            let kind = match effort.as_str() {
+                Some("none") => "disabled",
+                _ => "enabled",
+            };
+            optional.insert("thinking".into(), json!({"type": kind}));
+        }
+        drop_unsupported_tools(optional);
+        Ok(())
+    }
+
+    fn transform_messages(
+        &self,
+        messages: Vec<Value>,
+        optional: &Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<Vec<Value>, Error> {
+        let thinking = context.supports(self.name(), "supports_reasoning")
+            && optional
+                .get("thinking")
+                .and_then(|thinking| thinking.get("type"))
+                == Some(&json!("enabled"));
+        let messages: Vec<Value> = match thinking {
+            true => messages.into_iter().map(fill_reasoning_content).collect(),
+            false => messages,
+        };
+        let forward_images = messages
+            .iter()
+            .any(|message| message.get("content").is_some_and(Value::is_array))
+            && context.supports(self.name(), "supports_vision");
+        gpt_messages(
+            messages
+                .into_iter()
+                .map(|message| forward_or_collapse_content(message, forward_images))
+                .collect(),
+        )
+    }
+}
+
+/// `_drop_unsupported_tools`: DeepSeek takes only function tools.
+fn drop_unsupported_tools(optional: &mut Map<String, Value>) {
+    let Some(Value::Array(tools)) = optional.get("tools") else {
+        return;
+    };
+    let is_function = |tool: &Value| tool.get("type").and_then(Value::as_str) == Some("function");
+    if tools.is_empty() || tools.iter().all(is_function) {
+        return;
+    }
+    let functions: Vec<Value> = tools
+        .iter()
+        .filter(|tool| is_function(tool))
+        .cloned()
+        .collect();
+    if functions.is_empty() {
+        for key in ["tools", "tool_choice", "parallel_tool_calls"] {
+            optional.remove(key);
+        }
+        return;
+    }
+    let names: Vec<&str> = functions
+        .iter()
+        .filter_map(|tool| tool.get("function")?.get("name")?.as_str())
+        .collect();
+    let choice_matches = match optional.get("tool_choice") {
+        Some(Value::Object(choice)) => {
+            choice.get("type").and_then(Value::as_str) == Some("function")
+                && choice
+                    .get("function")
+                    .and_then(|function| function.get("name"))
+                    .and_then(Value::as_str)
+                    .is_some_and(|name| names.contains(&name))
+        }
+        _ => true,
+    };
+    if !choice_matches {
+        optional.remove("tool_choice");
+    }
+    optional.insert("tools".into(), Value::Array(functions));
+}
+
+/// `_fill_reasoning_content`.
+fn fill_reasoning_content(mut message: Value) -> Value {
+    if role(&message) != Some("assistant") || message.get("reasoning_content").is_some_and(truthy) {
+        return message;
+    }
+    let Some(object) = message.as_object_mut() else {
+        return message;
+    };
+    let stored = object
+        .get("provider_specific_fields")
+        .and_then(|fields| fields.get("reasoning_content"))
+        .filter(|stored| truthy(stored))
+        .cloned();
+    match stored {
+        Some(stored) => {
+            object.insert("reasoning_content".into(), stored);
+            if let Some(Value::Object(fields)) = object.get_mut("provider_specific_fields") {
+                fields.remove("reasoning_content");
+            }
+        }
+        None => {
+            object.insert("reasoning_content".into(), json!(" "));
+        }
+    }
+    message
+}
+
+fn forward_or_collapse_content(message: Value, forward_images: bool) -> Value {
+    if forward_images
+        && let Some(Value::Array(content)) = message.get("content")
+        && is_vision_forwardable(&message, content)
+    {
+        return with_search_results_text_block(message);
+    }
+    let collapsed = content_list_to_str(&message);
+    if collapsed.is_empty() || message.get("content") == Some(&Value::String(collapsed.clone())) {
+        return message;
+    }
+    let mut message = message;
+    if let Some(object) = message.as_object_mut() {
+        object.insert("content".into(), Value::String(collapsed));
+    }
+    message
+}
+
+fn is_vision_forwardable(message: &Value, content: &[Value]) -> bool {
+    let forwardable = |block: &Value| match block.get("type").and_then(Value::as_str) {
+        Some("image_url") => match block.get("image_url") {
+            Some(Value::String(url)) => !url.is_empty(),
+            Some(Value::Object(image)) => image
+                .get("url")
+                .and_then(Value::as_str)
+                .is_some_and(|url| !url.is_empty()),
+            _ => false,
+        },
+        Some("text") => block.get("text").is_some_and(Value::is_string),
+        _ => false,
+    };
+    role(message) == Some("user")
+        && content.iter().all(forwardable)
+        && content
+            .iter()
+            .any(|block| block.get("type").and_then(Value::as_str) == Some("image_url"))
+}
+
+fn with_search_results_text_block(mut message: Value) -> Value {
+    let text = search_results_text(message.get("search_results"));
+    if text.is_empty() {
+        return message;
+    }
+    if let Some(object) = message.as_object_mut() {
+        object.remove("search_results");
+        if let Some(Value::Array(content)) = object.get_mut("content") {
+            content.push(json!({"type": "text", "text": text}));
+        }
+    }
+    message
+}
+
+#[cfg(test)]
+mod tests {
+    use std::borrow::Cow;
+
+    use rstest::rstest;
+
+    use super::*;
+    use crate::{base_llm::model_info::ModelInfo, openai::chat::compatible::transform_request};
+
+    struct Capable;
+
+    impl ModelInfo for Capable {
+        fn entry(&self, model: &str) -> Option<Cow<'_, Map<String, Value>>> {
+            let entry = match model {
+                "deepseek/deepseek-reasoner" => json!({"supports_reasoning": true}),
+                "deepseek/deepseek-vl" => json!({"supports_vision": true}),
+                _ => return None,
+            };
+            let Value::Object(entry) = entry else {
+                unreachable!()
+            };
+            Some(Cow::Owned(entry))
+        }
+
+        fn declares(&self, _key: &str) -> bool {
+            true
+        }
+    }
+
+    fn send(model: &str, params: Value) -> Map<String, Value> {
+        let Value::Object(body) = params else {
+            panic!("an object")
+        };
+        transform_request(
+            &DEEPSEEK_CHAT_CONFIG,
+            &CompatibleContext {
+                model,
+                drop_params: false,
+                model_info: Some(&Capable),
+            },
+            body,
+        )
+        .unwrap()
+        .body
+    }
+
+    #[rstest]
+    #[case::base(
+        "https://api.deepseek.com/beta",
+        "https://api.deepseek.com/beta/chat/completions"
+    )]
+    #[case::named("https://h/chat/completions/", "https://h/chat/completions")]
+    fn the_url_ends_in_chat_completions_once(#[case] base: &str, #[case] expected: &str) {
+        assert_eq!(DEEPSEEK_CHAT_CONFIG.complete_url(base), expected);
+    }
+
+    #[rstest]
+    #[case::thinking_wins(json!({"thinking": {"type": "disabled", "budget_tokens": 9}, "reasoning_effort": "high"}), json!({"type": "disabled"}))]
+    #[case::effort_none(json!({"reasoning_effort": "none"}), json!({"type": "disabled"}))]
+    #[case::effort_on(json!({"reasoning_effort": "low"}), json!({"type": "enabled"}))]
+    fn reasoning_becomes_the_thinking_toggle(#[case] params: Value, #[case] thinking: Value) {
+        let Value::Object(mut params) = params else {
+            unreachable!()
+        };
+        params.insert("messages".into(), json!([]));
+        let sent = send("deepseek-chat", Value::Object(params));
+        assert_eq!(sent["thinking"], thinking);
+        assert!(!sent.contains_key("reasoning_effort"));
+    }
+
+    #[rstest]
+    fn only_function_tools_go_and_a_choice_naming_a_dropped_tool_goes_too() {
+        let sent = send(
+            "deepseek-chat",
+            json!({"messages": [],
+            "tools": [{"type": "function", "function": {"name": "f"}}, {"type": "namespace", "name": "n"}],
+            "tool_choice": {"type": "function", "function": {"name": "n"}}}),
+        );
+        assert_eq!(
+            sent["tools"],
+            json!([{"type": "function", "function": {"name": "f"}}])
+        );
+        assert!(!sent.contains_key("tool_choice"));
+    }
+
+    #[rstest]
+    fn a_thinking_turn_replays_reasoning_on_assistant_messages() {
+        let sent = send(
+            "deepseek-reasoner",
+            json!({"thinking": {"type": "enabled"}, "messages": [
+            {"role": "assistant", "content": "a", "provider_specific_fields": {"reasoning_content": "r"}},
+            {"role": "assistant", "content": "b"}]}),
+        );
+        assert_eq!(
+            sent["messages"],
+            json!([
+                {"role": "assistant", "content": "a", "provider_specific_fields": {}, "reasoning_content": "r"},
+                {"role": "assistant", "content": "b", "reasoning_content": " "}])
+        );
+    }
+
+    #[rstest]
+    #[case::collapsed("deepseek-chat", json!("look"))]
+    #[case::forwarded("deepseek-vl", json!([{"type": "text", "text": "look"}, {"type": "image_url", "image_url": {"url": "https://i"}}]))]
+    fn content_lists_collapse_unless_vision_takes_them(
+        #[case] model: &str,
+        #[case] content: Value,
+    ) {
+        let sent = send(
+            model,
+            json!({"messages": [{"role": "user", "content": [
+            {"type": "text", "text": "look"}, {"type": "image_url", "image_url": "https://i"}]}]}),
+        );
+        assert_eq!(sent["messages"][0]["content"], content);
+    }
+}

litellm-rust/crates/llms/src/deepseek/mod.rs

@@ -1 +1,2 @@
+pub mod chat;
 pub mod messages;

litellm-rust/crates/llms/src/fireworks_ai/chat/AGENTS.md

@@ -0,0 +1,12 @@
+# rules
+
+- Fireworks AI at `https://api.fireworks.ai/inference/v1`, the model named by its full resource (`resolve_fireworks_resource_name`)
+- Tool, tool choice and reasoning params are gated by the cost map, file parts and images on non-vision models are rejected
+- vLLM/NIM `extra_body` keys are translated (`truncate_prompt_tokens`, `chat_template_kwargs`, `guided_*`) or dropped
+- Tool calls Fireworks writes into the content go back as `tool_calls`; the answer's model gets the `fireworks_ai/` prefix
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://docs.fireworks.ai/api-reference/post-chatcompletions
+- litellm/llms/fireworks_ai/chat/transformation.py

litellm-rust/crates/llms/src/fireworks_ai/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/fireworks_ai/chat/transformation.rs

@@ -0,0 +1,680 @@
+//! `litellm/llms/fireworks_ai/chat/transformation.py`: Fireworks AI at
+//! `https://api.fireworks.ai/inference/v1`, with models named by their Fireworks resource,
+//! capability gated tool and reasoning params, vLLM style `extra_body` keys translated or
+//! dropped, and tool calls Fireworks writes into the content moved back to `tool_calls`.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error, ErrorDetail,
+    base_llm::chat::{
+        params::{python_dumps, truthy},
+        response_format::unpack_legacy_defs,
+    },
+    openai::chat::compatible::{
+        CompatibleContext, OpenAiCompatibleChatConfig, RESPONSE_FORMAT_TOOL_NAME, ResponseShape,
+        gpt_messages, messages_mut, role, strip_response_only_fields,
+    },
+};
+
+const SUPPORTED_PARAMS: &[&str] = &[
+    "stream",
+    "max_completion_tokens",
+    "max_tokens",
+    "temperature",
+    "top_p",
+    "top_k",
+    "frequency_penalty",
+    "presence_penalty",
+    "n",
+    "stop",
+    "response_format",
+    "user",
+    "logprobs",
+    "prompt_truncate_len",
+    "context_length_exceeded_behavior",
+    "seed",
+    "top_logprobs",
+    "min_p",
+    "typical_p",
+    "repetition_penalty",
+    "mirostat_target",
+    "mirostat_lr",
+    "logit_bias",
+    "echo",
+    "echo_last",
+    "ignore_eos",
+    "prompt_cache_key",
+    "prompt_cache_isolation_key",
+    "raw_output",
+    "perf_metrics_in_response",
+    "return_token_ids",
+    "safe_tokenization",
+    "service_tier",
+    "speculation",
+    "prediction",
+    "stream_options",
+    "sampling_mask",
+];
+const NIM_VLLM_STRIP_PARAMS: &[&str] = &[
+    "stop_token_ids",
+    "include_stop_str_in_output",
+    "skip_special_tokens",
+    "spaces_between_special_tokens",
+    "best_of",
+    "use_beam_search",
+    "guided_decoding_backend",
+    "guided_regex",
+    "add_generation_prompt",
+    "continue_final_message",
+    "add_special_tokens",
+    "detokenize",
+    "allowed_token_ids",
+    "bad_words",
+    "include_reasoning",
+    "nvext",
+];
+const EXTRA_BODY_CONSUMED_PARAMS: &[&str] = &[
+    "truncate_prompt_tokens",
+    "chat_template_kwargs",
+    "guided_json",
+    "guided_grammar",
+    "guided_choice",
+];
+const FIREROUTER: &str = "firerouter";
+const ROUTER_SHORT_NAMES: &[&str] = &[FIREROUTER, "auto", "auto-instant"];
+const SESSION_AFFINITY_HEADER: &str = "x-session-affinity";
+
+pub struct FireworksAiChatConfig;
+
+pub const FIREWORKS_AI_CHAT_CONFIG: FireworksAiChatConfig = FireworksAiChatConfig;
+
+impl OpenAiCompatibleChatConfig for FireworksAiChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::FireworksAi
+    }
+
+    fn label(&self) -> &'static str {
+        "Fireworks AI"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["FIREWORKS_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://api.fireworks.ai/inference/v1")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &[
+            "FIREWORKS_API_KEY",
+            "FIREWORKS_AI_API_KEY",
+            "FIREWORKSAI_API_KEY",
+            "FIREWORKS_AI_TOKEN",
+        ]
+    }
+
+    /// `with_fireworks_session_affinity`: the caller's session pins its calls to one node so
+    /// prompt caching hits.
+    fn headers(
+        &self,
+        body: &Map<String, Value>,
+        _lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Vec<(String, String)> {
+        let session = body
+            .get("litellm_session_id")
+            .filter(|id| truthy(id))
+            .or_else(|| {
+                body.get("metadata")
+                    .and_then(|metadata| metadata.get("session_id"))
+                    .filter(|id| truthy(id))
+            });
+        session
+            .map(|id| match id {
+                Value::String(id) => id.clone(),
+                other => other.to_string(),
+            })
+            .map(|id| vec![(SESSION_AFFINITY_HEADER.to_owned(), id)])
+            .unwrap_or_default()
+    }
+
+    fn wire_model(&self, model: &str) -> String {
+        resolve_fireworks_resource_name(model)
+    }
+
+    fn supported_params(&self, context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        let mut params = SUPPORTED_PARAMS.to_vec();
+        if exact_capability(context, "supports_function_calling") != Some(false) {
+            params.extend(["tools", "parallel_tool_calls"]);
+        }
+        if capability(context, "supports_tool_choice") {
+            params.push("tool_choice");
+        }
+        if capability(context, "supports_reasoning") {
+            params.extend(["reasoning_effort", "reasoning_history", "thinking"]);
+        }
+        params
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        if non_default.get("thinking").is_some_and(|v| !v.is_null())
+            && non_default
+                .get("reasoning_effort")
+                .is_some_and(|v| !v.is_null())
+        {
+            return Err(Error::InvalidRequest(ErrorDetail::Message(
+                "Fireworks AI chat completions does not support specifying both `thinking` and \
+                 `reasoning_effort` in the same request."
+                    .into(),
+            )));
+        }
+        let supported = self.supported_params(context);
+        let tools_set = non_default
+            .get("tools")
+            .is_some_and(|tools| !tools.is_null());
+        let mut response_format_tool = None;
+        for (name, value) in non_default {
+            match name.as_str() {
+                "tool_choice" => {
+                    let choice = match value.as_str() {
+                        Some("required") => json!("any"),
+                        _ => value.clone(),
+                    };
+                    optional.insert("tool_choice".into(), choice);
+                }
+                "response_format" if tools_set => {
+                    if let Some(schema) = response_schema(value) {
+                        response_format_tool = Some(json!({"type": "function",
+                            "function": {"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": schema}}));
+                        optional.insert("json_mode".into(), Value::Bool(true));
+                    }
+                }
+                "max_completion_tokens" => {
+                    optional.insert("max_tokens".into(), value.clone());
+                }
+                "reasoning_effort" => {
+                    if let Some(effort) = map_reasoning_effort(value) {
+                        optional.insert("reasoning_effort".into(), effort);
+                    }
+                }
+                name if supported.contains(&name) && !value.is_null() => {
+                    optional.insert(name.to_owned(), value.clone());
+                }
+                _ => {}
+            }
+        }
+        if let Some(tool) = response_format_tool
+            && let Some(Value::Array(tools)) = optional.get_mut("tools")
+        {
+            tools.push(tool);
+        }
+        Ok(())
+    }
+
+    /// `map_extra_body_params`.
+    fn map_extra_body(
+        &self,
+        extra_body: &mut Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        let mut promoted = Map::new();
+        if let Some(tokens) = extra_body
+            .get("truncate_prompt_tokens")
+            .filter(|v| !v.is_null())
+            && !extra_body.contains_key("prompt_truncate_len")
+            && !optional.contains_key("prompt_truncate_len")
+        {
+            promoted.insert("prompt_truncate_len".into(), tokens.clone());
+        }
+        let explicit_reasoning = ["reasoning_effort", "thinking"]
+            .iter()
+            .any(|key| optional.contains_key(*key) || extra_body.contains_key(*key));
+        if let Some(Value::Object(kwargs)) = extra_body.get("chat_template_kwargs")
+            && !explicit_reasoning
+            && let Some(effort) = effort_from_chat_template_kwargs(kwargs)
+            && context.supports("fireworks_ai", "supports_reasoning")
+        {
+            promoted.insert("reasoning_effort".into(), effort);
+        }
+        let has_response_format =
+            optional.contains_key("response_format") || extra_body.contains_key("response_format");
+        if !has_response_format && let Some(format) = guided_response_format(extra_body) {
+            promoted.insert("response_format".into(), format);
+        }
+        extra_body.retain(|key, _| {
+            !NIM_VLLM_STRIP_PARAMS.contains(&key.as_str())
+                && !EXTRA_BODY_CONSUMED_PARAMS.contains(&key.as_str())
+                && (key != "response_format" || !optional.contains_key("response_format"))
+        });
+        optional.extend(promoted);
+        Ok(())
+    }
+
+    /// `_transform_messages_helper`, then the base transform.
+    fn transform_messages(
+        &self,
+        messages: Vec<Value>,
+        _optional: &Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<Vec<Value>, Error> {
+        let vision = exact_capability(context, "supports_vision");
+        for message in messages
+            .iter()
+            .filter(|message| role(message) == Some("user"))
+        {
+            for kind in message
+                .get("content")
+                .and_then(Value::as_array)
+                .into_iter()
+                .flatten()
+                .filter_map(|part| part.get("type").and_then(Value::as_str))
+            {
+                if kind == "file" {
+                    return Err(Error::InvalidRequest(ErrorDetail::Message(
+                        "Fireworks AI chat completions does not support file content blocks. For \
+                         PDFs, convert pages to images and send image_url blocks to a Fireworks \
+                         vision model, or extract text before calling a text-only model."
+                            .into(),
+                    )));
+                }
+                if kind == "image_url" && vision == Some(false) {
+                    return Err(Error::InvalidRequest(ErrorDetail::Message(format!(
+                        "Fireworks AI model {} does not support image inputs. Use a Fireworks \
+                         vision model or remove image_url content blocks.",
+                        resolve_fireworks_resource_name(context.model)
+                    ))));
+                }
+            }
+        }
+        gpt_messages(
+            messages
+                .into_iter()
+                .map(|mut message| {
+                    strip_response_only_fields(&mut message);
+                    message
+                })
+                .collect(),
+        )
+    }
+
+    fn finish_request(
+        &self,
+        body: &mut Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        if let Some(Value::Array(tools)) = body.get_mut("tools") {
+            for function in tools
+                .iter_mut()
+                .filter(|tool| tool.get("type").and_then(Value::as_str) == Some("function"))
+                .filter_map(|tool| tool.get_mut("function")?.as_object_mut())
+            {
+                function.remove("strict");
+                if let Some(Value::Object(parameters)) = function.get_mut("parameters") {
+                    unpack_legacy_defs(parameters);
+                }
+            }
+        }
+        if body.get("stream").is_some_and(truthy) {
+            match body.get_mut("stream_options") {
+                Some(Value::Object(options)) => {
+                    if options.get("include_usage") != Some(&Value::Bool(false)) {
+                        options.insert("include_usage".into(), Value::Bool(true));
+                    }
+                }
+                _ => {
+                    body.insert("stream_options".into(), json!({"include_usage": true}));
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn fix_response(&self, body: &mut Map<String, Value>, shape: &ResponseShape) -> bool {
+        let mut changed = false;
+        if let Some(Value::String(model)) = body.get_mut("model") {
+            *model = format!("fireworks_ai/{model}");
+            changed = true;
+        }
+        if !shape.tools_sent {
+            return changed;
+        }
+        for message in messages_mut(body) {
+            changed |= content_tool_call_to_tool_calls(message, &shape.tool_names);
+        }
+        changed
+    }
+}
+
+/// `resolve_fireworks_resource_name`: the full Fireworks resource a short model name means.
+pub fn resolve_fireworks_resource_name(model: &str) -> String {
+    let stripped = model.strip_prefix("fireworks_ai/").unwrap_or(model);
+    if stripped.starts_with("accounts/") || stripped.starts_with("FW-") || stripped.contains('#') {
+        return stripped.to_owned();
+    }
+    if stripped.starts_with("routers/") || stripped.starts_with("models/") {
+        return format!("accounts/fireworks/{stripped}");
+    }
+    if stripped.ends_with("-fast")
+        || ROUTER_SHORT_NAMES.contains(&stripped)
+        || stripped.starts_with(&format!("{FIREROUTER}/"))
+    {
+        return format!("accounts/fireworks/routers/{stripped}");
+    }
+    format!("accounts/fireworks/models/{stripped}")
+}
+
+/// `_get_model_cost_capability_exact`.
+fn exact_capability(context: &CompatibleContext<'_>, key: &str) -> Option<bool> {
+    let short = context
+        .model
+        .strip_prefix("fireworks_ai/")
+        .unwrap_or(context.model);
+    let short = short
+        .strip_prefix("accounts/fireworks/models/")
+        .unwrap_or(short);
+    let firerouter = format!("accounts/fireworks/routers/{FIREROUTER}");
+    let mut candidates = vec![
+        context.model.to_owned(),
+        format!("fireworks_ai/{short}"),
+        format!("fireworks_ai/accounts/fireworks/models/{short}"),
+    ];
+    if resolve_fireworks_resource_name(context.model).starts_with(&format!("{firerouter}/")) {
+        candidates.push(format!("fireworks_ai/{firerouter}"));
+    }
+    context.capability_of(&candidates, key)
+}
+
+fn capability(context: &CompatibleContext<'_>, key: &str) -> bool {
+    exact_capability(context, key) == Some(true) || context.supports("fireworks_ai", key)
+}
+
+/// `_map_reasoning_effort`: booleans as levels, `auto` left to Fireworks.
+fn map_reasoning_effort(value: &Value) -> Option<Value> {
+    let effort = match value {
+        Value::Object(object) => object.get("effort").cloned().unwrap_or(Value::Null),
+        other => other.clone(),
+    };
+    match effort {
+        Value::Bool(true) => Some(json!("medium")),
+        Value::Bool(false) => Some(json!("none")),
+        Value::String(effort) if effort == "auto" => None,
+        Value::Null => None,
+        other => Some(other),
+    }
+}
+
+fn response_schema(value: &Value) -> Option<Value> {
+    let schema = match value.get("response_schema") {
+        Some(schema) => schema.clone(),
+        None => value.get("json_schema")?.get("schema")?.clone(),
+    };
+    truthy(&schema).then_some(schema)
+}
+
+/// `effort_from_chat_template_kwargs`.
+fn effort_from_chat_template_kwargs(kwargs: &Map<String, Value>) -> Option<Value> {
+    let flag = |keys: &[&str]| keys.iter().find_map(|key| kwargs.get(*key)?.as_bool());
+    if flag(&["enable_thinking", "thinking"]) == Some(false) {
+        return Some(json!("none"));
+    }
+    if let Some(budget) = kwargs.get("reasoning_budget").and_then(Value::as_f64)
+        && budget > 0.0
+    {
+        return Some(json!(budget as i64));
+    }
+    (flag(&["low_effort"]) == Some(true)).then(|| json!("low"))
+}
+
+/// `translate_guided_params`.
+fn guided_response_format(extra_body: &Map<String, Value>) -> Option<Value> {
+    let present = |key: &str| extra_body.get(key).filter(|value| !value.is_null());
+    let schema = |schema: &Value, name: &str| json!({"type": "json_schema", "json_schema": {"name": name, "schema": schema}});
+    if let Some(guided) = present("guided_json") {
+        return Some(schema(guided, "response"));
+    }
+    if let Some(grammar) = present("guided_grammar") {
+        return Some(json!({"type": "grammar", "grammar": grammar}));
+    }
+    present("guided_choice")
+        .map(|choices| schema(&json!({"type": "string", "enum": choices}), "choice"))
+}
+
+/// `_handle_message_content_with_tool_calls`: a call to a sent tool, written as the content.
+fn content_tool_call_to_tool_calls(
+    message: &mut Map<String, Value>,
+    tool_names: &[String],
+) -> bool {
+    if message
+        .get("tool_calls")
+        .is_some_and(|calls| !calls.is_null())
+    {
+        return false;
+    }
+    let Some(Value::Object(call)) = message
+        .get("content")
+        .and_then(Value::as_str)
+        .and_then(|content| serde_json::from_str::<Value>(content).ok())
+    else {
+        return false;
+    };
+    let Some(name) = call.get("name").and_then(Value::as_str) else {
+        return false;
+    };
+    if name == RESPONSE_FORMAT_TOOL_NAME || !tool_names.iter().any(|tool| tool == name) {
+        return false;
+    }
+    let arguments = match call.get("arguments") {
+        Some(Value::String(arguments)) => arguments.clone(),
+        Some(Value::Object(arguments)) => python_dumps(&Value::Object(arguments.clone())),
+        Some(Value::Null) | None => match call.get("parameters") {
+            Some(parameters @ Value::Object(_)) => python_dumps(parameters),
+            _ => String::new(),
+        },
+        Some(_) => return false,
+    };
+    message.insert(
+        "tool_calls".into(),
+        json!([{"id": uuid::Uuid::new_v4().to_string(), "type": "function",
+            "function": {"name": name, "arguments": arguments}}]),
+    );
+    message.insert("content".into(), Value::Null);
+    true
+}
+
+#[cfg(test)]
+mod tests {
+    use std::borrow::Cow;
+
+    use rstest::rstest;
+
+    use super::*;
+    use crate::{
+        base_llm::model_info::ModelInfo,
+        openai::chat::compatible::{CompatibleChatBody, fix_response, transform_request},
+    };
+
+    struct Registry;
+
+    impl ModelInfo for Registry {
+        fn entry(&self, model: &str) -> Option<Cow<'_, Map<String, Value>>> {
+            let entry = match model {
+                "fireworks_ai/accounts/fireworks/models/thinker" => {
+                    json!({"supports_reasoning": true, "supports_tool_choice": true})
+                }
+                "fireworks_ai/text-only" => {
+                    json!({"supports_vision": false, "supports_function_calling": false})
+                }
+                _ => return None,
+            };
+            let Value::Object(entry) = entry else {
+                unreachable!()
+            };
+            Some(Cow::Owned(entry))
+        }
+
+        fn declares(&self, _key: &str) -> bool {
+            true
+        }
+    }
+
+    fn send(model: &str, params: Value) -> Result<CompatibleChatBody, Error> {
+        let Value::Object(body) = params else {
+            panic!("an object")
+        };
+        transform_request(
+            &FIREWORKS_AI_CHAT_CONFIG,
+            &CompatibleContext {
+                model,
+                drop_params: false,
+                model_info: Some(&Registry),
+            },
+            body,
+        )
+    }
+
+    #[rstest]
+    #[case::short(
+        "llama-v3p1-8b-instruct",
+        "accounts/fireworks/models/llama-v3p1-8b-instruct"
+    )]
+    #[case::router("glm-4p6-fast", "accounts/fireworks/routers/glm-4p6-fast")]
+    #[case::models_prefix("models/x", "accounts/fireworks/models/x")]
+    #[case::full("accounts/me/models/x", "accounts/me/models/x")]
+    #[case::deployment("x#accounts/me/deployments/d", "x#accounts/me/deployments/d")]
+    fn models_are_named_by_their_resource(#[case] model: &str, #[case] resource: &str) {
+        assert_eq!(resolve_fireworks_resource_name(model), resource);
+    }
+
+    #[rstest]
+    fn reasoning_models_take_tool_choice_and_mapped_efforts() {
+        let sent = send("thinker", json!({"messages": [], "tool_choice": "required",
+            "tools": [{"type": "function", "function": {"name": "f", "strict": true,
+                "parameters": {"type": "object", "properties": {"a": {"$ref": "#/definitions/A"}}, "definitions": {"A": {"type": "string"}}}}}],
+            "reasoning_effort": true, "max_completion_tokens": 8}))
+        .unwrap();
+        assert_eq!(
+            Value::Object(sent.body),
+            json!({"model": "accounts/fireworks/models/thinker", "messages": [],
+                "tool_choice": "any",
+                "tools": [{"type": "function", "function": {"name": "f",
+                    "parameters": {"type": "object", "properties": {"a": {"type": "string"}}}}}],
+                "reasoning_effort": "medium", "max_tokens": 8})
+        );
+    }
+
+    #[rstest]
+    fn thinking_and_effort_together_are_rejected() {
+        assert!(matches!(
+            send(
+                "thinker",
+                json!({"messages": [], "thinking": {"type": "enabled"}, "reasoning_effort": "low"})
+            ),
+            Err(Error::InvalidRequest(ErrorDetail::Message(_)))
+        ));
+    }
+
+    #[rstest]
+    #[case::file(json!({"type": "file", "file": {"file_id": "f"}}))]
+    #[case::image_on_text_model(json!({"type": "image_url", "image_url": {"url": "https://i"}}))]
+    fn content_fireworks_cannot_read_is_rejected(#[case] part: Value) {
+        assert!(matches!(
+            send(
+                "text-only",
+                json!({"messages": [{"role": "user", "content": [part]}]})
+            ),
+            Err(Error::InvalidRequest(ErrorDetail::Message(_)))
+        ));
+    }
+
+    #[rstest]
+    fn tools_are_refused_where_the_registry_rules_out_function_calling() {
+        assert!(matches!(
+            send(
+                "text-only",
+                json!({"messages": [], "tools": [{"type": "function", "function": {"name": "f"}}]})
+            ),
+            Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams { .. }))
+        ));
+    }
+
+    #[rstest]
+    fn a_response_format_beside_tools_becomes_one_more_tool() {
+        let sent = send("unknown", json!({"messages": [],
+            "tools": [{"type": "function", "function": {"name": "f"}}],
+            "response_format": {"type": "json_schema", "json_schema": {"schema": {"type": "object"}}}}))
+        .unwrap();
+        assert_eq!(
+            sent.body["tools"],
+            json!([{"type": "function", "function": {"name": "f"}},
+                {"type": "function", "function": {"name": "json_tool_call", "parameters": {"type": "object"}}}])
+        );
+        assert!(!sent.body.contains_key("response_format"));
+    }
+
+    #[rstest]
+    fn vllm_extra_body_keys_are_translated_or_dropped() {
+        let sent = send(
+            "unknown",
+            json!({"messages": [], "truncate_prompt_tokens": 100,
+            "guided_choice": ["a", "b"], "stop_token_ids": [1], "custom": 1}),
+        )
+        .unwrap();
+        assert_eq!(
+            Value::Object(sent.body),
+            json!({"model": "accounts/fireworks/models/unknown", "messages": [],
+                "prompt_truncate_len": 100,
+                "response_format": {"type": "json_schema", "json_schema": {"name": "choice", "schema": {"type": "string", "enum": ["a", "b"]}}},
+                "custom": 1})
+        );
+    }
+
+    #[rstest]
+    fn a_session_pins_the_call() {
+        let body = Map::from_iter([("metadata".to_owned(), json!({"session_id": "s1"}))]);
+        assert_eq!(
+            FIREWORKS_AI_CHAT_CONFIG.headers(&body, &|_| None),
+            vec![("x-session-affinity".to_owned(), "s1".to_owned())]
+        );
+    }
+
+    #[rstest]
+    #[case::sent_tool("f", true)]
+    #[case::json_tool("json_tool_call", false)]
+    #[case::unknown_tool("g", false)]
+    fn tool_calls_in_the_content_move_to_tool_calls(#[case] name: &str, #[case] moved: bool) {
+        let content = json!({"name": name, "arguments": {"a": 1}}).to_string();
+        let mut body = Map::from_iter([
+            ("model".to_owned(), json!("accounts/fireworks/models/x")),
+            (
+                "choices".to_owned(),
+                json!([{"message": {"role": "assistant", "content": content}}]),
+            ),
+        ]);
+        let shape = ResponseShape {
+            tools_sent: true,
+            tool_names: vec!["f".into(), "json_tool_call".into()],
+            ..ResponseShape::default()
+        };
+        fix_response(&FIREWORKS_AI_CHAT_CONFIG, &mut body, &shape);
+        assert_eq!(
+            body["model"],
+            json!("fireworks_ai/accounts/fireworks/models/x")
+        );
+        let message = &body["choices"][0]["message"];
+        assert_eq!(message["content"].is_null(), moved);
+        if moved {
+            assert_eq!(
+                message["tool_calls"][0]["function"],
+                json!({"name": "f", "arguments": "{\"a\": 1}"})
+            );
+        }
+    }
+}

litellm-rust/crates/llms/src/fireworks_ai/mod.rs

@@ -0,0 +1 @@
+pub mod chat;

litellm-rust/crates/llms/src/groq/chat/AGENTS.md

@@ -0,0 +1,12 @@
+# rules
+
+- Groq at `https://api.groq.com/openai/v1`
+- A JSON schema on a model without native structured outputs goes as a forced `json_tool_call` tool, refused beside caller tools
+- A streamed `response_format` is refused: Python fakes that stream from a complete call
+- `web_search_options` adds the `browser_search` tool; `service_tier` in the answer is held to OpenAI's values
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://console.groq.com/docs/api-reference#chat
+- litellm/llms/groq/chat/transformation.py

litellm-rust/crates/llms/src/groq/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/groq/chat/transformation.rs

@@ -0,0 +1,368 @@
+//! `litellm/llms/groq/chat/transformation.py`: Groq at `https://api.groq.com/openai/v1`. A
+//! JSON schema a model cannot take natively goes as a forced `json_tool_call` tool whose call
+//! comes back as content, `web_search_options` turns on the `browser_search` tool, and the
+//! response's `service_tier` is held to the values OpenAI defines.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error, ErrorDetail,
+    openai::chat::compatible::{
+        CompatibleContext, GPT_BASE_PARAMS, OpenAiCompatibleChatConfig, RESPONSE_FORMAT_TOOL_NAME,
+        ResponseShape, copy_supported, gpt_messages, json_mode_tool_call_to_content, messages_mut,
+        role, sanitize_usage, strip_response_only_fields,
+    },
+};
+
+const COMPOUND_MODELS: &[&str] = &["compound", "compound-mini"];
+const SERVICE_TIERS: &[&str] = &["auto", "default", "flex"];
+
+pub struct GroqChatConfig;
+
+pub const GROQ_CHAT_CONFIG: GroqChatConfig = GroqChatConfig;
+
+impl OpenAiCompatibleChatConfig for GroqChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::Groq
+    }
+
+    fn label(&self) -> &'static str {
+        "Groq"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["GROQ_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://api.groq.com/openai/v1")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &["GROQ_API_KEY"]
+    }
+
+    fn supported_params(&self, context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        let web_search =
+            is_compound(context.model) || context.supports(self.name(), "supports_web_search");
+        let mut params: Vec<&'static str> = GPT_BASE_PARAMS
+            .iter()
+            .copied()
+            .filter(|param| *param != "max_retries")
+            .filter(|param| web_search || *param != "web_search_options")
+            .chain(["response_format"])
+            .collect();
+        if context.supports(self.name(), "supports_reasoning") {
+            params.push("reasoning_effort");
+        }
+        params
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        let mut params = non_default.clone();
+        if params.contains_key("response_format")
+            && params.get("stream") == Some(&Value::Bool(true))
+        {
+            return Err(Error::Unsupported(
+                "streaming a groq response_format, which Python fakes from a complete call",
+            ));
+        }
+        if let Some(schema) = response_schema(params.get("response_format"))
+            && context.capability(self.name(), "supports_response_schema") != Some(true)
+        {
+            if params.contains_key("tools") {
+                return Err(Error::InvalidRequest(ErrorDetail::Message(format!(
+                    "Groq model '{}' does not support native structured outputs. LiteLLM uses a \
+                     tool-calling workaround for structured outputs on this model, which is \
+                     incompatible with user-provided tools. Either use a model that supports \
+                     native structured outputs (e.g., gpt-oss-120b, llama-4, kimi-k2), or remove \
+                     the tools parameter. See: \
+                     https://console.groq.com/docs/structured-outputs#supported-models",
+                    context.model
+                ))));
+            }
+            optional.insert(
+                "tools".into(),
+                json!([{"type": "function", "function": {"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": schema}}]),
+            );
+            optional.insert(
+                "tool_choice".into(),
+                json!({"type": "function", "function": {"name": RESPONSE_FORMAT_TOOL_NAME}}),
+            );
+            optional.insert("json_mode".into(), Value::Bool(true));
+            params.remove("response_format");
+        }
+        let web_search = params.remove("web_search_options");
+        copy_supported(&params, optional, &self.supported_params(context));
+        if let Some(max_tokens) = optional.remove("max_completion_tokens") {
+            optional.insert("max_tokens".into(), max_tokens);
+        }
+        if web_search.is_none() || is_compound(context.model) {
+            return Ok(());
+        }
+        let tools = optional
+            .entry("tools")
+            .or_insert_with(|| Value::Array(Vec::new()));
+        if tools.is_null() {
+            *tools = Value::Array(Vec::new());
+        }
+        if let Value::Array(tools) = tools
+            && !tools
+                .iter()
+                .any(|tool| tool.get("type").and_then(Value::as_str) == Some("browser_search"))
+        {
+            tools.push(json!({"type": "browser_search"}));
+        }
+        Ok(())
+    }
+
+    fn transform_messages(
+        &self,
+        messages: Vec<Value>,
+        _optional: &Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<Vec<Value>, Error> {
+        gpt_messages(
+            messages
+                .into_iter()
+                .map(|mut message| {
+                    if role(&message) == Some("assistant")
+                        && let Value::Object(object) = &mut message
+                    {
+                        object.retain(|_, value| !value.is_null());
+                        strip_response_only_fields(&mut message);
+                    }
+                    message
+                })
+                .collect(),
+        )
+    }
+
+    fn fix_response(&self, body: &mut Map<String, Value>, shape: &ResponseShape) -> bool {
+        let mut changed = sanitize_usage(body);
+        if shape.json_mode {
+            changed |= json_mode_tool_call_to_content(body);
+        }
+        let tier = body.get("service_tier").and_then(Value::as_str);
+        if !tier.is_some_and(|tier| SERVICE_TIERS.contains(&tier)) {
+            body.insert("service_tier".into(), json!("auto"));
+            changed = true;
+        }
+        changed | add_web_search_usage(body)
+    }
+}
+
+fn is_compound(model: &str) -> bool {
+    COMPOUND_MODELS.contains(&model.strip_prefix("groq/").unwrap_or(model))
+}
+
+fn response_schema(response_format: Option<&Value>) -> Option<Value> {
+    let format = response_format?.as_object()?;
+    match format.get("response_schema") {
+        Some(schema) => Some(schema.clone()),
+        None => format.get("json_schema")?.get("schema").cloned(),
+    }
+    .filter(|schema| !schema.is_null())
+}
+
+/// `_add_web_search_usage`: the searches and page opens Groq ran, from each message's
+/// `executed_tools`.
+fn add_web_search_usage(body: &mut Map<String, Value>) -> bool {
+    if !body.get("usage").is_some_and(Value::is_object) {
+        return false;
+    }
+    let actions: Vec<String> = messages_mut(body)
+        .filter_map(|message| message.get("executed_tools")?.as_array())
+        .flatten()
+        .filter_map(|tool| {
+            tool.get("name")
+                .and_then(Value::as_str)
+                .filter(|name| !name.is_empty())
+                .or_else(|| tool.get("type").and_then(Value::as_str))
+                .map(str::to_owned)
+        })
+        .collect();
+    let count = |names: &[&str]| {
+        actions
+            .iter()
+            .filter(|a| names.contains(&a.as_str()))
+            .count()
+    };
+    let searches = count(&["browser.search", "browser_search"]);
+    let opens = count(&["browser.open"]);
+    if searches == 0 && opens == 0 {
+        return false;
+    }
+    if let Some(Value::Object(usage)) = body.get_mut("usage") {
+        usage.insert(
+            "server_tool_use".into(),
+            json!({"web_search_requests": searches, "browser_open_requests": opens}),
+        );
+    }
+    true
+}
+
+#[cfg(test)]
+mod tests {
+    use std::borrow::Cow;
+
+    use rstest::rstest;
+
+    use super::*;
+    use crate::{
+        base_llm::model_info::ModelInfo,
+        openai::chat::compatible::{fix_response, transform_request},
+    };
+
+    fn object(value: Value) -> Map<String, Value> {
+        let Value::Object(map) = value else {
+            panic!("an object")
+        };
+        map
+    }
+
+    fn send(model: &str, params: Value) -> Result<Map<String, Value>, Error> {
+        transform_request(
+            &GROQ_CHAT_CONFIG,
+            &CompatibleContext {
+                model,
+                drop_params: false,
+                model_info: None,
+            },
+            object(params),
+        )
+        .map(|sent| sent.body)
+    }
+
+    #[rstest]
+    fn a_schema_without_native_support_goes_as_a_forced_tool() {
+        let sent = transform_request(
+            &GROQ_CHAT_CONFIG,
+            &CompatibleContext {
+                model: "llama-3.1-8b-instant",
+                drop_params: false,
+                model_info: None,
+            },
+            object(json!({"messages": [], "max_completion_tokens": 3,
+                "response_format": {"type": "json_schema", "json_schema": {"name": "x", "schema": {"type": "object"}}}})),
+        )
+        .unwrap();
+        assert_eq!(
+            Value::Object(sent.body),
+            json!({"model": "llama-3.1-8b-instant", "messages": [],
+                "tools": [{"type": "function", "function": {"name": "json_tool_call", "parameters": {"type": "object"}}}],
+                "tool_choice": {"type": "function", "function": {"name": "json_tool_call"}},
+                "max_tokens": 3})
+        );
+        assert!(sent.shape.json_mode);
+    }
+
+    #[rstest]
+    fn a_schema_workaround_with_user_tools_is_rejected() {
+        assert!(matches!(
+            send(
+                "llama-3.1-8b-instant",
+                json!({"messages": [], "tools": [{"type": "function", "function": {"name": "f"}}],
+                "response_format": {"type": "json_schema", "json_schema": {"schema": {}}}})
+            ),
+            Err(Error::InvalidRequest(ErrorDetail::Message(_)))
+        ));
+    }
+
+    #[rstest]
+    fn a_streamed_response_format_is_refused() {
+        assert!(matches!(
+            send(
+                "llama-3.1-8b-instant",
+                json!({"messages": [], "stream": true, "response_format": {"type": "json_object"}})
+            ),
+            Err(Error::Unsupported(_))
+        ));
+    }
+
+    struct WebSearch;
+
+    impl ModelInfo for WebSearch {
+        fn entry(&self, model: &str) -> Option<Cow<'_, Map<String, Value>>> {
+            (model == "groq/openai/gpt-oss-20b").then(|| {
+                Cow::Owned(Map::from_iter([(
+                    "supports_web_search".to_owned(),
+                    json!(true),
+                )]))
+            })
+        }
+
+        fn declares(&self, _key: &str) -> bool {
+            true
+        }
+    }
+
+    #[rstest]
+    #[case::adds_browser_search("openai/gpt-oss-20b", json!([{"type": "browser_search"}]))]
+    #[case::compound_searches_itself("compound", Value::Null)]
+    fn web_search_turns_on_browser_search(#[case] model: &str, #[case] tools: Value) {
+        let sent = transform_request(
+            &GROQ_CHAT_CONFIG,
+            &CompatibleContext {
+                model,
+                drop_params: false,
+                model_info: Some(&WebSearch),
+            },
+            object(json!({"messages": [], "web_search_options": {"search_context_size": "low"}})),
+        )
+        .unwrap()
+        .body;
+        assert_eq!(sent.get("tools").cloned().unwrap_or(Value::Null), tools);
+        assert!(!sent.contains_key("web_search_options"));
+    }
+
+    #[rstest]
+    fn assistant_turns_lose_nulls_and_response_only_fields() {
+        let sent = send(
+            "llama-3.1-8b-instant",
+            json!({"messages": [
+            {"role": "assistant", "content": "a", "function_call": null, "thinking_blocks": []}]}),
+        )
+        .unwrap();
+        assert_eq!(
+            sent["messages"],
+            json!([{"role": "assistant", "content": "a"}])
+        );
+    }
+
+    #[rstest]
+    #[case::unknown(json!("on_demand"), json!("auto"))]
+    #[case::missing(Value::Null, json!("auto"))]
+    #[case::kept(json!("flex"), json!("flex"))]
+    fn service_tier_is_held_to_openai_values(#[case] tier: Value, #[case] expected: Value) {
+        let mut body =
+            object(json!({"choices": [], "service_tier": tier, "usage": {"prompt_tokens": null}}));
+        fix_response(&GROQ_CHAT_CONFIG, &mut body, &ResponseShape::default());
+        assert_eq!(body["service_tier"], expected);
+        assert_eq!(body["usage"]["prompt_tokens"], json!(0));
+    }
+
+    #[rstest]
+    fn executed_browser_tools_reach_the_usage() {
+        let mut body = object(
+            json!({"service_tier": "auto", "usage": {"prompt_tokens": 1},
+            "choices": [{"message": {"role": "assistant", "content": "x",
+                "executed_tools": [{"type": "browser_search"}, {"name": "browser.open"}, {"name": "browser.search"}]}}]}),
+        );
+        assert!(fix_response(
+            &GROQ_CHAT_CONFIG,
+            &mut body,
+            &ResponseShape::default()
+        ));
+        assert_eq!(
+            body["usage"]["server_tool_use"],
+            json!({"web_search_requests": 2, "browser_open_requests": 1})
+        );
+    }
+}

litellm-rust/crates/llms/src/groq/mod.rs

@@ -0,0 +1 @@
+pub mod chat;

litellm-rust/crates/llms/src/hosted_vllm/chat/AGENTS.md

@@ -0,0 +1,11 @@
+# rules
+
+- vLLM's OpenAI-compatible server at the deployment's `api_base`; there is no default host, and an unset key sends `fake-api-key`
+- Tools lose `additionalProperties: false` and `strict`, custom tools go as function tools, `thinking` becomes `reasoning_effort`
+- Replayed assistant turns lose `thinking_blocks` and non-string `reasoning_content`, content lists flatten to text
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html
+- litellm/llms/hosted_vllm/chat/transformation.py

litellm-rust/crates/llms/src/hosted_vllm/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/hosted_vllm/chat/transformation.rs

@@ -0,0 +1,385 @@
+//! `litellm/llms/hosted_vllm/chat/transformation.py`: a vLLM server at the deployment's
+//! `api_base`. Tools lose `additionalProperties: false` and `strict` and custom tools become
+//! function tools, `thinking` becomes a `reasoning_effort`, replayed assistant turns lose the
+//! fields vLLM rejects with their content lists flattened, and video files go as `video_url`.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error,
+    base_llm::chat::params::python_dumps,
+    openai::chat::compatible::{
+        CompatibleContext, GPT_BASE_PARAMS, OpenAiCompatibleChatConfig, copy_supported,
+        gpt_messages, reasoning_effort_from_thinking_budget, role,
+    },
+    vertex_ai::gemini::schema::{remove_additional_properties, remove_strict_from_schema},
+};
+
+pub struct HostedVllmChatConfig;
+
+pub const HOSTED_VLLM_CHAT_CONFIG: HostedVllmChatConfig = HostedVllmChatConfig;
+
+impl OpenAiCompatibleChatConfig for HostedVllmChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::HostedVllm
+    }
+
+    fn label(&self) -> &'static str {
+        "hosted vLLM"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["HOSTED_VLLM_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        None
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &["HOSTED_VLLM_API_KEY"]
+    }
+
+    /// vLLM needs no key, and Python sends this one when none is set.
+    fn default_api_key(&self) -> Option<&'static str> {
+        Some("fake-api-key")
+    }
+
+    fn supported_params(&self, _context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        GPT_BASE_PARAMS
+            .iter()
+            .copied()
+            .chain(["response_format", "reasoning_effort", "thinking"])
+            .collect()
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        let mut params = non_default.clone();
+        if let Some(mut tools) = params.remove("tools").filter(|tools| !tools.is_null()) {
+            remove_additional_properties(&mut tools);
+            remove_strict_from_schema(&mut tools);
+            if let Value::Array(items) = tools {
+                tools = Value::Array(
+                    items
+                        .into_iter()
+                        .enumerate()
+                        .map(|(index, tool)| custom_tool_as_function(index, tool))
+                        .collect(),
+                );
+            }
+            params.insert("tools".into(), tools);
+        }
+        if let Some(Value::Object(thinking)) = params.remove("thinking")
+            && thinking.get("type").and_then(Value::as_str) == Some("enabled")
+            && !params.contains_key("reasoning_effort")
+        {
+            let budget = thinking
+                .get("budget_tokens")
+                .and_then(Value::as_i64)
+                .unwrap_or(0);
+            params.insert(
+                "reasoning_effort".into(),
+                json!(reasoning_effort_from_thinking_budget(budget)),
+            );
+        }
+        copy_supported(&params, optional, &self.supported_params(context));
+        Ok(())
+    }
+
+    fn transform_messages(
+        &self,
+        messages: Vec<Value>,
+        _optional: &Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<Vec<Value>, Error> {
+        let messages = messages
+            .into_iter()
+            .map(|mut message| {
+                match role(&message) {
+                    Some("assistant") => assistant_for_vllm(&mut message),
+                    Some("user") => video_files_as_video_urls(&mut message),
+                    _ => {}
+                }
+                message
+            })
+            .collect();
+        gpt_messages(messages)
+    }
+}
+
+/// `_convert_custom_tools_to_function_tools` for one tool.
+fn custom_tool_as_function(index: usize, tool: Value) -> Value {
+    if tool.get("type").and_then(Value::as_str) != Some("custom") {
+        return tool;
+    }
+    let custom = tool
+        .get("custom")
+        .filter(|custom| custom.is_object())
+        .cloned()
+        .unwrap_or_else(|| json!({}));
+    let pick = |key: &str| {
+        custom
+            .get(key)
+            .filter(|value| crate::base_llm::chat::params::truthy(value))
+            .or_else(|| tool.get(key).filter(|value| !value.is_null()))
+            .cloned()
+    };
+    let name = match pick("name") {
+        Some(Value::String(name)) => name,
+        Some(other) => python_str(&other),
+        None => format!("custom_tool_{index}"),
+    };
+    let parameters = match pick("input_schema") {
+        Some(schema @ Value::Object(_)) => schema,
+        _ => json!({
+            "type": "object",
+            "properties": {"input": {"type": "string", "description": "Raw tool input payload."}},
+            "required": ["input"],
+        }),
+    };
+    let mut function = Map::from_iter([
+        ("name".to_owned(), Value::String(name)),
+        ("parameters".to_owned(), parameters),
+    ]);
+    if let Some(Value::String(description)) = pick("description") {
+        function.insert("description".into(), Value::String(description));
+    }
+    json!({"type": "function", "function": function})
+}
+
+fn python_str(value: &Value) -> String {
+    match value {
+        Value::String(text) => text.clone(),
+        other => python_dumps(other),
+    }
+}
+
+/// The assistant half of `HostedVLLMChatConfig._transform_messages`.
+fn assistant_for_vllm(message: &mut Value) {
+    let Some(object) = message.as_object_mut() else {
+        return;
+    };
+    object.remove("thinking_blocks");
+    if !object
+        .get("reasoning_content")
+        .is_some_and(Value::is_string)
+    {
+        object.remove("reasoning_content");
+    }
+    let Some(Value::Array(parts)) = object.get("content").cloned() else {
+        return;
+    };
+    let mut texts = Vec::new();
+    let mut tool_calls = Vec::new();
+    let mut blocks = Vec::new();
+    let mut structured = false;
+    for part in parts {
+        match part.get("type").and_then(Value::as_str) {
+            Some("text") if part.is_object() => {
+                texts.push(
+                    part.get("text")
+                        .and_then(Value::as_str)
+                        .unwrap_or_default()
+                        .to_owned(),
+                );
+                blocks.push(part);
+            }
+            Some("tool_use") if part.is_object() => {
+                let input = part.get("input").cloned().unwrap_or_else(|| json!({}));
+                let arguments = match input {
+                    Value::String(text) => text,
+                    other => python_dumps(&other),
+                };
+                tool_calls.push(json!({
+                    "id": part.get("id").cloned().unwrap_or(Value::Null),
+                    "type": "function",
+                    "function": {"name": part.get("name").cloned().unwrap_or(Value::Null), "arguments": arguments},
+                }));
+            }
+            _ => {
+                blocks.push(part);
+                structured = true;
+            }
+        }
+    }
+    if !tool_calls.is_empty() {
+        match object.get_mut("tool_calls") {
+            Some(Value::Array(existing)) => {
+                let known: Vec<Value> = existing
+                    .iter()
+                    .filter_map(|call| call.get("id").filter(|id| !id.is_null()).cloned())
+                    .collect();
+                existing.extend(
+                    tool_calls
+                        .into_iter()
+                        .filter(|call| !known.contains(&call["id"])),
+                );
+            }
+            _ => {
+                object.insert("tool_calls".into(), Value::Array(tool_calls));
+            }
+        }
+    }
+    let content = match structured {
+        true => Value::Array(blocks),
+        false => Value::String(texts.join("\n")),
+    };
+    object.insert("content".into(), content);
+}
+
+/// `_is_video_file`: a `file` part whose format, data URL or file id names a video.
+fn is_video_file(part: &Value) -> bool {
+    if part.get("type").and_then(Value::as_str) != Some("file") {
+        return false;
+    }
+    let file = part.get("file");
+    let field = |key: &str| {
+        file.and_then(|file| file.get(key))
+            .and_then(Value::as_str)
+            .filter(|value| !value.is_empty())
+    };
+    if field("format").is_some_and(|format| format.starts_with("video/")) {
+        return true;
+    }
+    if let Some(data) = field("file_data") {
+        return data_url_mime(data).is_some_and(|mime| mime.starts_with("video/"));
+    }
+    if let Some(id) = field("file_id") {
+        return url_video_mime(id);
+    }
+    false
+}
+
+/// `parse_mime_type`: the media type of a `data:` URL.
+fn data_url_mime(data: &str) -> Option<&str> {
+    let rest = data.strip_prefix("data:")?;
+    let end = rest.find([';', ','])?;
+    Some(&rest[..end])
+}
+
+/// `get_image_mime_type_from_url` for the video extensions it knows.
+fn url_video_mime(url: &str) -> bool {
+    let path = url.split(['?', '#']).next().unwrap_or(url).to_lowercase();
+    [
+        ".mp4", ".mov", ".avi", ".flv", ".mpeg", ".mpg", ".webm", ".wmv", ".3gpp",
+    ]
+    .iter()
+    .any(|extension| path.ends_with(extension))
+}
+
+fn video_files_as_video_urls(message: &mut Value) {
+    let Some(Value::Array(parts)) = message.get_mut("content") else {
+        return;
+    };
+    for part in parts.iter_mut() {
+        if !is_video_file(part) {
+            continue;
+        }
+        let file = part.get("file").cloned().unwrap_or_default();
+        let url = ["file_id", "file_data"]
+            .iter()
+            .find_map(|key| {
+                file.get(*key)
+                    .filter(|value| crate::base_llm::chat::params::truthy(value))
+            })
+            .cloned();
+        if let Some(url) = url {
+            *part = json!({"type": "video_url", "video_url": {"url": url}});
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+    use crate::openai::chat::compatible::transform_request;
+
+    fn send(params: Value) -> Map<String, Value> {
+        let Value::Object(body) = params else {
+            panic!("an object")
+        };
+        transform_request(
+            &HOSTED_VLLM_CHAT_CONFIG,
+            &CompatibleContext {
+                model: "Qwen/Qwen3-8B",
+                drop_params: false,
+                model_info: None,
+            },
+            body,
+        )
+        .unwrap()
+        .body
+    }
+
+    #[rstest]
+    fn tools_lose_strict_and_closed_objects_and_custom_tools_become_functions() {
+        let sent = send(json!({"messages": [], "tools": [
+            {"type": "function", "function": {"name": "f", "strict": true,
+                "parameters": {"type": "object", "additionalProperties": false, "properties": {}}}},
+            {"type": "custom", "custom": {"name": "grep", "description": "search"}},
+        ]}));
+        assert_eq!(
+            sent["tools"],
+            json!([
+                {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}},
+                {"type": "function", "function": {"name": "grep", "description": "search", "parameters": {
+                    "type": "object",
+                    "properties": {"input": {"type": "string", "description": "Raw tool input payload."}},
+                    "required": ["input"]}}},
+            ])
+        );
+    }
+
+    #[rstest]
+    #[case::budget(json!({"thinking": {"type": "enabled", "budget_tokens": 3000}}), Some("medium"))]
+    #[case::effort_wins(json!({"thinking": {"type": "enabled", "budget_tokens": 3000}, "reasoning_effort": "low"}), Some("low"))]
+    #[case::disabled(json!({"thinking": {"type": "disabled"}}), None)]
+    fn thinking_becomes_a_reasoning_effort(#[case] params: Value, #[case] effort: Option<&str>) {
+        let mut body = json!({"messages": []});
+        body.as_object_mut()
+            .unwrap()
+            .extend(params.as_object().unwrap().clone());
+        let sent = send(body);
+        assert!(!sent.contains_key("thinking"));
+        assert_eq!(sent.get("reasoning_effort").and_then(Value::as_str), effort);
+    }
+
+    #[rstest]
+    fn assistant_turns_are_flattened_for_vllm() {
+        let sent = send(json!({"messages": [
+            {"role": "assistant", "thinking_blocks": [{"type": "thinking"}], "reasoning_content": {"x": 1},
+             "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"},
+                {"type": "tool_use", "id": "t1", "name": "f", "input": {"q": 1}}]},
+            {"role": "user", "content": [{"type": "file", "file": {"file_data": "data:video/mp4;base64,AA"}}]},
+        ]}));
+        assert_eq!(
+            sent["messages"],
+            json!([
+                {"role": "assistant", "content": "a\nb", "tool_calls": [
+                    {"id": "t1", "type": "function", "function": {"name": "f", "arguments": "{\"q\": 1}"}}]},
+                {"role": "user", "content": [{"type": "video_url", "video_url": {"url": "data:video/mp4;base64,AA"}}]},
+            ])
+        );
+    }
+
+    #[rstest]
+    fn the_key_defaults_for_a_server_that_takes_none() {
+        assert_eq!(
+            HOSTED_VLLM_CHAT_CONFIG.api_key(None, &|_| None).unwrap(),
+            "fake-api-key"
+        );
+    }
+
+    #[rstest]
+    fn an_api_base_is_required() {
+        assert!(HOSTED_VLLM_CHAT_CONFIG.api_base(None, &|_| None).is_err());
+    }
+}

litellm-rust/crates/llms/src/hosted_vllm/mod.rs

@@ -0,0 +1 @@
+pub mod chat;

litellm-rust/crates/llms/src/lib.rs

@@ -5,18 +5,26 @@ pub mod azure_ai;
 pub mod base_llm;
 pub mod bedrock;
 pub mod bedrock_mantle;
+pub mod cerebras;
 pub mod cohere;
 pub mod deepseek;
 pub mod edenai;
 mod error;
+pub mod fireworks_ai;
 pub mod gemini;
+pub mod groq;
+pub mod hosted_vllm;
 pub mod minimax;
 pub mod mistral;
+pub mod nvidia_nim;
 pub mod openai;
 pub mod openai_like;
 pub mod openrouter;
+pub mod perplexity;
 pub mod reducto;
 pub mod tencent;
+pub mod together_ai;
 pub mod vertex_ai;
+pub mod xai;
 
 pub use error::{Error, ErrorDetail, ErrorSource};

litellm-rust/crates/llms/src/mistral/chat/AGENTS.md

@@ -0,0 +1,11 @@
+# rules
+
+- Mistral at `https://api.mistral.ai/v1`; a base without `/v1` gets it appended
+- `tool_choice` narrows to `auto`/`any`/`none`, `seed` goes as `random_seed`, magistral reasoning adds Mistral's system prompt
+- Content blocks in answers and chunks become text plus `reasoning_content`
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://docs.mistral.ai/api/#tag/chat
+- litellm/llms/mistral/chat/transformation.py

litellm-rust/crates/llms/src/mistral/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/mistral/chat/transformation.rs

@@ -0,0 +1,605 @@
+//! `litellm/llms/mistral/chat/transformation.py`: Mistral at `https://api.mistral.ai/v1`.
+//! Tool choices narrow to Mistral's `auto`/`any`/`none`, `seed` goes as `random_seed`,
+//! magistral models get Mistral's reasoning system prompt, replayed messages lose what
+//! Mistral's schema forbids, and content lists in the answer become text plus
+//! `reasoning_content`.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error,
+    openai::chat::compatible::{
+        CompatibleContext, OpenAiCompatibleChatConfig, ResponseShape, content_list_to_str,
+        gpt_messages, messages_mut, nearest_declared_reasoning_effort, role, strip_internal_fields,
+    },
+};
+
+const REASONING_PROMPT_FLAG: &str = "_add_reasoning_prompt";
+/// `_get_mistral_reasoning_system_prompt`, verbatim with its indentation.
+const REASONING_SYSTEM_PROMPT: &str = r"
+        <s>[SYSTEM_PROMPT]system_prompt
+        A user will ask you to solve a task. You should first draft your thinking process (inner monologue) until you have derived the final answer. Afterwards, write a self-contained summary of your thoughts (i.e. your summary should be succinct but contain all the critical steps you needed to reach the conclusion). You should use Markdown to format your response. Write both your thoughts and summary in the same language as the task posed by the user. NEVER use \boxed{} in your response.
+
+        Your thinking process must follow the template below:
+        <think>
+        Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct answer.
+        </think>
+
+        Here, provide a concise summary that reflects your reasoning and presents a clear final answer to the user. Don't mention that this is a summary.
+
+        Problem:
+
+        [/SYSTEM_PROMPT][INST]user_message[/INST]<think>
+        reasoning_traces
+        </think>
+        assistant_response</s>[INST]user_message[/INST]
+        ";
+/// The keys of `ChatCompletionAssistantMessage` besides `role`.
+const ASSISTANT_MESSAGE_KEYS: &[&str] = &[
+    "content",
+    "name",
+    "tool_calls",
+    "function_call",
+    "reasoning_content",
+    "cache_control",
+    "thinking_blocks",
+    "reasoning_items",
+];
+
+pub struct MistralChatConfig;
+
+pub const MISTRAL_CHAT_CONFIG: MistralChatConfig = MistralChatConfig;
+
+impl OpenAiCompatibleChatConfig for MistralChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::Mistral
+    }
+
+    fn label(&self) -> &'static str {
+        "Mistral"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["MISTRAL_AZURE_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://api.mistral.ai/v1")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &["MISTRAL_AZURE_API_KEY", "MISTRAL_API_KEY"]
+    }
+
+    /// Mistral always needs `/v1` at the end of the base.
+    fn api_base(
+        &self,
+        configured: Option<&str>,
+        lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        let base = configured
+            .filter(|base| !base.is_empty())
+            .map(str::to_owned)
+            .or_else(|| lookup("MISTRAL_AZURE_API_BASE"))
+            .unwrap_or_else(|| "https://api.mistral.ai/v1".to_owned());
+        Ok(match base.ends_with("/v1") {
+            true => base,
+            false => format!("{base}/v1"),
+        })
+    }
+
+    fn supported_params(&self, context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        let magistral = is_magistral(context.model);
+        let mut params = vec![
+            "stream",
+            "temperature",
+            "top_p",
+            "max_tokens",
+            "max_completion_tokens",
+            "tools",
+            "tool_choice",
+            "seed",
+            "stop",
+            "response_format",
+            "parallel_tool_calls",
+        ];
+        if magistral {
+            params.push("thinking");
+        }
+        if magistral || context.supports(self.name(), "supports_reasoning") {
+            params.push("reasoning_effort");
+        }
+        params
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        let magistral = is_magistral(context.model);
+        for (name, value) in non_default {
+            match name.as_str() {
+                "max_tokens" if !non_default.contains_key("max_completion_tokens") => {
+                    optional.insert("max_tokens".into(), value.clone());
+                }
+                "max_completion_tokens" => {
+                    optional.insert("max_tokens".into(), value.clone());
+                }
+                "tools" => {
+                    let mut tools = value.clone();
+                    remove_json_schema_refs(&mut tools, 10);
+                    optional.insert("tools".into(), tools);
+                }
+                "stream" if value == &Value::Bool(true) => {
+                    optional.insert("stream".into(), value.clone());
+                }
+                "temperature" | "top_p" | "stop" | "response_format" | "parallel_tool_calls" => {
+                    optional.insert(name.clone(), value.clone());
+                }
+                "tool_choice" => {
+                    if let Some(choice) = value.as_str() {
+                        let choice = match choice {
+                            "auto" | "none" => choice,
+                            _ => "any",
+                        };
+                        optional.insert("tool_choice".into(), json!(choice));
+                    }
+                }
+                "seed" => {
+                    optional.insert("extra_body".into(), json!({"random_seed": value}));
+                }
+                "reasoning_effort" | "thinking" if magistral => {
+                    optional.insert(REASONING_PROMPT_FLAG.into(), Value::Bool(true));
+                }
+                "reasoning_effort" => {
+                    let effort = match (
+                        value.as_str(),
+                        context.declared_reasoning_efforts(self.name()),
+                    ) {
+                        (Some(requested), Some(declared)) => {
+                            json!(nearest_declared_reasoning_effort(requested, &declared))
+                        }
+                        _ => value.clone(),
+                    };
+                    optional.insert("reasoning_effort".into(), effort);
+                }
+                _ => {}
+            }
+        }
+        Ok(())
+    }
+
+    fn map_extra_body(
+        &self,
+        extra_body: &mut Map<String, Value>,
+        _optional: &mut Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        extra_body.remove("client_metadata");
+        Ok(())
+    }
+
+    fn transform_messages(
+        &self,
+        messages: Vec<Value>,
+        optional: &Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<Vec<Value>, Error> {
+        let mut messages = messages;
+        if is_magistral(context.model) && optional.contains_key(REASONING_PROMPT_FLAG) {
+            add_reasoning_system_prompt(&mut messages);
+        }
+        for message in messages.iter_mut() {
+            if role(message) == Some("assistant") {
+                strip_internal_fields(message);
+            }
+        }
+        let has_media = messages.iter().any(|message| {
+            message
+                .get("content")
+                .and_then(Value::as_array)
+                .is_some_and(|parts| {
+                    parts.iter().any(|part| {
+                        matches!(
+                            part.get("type").and_then(Value::as_str),
+                            Some("image_url" | "file")
+                        )
+                    })
+                })
+        });
+        if has_media {
+            let mut messages = gpt_messages(messages)?;
+            messages.iter_mut().for_each(file_parts_to_file_ids);
+            return Ok(messages);
+        }
+        let messages = messages
+            .into_iter()
+            .map(|mut message| {
+                let text = content_list_to_str(&message);
+                if let Some(object) = message.as_object_mut() {
+                    if !text.is_empty() {
+                        object.insert("content".into(), Value::String(text));
+                    }
+                    keep_name_on_tool_messages(object);
+                    rebuild_tool_calls(object);
+                }
+                message
+            })
+            .filter(|message| !is_empty_assistant_message(message))
+            .map(|mut message| {
+                if let Some(object) = message.as_object_mut() {
+                    object.retain(|_, value| !value.is_null());
+                }
+                message
+            })
+            .collect();
+        gpt_messages(messages)
+    }
+
+    fn finish_request(
+        &self,
+        body: &mut Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        body.remove(REASONING_PROMPT_FLAG);
+        body.remove("client_metadata");
+        Ok(())
+    }
+
+    fn fix_response(&self, body: &mut Map<String, Value>, _shape: &ResponseShape) -> bool {
+        let mut changed = false;
+        for message in messages_mut(body) {
+            match message.get("content") {
+                Some(Value::String(content)) if content.is_empty() => {
+                    message.insert("content".into(), Value::Null);
+                    changed = true;
+                }
+                Some(Value::Array(blocks)) if !blocks.is_empty() => {
+                    let (text, thinking) = split_content_blocks(blocks);
+                    message.insert("content".into(), Value::String(text));
+                    if !thinking.is_empty() {
+                        message.insert("reasoning_content".into(), Value::String(thinking));
+                    }
+                    changed = true;
+                }
+                _ => {}
+            }
+        }
+        changed
+    }
+
+    fn fix_chunk(&self, chunk: &mut Map<String, Value>, _shape: &ResponseShape) -> bool {
+        let mut changed = false;
+        let Some(Value::Array(choices)) = chunk.get_mut("choices") else {
+            return false;
+        };
+        for delta in choices
+            .iter_mut()
+            .filter_map(|choice| choice.get_mut("delta")?.as_object_mut())
+        {
+            let Some(Value::Array(blocks)) = delta.get("content") else {
+                continue;
+            };
+            let (text, thinking_blocks, reasoning) = normalize_stream_blocks(blocks);
+            delta.insert("content".into(), text.map_or(Value::Null, Value::String));
+            match reasoning {
+                Some(reasoning) if !thinking_blocks.is_empty() => {
+                    delta.insert("thinking_blocks".into(), Value::Array(thinking_blocks));
+                    delta.insert("reasoning_content".into(), Value::String(reasoning));
+                }
+                _ => {
+                    delta.remove("thinking_blocks");
+                    delta.remove("reasoning_content");
+                }
+            }
+            changed = true;
+        }
+        changed
+    }
+}
+
+fn is_magistral(model: &str) -> bool {
+    model.to_lowercase().contains("magistral")
+}
+
+/// `remove_json_schema_refs`.
+fn remove_json_schema_refs(value: &mut Value, depth: usize) {
+    if depth == 0 {
+        return;
+    }
+    match value {
+        Value::Object(object) => {
+            object.remove("$id");
+            object.remove("$schema");
+            object
+                .values_mut()
+                .for_each(|nested| remove_json_schema_refs(nested, depth - 1));
+        }
+        Value::Array(items) => items
+            .iter_mut()
+            .for_each(|item| remove_json_schema_refs(item, depth - 1)),
+        _ => {}
+    }
+}
+
+/// `_add_reasoning_system_prompt_if_needed`.
+fn add_reasoning_system_prompt(messages: &mut Vec<Value>) {
+    let Some(system) = messages
+        .iter_mut()
+        .find(|message| role(message) == Some("system"))
+        .and_then(Value::as_object_mut)
+    else {
+        messages.insert(
+            0,
+            json!({"role": "system", "content": REASONING_SYSTEM_PROMPT}),
+        );
+        return;
+    };
+    let content = match system.remove("content") {
+        Some(Value::String(existing)) => json!(format!("{REASONING_SYSTEM_PROMPT}\n\n{existing}")),
+        Some(Value::Array(existing)) => Value::Array(
+            std::iter::once(
+                json!({"type": "text", "text": format!("{REASONING_SYSTEM_PROMPT}\n\n")}),
+            )
+            .chain(existing)
+            .collect(),
+        ),
+        Some(Value::Null) => json!(format!("{REASONING_SYSTEM_PROMPT}\n\nNone")),
+        Some(other) => json!(format!("{REASONING_SYSTEM_PROMPT}\n\n{other}")),
+        None => json!(format!("{REASONING_SYSTEM_PROMPT}\n\n")),
+    };
+    system.insert("content".into(), content);
+}
+
+/// `_handle_message_with_file`: Mistral takes a file part by its `file_id`.
+fn file_parts_to_file_ids(message: &mut Value) {
+    let Some(Value::Array(parts)) = message.get_mut("content") else {
+        return;
+    };
+    for part in parts
+        .iter_mut()
+        .filter_map(Value::as_object_mut)
+        .filter(|part| part.get("type").and_then(Value::as_str) == Some("file"))
+    {
+        let file_id = part
+            .get("file")
+            .and_then(|file| file.get("file_id"))
+            .filter(|id| crate::base_llm::chat::params::truthy(id))
+            .cloned();
+        if let Some(file_id) = file_id {
+            part.insert("file_id".into(), file_id);
+            part.remove("file");
+        }
+    }
+}
+
+/// `_handle_name_in_message`: Mistral takes `name` only on tool messages.
+fn keep_name_on_tool_messages(message: &mut Map<String, Value>) {
+    let Some(name) = message.get("name").filter(|name| !name.is_null()) else {
+        return;
+    };
+    let blank = name.as_str().is_some_and(|name| name.trim().is_empty());
+    if message.get("role").and_then(Value::as_str) != Some("tool") || blank {
+        message.remove("name");
+    }
+}
+
+/// `_handle_tool_call_message`: each call as `{id, type: "function", function}`.
+fn rebuild_tool_calls(message: &mut Map<String, Value>) {
+    let Some(Value::Array(calls)) = message.get_mut("tool_calls") else {
+        return;
+    };
+    for call in calls.iter_mut() {
+        *call = json!({
+            "id": call.get("id").cloned().unwrap_or(Value::Null),
+            "type": "function",
+            "function": call.get("function").cloned().unwrap_or(Value::Null),
+        });
+    }
+}
+
+/// `_is_empty_assistant_message`.
+fn is_empty_assistant_message(message: &Value) -> bool {
+    role(message) == Some("assistant")
+        && ASSISTANT_MESSAGE_KEYS
+            .iter()
+            .all(|key| match message.get(*key) {
+                None | Some(Value::Null) => true,
+                Some(Value::String(content)) => *key == "content" && content.is_empty(),
+                Some(_) => false,
+            })
+}
+
+/// `_handle_content_list_to_str_conversion` for one message: the last text block, and the
+/// last thinking block's text parts joined by newlines.
+fn split_content_blocks(blocks: &[Value]) -> (String, String) {
+    let mut text = String::new();
+    let mut thinking = String::new();
+    for block in blocks {
+        match block.get("type").and_then(Value::as_str) {
+            Some("thinking") => thinking = thinking_texts(block).join("\n"),
+            Some("text") => {
+                text = block
+                    .get("text")
+                    .and_then(Value::as_str)
+                    .unwrap_or_default()
+                    .to_owned();
+            }
+            _ => {}
+        }
+    }
+    (text, thinking)
+}
+
+fn thinking_texts(block: &Value) -> Vec<&str> {
+    block
+        .get("thinking")
+        .and_then(Value::as_array)
+        .into_iter()
+        .flatten()
+        .filter(|part| part.get("type").and_then(Value::as_str) == Some("text"))
+        .map(|part| part.get("text").and_then(Value::as_str).unwrap_or_default())
+        .collect()
+}
+
+/// `MistralChatResponseIterator._normalize_content_blocks`.
+fn normalize_stream_blocks(blocks: &[Value]) -> (Option<String>, Vec<Value>, Option<String>) {
+    let mut text: Option<String> = None;
+    let mut thinking_blocks = Vec::new();
+    let mut reasoning = Vec::new();
+    for block in blocks {
+        match block.get("type").and_then(Value::as_str) {
+            Some("thinking") => {
+                let thought = thinking_texts(block).concat();
+                if !thought.is_empty() {
+                    thinking_blocks.push(
+                        json!({"type": "thinking", "thinking": thought, "signature": "mistral"}),
+                    );
+                    reasoning.push(thought);
+                }
+            }
+            Some("text") => text.get_or_insert_with(String::new).push_str(
+                block
+                    .get("text")
+                    .and_then(Value::as_str)
+                    .unwrap_or_default(),
+            ),
+            _ => {}
+        }
+    }
+    let reasoning = (!reasoning.is_empty()).then(|| reasoning.join("\n"));
+    (text, thinking_blocks, reasoning)
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+    use crate::openai::chat::compatible::{fix_response, transform_request};
+
+    fn object(value: Value) -> Map<String, Value> {
+        let Value::Object(map) = value else {
+            panic!("an object")
+        };
+        map
+    }
+
+    fn send(model: &str, params: Value) -> Map<String, Value> {
+        transform_request(
+            &MISTRAL_CHAT_CONFIG,
+            &CompatibleContext {
+                model,
+                drop_params: false,
+                model_info: None,
+            },
+            object(params),
+        )
+        .unwrap()
+        .body
+    }
+
+    #[rstest]
+    #[case::bare("https://example.ai", "https://example.ai/v1")]
+    #[case::versioned("https://example.ai/v1", "https://example.ai/v1")]
+    fn the_base_always_ends_in_v1(#[case] configured: &str, #[case] expected: &str) {
+        assert_eq!(
+            MISTRAL_CHAT_CONFIG
+                .api_base(Some(configured), &|_| None)
+                .unwrap(),
+            expected
+        );
+    }
+
+    #[rstest]
+    fn params_take_mistral_names() {
+        let sent = send(
+            "mistral-large-latest",
+            json!({"messages": [], "max_tokens": 1,
+            "max_completion_tokens": 9, "seed": 3, "stream": false, "tool_choice": "required",
+            "tools": [{"type": "function", "function": {"name": "f", "parameters": {"$schema": "s", "$id": "i", "type": "object"}}}]}),
+        );
+        assert_eq!(
+            Value::Object(sent),
+            json!({"model": "mistral-large-latest", "messages": [], "max_tokens": 9,
+                "tools": [{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}],
+                "tool_choice": "any", "random_seed": 3})
+        );
+    }
+
+    #[rstest]
+    fn magistral_reasoning_adds_the_system_prompt_instead_of_the_param() {
+        let sent = send(
+            "magistral-medium-latest",
+            json!({"reasoning_effort": "high",
+            "messages": [{"role": "system", "content": "be brief"}]}),
+        );
+        assert_eq!(
+            sent["messages"],
+            json!([{"role": "system", "content": format!("{REASONING_SYSTEM_PROMPT}\n\nbe brief")}])
+        );
+        assert!(!sent.contains_key("reasoning_effort"));
+        assert!(!sent.contains_key(REASONING_PROMPT_FLAG));
+    }
+
+    #[rstest]
+    fn replayed_messages_lose_what_mistral_forbids() {
+        let sent = send(
+            "mistral-small",
+            json!({"messages": [
+            {"role": "user", "name": "u", "content": [{"type": "text", "text": "hi"}]},
+            {"role": "assistant", "content": "", "reasoning_content": "r", "thinking_blocks": []},
+            {"role": "assistant", "content": null, "tool_calls": [{"index": 0, "id": "c", "type": "function", "function": {"name": "f", "arguments": "{}"}}]},
+            {"role": "tool", "name": "f", "tool_call_id": "c", "content": "ok"}]}),
+        );
+        assert_eq!(
+            sent["messages"],
+            json!([
+                {"role": "user", "content": "hi"},
+                {"role": "assistant", "tool_calls": [{"id": "c", "type": "function", "function": {"name": "f", "arguments": "{}"}}]},
+                {"role": "tool", "name": "f", "tool_call_id": "c", "content": "ok"}])
+        );
+    }
+
+    #[rstest]
+    fn file_parts_go_by_file_id() {
+        let sent = send(
+            "mistral-small",
+            json!({"messages": [{"role": "user", "content": [
+            {"type": "text", "text": "read"}, {"type": "file", "file": {"file_id": "abc"}}]}]}),
+        );
+        assert_eq!(
+            sent["messages"][0]["content"],
+            json!([{"type": "text", "text": "read"}, {"type": "file", "file_id": "abc"}])
+        );
+    }
+
+    #[rstest]
+    #[case::empty(json!(""), json!({"role": "assistant", "content": null}))]
+    #[case::blocks(json!([{"type": "thinking", "thinking": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}, {"type": "text", "text": "done"}]),
+        json!({"role": "assistant", "content": "done", "reasoning_content": "a\nb"}))]
+    fn answers_become_text_and_reasoning(#[case] content: Value, #[case] message: Value) {
+        let mut body =
+            object(json!({"choices": [{"message": {"role": "assistant", "content": content}}]}));
+        assert!(fix_response(
+            &MISTRAL_CHAT_CONFIG,
+            &mut body,
+            &ResponseShape::default()
+        ));
+        assert_eq!(body["choices"][0]["message"], message);
+    }
+
+    #[rstest]
+    fn streamed_blocks_become_text_and_thinking() {
+        let mut chunk = object(json!({"choices": [{"index": 0, "delta": {"content": [
+            {"type": "thinking", "thinking": [{"type": "text", "text": "hm"}]}, {"type": "text", "text": "ok"}]}}]}));
+        assert!(MISTRAL_CHAT_CONFIG.fix_chunk(&mut chunk, &ResponseShape::default()));
+        assert_eq!(
+            chunk["choices"][0]["delta"],
+            json!({"content": "ok", "thinking_blocks": [{"type": "thinking", "thinking": "hm", "signature": "mistral"}], "reasoning_content": "hm"})
+        );
+    }
+}

litellm-rust/crates/llms/src/mistral/mod.rs

@@ -1 +1,2 @@
+pub mod chat;
 pub mod ocr;

litellm-rust/crates/llms/src/nvidia_nim/chat/AGENTS.md

@@ -0,0 +1,11 @@
+# rules
+
+- NIM's hosted API through the OpenAI SDK URL rule: `{api_base}/chat/completions`
+- The supported params depend on the model family; `max_completion_tokens` goes as `max_tokens`
+- No `stream_options`, so stream usage is only what NIM sends unasked
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://docs.api.nvidia.com/nim/reference/llm-apis
+- litellm/llms/nvidia_nim/chat/transformation.py

litellm-rust/crates/llms/src/nvidia_nim/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/nvidia_nim/chat/transformation.rs

@@ -0,0 +1,196 @@
+//! `litellm/llms/nvidia_nim/chat/transformation.py`: NVIDIA NIM through the OpenAI SDK at
+//! `https://integrate.api.nvidia.com/v1`, with the params each model family takes and
+//! `max_completion_tokens` sent as `max_tokens`.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value};
+
+use crate::{
+    Error,
+    openai::chat::compatible::{CompatibleContext, OpenAiCompatibleChatConfig, sdk_complete_url},
+};
+
+const GEMMA_MODELS: &[&str] = &[
+    "google/recurrentgemma-2b",
+    "google/gemma-2-27b-it",
+    "google/gemma-2-9b-it",
+    "gemma-2-9b-it",
+];
+
+pub struct NvidiaNimChatConfig;
+
+pub const NVIDIA_NIM_CHAT_CONFIG: NvidiaNimChatConfig = NvidiaNimChatConfig;
+
+impl OpenAiCompatibleChatConfig for NvidiaNimChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::NvidiaNim
+    }
+
+    fn label(&self) -> &'static str {
+        "NVIDIA NIM"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["NVIDIA_NIM_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://integrate.api.nvidia.com/v1")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &["NVIDIA_NIM_API_KEY"]
+    }
+
+    fn complete_url(&self, api_base: &str) -> String {
+        sdk_complete_url(api_base)
+    }
+
+    fn supported_params(&self, context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        let model = context.model;
+        if GEMMA_MODELS.contains(&model) {
+            return vec![
+                "stream",
+                "temperature",
+                "top_p",
+                "max_tokens",
+                "stop",
+                "seed",
+            ];
+        }
+        match model {
+            "nvidia/nemotron-4-340b-instruct" => vec![
+                "stream",
+                "temperature",
+                "top_p",
+                "max_tokens",
+                "max_completion_tokens",
+            ],
+            "nvidia/nemotron-4-340b-reward" => vec!["stream"],
+            "google/codegemma-1.1-7b" => vec![
+                "stream",
+                "temperature",
+                "top_p",
+                "frequency_penalty",
+                "presence_penalty",
+                "max_tokens",
+                "max_completion_tokens",
+                "stop",
+            ],
+            _ => vec![
+                "stream",
+                "temperature",
+                "top_p",
+                "frequency_penalty",
+                "presence_penalty",
+                "max_tokens",
+                "max_completion_tokens",
+                "stop",
+                "seed",
+                "tools",
+                "tool_choice",
+                "parallel_tool_calls",
+                "response_format",
+            ],
+        }
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        let supported = self.supported_params(context);
+        for (name, value) in non_default {
+            if name == "max_completion_tokens" {
+                optional.insert("max_tokens".into(), value.clone());
+            } else if supported.contains(&name.as_str()) {
+                optional.insert(name.clone(), value.clone());
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+    use crate::{ErrorDetail, openai::chat::compatible::transform_request};
+
+    fn context(model: &str, drop_params: bool) -> CompatibleContext<'_> {
+        CompatibleContext {
+            model,
+            drop_params,
+            model_info: None,
+        }
+    }
+
+    fn body(value: Value) -> Map<String, Value> {
+        let Value::Object(map) = value else {
+            panic!("an object")
+        };
+        map
+    }
+
+    #[rstest]
+    fn max_completion_tokens_goes_as_max_tokens() {
+        let sent = transform_request(
+            &NVIDIA_NIM_CHAT_CONFIG,
+            &context("meta/llama3-70b-instruct", false),
+            body(
+                json!({"model": "x", "messages": [{"role": "user", "content": "hi"}],
+                "max_completion_tokens": 9, "seed": 1, "top_k": 4}),
+            ),
+        )
+        .unwrap();
+        assert_eq!(
+            Value::Object(sent.body),
+            json!({"model": "meta/llama3-70b-instruct", "messages": [{"role": "user", "content": "hi"}],
+                "max_tokens": 9, "seed": 1, "top_k": 4})
+        );
+    }
+
+    #[rstest]
+    #[case::refused(false, true)]
+    #[case::dropped(true, false)]
+    fn a_param_the_model_family_lacks_is_refused_or_dropped(
+        #[case] drop_params: bool,
+        #[case] refused: bool,
+    ) {
+        let result = transform_request(
+            &NVIDIA_NIM_CHAT_CONFIG,
+            &context("google/gemma-2-9b-it", drop_params),
+            body(
+                json!({"messages": [], "tools": [{"type": "function", "function": {"name": "f"}}]}),
+            ),
+        );
+        match refused {
+            true => assert!(matches!(
+                result,
+                Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams {
+                    provider: "nvidia_nim",
+                    ..
+                }))
+            )),
+            false => assert!(!result.unwrap().body.contains_key("tools")),
+        }
+    }
+
+    #[rstest]
+    fn stream_options_are_not_sent_since_nim_does_not_list_them() {
+        let sent = transform_request(
+            &NVIDIA_NIM_CHAT_CONFIG,
+            &context("meta/llama3-70b-instruct", false),
+            body(
+                json!({"messages": [], "stream": true, "stream_options": {"include_usage": true}}),
+            ),
+        )
+        .unwrap();
+        assert!(!sent.body.contains_key("stream_options"));
+        assert!(sent.shape.client_usage);
+    }
+}

litellm-rust/crates/llms/src/nvidia_nim/mod.rs

@@ -0,0 +1 @@
+pub mod chat;

litellm-rust/crates/llms/src/openai/chat/AGENTS.md

@@ -4,3 +4,4 @@
 - Passthrough, not a `BaseConfig`: that contract parses messages and normalizes the response into `ChatCompletionsResponse`, which drops tool calls, ids and every field it does not type. OpenAI's own body and SSE bytes reach the caller as sent
 - The request changes only by the deployment model name, the `extra_body` merge the OpenAI SDK does, and dropping the LiteLLM-owned keys Python consumes before the call
 - Endpoint and credential fallbacks are shared with Responses in `../common_utils.rs`
+- `compatible.rs` is the request flow and response fixups the OpenAI-compatible providers share. Each provider's config lives in `src/<provider>/chat/`, and its body and SSE bytes change only where its config's fixups change them

litellm-rust/crates/llms/src/openai/chat/compatible.rs

@@ -0,0 +1,985 @@
+//! The chat flow every OpenAI-compatible provider shares, `litellm/llms/openai/chat/gpt_transformation.py`
+//! (`OpenAIGPTConfig`) as the base each provider config overrides, run over the caller's
+//! OpenAI-shaped body: Python's `get_optional_params` sorts the request's OpenAI params from the
+//! rest, refuses or drops what the provider does not take, maps the rest by the provider's
+//! rules, and passes every other key through as `extra_body` does. The messages go through the
+//! provider's `_transform_messages`, and the response reaches the caller as the provider sent
+//! it, changed only by the fixups each provider's `transform_response` and stream iterator make.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error, ErrorDetail,
+    base_llm::{
+        chat::params::{ChatRequestContext, sort_params, truthy},
+        model_info::ModelInfo,
+    },
+    openai::chat::transformation::{OPENAI_CHAT_COMPLETIONS_PATH, is_stream, take_extra_body},
+};
+
+/// The `base_params` of `OpenAIGPTConfig.get_supported_openai_params`.
+pub const GPT_BASE_PARAMS: &[&str] = &[
+    "frequency_penalty",
+    "logit_bias",
+    "logprobs",
+    "top_logprobs",
+    "max_tokens",
+    "max_completion_tokens",
+    "modalities",
+    "prediction",
+    "n",
+    "presence_penalty",
+    "seed",
+    "stop",
+    "stream",
+    "stream_options",
+    "temperature",
+    "top_p",
+    "tools",
+    "tool_choice",
+    "function_call",
+    "functions",
+    "max_retries",
+    "extra_headers",
+    "parallel_tool_calls",
+    "audio",
+    "web_search_options",
+    "service_tier",
+    "safety_identifier",
+    "prompt_cache_key",
+    "prompt_cache_retention",
+    "store",
+];
+
+/// `LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS`: what LiteLLM adds to a returned message that no
+/// OpenAI-compatible endpoint takes back.
+const RESPONSE_ONLY_MESSAGE_FIELDS: &[&str] = &[
+    "thinking_blocks",
+    "reasoning_items",
+    "provider_specific_fields",
+];
+/// `LITELLM_INTERNAL_MESSAGE_FIELDS`: the response-only fields and `reasoning_content`.
+const INTERNAL_MESSAGE_FIELDS: &[&str] = &[
+    "thinking_blocks",
+    "reasoning_items",
+    "provider_specific_fields",
+    "reasoning_content",
+];
+/// `DEFAULT_MAX_RECURSE_DEPTH`.
+const MAX_RECURSE_DEPTH: usize = 100;
+const STREAM_OPTIONS: &str = "stream_options";
+const INCLUDE_USAGE: &str = "include_usage";
+/// `RESPONSE_FORMAT_TOOL_NAME`.
+pub const RESPONSE_FORMAT_TOOL_NAME: &str = "json_tool_call";
+const POTENTIAL_PDF_URL_STARTS: &[&str] = &["https://", "http://", "www."];
+/// The content part types `_content_parts_contain_image` looks for.
+const IMAGE_PART_TYPES: &[&str] = &["image_url", "input_image", "image"];
+/// `DEFAULT_REASONING_EFFORT_{LOW,MEDIUM,HIGH}_THINKING_BUDGET` at their defaults.
+const THINKING_BUDGETS: (u64, u64, u64) = (1024, 2048, 4096);
+/// `REASONING_EFFORT_STRENGTH_ORDER`.
+const EFFORT_STRENGTH_ORDER: &[&str] = &["minimal", "low", "medium", "high", "xhigh", "max"];
+
+/// What a provider's param rules read about the call.
+#[derive(Clone, Copy)]
+pub struct CompatibleContext<'a> {
+    /// The model as the provider names it, without the route prefix.
+    pub model: &'a str,
+    /// `drop_params` from the deployment, the proxy settings or the request.
+    pub drop_params: bool,
+    pub model_info: Option<&'a dyn ModelInfo>,
+}
+
+impl CompatibleContext<'_> {
+    /// `supports_factory`: the `<provider>/<model>` entry's flag, else the bare model's.
+    pub fn capability(&self, provider: &str, key: &str) -> Option<bool> {
+        self.capability_of(
+            &[format!("{provider}/{}", self.model), self.model.to_string()],
+            key,
+        )
+    }
+
+    /// The first of `candidates` whose cost map entry sets `key`.
+    pub fn capability_of(&self, candidates: &[String], key: &str) -> Option<bool> {
+        let info = self.model_info?;
+        candidates
+            .iter()
+            .filter_map(|candidate| info.entry(candidate))
+            .find_map(|entry| entry.get(key).and_then(Value::as_bool))
+    }
+
+    pub fn supports(&self, provider: &str, key: &str) -> bool {
+        self.capability(provider, key) == Some(true)
+    }
+
+    /// `declared_reasoning_efforts_for_model`: the `reasoning_effort_levels` the entry for the
+    /// model declares, read through its bare twin. `None` when the map says nothing.
+    pub fn declared_reasoning_efforts(&self, provider: &str) -> Option<Vec<String>> {
+        let info = self.model_info?;
+        let prefixed = format!("{provider}/{}", self.model);
+        let (entry, prefixed_hit) = match info.entry(&prefixed) {
+            Some(entry) => (entry, true),
+            None => (info.entry(self.model)?, false),
+        };
+        let levels = match entry.get("reasoning_effort_levels") {
+            Some(levels) if !levels.is_null() => levels.clone(),
+            _ if prefixed_hit => info
+                .entry(self.model)
+                .and_then(|bare| bare.get("reasoning_effort_levels").cloned())
+                .unwrap_or(Value::Null),
+            _ => Value::Null,
+        };
+        let Value::Array(levels) = levels else {
+            return None;
+        };
+        Some(
+            levels
+                .into_iter()
+                .filter_map(|level| level.as_str().map(str::to_owned))
+                .collect(),
+        )
+    }
+}
+
+/// `nearest_declared_reasoning_effort`: the weakest declared level at least as strong as the
+/// request, else the strongest declared one. `none` and unknown levels go as asked.
+pub fn nearest_declared_reasoning_effort(requested: &str, declared: &[String]) -> String {
+    let rank = |effort: &str| EFFORT_STRENGTH_ORDER.iter().position(|e| *e == effort);
+    let mut ranked: Vec<&str> = declared
+        .iter()
+        .map(String::as_str)
+        .filter(|effort| rank(effort).is_some())
+        .collect();
+    ranked.sort_by_key(|effort| rank(effort));
+    let Some(requested_rank) = rank(requested) else {
+        return requested.to_owned();
+    };
+    if ranked.is_empty() || ranked.contains(&requested) {
+        return requested.to_owned();
+    }
+    ranked
+        .iter()
+        .find(|effort| rank(effort) >= Some(requested_rank))
+        .or(ranked.last())
+        .map_or_else(|| requested.to_owned(), |effort| (*effort).to_owned())
+}
+
+/// `reasoning_effort_from_thinking_budget` at the default budget thresholds.
+pub fn reasoning_effort_from_thinking_budget(budget_tokens: i64) -> &'static str {
+    let (low, medium, high) = THINKING_BUDGETS;
+    match budget_tokens {
+        tokens if tokens >= high as i64 => "high",
+        tokens if tokens >= medium as i64 => "medium",
+        tokens if tokens >= low as i64 => "low",
+        _ => "minimal",
+    }
+}
+
+/// What a request decided about its answer, which the response and stream fixups read.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ResponseShape {
+    pub stream: bool,
+    /// The client asked to see the usage on a stream. When it did not, the usage the call asked
+    /// for on its behalf is recorded and kept from the client.
+    pub client_usage: bool,
+    /// The provider answers a `response_format` with a `json_tool_call` tool call, which goes
+    /// back as content.
+    pub json_mode: bool,
+    /// The names of the function tools the request sent.
+    pub tool_names: Vec<String>,
+    /// Whether the request sent `tools` at all.
+    pub tools_sent: bool,
+}
+
+/// A request ready for an OpenAI-compatible provider.
+#[derive(Debug, PartialEq)]
+pub struct CompatibleChatBody {
+    pub body: Map<String, Value>,
+    pub shape: ResponseShape,
+}
+
+/// One OpenAI-compatible provider's config: its endpoint and credential fallbacks, and the
+/// overrides it makes to `OpenAIGPTConfig`.
+pub trait OpenAiCompatibleChatConfig: Sync {
+    fn provider(&self) -> LlmProviders;
+
+    /// The provider as a missing key error names it.
+    fn label(&self) -> &'static str;
+
+    /// The variables an unset `api_base` falls back to, in order.
+    fn api_base_env(&self) -> &'static [&'static str];
+
+    fn default_api_base(&self) -> Option<&'static str>;
+
+    /// The variables an unset key falls back to, in order.
+    fn api_key_env(&self) -> &'static [&'static str];
+
+    /// The key sent when neither the deployment nor the environment has one; `None` means the
+    /// call needs one.
+    fn default_api_key(&self) -> Option<&'static str> {
+        None
+    }
+
+    /// Other variables the provider's headers read.
+    fn header_env(&self) -> &'static [&'static str] {
+        &[]
+    }
+
+    fn name(&self) -> &'static str {
+        self.provider().into()
+    }
+
+    /// Every variable a call may read.
+    fn secret_names(&self) -> Vec<&'static str> {
+        self.api_base_env()
+            .iter()
+            .chain(self.api_key_env())
+            .chain(self.header_env())
+            .copied()
+            .collect()
+    }
+
+    /// The deployment's `api_base`, else the environment's, else the provider's own.
+    fn api_base(
+        &self,
+        configured: Option<&str>,
+        lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        configured
+            .filter(|base| !base.is_empty())
+            .map(str::to_owned)
+            .or_else(|| self.api_base_env().iter().find_map(|name| lookup(name)))
+            .or_else(|| self.default_api_base().map(str::to_owned))
+            .ok_or_else(|| {
+                Error::InvalidRequest(ErrorDetail::Message(format!(
+                    "Missing API Base - set api_base on the {} deployment or {}",
+                    self.name(),
+                    self.api_base_env().join(" or ")
+                )))
+            })
+    }
+
+    /// `OpenAIGPTConfig.get_complete_url`: `/chat/completions` appended unless the base already
+    /// names it.
+    fn complete_url(&self, api_base: &str) -> String {
+        gpt_complete_url(api_base)
+    }
+
+    /// The deployment's key, else the environment's, else the provider's default.
+    fn api_key(
+        &self,
+        configured: Option<&str>,
+        lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Result<String, Error> {
+        configured
+            .filter(|key| !key.is_empty())
+            .map(str::to_owned)
+            .or_else(|| {
+                self.api_key_env()
+                    .iter()
+                    .find_map(|name| lookup(name).filter(|key| !key.is_empty()))
+            })
+            .or_else(|| self.default_api_key().map(str::to_owned))
+            .ok_or(Error::Auth(litellm_auth::Error::MissingApiKey {
+                provider: self.label(),
+                environment_variable: self.api_key_env().first().copied().unwrap_or_default(),
+            }))
+    }
+
+    /// Headers the provider adds to every call, read off the caller's body before it is
+    /// sorted.
+    fn headers(
+        &self,
+        _body: &Map<String, Value>,
+        _lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Vec<(String, String)> {
+        Vec::new()
+    }
+
+    /// The model the body names.
+    fn wire_model(&self, model: &str) -> String {
+        model.to_owned()
+    }
+
+    /// `get_supported_openai_params`.
+    fn supported_params(&self, context: &CompatibleContext<'_>) -> Vec<&'static str>;
+
+    /// `map_openai_params`: `non_default` holds only supported params.
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        copy_supported(non_default, optional, &self.supported_params(context));
+        Ok(())
+    }
+
+    /// The keys sent as `extra_body`, which a provider may rewrite.
+    fn map_extra_body(
+        &self,
+        _extra_body: &mut Map<String, Value>,
+        _optional: &mut Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        Ok(())
+    }
+
+    /// `_transform_messages`.
+    fn transform_messages(
+        &self,
+        messages: Vec<Value>,
+        _optional: &Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<Vec<Value>, Error> {
+        gpt_messages(messages)
+    }
+
+    /// Whether `cache_control` marks reach the provider; `OpenAIGPTConfig` strips them.
+    fn keeps_cache_control(&self, _context: &CompatibleContext<'_>) -> bool {
+        false
+    }
+
+    /// The provider's `transform_request` changes over the assembled body, before the
+    /// `extra_body` keys are merged over it.
+    fn finish_request(
+        &self,
+        _body: &mut Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        Ok(())
+    }
+
+    /// `transform_response`'s fixups to a complete response. Returns whether it changed.
+    fn fix_response(&self, _body: &mut Map<String, Value>, _shape: &ResponseShape) -> bool {
+        false
+    }
+
+    /// The stream iterator's fixups to one chunk. Returns whether it changed.
+    fn fix_chunk(&self, chunk: &mut Map<String, Value>, _shape: &ResponseShape) -> bool {
+        rename_reasoning_in_deltas(chunk)
+    }
+}
+
+/// `OpenAIGPTConfig.get_complete_url`.
+pub fn gpt_complete_url(api_base: &str) -> String {
+    let base = api_base.trim_end_matches('/');
+    match base.contains("chat/completions") {
+        true => base.to_owned(),
+        false => format!("{base}{OPENAI_CHAT_COMPLETIONS_PATH}"),
+    }
+}
+
+/// The URL the OpenAI SDK calls for a `base_url`.
+pub fn sdk_complete_url(api_base: &str) -> String {
+    format!(
+        "{}{OPENAI_CHAT_COMPLETIONS_PATH}",
+        api_base.trim_end_matches('/')
+    )
+}
+
+/// `OpenAIGPTConfig._map_openai_params`: every supported param as sent.
+pub fn copy_supported(
+    non_default: &Map<String, Value>,
+    optional: &mut Map<String, Value>,
+    supported: &[&str],
+) {
+    for (name, value) in non_default {
+        if supported.contains(&name.as_str()) {
+            optional.insert(name.clone(), value.clone());
+        }
+    }
+}
+
+/// The request for `config`, built from the caller's body.
+pub fn transform_request(
+    config: &dyn OpenAiCompatibleChatConfig,
+    context: &CompatibleContext<'_>,
+    body: Map<String, Value>,
+) -> Result<CompatibleChatBody, Error> {
+    let mut body = body;
+    body.remove("model");
+    let messages = match body.remove("messages") {
+        None | Some(Value::Null) => Vec::new(),
+        Some(Value::Array(messages)) => messages,
+        Some(other) => {
+            return Err(Error::InvalidRequest(ErrorDetail::InvalidValue {
+                field: "messages",
+                expected: "an array",
+                actual: other,
+            }));
+        }
+    };
+    let mut user_extra_body = take_extra_body(&mut body)?;
+    user_extra_body.remove("model");
+    let stream = is_stream(&body)?;
+    let client_usage = stream
+        && body
+            .get(STREAM_OPTIONS)
+            .and_then(|options| options.get(INCLUDE_USAGE))
+            == Some(&Value::Bool(true));
+    let supported = config.supported_params(context);
+    let sorted = sort_params(
+        body,
+        &[],
+        &ChatRequestContext {
+            drop_params: Some(context.drop_params),
+            model_info: context.model_info,
+        },
+        |_| Ok(supported.clone()),
+        config.name(),
+        context.model,
+    )?;
+    let mut non_default = sorted.openai;
+    // The proxy's stream usage tracking: a stream the provider can report usage on asks for
+    // it, so the call is accounted for, and the client sees it only if it asked.
+    if stream && !client_usage && supported.contains(&STREAM_OPTIONS) {
+        let mut options = match non_default.remove(STREAM_OPTIONS) {
+            Some(Value::Object(options)) => options,
+            _ => Map::new(),
+        };
+        options.insert(INCLUDE_USAGE.into(), Value::Bool(true));
+        non_default.insert(STREAM_OPTIONS.into(), Value::Object(options));
+    }
+
+    let mut optional = Map::new();
+    config.map_params(&non_default, &mut optional, context)?;
+    // `apply_openai_param_overrides`: what the caller allowed goes as sent.
+    for name in &sorted.allowed {
+        if !optional.contains_key(name)
+            && let Some(value) = non_default.get(name)
+        {
+            optional.insert(name.clone(), value.clone());
+        }
+    }
+    let json_mode = optional
+        .remove("json_mode")
+        .is_some_and(|value| value == Value::Bool(true));
+    let mut extra_body = match optional.remove("extra_body") {
+        Some(Value::Object(extra)) => extra,
+        _ => Map::new(),
+    };
+    extra_body.extend(user_extra_body);
+    extra_body.extend(sorted.provider);
+    config.map_extra_body(&mut extra_body, &mut optional, context)?;
+
+    let mut messages = config.transform_messages(messages, &optional, context)?;
+    if !config.keeps_cache_control(context) {
+        for message in &mut messages {
+            filter_value(message, "cache_control", 0);
+        }
+        if let Some(Value::Array(tools)) = optional.get_mut("tools") {
+            for tool in tools {
+                filter_value(tool, "cache_control", 0);
+            }
+        }
+    }
+    if context.drop_params {
+        for message in &mut messages {
+            without_prompt_cache_breakpoints(message);
+        }
+        if let Some(Value::Array(tools)) = optional.get_mut("tools") {
+            for tool in tools {
+                without_prompt_cache_breakpoints(tool);
+            }
+        }
+    }
+    optional.remove("max_retries");
+    if !optional.get("tools").is_some_and(truthy) && !optional.get("functions").is_some_and(truthy)
+    {
+        optional.remove("tool_choice");
+    }
+
+    let mut sent = Map::new();
+    sent.insert(
+        "model".into(),
+        Value::String(config.wire_model(context.model)),
+    );
+    sent.insert("messages".into(), Value::Array(messages));
+    sent.extend(optional);
+    config.finish_request(&mut sent, context)?;
+    extra_body.remove("model");
+    sent.extend(extra_body);
+
+    let tools = sent.get("tools").and_then(Value::as_array);
+    let shape = ResponseShape {
+        stream,
+        client_usage,
+        json_mode,
+        tool_names: tools
+            .into_iter()
+            .flatten()
+            .filter_map(|tool| tool.get("function")?.get("name")?.as_str())
+            .map(str::to_owned)
+            .collect(),
+        tools_sent: sent.get("tools").is_some_and(|tools| !tools.is_null()),
+    };
+    Ok(CompatibleChatBody { body: sent, shape })
+}
+
+/// A complete response as the caller receives it. Returns whether it changed.
+pub fn fix_response(
+    config: &dyn OpenAiCompatibleChatConfig,
+    body: &mut Map<String, Value>,
+    shape: &ResponseShape,
+) -> bool {
+    let renamed = rename_reasoning_in_messages(body);
+    config.fix_response(body, shape) || renamed
+}
+
+/// `OpenAIGPTConfig._transform_messages`: `tool_reference` parts dropped from tool messages,
+/// and each user content part normalized.
+pub fn gpt_messages(messages: Vec<Value>) -> Result<Vec<Value>, Error> {
+    if messages.iter().any(tool_message_carries_image) {
+        // `hoist_images_from_tool_messages` moves them into an inserted user message.
+        return Err(Error::Unsupported(
+            "images in tool messages, which Python moves into a user message",
+        ));
+    }
+    messages
+        .into_iter()
+        .map(drop_tool_reference_parts)
+        .map(|mut message| {
+            if role(&message) == Some("user")
+                && let Some(Value::Array(parts)) = message.get_mut("content")
+            {
+                for part in parts.iter_mut() {
+                    transform_content_item(part)?;
+                }
+            }
+            Ok(message)
+        })
+        .collect()
+}
+
+pub fn role(message: &Value) -> Option<&str> {
+    message.get("role").and_then(Value::as_str)
+}
+
+fn is_tool_reference(part: &Value) -> bool {
+    part.get("type").and_then(Value::as_str) == Some("tool_reference")
+}
+
+/// `drop_tool_reference_parts_from_tool_messages` for one message.
+fn drop_tool_reference_parts(mut message: Value) -> Value {
+    if role(&message) != Some("tool") {
+        return message;
+    }
+    if let Some(content) = message.get_mut("content")
+        && let Value::Array(parts) = content
+        && parts.iter().any(is_tool_reference)
+    {
+        parts.retain(|part| !is_tool_reference(part));
+        if parts.is_empty() {
+            *content = Value::String(String::new());
+        }
+    }
+    message
+}
+
+fn tool_message_carries_image(message: &Value) -> bool {
+    fn has_image(parts: &[Value], depth: usize) -> bool {
+        parts.iter().any(|part| {
+            part.get("type")
+                .and_then(Value::as_str)
+                .is_some_and(|kind| IMAGE_PART_TYPES.contains(&kind))
+                || (depth < 4
+                    && part
+                        .get("content")
+                        .and_then(Value::as_array)
+                        .is_some_and(|nested| has_image(nested, depth + 1)))
+        })
+    }
+    role(message) == Some("tool")
+        && message
+            .get("content")
+            .and_then(Value::as_array)
+            .is_some_and(|parts| has_image(parts, 1))
+}
+
+/// `OpenAIGPTConfig._transform_content_item`.
+fn transform_content_item(part: &mut Value) -> Result<(), Error> {
+    let Some(object) = part.as_object_mut() else {
+        return Ok(());
+    };
+    match object.get("type").and_then(Value::as_str) {
+        Some("image_url") => match object.get_mut("image_url") {
+            Some(Value::String(url)) => {
+                let url = std::mem::take(url);
+                object.insert("image_url".into(), json!({"url": url}));
+            }
+            Some(Value::Object(image)) => {
+                image.remove("format");
+            }
+            _ => {}
+        },
+        Some("file") => {
+            let Some(Value::Object(file)) = object.get_mut("file") else {
+                return Err(Error::InvalidRequest(ErrorDetail::Message(
+                    "Content block has type='file' but is missing the required 'file' field".into(),
+                )));
+            };
+            file.remove("format");
+            if file
+                .get("file_id")
+                .and_then(Value::as_str)
+                .is_some_and(|id| POTENTIAL_PDF_URL_STARTS.iter().any(|s| id.starts_with(s)))
+            {
+                // `_handle_pdf_url` downloads it into `file_data`.
+                return Err(Error::Unsupported(
+                    "a file_id URL Python downloads into file_data",
+                ));
+            }
+            if file.get("file_data").is_some_and(|data| !data.is_null())
+                && file.get("filename").is_none_or(Value::is_null)
+            {
+                file.insert("filename".into(), Value::String("my_file.pdf".into()));
+            }
+        }
+        _ => {}
+    }
+    Ok(())
+}
+
+/// `filter_value_from_dict`: `key` removed from every object, at any depth.
+pub fn filter_value(value: &mut Value, key: &str, depth: usize) {
+    if depth > MAX_RECURSE_DEPTH {
+        return;
+    }
+    if let Value::Object(object) = value {
+        object.remove(key);
+        for nested in object.values_mut() {
+            match nested {
+                Value::Object(_) => filter_value(nested, key, depth + 1),
+                Value::Array(items) => items
+                    .iter_mut()
+                    .filter(|item| item.is_object())
+                    .for_each(|item| filter_value(item, key, depth + 1)),
+                _ => {}
+            }
+        }
+    }
+}
+
+/// `without_prompt_cache_breakpoints`: the marker removed from a message or tool, its content
+/// parts, its tool calls and its `function`.
+fn without_prompt_cache_breakpoints(block: &mut Value) {
+    const MARKER: &str = "prompt_cache_breakpoint";
+    let Some(object) = block.as_object_mut() else {
+        return;
+    };
+    object.remove(MARKER);
+    for key in ["content", "tool_calls"] {
+        if let Some(Value::Array(items)) = object.get_mut(key) {
+            for item in items.iter_mut().filter_map(Value::as_object_mut) {
+                item.remove(MARKER);
+            }
+        }
+    }
+    if let Some(Value::Object(function)) = object.get_mut("function") {
+        function.remove(MARKER);
+    }
+}
+
+fn without_fields(message: &mut Value, fields: &[&str]) {
+    let Some(object) = message.as_object_mut() else {
+        return;
+    };
+    for field in fields {
+        object.remove(*field);
+    }
+    if let Some(Value::Array(calls)) = object.get_mut("tool_calls") {
+        for call in calls.iter_mut().filter_map(Value::as_object_mut) {
+            call.remove("provider_specific_fields");
+            if let Some(Value::Object(function)) = call.get_mut("function") {
+                function.remove("provider_specific_fields");
+            }
+        }
+    }
+}
+
+/// `strip_litellm_response_only_fields`.
+pub fn strip_response_only_fields(message: &mut Value) {
+    without_fields(message, RESPONSE_ONLY_MESSAGE_FIELDS);
+}
+
+/// `strip_litellm_internal_message_fields`.
+pub fn strip_internal_fields(message: &mut Value) {
+    without_fields(message, INTERNAL_MESSAGE_FIELDS);
+}
+
+/// `extract_search_results_text`.
+pub fn search_results_text(search_results: Option<&Value>) -> String {
+    let Some(Value::Array(results)) = search_results else {
+        return String::new();
+    };
+    let mut text = String::new();
+    for result in results.iter().filter_map(Value::as_object) {
+        for key in ["source", "title"] {
+            if let Some(value) = result.get(key).and_then(Value::as_str) {
+                text.push_str(value);
+            }
+        }
+        if let Some(Value::Array(blocks)) = result.get("content") {
+            for block in blocks {
+                if let Some(value) = block.get("text").and_then(Value::as_str) {
+                    text.push_str(value);
+                }
+            }
+        }
+        if let Some(citations) = result.get("citations").filter(|c| !c.is_null()) {
+            text.push_str(&serde_json::to_string(citations).unwrap_or_default());
+        }
+    }
+    text
+}
+
+/// `convert_content_list_to_str`: the text parts joined, then any `search_results` text.
+pub fn content_list_to_str(message: &Value) -> String {
+    let mut text = String::new();
+    match message.get("content") {
+        Some(Value::Array(parts)) => {
+            for part in parts {
+                if let Some(value) = part.get("text").and_then(Value::as_str) {
+                    text.push_str(value);
+                }
+            }
+        }
+        Some(Value::String(content)) => text.push_str(content),
+        _ => {}
+    }
+    text.push_str(&search_results_text(message.get("search_results")));
+    text
+}
+
+/// The `reasoning` some OpenAI-compatible servers send, under LiteLLM's `reasoning_content`.
+pub fn rename_reasoning(message: &mut Map<String, Value>) -> bool {
+    if message.contains_key("reasoning_content") || !message.contains_key("reasoning") {
+        return false;
+    }
+    if let Some(reasoning) = message.remove("reasoning") {
+        message.insert("reasoning_content".into(), reasoning);
+    }
+    true
+}
+
+/// `_map_reasoning_to_reasoning_content` over a chunk's deltas.
+pub fn rename_reasoning_in_deltas(chunk: &mut Map<String, Value>) -> bool {
+    let mut changed = false;
+    if let Some(Value::Array(choices)) = chunk.get_mut("choices") {
+        for delta in choices
+            .iter_mut()
+            .filter_map(|choice| choice.get_mut("delta")?.as_object_mut())
+        {
+            changed |= rename_reasoning(delta);
+        }
+    }
+    changed
+}
+
+/// `extract_reasoning_content` over a complete response's messages.
+pub fn rename_reasoning_in_messages(body: &mut Map<String, Value>) -> bool {
+    let mut changed = false;
+    for message in messages_mut(body) {
+        changed |= rename_reasoning(message);
+    }
+    changed
+}
+
+/// Each choice's message in a complete response.
+pub fn messages_mut(
+    body: &mut Map<String, Value>,
+) -> impl Iterator<Item = &mut Map<String, Value>> {
+    body.get_mut("choices")
+        .and_then(Value::as_array_mut)
+        .into_iter()
+        .flatten()
+        .filter_map(|choice| choice.get_mut("message")?.as_object_mut())
+}
+
+/// `OpenAILikeChatConfig._sanitize_usage_obj`: a null `*_tokens` usage count becomes 0.
+pub fn sanitize_usage(body: &mut Map<String, Value>) -> bool {
+    let mut changed = false;
+    if let Some(Value::Object(usage)) = body.get_mut("usage") {
+        for (key, value) in usage.iter_mut() {
+            if key.ends_with("_tokens") && value.is_null() {
+                *value = json!(0);
+                changed = true;
+            }
+        }
+    }
+    changed
+}
+
+/// `_json_mode_convert_tool_response_to_message`: a lone tool call's arguments become the
+/// content.
+pub fn json_mode_tool_call_to_content(body: &mut Map<String, Value>) -> bool {
+    let mut changed = false;
+    for message in messages_mut(body) {
+        let arguments = match message.get("tool_calls") {
+            Some(Value::Array(calls)) if calls.len() == 1 => calls[0]
+                .get("function")
+                .and_then(|function| function.get("arguments"))
+                .and_then(Value::as_str)
+                .unwrap_or_default()
+                .to_owned(),
+            _ => continue,
+        };
+        message.insert("content".into(), Value::String(arguments));
+        message.insert("tool_calls".into(), Value::Null);
+        changed = true;
+    }
+    changed
+}
+
+/// Python's `str` indices of `[n]` citation markers in `text`: `(number, start, end)` in code
+/// points, as `re.finditer(r"\[(\d+)\]")` reports them.
+pub fn citation_markers(text: &str) -> Vec<(usize, usize, usize)> {
+    let chars: Vec<char> = text.chars().collect();
+    let mut markers = Vec::new();
+    let mut index = 0;
+    while index < chars.len() {
+        if chars[index] == '[' {
+            let digits = chars[index + 1..]
+                .iter()
+                .take_while(|c| c.is_ascii_digit())
+                .count();
+            let close = index + 1 + digits;
+            if digits > 0 && chars.get(close) == Some(&']') {
+                let number: String = chars[index + 1..close].iter().collect();
+                if let Ok(number) = number.parse() {
+                    markers.push((number, index, close + 1));
+                }
+                index = close + 1;
+                continue;
+            }
+        }
+        index += 1;
+    }
+    markers
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    fn object(value: Value) -> Map<String, Value> {
+        match value {
+            Value::Object(map) => map,
+            other => panic!("expected an object, got {other}"),
+        }
+    }
+
+    #[rstest]
+    #[case::bare("https://h/v1", "https://h/v1/chat/completions")]
+    #[case::slash("https://h/v1/", "https://h/v1/chat/completions")]
+    #[case::named("https://h/v1/chat/completions", "https://h/v1/chat/completions")]
+    fn the_gpt_url_appends_the_path_once(#[case] base: &str, #[case] expected: &str) {
+        assert_eq!(gpt_complete_url(base), expected);
+    }
+
+    #[rstest]
+    fn user_content_parts_are_normalized() {
+        let messages = gpt_messages(vec![json!({"role": "user", "content": [
+            {"type": "image_url", "image_url": "https://img"},
+            {"type": "image_url", "image_url": {"url": "u", "format": "image/png", "detail": "low"}},
+            {"type": "file", "file": {"file_data": "data:application/pdf;base64,AA", "format": "application/pdf"}},
+        ]})])
+        .unwrap();
+        assert_eq!(
+            messages[0]["content"],
+            json!([
+                {"type": "image_url", "image_url": {"url": "https://img"}},
+                {"type": "image_url", "image_url": {"url": "u", "detail": "low"}},
+                {"type": "file", "file": {"file_data": "data:application/pdf;base64,AA", "filename": "my_file.pdf"}},
+            ])
+        );
+    }
+
+    #[rstest]
+    #[case::pdf_url(json!({"role": "user", "content": [{"type": "file", "file": {"file_id": "https://x/a.pdf"}}]}))]
+    #[case::tool_image(json!({"role": "tool", "tool_call_id": "c", "content": [{"type": "image_url", "image_url": {"url": "u"}}]}))]
+    fn shapes_python_rewrites_with_io_are_refused(#[case] message: Value) {
+        assert!(matches!(
+            gpt_messages(vec![message]),
+            Err(Error::Unsupported(_))
+        ));
+    }
+
+    #[rstest]
+    fn a_file_part_without_its_file_is_rejected() {
+        assert!(matches!(
+            gpt_messages(vec![json!({"role": "user", "content": [{"type": "file"}]})]),
+            Err(Error::InvalidRequest(_))
+        ));
+    }
+
+    #[rstest]
+    fn tool_reference_parts_leave_tool_messages() {
+        let messages = gpt_messages(vec![json!({"role": "tool", "tool_call_id": "c",
+            "content": [{"type": "tool_reference", "tool_name": "f"}]})])
+        .unwrap();
+        assert_eq!(messages[0]["content"], json!(""));
+    }
+
+    #[rstest]
+    fn cache_control_is_filtered_at_any_depth() {
+        let mut message = json!({"role": "user", "cache_control": {"type": "ephemeral"},
+            "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]});
+        filter_value(&mut message, "cache_control", 0);
+        assert_eq!(
+            message,
+            json!({"role": "user", "content": [{"type": "text", "text": "hi"}]})
+        );
+    }
+
+    #[rstest]
+    #[case::exact(1024, "low")]
+    #[case::medium(3000, "medium")]
+    #[case::high(5000, "high")]
+    #[case::minimal(10, "minimal")]
+    fn thinking_budgets_bucket_into_efforts(#[case] budget: i64, #[case] effort: &str) {
+        assert_eq!(reasoning_effort_from_thinking_budget(budget), effort);
+    }
+
+    #[rstest]
+    #[case::declared("high", &["low", "high"], "high")]
+    #[case::rounds_up("medium", &["low", "high"], "high")]
+    #[case::rounds_down("max", &["low", "high"], "high")]
+    #[case::none("none", &["low"], "none")]
+    fn efforts_round_to_a_declared_level(
+        #[case] requested: &str,
+        #[case] declared: &[&str],
+        #[case] expected: &str,
+    ) {
+        let declared: Vec<String> = declared.iter().map(|d| (*d).to_owned()).collect();
+        assert_eq!(
+            nearest_declared_reasoning_effort(requested, &declared),
+            expected
+        );
+    }
+
+    #[rstest]
+    fn citation_markers_count_code_points() {
+        assert_eq!(
+            citation_markers("é [1] and [22][x]"),
+            vec![(1, 2, 5), (22, 10, 14)]
+        );
+    }
+
+    #[rstest]
+    fn json_mode_moves_a_lone_tool_call_into_the_content() {
+        let mut body = object(
+            json!({"choices": [{"message": {"role": "assistant", "content": null,
+            "tool_calls": [{"id": "c", "type": "function", "function": {"name": "json_tool_call", "arguments": "{\"a\":1}"}}]},
+            "finish_reason": "tool_calls"}]}),
+        );
+        assert!(json_mode_tool_call_to_content(&mut body));
+        assert_eq!(
+            body["choices"][0]["message"],
+            json!({"role": "assistant", "content": "{\"a\":1}", "tool_calls": null})
+        );
+    }
+}

litellm-rust/crates/llms/src/openai/chat/mod.rs

@@ -1 +1,2 @@
+pub mod compatible;
 pub mod transformation;

litellm-rust/crates/llms/src/openrouter/chat/AGENTS.md

@@ -0,0 +1,11 @@
+# rules
+
+- OpenRouter at `https://openrouter.ai/api/v1` with the `HTTP-Referer` and `X-Title` attribution headers
+- `cache_control` is kept for caching model families and moved into the last content block
+- `usage: {include: true}` is always asked for unless the caller set `usage`
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://openrouter.ai/docs/api-reference/chat-completion
+- litellm/llms/openrouter/chat/transformation.py

litellm-rust/crates/llms/src/openrouter/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/openrouter/chat/transformation.rs

@@ -0,0 +1,231 @@
+//! `litellm/llms/openrouter/chat/transformation.py`: OpenRouter at
+//! `https://openrouter.ai/api/v1`, with LiteLLM's attribution headers, `cache_control` kept
+//! for the model families that cache, moved into the last content block, and the usage
+//! accounting OpenRouter reports cost in always asked for.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error,
+    openai::chat::compatible::{
+        CompatibleContext, GPT_BASE_PARAMS, OpenAiCompatibleChatConfig, copy_supported,
+        gpt_messages,
+    },
+};
+
+const CACHE_CONTROL_MODEL_FAMILIES: &[&str] = &["claude", "gemini", "minimax", "glm", "z-ai"];
+
+pub struct OpenRouterChatConfig;
+
+pub const OPENROUTER_CHAT_CONFIG: OpenRouterChatConfig = OpenRouterChatConfig;
+
+impl OpenAiCompatibleChatConfig for OpenRouterChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::Openrouter
+    }
+
+    fn label(&self) -> &'static str {
+        "OpenRouter"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["OPENROUTER_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://openrouter.ai/api/v1")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &["OPENROUTER_API_KEY", "OR_API_KEY"]
+    }
+
+    fn header_env(&self) -> &'static [&'static str] {
+        &["OR_SITE_URL", "OR_APP_NAME"]
+    }
+
+    fn headers(
+        &self,
+        _body: &Map<String, Value>,
+        lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Vec<(String, String)> {
+        let setting = |name: &str, default: &str| {
+            lookup(name)
+                .filter(|value| !value.is_empty())
+                .unwrap_or_else(|| default.to_owned())
+        };
+        vec![
+            (
+                "HTTP-Referer".into(),
+                setting("OR_SITE_URL", "https://litellm.ai"),
+            ),
+            ("X-Title".into(), setting("OR_APP_NAME", "liteLLM")),
+        ]
+    }
+
+    fn supported_params(&self, _context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        GPT_BASE_PARAMS
+            .iter()
+            .copied()
+            .chain(["response_format", "reasoning_effort", "thinking"])
+            .collect()
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        copy_supported(non_default, optional, &self.supported_params(context));
+        if optional.get("reasoning_effort") == Some(&json!("max")) {
+            optional.insert("reasoning_effort".into(), json!("xhigh"));
+        }
+        Ok(())
+    }
+
+    fn keeps_cache_control(&self, context: &CompatibleContext<'_>) -> bool {
+        caches(context.model)
+    }
+
+    fn transform_messages(
+        &self,
+        messages: Vec<Value>,
+        _optional: &Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<Vec<Value>, Error> {
+        match caches(context.model) {
+            true => gpt_messages(
+                messages
+                    .into_iter()
+                    .map(move_cache_control_to_content)
+                    .collect(),
+            ),
+            false => gpt_messages(messages),
+        }
+    }
+
+    fn finish_request(
+        &self,
+        body: &mut Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        body.entry("usage")
+            .or_insert_with(|| json!({"include": true}));
+        Ok(())
+    }
+}
+
+fn caches(model: &str) -> bool {
+    let model = model.to_lowercase();
+    CACHE_CONTROL_MODEL_FAMILIES
+        .iter()
+        .any(|family| model.contains(family))
+}
+
+/// `_move_cache_control_to_content`: only the last block carries it, to stay within
+/// Anthropic's four cache breakpoints.
+fn move_cache_control_to_content(mut message: Value) -> Value {
+    let Some(object) = message.as_object_mut() else {
+        return message;
+    };
+    let Some(cache_control) = object.remove("cache_control").filter(|c| !c.is_null()) else {
+        return message;
+    };
+    match object.get_mut("content") {
+        Some(Value::Array(blocks)) => {
+            if let Some(Value::Object(last)) = blocks.last_mut() {
+                last.insert("cache_control".into(), cache_control);
+            }
+        }
+        content => {
+            let text = content.map(Value::take).unwrap_or(Value::Null);
+            object.insert(
+                "content".into(),
+                json!([{"type": "text", "text": text, "cache_control": cache_control}]),
+            );
+        }
+    }
+    message
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+    use crate::openai::chat::compatible::transform_request;
+
+    fn send(model: &str, params: Value) -> Map<String, Value> {
+        let Value::Object(body) = params else {
+            panic!("an object")
+        };
+        transform_request(
+            &OPENROUTER_CHAT_CONFIG,
+            &CompatibleContext {
+                model,
+                drop_params: false,
+                model_info: None,
+            },
+            body,
+        )
+        .unwrap()
+        .body
+    }
+
+    #[rstest]
+    fn openrouter_params_and_usage_accounting_go_along() {
+        let sent = send(
+            "openai/gpt-4o",
+            json!({"messages": [], "reasoning_effort": "max",
+            "transforms": ["middle-out"], "route": "fallback"}),
+        );
+        assert_eq!(
+            Value::Object(sent),
+            json!({"model": "openai/gpt-4o", "messages": [], "reasoning_effort": "xhigh",
+                "usage": {"include": true}, "transforms": ["middle-out"], "route": "fallback"})
+        );
+    }
+
+    #[rstest]
+    fn a_callers_usage_setting_wins() {
+        let sent = send(
+            "openai/gpt-4o",
+            json!({"messages": [], "usage": {"include": false}}),
+        );
+        assert_eq!(sent["usage"], json!({"include": false}));
+    }
+
+    #[rstest]
+    #[case::caching_family("anthropic/claude-sonnet-4", json!([
+        {"role": "system", "content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}]},
+        {"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b", "cache_control": {"type": "ephemeral"}}]}]))]
+    #[case::other_family("openai/gpt-4o", json!([
+        {"role": "system", "content": "s"},
+        {"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}]))]
+    fn cache_control_reaches_only_families_that_cache(
+        #[case] model: &str,
+        #[case] messages: Value,
+    ) {
+        let sent = send(
+            model,
+            json!({"messages": [
+            {"role": "system", "content": "s", "cache_control": {"type": "ephemeral"}},
+            {"role": "user", "cache_control": {"type": "ephemeral"}, "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}]}),
+        );
+        assert_eq!(sent["messages"], messages);
+    }
+
+    #[rstest]
+    fn attribution_headers_default_to_litellm() {
+        assert_eq!(
+            OPENROUTER_CHAT_CONFIG.headers(&Map::new(), &|name| (name == "OR_APP_NAME")
+                .then(|| "app".to_owned())),
+            vec![
+                ("HTTP-Referer".to_owned(), "https://litellm.ai".to_owned()),
+                ("X-Title".to_owned(), "app".to_owned()),
+            ]
+        );
+    }
+}

litellm-rust/crates/llms/src/openrouter/mod.rs

@@ -1,2 +1,3 @@
+pub mod chat;
 pub mod messages;
 pub mod responses;

litellm-rust/crates/llms/src/perplexity/chat/AGENTS.md

@@ -0,0 +1,10 @@
+# rules
+
+- Perplexity through the OpenAI SDK URL rule at `https://api.perplexity.ai`, with `X-Pplx-Integration: litellm`
+- Citations become `url_citation` annotations on the first choice, search counts reach `prompt_tokens_details`
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://docs.perplexity.ai/api-reference/chat-completions-post
+- litellm/llms/perplexity/chat/transformation.py

litellm-rust/crates/llms/src/perplexity/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/perplexity/chat/transformation.rs

@@ -0,0 +1,283 @@
+//! `litellm/llms/perplexity/chat/transformation.py`: Perplexity through the OpenAI SDK at
+//! `https://api.perplexity.ai`, with the subset of OpenAI params it takes, LiteLLM's
+//! `X-Pplx-Integration` attribution header, and the citations of an answer added to its
+//! message as `url_citation` annotations with the search counts in the usage.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::openai::chat::compatible::{
+    CompatibleContext, OpenAiCompatibleChatConfig, ResponseShape, citation_markers,
+    sdk_complete_url,
+};
+
+const SUPPORTED_PARAMS: &[&str] = &[
+    "frequency_penalty",
+    "max_tokens",
+    "max_completion_tokens",
+    "presence_penalty",
+    "response_format",
+    "stream",
+    "temperature",
+    "top_p",
+    "max_retries",
+    "extra_headers",
+];
+const ATTRIBUTION_HEADER: (&str, &str) = ("X-Pplx-Integration", "litellm");
+
+pub struct PerplexityChatConfig;
+
+pub const PERPLEXITY_CHAT_CONFIG: PerplexityChatConfig = PerplexityChatConfig;
+
+impl OpenAiCompatibleChatConfig for PerplexityChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::Perplexity
+    }
+
+    fn label(&self) -> &'static str {
+        "Perplexity"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["PERPLEXITY_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://api.perplexity.ai")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &["PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"]
+    }
+
+    fn complete_url(&self, api_base: &str) -> String {
+        sdk_complete_url(api_base)
+    }
+
+    fn headers(
+        &self,
+        _body: &Map<String, Value>,
+        _lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Vec<(String, String)> {
+        vec![(ATTRIBUTION_HEADER.0.into(), ATTRIBUTION_HEADER.1.into())]
+    }
+
+    fn supported_params(&self, context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        let mut params = SUPPORTED_PARAMS.to_vec();
+        if context.supports(self.name(), "supports_reasoning") {
+            params.push("reasoning_effort");
+        }
+        if context.supports(self.name(), "supports_web_search") {
+            params.push("web_search_options");
+        }
+        params
+    }
+
+    /// `_enhance_usage_with_perplexity_fields` and `_add_citations_as_annotations`.
+    fn fix_response(&self, body: &mut Map<String, Value>, _shape: &ResponseShape) -> bool {
+        let usage_changed = enhance_usage(body);
+        add_citations_as_annotations(body) || usage_changed
+    }
+}
+
+/// The search count Perplexity reports, wherever it put it.
+fn search_queries(body: &Map<String, Value>) -> Option<u64> {
+    let usage = body.get("usage");
+    let field = |value: Option<&Value>, key: &str| value.and_then(|v| v.get(key)).cloned();
+    [
+        field(usage, "num_search_queries"),
+        body.get("num_search_queries").cloned(),
+        field(usage, "search_queries"),
+        body.get("search_queries").cloned(),
+    ]
+    .into_iter()
+    .flatten()
+    .find(|value| !value.is_null())
+    .and_then(|value| value.as_u64())
+}
+
+fn enhance_usage(body: &mut Map<String, Value>) -> bool {
+    let citation_chars: usize = match body.get("citations") {
+        Some(Value::Array(citations)) => citations
+            .iter()
+            .filter(|citation| crate::base_llm::chat::params::truthy(citation))
+            .map(|citation| match citation {
+                Value::String(text) => text.chars().count(),
+                other => other.to_string().chars().count(),
+            })
+            .sum(),
+        _ => 0,
+    };
+    let citation_tokens = match citation_chars {
+        0 => 0,
+        chars => (chars / 4).max(1),
+    };
+    let searches = search_queries(body).filter(|count| *count > 0);
+    if citation_tokens == 0 && searches.is_none() {
+        return false;
+    }
+    let usage = body
+        .entry("usage")
+        .or_insert_with(|| json!({"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}));
+    if usage.is_null() {
+        *usage = json!({"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0});
+    }
+    let Some(usage) = usage.as_object_mut() else {
+        return false;
+    };
+    if citation_tokens > 0 {
+        usage.insert("citation_tokens".into(), json!(citation_tokens));
+    }
+    if let Some(searches) = searches {
+        let details = usage
+            .entry("prompt_tokens_details")
+            .or_insert_with(|| json!({}));
+        if !details.is_object() {
+            *details = json!({});
+        }
+        if let Some(details) = details.as_object_mut() {
+            details.insert("web_search_requests".into(), json!(searches));
+        }
+    }
+    true
+}
+
+fn add_citations_as_annotations(body: &mut Map<String, Value>) -> bool {
+    let citations: Vec<Option<String>> = match body.get("citations") {
+        Some(Value::Array(citations)) => citations
+            .iter()
+            .map(|citation| citation.as_str().map(str::to_owned))
+            .collect(),
+        _ => Vec::new(),
+    };
+    let titles: Vec<(String, String)> = match body.get("search_results") {
+        Some(Value::Array(results)) => results
+            .iter()
+            .filter_map(|result| {
+                Some((
+                    result.get("url")?.as_str()?.to_owned(),
+                    result.get("title")?.as_str()?.to_owned(),
+                ))
+            })
+            .collect(),
+        _ => Vec::new(),
+    };
+    let Some(message) = body
+        .get_mut("choices")
+        .and_then(Value::as_array_mut)
+        .and_then(|choices| choices.first_mut())
+        .and_then(|choice| choice.get_mut("message"))
+        .and_then(Value::as_object_mut)
+    else {
+        return false;
+    };
+    let Some(content) = message
+        .get("content")
+        .and_then(Value::as_str)
+        .filter(|content| !content.is_empty())
+    else {
+        return false;
+    };
+    let annotations: Vec<Value> = citation_markers(content)
+        .into_iter()
+        .filter_map(|(number, start, end)| {
+            let url = citations.get(number.checked_sub(1)?)?.as_ref()?;
+            let title = titles
+                .iter()
+                .rev()
+                .find(|(known, _)| known == url)
+                .map_or("", |(_, title)| title.as_str());
+            Some(json!({"type": "url_citation", "url_citation": {
+                "url": url, "title": title, "start_index": start, "end_index": end,
+            }}))
+        })
+        .collect();
+    if annotations.is_empty() {
+        return false;
+    }
+    match message.get_mut("annotations") {
+        Some(Value::Array(existing)) => existing.extend(annotations),
+        _ => {
+            message.insert("annotations".into(), Value::Array(annotations));
+        }
+    }
+    true
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+    use crate::{
+        Error, ErrorDetail,
+        openai::chat::compatible::{fix_response, transform_request},
+    };
+
+    fn object(value: Value) -> Map<String, Value> {
+        let Value::Object(map) = value else {
+            panic!("an object")
+        };
+        map
+    }
+
+    fn send(params: Value) -> Result<Map<String, Value>, Error> {
+        transform_request(
+            &PERPLEXITY_CHAT_CONFIG,
+            &CompatibleContext {
+                model: "sonar",
+                drop_params: false,
+                model_info: None,
+            },
+            object(params),
+        )
+        .map(|sent| sent.body)
+    }
+
+    #[rstest]
+    fn max_completion_tokens_goes_as_sent() {
+        let sent = send(
+            json!({"messages": [], "max_completion_tokens": 7, "search_domain_filter": ["a.com"]}),
+        )
+        .unwrap();
+        assert_eq!(
+            Value::Object(sent),
+            json!({"model": "sonar", "messages": [], "max_completion_tokens": 7, "search_domain_filter": ["a.com"]})
+        );
+    }
+
+    #[rstest]
+    #[case::tools(json!({"messages": [], "tools": [{"type": "function", "function": {"name": "f"}}]}))]
+    #[case::seed(json!({"messages": [], "seed": 1}))]
+    fn params_perplexity_lacks_are_refused(#[case] params: Value) {
+        assert!(matches!(
+            send(params),
+            Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams { .. }))
+        ));
+    }
+
+    #[rstest]
+    fn citations_become_annotations_and_searches_reach_the_usage() {
+        let mut body = object(json!({
+            "model": "sonar",
+            "citations": ["https://a.example", "https://b.example"],
+            "search_results": [{"url": "https://b.example", "title": "B"}],
+            "choices": [{"index": 0, "message": {"role": "assistant", "content": "é see [2] and [3]"}, "finish_reason": "stop"}],
+            "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7, "num_search_queries": 2},
+        }));
+        assert!(fix_response(
+            &PERPLEXITY_CHAT_CONFIG,
+            &mut body,
+            &ResponseShape::default()
+        ));
+        assert_eq!(
+            body["choices"][0]["message"]["annotations"],
+            json!([{"type": "url_citation", "url_citation": {"url": "https://b.example", "title": "B", "start_index": 6, "end_index": 9}}])
+        );
+        assert_eq!(body["usage"]["citation_tokens"], json!(8));
+        assert_eq!(
+            body["usage"]["prompt_tokens_details"],
+            json!({"web_search_requests": 2})
+        );
+    }
+}

litellm-rust/crates/llms/src/perplexity/mod.rs

@@ -0,0 +1 @@
+pub mod chat;

litellm-rust/crates/llms/src/together_ai/chat/AGENTS.md

@@ -0,0 +1,11 @@
+# rules
+
+- Together AI at `https://api.together.ai/v1`
+- Tool and `response_format` params are refused or dropped only when the `together_ai/<model>` cost map entry rules them out
+- `reasoning_effort` is translated per model family, `none` turns reasoning off with `reasoning.enabled: false`
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://docs.together.ai/docs/chat-overview
+- litellm/llms/together_ai/chat/transformation.py

litellm-rust/crates/llms/src/together_ai/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/together_ai/chat/transformation.rs

@@ -0,0 +1,324 @@
+//! `litellm/llms/together_ai/chat/transformation.py`: Together AI at
+//! `https://api.together.ai/v1`. Tool and structured output params the cost map rules out are
+//! refused or dropped, and `reasoning_effort` is translated to the levels each reasoning model
+//! family takes.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error, ErrorDetail,
+    openai::chat::compatible::{
+        CompatibleContext, GPT_BASE_PARAMS, OpenAiCompatibleChatConfig, copy_supported,
+        gpt_messages, role, strip_response_only_fields,
+    },
+};
+
+const TOOL_CALLING_PARAMS: &[&str] = &["tools", "tool_choice", "function_call"];
+const ADJUSTABLE_EFFORT_REASONING_MODELS: &[&str] = &["openai/gpt-oss-120b", "openai/gpt-oss-20b"];
+const HYBRID_REASONING_MODELS: &[&str] = &[
+    "MiniMaxAI/MiniMax-M3",
+    "Qwen/Qwen3.5-9B",
+    "Qwen/Qwen3.6-Plus",
+    "deepseek-ai/DeepSeek-V4-Pro",
+    "moonshotai/Kimi-K3",
+    "nvidia/nemotron-3-ultra-550b-a55b",
+    "zai-org/GLM-5.2",
+];
+const HIGH_MAX_EFFORT_MODEL_PREFIX: &str = "deepseek-ai/DeepSeek-V4-Pro";
+
+pub struct TogetherAiChatConfig;
+
+pub const TOGETHER_AI_CHAT_CONFIG: TogetherAiChatConfig = TogetherAiChatConfig;
+
+impl OpenAiCompatibleChatConfig for TogetherAiChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::TogetherAi
+    }
+
+    fn label(&self) -> &'static str {
+        "Together AI"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["TOGETHER_AI_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://api.together.ai/v1")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &[
+            "TOGETHER_API_KEY",
+            "TOGETHER_AI_API_KEY",
+            "TOGETHERAI_API_KEY",
+            "TOGETHER_AI_TOKEN",
+        ]
+    }
+
+    fn supported_params(&self, context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        let mut params: Vec<&'static str> = GPT_BASE_PARAMS
+            .iter()
+            .copied()
+            .chain(["response_format"])
+            .collect();
+        if supports_reasoning(context) {
+            params.push("reasoning_effort");
+        }
+        params
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        copy_supported(non_default, optional, &self.supported_params(context));
+        let passed_tools: Vec<&str> = TOOL_CALLING_PARAMS
+            .iter()
+            .copied()
+            .filter(|param| optional.contains_key(*param))
+            .collect();
+        if !passed_tools.is_empty()
+            && registry_rules_out(context, "supports_function_calling", &passed_tools)?
+        {
+            for param in passed_tools {
+                optional.remove(param);
+            }
+        }
+        if optional.contains_key("response_format")
+            && registry_rules_out(context, "supports_response_schema", &["response_format"])?
+        {
+            optional.remove("response_format");
+        }
+        let Some(Value::String(effort)) = optional.get("reasoning_effort").cloned() else {
+            return Ok(());
+        };
+        optional.remove("reasoning_effort");
+        for (key, value) in reasoning_effort_payload(&effort, context) {
+            optional.entry(key).or_insert(value);
+        }
+        Ok(())
+    }
+
+    fn transform_messages(
+        &self,
+        messages: Vec<Value>,
+        _optional: &Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<Vec<Value>, Error> {
+        gpt_messages(
+            messages
+                .into_iter()
+                .map(|mut message| {
+                    if role(&message) == Some("assistant") {
+                        strip_response_only_fields(&mut message);
+                    }
+                    message
+                })
+                .collect(),
+        )
+    }
+}
+
+fn supports_reasoning(context: &CompatibleContext<'_>) -> bool {
+    let model = context.model;
+    ADJUSTABLE_EFFORT_REASONING_MODELS.contains(&model)
+        || HYBRID_REASONING_MODELS.contains(&model)
+        || model.starts_with(HIGH_MAX_EFFORT_MODEL_PREFIX)
+        || context.supports("together_ai", "supports_reasoning")
+}
+
+/// `_registry_verdict`: true when the `together_ai/<model>` entry sets `flag` to false and the
+/// params are to be dropped. Without `drop_params` that is a refusal.
+fn registry_rules_out(
+    context: &CompatibleContext<'_>,
+    flag: &str,
+    params: &[&str],
+) -> Result<bool, Error> {
+    if context.supports("together_ai", flag)
+        || context.capability_of(&[format!("together_ai/{}", context.model)], flag) != Some(false)
+    {
+        return Ok(false);
+    }
+    if context.drop_params {
+        return Ok(true);
+    }
+    Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams {
+        provider: "together_ai",
+        params: params.iter().map(|param| (*param).to_owned()).collect(),
+        model: context.model.to_owned(),
+    }))
+}
+
+fn reasoning_effort_payload(effort: &str, context: &CompatibleContext<'_>) -> Vec<(String, Value)> {
+    let model = context.model;
+    let effort_only = |effort: &str| vec![("reasoning_effort".to_owned(), json!(effort))];
+    if effort == "default" {
+        return Vec::new();
+    }
+    if ADJUSTABLE_EFFORT_REASONING_MODELS.contains(&model) {
+        return effort_only(match effort {
+            "none" => "low",
+            other => translate(other),
+        });
+    }
+    if effort == "none" {
+        return vec![("reasoning".to_owned(), json!({"enabled": false}))];
+    }
+    if context
+        .declared_reasoning_efforts("together_ai")
+        .is_some_and(|declared| declared.iter().any(|level| level == effort))
+    {
+        return effort_only(effort);
+    }
+    if model.starts_with(HIGH_MAX_EFFORT_MODEL_PREFIX) {
+        return effort_only(match effort {
+            "minimal" | "low" | "medium" => "high",
+            "xhigh" => "max",
+            other => other,
+        });
+    }
+    effort_only(translate(effort))
+}
+
+fn translate(effort: &str) -> &str {
+    match effort {
+        "minimal" => "low",
+        "xhigh" | "max" => "high",
+        other => other,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::borrow::Cow;
+
+    use rstest::rstest;
+
+    use super::*;
+    use crate::{base_llm::model_info::ModelInfo, openai::chat::compatible::transform_request};
+
+    struct Registry;
+
+    impl ModelInfo for Registry {
+        fn entry(&self, model: &str) -> Option<Cow<'_, Map<String, Value>>> {
+            let entry = match model {
+                "together_ai/no-tools" => {
+                    json!({"supports_function_calling": false, "supports_response_schema": false})
+                }
+                "together_ai/declared" => {
+                    json!({"supports_reasoning": true, "reasoning_effort_levels": ["low", "xhigh"]})
+                }
+                _ => return None,
+            };
+            let Value::Object(entry) = entry else {
+                unreachable!()
+            };
+            Some(Cow::Owned(entry))
+        }
+
+        fn declares(&self, _key: &str) -> bool {
+            true
+        }
+    }
+
+    fn send(model: &str, drop_params: bool, params: Value) -> Result<Map<String, Value>, Error> {
+        let Value::Object(body) = params else {
+            panic!("an object")
+        };
+        transform_request(
+            &TOGETHER_AI_CHAT_CONFIG,
+            &CompatibleContext {
+                model,
+                drop_params,
+                model_info: Some(&Registry),
+            },
+            body,
+        )
+        .map(|sent| sent.body)
+    }
+
+    #[rstest]
+    #[case::adjustable_none("openai/gpt-oss-20b", "none", json!({"reasoning_effort": "low"}))]
+    #[case::adjustable_max("openai/gpt-oss-20b", "max", json!({"reasoning_effort": "high"}))]
+    #[case::hybrid_off("zai-org/GLM-5.2", "none", json!({"reasoning": {"enabled": false}}))]
+    #[case::declared("declared", "xhigh", json!({"reasoning_effort": "xhigh"}))]
+    #[case::high_max_floor("deepseek-ai/DeepSeek-V4-Pro", "low", json!({"reasoning_effort": "high"}))]
+    #[case::high_max_top("deepseek-ai/DeepSeek-V4-Pro", "xhigh", json!({"reasoning_effort": "max"}))]
+    #[case::hybrid_minimal("zai-org/GLM-5.2", "minimal", json!({"reasoning_effort": "low"}))]
+    #[case::default("zai-org/GLM-5.2", "default", json!({}))]
+    fn reasoning_effort_fits_the_model_family(
+        #[case] model: &str,
+        #[case] effort: &str,
+        #[case] expected: Value,
+    ) {
+        let mut sent = send(
+            model,
+            false,
+            json!({"messages": [], "reasoning_effort": effort}),
+        )
+        .unwrap();
+        sent.remove("model");
+        sent.remove("messages");
+        assert_eq!(Value::Object(sent), expected);
+    }
+
+    #[rstest]
+    #[case::refused(false)]
+    #[case::dropped(true)]
+    fn params_the_registry_rules_out_are_refused_or_dropped(#[case] drop_params: bool) {
+        let result = send(
+            "no-tools",
+            drop_params,
+            json!({"messages": [],
+            "tools": [{"type": "function", "function": {"name": "f"}}], "tool_choice": "auto",
+            "response_format": {"type": "json_object"}}),
+        );
+        match drop_params {
+            false => assert!(matches!(
+                result,
+                Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams {
+                    provider: "together_ai",
+                    ..
+                }))
+            )),
+            true => assert_eq!(
+                Value::Object(result.unwrap()),
+                json!({"model": "no-tools", "messages": []})
+            ),
+        }
+    }
+
+    #[rstest]
+    fn unknown_models_pass_tools_through() {
+        let sent = send(
+            "unknown",
+            false,
+            json!({"messages": [],
+            "tools": [{"type": "function", "function": {"name": "f"}}]}),
+        )
+        .unwrap();
+        assert_eq!(
+            sent["tools"],
+            json!([{"type": "function", "function": {"name": "f"}}])
+        );
+    }
+
+    #[rstest]
+    fn replayed_reasoning_stays_while_response_only_fields_go() {
+        let sent = send(
+            "unknown",
+            false,
+            json!({"messages": [{"role": "assistant", "content": "a",
+            "reasoning_content": "r", "thinking_blocks": [], "provider_specific_fields": {}}]}),
+        )
+        .unwrap();
+        assert_eq!(
+            sent["messages"],
+            json!([{"role": "assistant", "content": "a", "reasoning_content": "r"}])
+        );
+    }
+}

litellm-rust/crates/llms/src/together_ai/mod.rs

@@ -0,0 +1 @@
+pub mod chat;

litellm-rust/crates/llms/src/xai/chat/AGENTS.md

@@ -0,0 +1,11 @@
+# rules
+
+- xAI at `https://api.x.ai/v1`
+- `web_search_options` and non-user `name`s are dropped, tools lose `strict`
+- Reasoning tokens xAI counts apart are folded into `completion_tokens`, in answers and stream chunks
+- Shared flow and response handling: `../../openai/chat/compatible.rs`
+
+# references
+
+- https://docs.x.ai/docs/api-reference#chat-completions
+- litellm/llms/xai/chat/transformation.py

litellm-rust/crates/llms/src/xai/chat/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/llms/src/xai/chat/transformation.rs

@@ -0,0 +1,272 @@
+//! `litellm/llms/xai/chat/transformation.py`: xAI at `https://api.x.ai/v1`. Tools lose
+//! `strict`, `web_search_options` and non-user `name`s are dropped, and the usage folds the
+//! separately counted reasoning tokens into `completion_tokens` as OpenAI counts them.
+
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use serde_json::{Map, Value, json};
+
+use crate::{
+    Error,
+    openai::chat::compatible::{
+        CompatibleContext, OpenAiCompatibleChatConfig, ResponseShape, filter_value, role,
+    },
+};
+
+const SUPPORTED_PARAMS: &[&str] = &[
+    "logit_bias",
+    "logprobs",
+    "max_completion_tokens",
+    "max_tokens",
+    "n",
+    "parallel_tool_calls",
+    "presence_penalty",
+    "response_format",
+    "seed",
+    "stream",
+    "stream_options",
+    "temperature",
+    "tool_choice",
+    "tools",
+    "top_logprobs",
+    "top_p",
+    "user",
+    "web_search_options",
+];
+
+pub struct XaiChatConfig;
+
+pub const XAI_CHAT_CONFIG: XaiChatConfig = XaiChatConfig;
+
+impl OpenAiCompatibleChatConfig for XaiChatConfig {
+    fn provider(&self) -> LlmProviders {
+        LlmProviders::Xai
+    }
+
+    fn label(&self) -> &'static str {
+        "xAI"
+    }
+
+    fn api_base_env(&self) -> &'static [&'static str] {
+        &["XAI_API_BASE"]
+    }
+
+    fn default_api_base(&self) -> Option<&'static str> {
+        Some("https://api.x.ai/v1")
+    }
+
+    fn api_key_env(&self) -> &'static [&'static str] {
+        &["XAI_API_KEY"]
+    }
+
+    fn supported_params(&self, context: &CompatibleContext<'_>) -> Vec<&'static str> {
+        let model = context.model;
+        let mut params = SUPPORTED_PARAMS.to_vec();
+        if !["grok-3-mini", "grok-4", "grok-code-fast"]
+            .iter()
+            .any(|family| model.contains(family))
+        {
+            params.push("stop");
+        }
+        if !["grok-4", "grok-code-fast"]
+            .iter()
+            .any(|family| model.contains(family))
+        {
+            params.push("frequency_penalty");
+        }
+        if context.supports(self.name(), "supports_reasoning") {
+            params.push("reasoning_effort");
+        }
+        params
+    }
+
+    fn map_params(
+        &self,
+        non_default: &Map<String, Value>,
+        optional: &mut Map<String, Value>,
+        context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        let supported = self.supported_params(context);
+        for (name, value) in non_default {
+            match name.as_str() {
+                "max_completion_tokens" => {
+                    optional.insert("max_tokens".into(), value.clone());
+                }
+                "tools" => {
+                    let Value::Array(tools) = value else { continue };
+                    let tools: Vec<Value> = tools
+                        .iter()
+                        .cloned()
+                        .map(|mut tool| {
+                            filter_value(&mut tool, "strict", 0);
+                            tool
+                        })
+                        .collect();
+                    if !tools.is_empty() {
+                        optional.insert("tools".into(), Value::Array(tools));
+                    }
+                }
+                name if supported.contains(&name) && !value.is_null() => {
+                    optional.insert(name.to_owned(), value.clone());
+                }
+                _ => {}
+            }
+        }
+        Ok(())
+    }
+
+    fn finish_request(
+        &self,
+        body: &mut Map<String, Value>,
+        _context: &CompatibleContext<'_>,
+    ) -> Result<(), Error> {
+        body.remove("web_search_options");
+        if let Some(Value::Array(messages)) = body.get_mut("messages") {
+            for message in messages.iter_mut() {
+                if role(message) != Some("user")
+                    && let Some(object) = message.as_object_mut()
+                {
+                    object.remove("name");
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn fix_response(&self, body: &mut Map<String, Value>, _shape: &ResponseShape) -> bool {
+        let mut changed = false;
+        if let Some(Value::Array(choices)) = body.get_mut("choices") {
+            for choice in choices.iter_mut().filter_map(Value::as_object_mut) {
+                let has_tool_calls = choice
+                    .get("message")
+                    .and_then(|message| message.get("tool_calls"))
+                    .and_then(Value::as_array)
+                    .is_some_and(|calls| !calls.is_empty());
+                if has_tool_calls && choice.get("finish_reason") == Some(&json!("")) {
+                    choice.insert("finish_reason".into(), json!("tool_calls"));
+                    changed = true;
+                }
+            }
+        }
+        changed | fix_usage(body)
+    }
+
+    fn fix_chunk(&self, chunk: &mut Map<String, Value>, _shape: &ResponseShape) -> bool {
+        fix_usage(chunk)
+    }
+}
+
+fn count(usage: &Map<String, Value>, key: &str) -> i64 {
+    usage.get(key).and_then(Value::as_i64).unwrap_or(0)
+}
+
+/// `fold_reasoning_tokens_into_completion` then `_normalize_openai_compatible_usage_totals`.
+fn fix_usage(body: &mut Map<String, Value>) -> bool {
+    let Some(Value::Object(usage)) = body.get_mut("usage") else {
+        return false;
+    };
+    let mut changed = false;
+    let reasoning = usage
+        .get("completion_tokens_details")
+        .and_then(|details| details.get("reasoning_tokens"))
+        .and_then(Value::as_i64)
+        .unwrap_or(0);
+    let (prompt, completion, total) = (
+        count(usage, "prompt_tokens"),
+        count(usage, "completion_tokens"),
+        count(usage, "total_tokens"),
+    );
+    if reasoning > 0 && total != prompt + completion && total == prompt + completion + reasoning {
+        usage.insert("completion_tokens".into(), json!(completion + reasoning));
+        changed = true;
+    }
+    let expected = prompt + count(usage, "completion_tokens");
+    if count(usage, "total_tokens") < expected {
+        usage.insert("total_tokens".into(), json!(expected));
+        changed = true;
+    }
+    changed
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+    use crate::openai::chat::compatible::{fix_response, transform_request};
+
+    fn object(value: Value) -> Map<String, Value> {
+        let Value::Object(map) = value else {
+            panic!("an object")
+        };
+        map
+    }
+
+    fn send(model: &str, params: Value) -> Result<Map<String, Value>, Error> {
+        transform_request(
+            &XAI_CHAT_CONFIG,
+            &CompatibleContext {
+                model,
+                drop_params: false,
+                model_info: None,
+            },
+            object(params),
+        )
+        .map(|sent| sent.body)
+    }
+
+    #[rstest]
+    fn tools_lose_strict_and_web_search_and_names_are_dropped() {
+        let sent = send("grok-3", json!({"messages": [
+                {"role": "user", "name": "u", "content": "hi"},
+                {"role": "assistant", "name": "a", "content": "yo"}],
+            "tools": [{"type": "function", "function": {"name": "f", "strict": true, "parameters": {"type": "object"}}}],
+            "web_search_options": {}, "max_completion_tokens": 4, "stop": ["x"]}))
+        .unwrap();
+        assert_eq!(
+            Value::Object(sent),
+            json!({"model": "grok-3", "messages": [
+                    {"role": "user", "name": "u", "content": "hi"},
+                    {"role": "assistant", "content": "yo"}],
+                "tools": [{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}],
+                "max_tokens": 4, "stop": ["x"]})
+        );
+    }
+
+    #[rstest]
+    #[case::stop("grok-4", json!({"messages": [], "stop": ["x"]}))]
+    #[case::frequency_penalty("grok-code-fast-1", json!({"messages": [], "frequency_penalty": 1}))]
+    fn params_some_grok_models_reject_are_refused(#[case] model: &str, #[case] params: Value) {
+        assert!(send(model, params).is_err());
+    }
+
+    #[rstest]
+    #[case::folds(json!({"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 9,
+        "completion_tokens_details": {"reasoning_tokens": 4}}), json!(7), json!(9))]
+    #[case::already_folded(json!({"prompt_tokens": 2, "completion_tokens": 7, "total_tokens": 9,
+        "completion_tokens_details": {"reasoning_tokens": 4}}), json!(7), json!(9))]
+    #[case::raises_total(json!({"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 1}), json!(3), json!(5))]
+    fn usage_counts_reasoning_as_completion(
+        #[case] usage: Value,
+        #[case] completion: Value,
+        #[case] total: Value,
+    ) {
+        let mut body = object(json!({"choices": [], "usage": usage}));
+        XAI_CHAT_CONFIG.fix_chunk(&mut body, &ResponseShape::default());
+        assert_eq!(body["usage"]["completion_tokens"], completion);
+        assert_eq!(body["usage"]["total_tokens"], total);
+    }
+
+    #[rstest]
+    fn an_empty_finish_reason_with_tool_calls_becomes_tool_calls() {
+        let mut body = object(
+            json!({"choices": [{"finish_reason": "", "message": {"role": "assistant",
+            "tool_calls": [{"id": "c", "type": "function", "function": {"name": "f", "arguments": "{}"}}]}}]}),
+        );
+        assert!(fix_response(
+            &XAI_CHAT_CONFIG,
+            &mut body,
+            &ResponseShape::default()
+        ));
+        assert_eq!(body["choices"][0]["finish_reason"], json!("tool_calls"));
+    }
+}

litellm-rust/crates/llms/src/xai/mod.rs

@@ -0,0 +1 @@
+pub mod chat;

litellm-rust/docs/customer-proxy/openai-compat-chat.svg

@@ -0,0 +1,98 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="980" height="820" viewBox="0 0 980 820" font-family="Helvetica, Arial, sans-serif" font-size="13">
+  <defs>
+    <marker id="a" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M0,0 L10,5 L0,10 z" fill="#333"/>
+    </marker>
+  </defs>
+  <rect x="0" y="0" width="980" height="820" fill="#ffffff"/>
+  <text x="20" y="30" font-size="17" font-weight="bold" fill="#111">POST /v1/chat/completions to an OpenAI-compatible deployment</text>
+
+  <!-- client -->
+  <rect x="20" y="60" width="150" height="56" rx="6" fill="#eef3fb" stroke="#4a6fa5"/>
+  <text x="95" y="84" text-anchor="middle" fill="#111" font-weight="bold">Client</text>
+  <text x="95" y="102" text-anchor="middle" fill="#333">OpenAI SDK body</text>
+
+  <!-- gateway handle -->
+  <rect x="220" y="50" width="330" height="94" rx="6" fill="#f5f5f5" stroke="#666"/>
+  <text x="385" y="72" text-anchor="middle" fill="#111" font-weight="bold">gateway-inference chat_completions</text>
+  <text x="385" y="92" text-anchor="middle" fill="#333">auth, hooks admit, route deployment</text>
+  <text x="385" y="110" text-anchor="middle" fill="#333">serves_openai: OpenAI passthrough machine,</text>
+  <text x="385" y="128" text-anchor="middle" fill="#333">a ReportedUsage rides on the response</text>
+  <line x1="170" y1="88" x2="218" y2="88" stroke="#333" marker-end="url(#a)"/>
+
+  <rect x="590" y="50" width="370" height="94" rx="6" fill="#fff8e6" stroke="#b8860b"/>
+  <text x="775" y="72" text-anchor="middle" fill="#111" font-weight="bold">inference-chat compatible_chat_config</text>
+  <text x="775" y="92" text-anchor="middle" fill="#333">hosted_vllm, nvidia_nim, groq, together_ai,</text>
+  <text x="775" y="110" text-anchor="middle" fill="#333">fireworks_ai, xai, mistral, deepseek,</text>
+  <text x="775" y="128" text-anchor="middle" fill="#333">openrouter, cerebras, perplexity</text>
+  <line x1="550" y1="97" x2="588" y2="97" stroke="#333" marker-end="url(#a)"/>
+
+  <!-- shared request build -->
+  <rect x="20" y="176" width="560" height="256" rx="6" fill="#eef7ee" stroke="#3c8d3c"/>
+  <text x="300" y="198" text-anchor="middle" fill="#111" font-weight="bold">llms openai/chat/compatible transform_request</text>
+  <text x="36" y="222" fill="#333">1. sort_params: the provider's supported params for the model,</text>
+  <text x="54" y="240" fill="#333">unsupported refused (400) or dropped with drop_params</text>
+  <text x="36" y="260" fill="#333">2. a stream asks for stream_options.include_usage when the</text>
+  <text x="54" y="278" fill="#333">provider takes it and the client did not</text>
+  <text x="36" y="298" fill="#333">3. map_params: max_tokens names, tool_choice, reasoning,</text>
+  <text x="54" y="316" fill="#333">json_tool_call workaround, web search tools</text>
+  <text x="36" y="336" fill="#333">4. extra_body keys merged, the provider's own rewritten</text>
+  <text x="36" y="356" fill="#333">5. messages: response-only fields, content lists, names,</text>
+  <text x="54" y="374" fill="#333">cache_control kept only where the provider caches</text>
+  <text x="36" y="398" fill="#555">refused (501): groq streamed response_format, images in</text>
+  <text x="36" y="416" fill="#555">tool messages, file_id URLs Python downloads</text>
+  <line x1="300" y1="144" x2="300" y2="174" stroke="#333" marker-end="url(#a)"/>
+
+  <!-- urls and auth -->
+  <rect x="610" y="176" width="350" height="256" rx="6" fill="#ffffff" stroke="#4a6fa5"/>
+  <text x="785" y="198" text-anchor="middle" fill="#111" font-weight="bold">URL and credential</text>
+  <text x="626" y="222" fill="#333">api_base: deployment, then the provider's</text>
+  <text x="644" y="240" fill="#333">env var, then its default host</text>
+  <text x="644" y="258" fill="#333">(hosted_vllm has no default host)</text>
+  <text x="626" y="278" fill="#333">{api_base}/chat/completions, once</text>
+  <text x="644" y="296" fill="#333">mistral adds /v1 to the base first</text>
+  <text x="626" y="316" fill="#333">Bearer: deployment key, then env vars</text>
+  <text x="644" y="334" fill="#333">hosted_vllm falls back to fake-api-key</text>
+  <text x="626" y="354" fill="#333">headers: openrouter HTTP-Referer, X-Title,</text>
+  <text x="644" y="372" fill="#333">perplexity X-Pplx-Integration,</text>
+  <text x="644" y="390" fill="#333">fireworks x-session-affinity</text>
+  <text x="626" y="414" fill="#555">os.environ/ refs resolved via the secret source</text>
+  <line x1="580" y1="304" x2="608" y2="304" stroke="#333" marker-end="url(#a)"/>
+
+  <!-- upstream -->
+  <rect x="610" y="464" width="350" height="96" rx="6" fill="#eef3fb" stroke="#4a6fa5"/>
+  <text x="785" y="488" text-anchor="middle" fill="#111" font-weight="bold">Provider</text>
+  <text x="785" y="508" text-anchor="middle" fill="#333">OpenAI chat.completion JSON</text>
+  <text x="785" y="526" text-anchor="middle" fill="#333">stream: chat.completion.chunk SSE</text>
+  <text x="785" y="544" text-anchor="middle" fill="#555">data: frames closed by [DONE]</text>
+  <line x1="785" y1="432" x2="785" y2="462" stroke="#333" marker-end="url(#a)"/>
+
+  <!-- response -->
+  <rect x="20" y="464" width="560" height="160" rx="6" fill="#eef7ee" stroke="#3c8d3c"/>
+  <text x="300" y="486" text-anchor="middle" fill="#111" font-weight="bold">response and stream (ResponseAdapter)</text>
+  <text x="36" y="510" fill="#333">bytes pass through, re-encoded only when a fixup changes them</text>
+  <text x="36" y="530" fill="#333">reasoning to reasoning_content, json_tool_call to content</text>
+  <text x="36" y="550" fill="#333">groq service_tier, xai reasoning in completion_tokens,</text>
+  <text x="54" y="568" fill="#333">perplexity citations, mistral content blocks, fireworks</text>
+  <text x="54" y="586" fill="#333">tool calls in content</text>
+  <text x="36" y="606" fill="#333">stream: usage recorded, hidden unless the client asked</text>
+  <line x1="608" y1="512" x2="580" y2="512" stroke="#333" marker-end="url(#a)"/>
+
+  <!-- bottom boxes -->
+  <rect x="20" y="656" width="440" height="144" rx="6" fill="#ffffff" stroke="#b8860b"/>
+  <text x="240" y="678" text-anchor="middle" fill="#111" font-weight="bold">usage and cost (gateway-inference Finish)</text>
+  <text x="36" y="702" fill="#333">complete: the usage object of the body</text>
+  <text x="36" y="722" fill="#333">stream: the ReportedUsage the relay recorded,</text>
+  <text x="54" y="740" fill="#333">even from a chunk the client never saw</text>
+  <text x="36" y="760" fill="#333">cached_tokens, reasoning, server_tool_use kept</text>
+  <text x="36" y="780" fill="#555">config prices win, then the cost map entry</text>
+
+  <rect x="500" y="656" width="460" height="144" rx="6" fill="#ffffff" stroke="#3c8d3c"/>
+  <text x="730" y="678" text-anchor="middle" fill="#111" font-weight="bold">gateway startup (hooks::install)</text>
+  <text x="516" y="702" fill="#333">an OpenAI-compatible deployment loads the cost map</text>
+  <text x="516" y="722" fill="#333">and installs it as ModelInfo, which answers</text>
+  <text x="516" y="742" fill="#333">supports_reasoning, supports_web_search, vision,</text>
+  <text x="516" y="762" fill="#333">function calling, reasoning_effort_levels</text>
+  <text x="516" y="782" fill="#555">for {provider}/{m} or {m}</text>
+  <line x1="240" y1="624" x2="240" y2="654" stroke="#333" marker-end="url(#a)"/>
+</svg>