diff --git a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs index c3ee369a4d8..2e88afdd8ff 100644 --- a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs +++ b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs @@ -26,6 +26,8 @@ pub enum LlmProviders { Deepseek, #[strum(serialize = "edenai")] Edenai, + #[strum(serialize = "fireworks_ai")] + FireworksAi, #[strum(serialize = "hosted_vllm")] HostedVllm, #[strum(serialize = "minimax")] diff --git a/litellm-rust/crates/inference-responses/src/prepare.rs b/litellm-rust/crates/inference-responses/src/prepare.rs index 8bcf828106d..c7f72372b51 100644 --- a/litellm-rust/crates/inference-responses/src/prepare.rs +++ b/litellm-rust/crates/inference-responses/src/prepare.rs @@ -5,6 +5,7 @@ use litellm_host::interceptors::{ProviderIdentity, RequestContext}; use litellm_llms::{ azure::responses::transformation::AZURE_RESPONSES_CONFIG, base_llm::responses::transformation::BaseResponsesApiConfig, + fireworks_ai::responses::transformation::FIREWORKS_AI_RESPONSES_CONFIG, hosted_vllm::responses::transformation::HOSTED_VLLM_RESPONSES_CONFIG, openai::responses::transformation::OpenAiResponsesApiConfig, openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG, @@ -98,6 +99,7 @@ pub(super) fn resolve_provider( Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()), Ok(LlmProviders::Azure) => (&AZURE_RESPONSES_CONFIG, resolved.into()), Ok(LlmProviders::Xai) => (&XAI_RESPONSES_CONFIG, resolved.into()), + Ok(LlmProviders::FireworksAi) => (&FIREWORKS_AI_RESPONSES_CONFIG, resolved.into()), Ok(LlmProviders::HostedVllm) => (&HOSTED_VLLM_RESPONSES_CONFIG, resolved.into()), _ => return Err(Error::Unsupported("native HTTP responses provider")), }; @@ -148,6 +150,12 @@ mod tests { #[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")] #[case::azure_prefix("azure/my-deployment", None, "azure", "my-deployment")] #[case::azure_explicit("my-deployment", Some("azure"), "azure", "my-deployment")] + #[case::fireworks_prefix( + "fireworks_ai/accounts/fireworks/models/qwen3", + None, + "fireworks_ai", + "accounts/fireworks/models/qwen3" + )] #[case::xai_prefix("xai/grok-4", None, "xai", "grok-4")] #[case::hosted_vllm_keeps_the_vendor( "hosted_vllm/meta-llama/Llama-3.1-8B", diff --git a/litellm-rust/crates/llms/AGENTS.md b/litellm-rust/crates/llms/AGENTS.md index d68e488486c..245b6756a50 100644 --- a/litellm-rust/crates/llms/AGENTS.md +++ b/litellm-rust/crates/llms/AGENTS.md @@ -58,7 +58,7 @@ - `AnthropicConfig`, `AmazonConverseConfig` and the non-OCR base traits are partial. Preserve their acceptance gates, passthrough and host fallback contracts - `OpenAiResponsesApiConfig` implements WebSocket transformations and direct HTTP Responses, without Python's model-specific param rewriting or Responses-to-Chat emulation -- OpenAI-compatible Responses hosts (`openrouter`, `azure`, `xai`, `hosted_vllm`) reuse the OpenAI request and response transforms as free functions and own only their credentials, URL and host-specific item rules +- OpenAI-compatible Responses hosts (`openrouter`, `azure`, `xai`, `hosted_vllm`, `fireworks_ai`) reuse the OpenAI request and response transforms as free functions and own only their credentials, URL and host-specific item rules - Chat `supported_openai_param_mappings` lists OpenAI/provider name pairs, unlike Python's name-only list. Audio `map_transcription_params` is a Rust filtering helper - Auth resolution is split between configs and route preparation in `inference-` diff --git a/litellm-rust/crates/llms/src/fireworks_ai/mod.rs b/litellm-rust/crates/llms/src/fireworks_ai/mod.rs new file mode 100644 index 00000000000..b396b037bc5 --- /dev/null +++ b/litellm-rust/crates/llms/src/fireworks_ai/mod.rs @@ -0,0 +1 @@ +pub mod responses; diff --git a/litellm-rust/crates/llms/src/fireworks_ai/responses/AGENTS.md b/litellm-rust/crates/llms/src/fireworks_ai/responses/AGENTS.md new file mode 100644 index 00000000000..a4d8e325e62 --- /dev/null +++ b/litellm-rust/crates/llms/src/fireworks_ai/responses/AGENTS.md @@ -0,0 +1,11 @@ +# rules + +- Send the OpenAI Responses payload to `{api_base}/responses`, defaulting to `https://api.fireworks.ai/inference/v1`, with a bearer token from `api_key` or the Fireworks key variables +- Send the model as its Fireworks resource name, such as `accounts/fireworks/models/` +- Fold leading text-only system and developer items into `instructions`, and replay other developer items as `system` messages, since Fireworks has no developer role +- Session affinity and forwarded user ids stay on Python +- No native WebSocket mode + +# references + +- https://docs.fireworks.ai/api-reference/post-responses diff --git a/litellm-rust/crates/llms/src/fireworks_ai/responses/mod.rs b/litellm-rust/crates/llms/src/fireworks_ai/responses/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/fireworks_ai/responses/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/fireworks_ai/responses/transformation.rs b/litellm-rust/crates/llms/src/fireworks_ai/responses/transformation.rs new file mode 100644 index 00000000000..ba32cf10708 --- /dev/null +++ b/litellm-rust/crates/llms/src/fireworks_ai/responses/transformation.rs @@ -0,0 +1,405 @@ +//! Fireworks AI Responses, matching `litellm/llms/fireworks_ai/responses/transformation.py` + +use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_llms_types::formats::responses::ResponsesApiResponse; +use serde_json::{Map, Value, json}; + +use crate::{ + Error, + base_llm::{ + auth::{AuthScheme, ValidatedEnvironment}, + responses::transformation::BaseResponsesApiConfig, + }, + openai::responses::{ + request::ResponsesHost, + transformation::{responses_request, responses_response}, + }, +}; + +const API_KEY_ENVS: [&str; 4] = [ + "FIREWORKS_API_KEY", + "FIREWORKS_AI_API_KEY", + "FIREWORKSAI_API_KEY", + "FIREWORKS_AI_TOKEN", +]; +const API_BASE_ENV: &str = "FIREWORKS_API_BASE"; +const DEFAULT_BASE: &str = "https://api.fireworks.ai/inference/v1"; +const INSTRUCTION_ROLES: [&str; 2] = ["system", "developer"]; +const AZURE_FOUNDRY_MODEL_ID_PREFIX: &str = "FW-"; +const FIREROUTER: &str = "firerouter"; +const ROUTER_SHORT_NAMES: [&str; 3] = [FIREROUTER, "auto", "auto-instant"]; + +pub struct FireworksAiResponsesApiConfig; +pub const FIREWORKS_AI_RESPONSES_CONFIG: FireworksAiResponsesApiConfig = + FireworksAiResponsesApiConfig; + +impl BaseResponsesApiConfig for FireworksAiResponsesApiConfig { + fn secret_names( + &self, + api_key: Option<&str>, + api_base: Option<&str>, + ) -> &'static [&'static str] { + match ( + api_key.is_some_and(|key| !key.is_empty()), + api_base.is_some_and(|base| !base.is_empty()), + ) { + (true, true) => &[], + (true, false) => &[API_BASE_ENV], + (false, true) => &API_KEY_ENVS, + (false, false) => &[ + "FIREWORKS_API_KEY", + "FIREWORKS_AI_API_KEY", + "FIREWORKSAI_API_KEY", + "FIREWORKS_AI_TOKEN", + API_BASE_ENV, + ], + } + } + + fn validate_environment( + &self, + headers: Vec<(String, String)>, + api_key: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let key = api_key + .filter(|key| !key.is_empty()) + .map(str::to_owned) + .or_else(|| { + API_KEY_ENVS + .iter() + .find_map(|name| lookup(name).filter(|key| !key.is_empty())) + }) + .ok_or(litellm_auth::Error::MissingApiKey { + provider: "Fireworks AI", + environment_variable: API_KEY_ENVS[0], + })?; + Ok(ValidatedEnvironment { + headers: litellm_http::request::with_default_headers( + headers, + &[("content-type", "application/json")], + ), + auth: AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret: SecretValue::new(key), + }, + }) + } + + fn get_complete_url( + &self, + api_base: Option<&str>, + _api_version: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let api_base = api_base + .filter(|base| !base.is_empty()) + .map(str::to_owned) + .or_else(|| lookup(API_BASE_ENV).filter(|base| !base.is_empty())) + .unwrap_or_else(|| DEFAULT_BASE.into()); + Ok(format!("{}/responses", api_base.trim_end_matches('/'))) + } + + fn transform_responses_api_request( + &self, + model: &str, + input: Value, + mut params: Map, + ) -> Result { + let input = match params.remove("instructions") { + None | Some(Value::Null) => fold_instruction_items(input, None, &mut params), + Some(Value::String(instructions)) => { + fold_instruction_items(input, Some(instructions), &mut params) + } + Some(other) => { + params.insert("instructions".into(), other); + developer_items_as_system(input) + } + }; + responses_request( + &resource_name(model), + input, + params, + ResponsesHost::Compatible, + ) + } + + fn transform_response_api_response(&self, body: Value) -> Result { + responses_response(body) + } +} + +/// Python's `resolve_fireworks_resource_name`: a bare model id becomes its account path +fn resource_name(model: &str) -> String { + let stripped = model.strip_prefix("fireworks_ai/").unwrap_or(model); + if stripped.starts_with("accounts/") + || stripped.starts_with(AZURE_FOUNDRY_MODEL_ID_PREFIX) + || stripped.contains('#') + { + stripped.into() + } else if stripped.starts_with("routers/") || stripped.starts_with("models/") { + format!("accounts/fireworks/{stripped}") + } else if stripped.ends_with("-fast") + || ROUTER_SHORT_NAMES.contains(&stripped) + || stripped.starts_with(&format!("{FIREROUTER}/")) + { + format!("accounts/fireworks/routers/{stripped}") + } else { + format!("accounts/fireworks/models/{stripped}") + } +} + +fn role(item: &Value) -> Option<&str> { + item.get("role").and_then(Value::as_str) +} + +fn is_instruction_role(role: Option<&str>) -> bool { + role.is_some_and(|role| INSTRUCTION_ROLES.contains(&role)) +} + +/// Fireworks takes no `developer` role, so those items replay as plain `system` messages +fn developer_item_as_system(item: Value) -> Value { + match role(&item) { + Some("developer") => json!({ + "role": "system", + "content": item.get("content").cloned().unwrap_or(Value::Null), + "type": "message", + }), + _ => item, + } +} + +fn developer_items_as_system(input: Value) -> Value { + match input { + Value::Array(items) => { + Value::Array(items.into_iter().map(developer_item_as_system).collect()) + } + other => other, + } +} + +/// The text of a text-only system or developer item, with its parts joined by blank lines +fn text_only_content(item: &Value) -> Option { + if !is_instruction_role(role(item)) { + return None; + } + match item.get("content")? { + Value::String(text) => Some(text.clone()), + Value::Array(parts) => { + let texts = parts + .iter() + .map(|part| match part.get("type").and_then(Value::as_str) { + Some("input_text") => part.get("text").and_then(Value::as_str), + _ => None, + }) + .collect::>>()?; + Some( + texts + .into_iter() + .filter(|text| !text.is_empty()) + .collect::>() + .join("\n\n"), + ) + } + _ => None, + } +} + +/// Python's `_hoisted_indices`: the leading instruction block, plus every developer item before +/// a closing instruction block that follows the last assistant turn +fn hoisted_indices(roles: &[Option<&str>]) -> Vec { + let leading = roles + .iter() + .position(|role| !is_instruction_role(*role)) + .unwrap_or(roles.len()); + let closing = match (leading..roles.len()) + .rev() + .find(|&index| !is_instruction_role(roles[index])) + { + Some(index) if roles[index] == Some("assistant") => index + 1, + _ => roles.len(), + }; + (0..closing) + .filter(|&index| index < leading || roles[index] == Some("developer")) + .collect() +} + +/// Python's `_with_instruction_items_folded`: hoisted text-only instruction items join +/// `instructions`, and the remaining developer items become system messages +fn fold_instruction_items( + input: Value, + instructions: Option, + params: &mut Map, +) -> Value { + let (instructions, input) = match input { + Value::Array(items) => { + let roles: Vec> = items.iter().map(role).collect(); + let folded: Vec<(usize, String)> = hoisted_indices(&roles) + .into_iter() + .filter_map(|index| Some((index, text_only_content(&items[index])?))) + .collect(); + let instructions = match folded.is_empty() { + true => instructions, + false => { + let joined = instructions + .iter() + .map(String::as_str) + .chain(folded.iter().map(|(_, text)| text.as_str())) + .filter(|chunk| !chunk.is_empty()) + .collect::>() + .join("\n\n"); + (!joined.is_empty()).then_some(joined) + } + }; + let kept = items + .into_iter() + .enumerate() + .filter(|(index, _)| !folded.iter().any(|(folded, _)| folded == index)) + .map(|(_, item)| developer_item_as_system(item)) + .collect(); + (instructions, Value::Array(kept)) + } + other => (instructions, other), + }; + if let Some(instructions) = instructions { + params.insert("instructions".into(), Value::String(instructions)); + } + input +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + pairs + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + } + } + + #[rstest] + #[case::default(None, &[], "https://api.fireworks.ai/inference/v1/responses")] + #[case::explicit(Some("https://proxy.test/v1/"), &[], "https://proxy.test/v1/responses")] + #[case::secret(None, &[("FIREWORKS_API_BASE", "https://secret.test/v1")], "https://secret.test/v1/responses")] + fn url_is_the_base_plus_responses( + #[case] api_base: Option<&str>, + #[case] secrets: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + assert_eq!( + FIREWORKS_AI_RESPONSES_CONFIG + .get_complete_url(api_base, None, &env(secrets)) + .unwrap(), + expected + ); + } + + #[rstest] + #[case::explicit(Some("explicit"), &[("FIREWORKS_API_KEY", "managed")], "explicit")] + #[case::first_secret(None, &[("FIREWORKS_AI_API_KEY", "b"), ("FIREWORKS_API_KEY", "a")], "a")] + #[case::token(None, &[("FIREWORKS_AI_TOKEN", "token")], "token")] + fn bearer_comes_from_the_call_then_the_fireworks_secrets( + #[case] api_key: Option<&str>, + #[case] secrets: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + let environment = FIREWORKS_AI_RESPONSES_CONFIG + .validate_environment(vec![], api_key, &env(secrets)) + .unwrap(); + let AuthScheme::Credential { placement, secret } = environment.auth else { + panic!("expected a credential"); + }; + assert_eq!(placement, CredentialPlacement::Bearer); + assert_eq!(secret.expose(), expected); + } + + #[rstest] + #[case::bare( + "llama-v3p1-8b-instruct", + "accounts/fireworks/models/llama-v3p1-8b-instruct" + )] + #[case::prefixed("fireworks_ai/qwen3", "accounts/fireworks/models/qwen3")] + #[case::account("accounts/me/models/x", "accounts/me/models/x")] + #[case::azure_foundry("FW-x", "FW-x")] + #[case::deployment("x#accounts/me/deployments/d", "x#accounts/me/deployments/d")] + #[case::models_path("models/x", "accounts/fireworks/models/x")] + #[case::router_path("routers/x", "accounts/fireworks/routers/x")] + #[case::fast("glm-fast", "accounts/fireworks/routers/glm-fast")] + #[case::short_router("auto", "accounts/fireworks/routers/auto")] + #[case::firerouter("firerouter/code", "accounts/fireworks/routers/firerouter/code")] + fn model_is_the_fireworks_resource_name(#[case] model: &str, #[case] expected: &str) { + assert_eq!(resource_name(model), expected); + } + + fn request(input: Value, params: Value) -> Value { + let Value::Object(params) = params else { + unreachable!() + }; + FIREWORKS_AI_RESPONSES_CONFIG + .transform_responses_api_request("qwen3", input, params) + .unwrap() + } + + #[test] + fn leading_instruction_items_fold_into_instructions() { + let body = request( + json!([ + {"role": "system", "content": "Be brief"}, + {"role": "developer", "content": [{"type": "input_text", "text": "Use French"}]}, + {"role": "user", "content": "hi"}, + ]), + json!({"instructions": "Top"}), + ); + assert_eq!(body["instructions"], json!("Top\n\nBe brief\n\nUse French")); + assert_eq!(body["input"], json!([{"role": "user", "content": "hi"}])); + assert_eq!(body["model"], json!("accounts/fireworks/models/qwen3")); + } + + #[test] + fn developer_items_after_the_last_assistant_turn_stay_as_system_messages() { + let body = request( + json!([ + {"role": "user", "content": "hi"}, + {"role": "developer", "content": "mid"}, + {"role": "assistant", "content": "hello"}, + {"role": "developer", "content": "late", "id": "x"}, + ]), + json!({}), + ); + assert_eq!(body["instructions"], json!("mid")); + assert_eq!( + body["input"], + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "system", "content": "late", "type": "message"}, + ]) + ); + } + + #[test] + fn non_text_instruction_items_are_kept_and_non_string_instructions_only_rename_roles() { + let image = + json!({"role": "system", "content": [{"type": "input_image", "image_url": "u"}]}); + let body = request( + json!([image.clone(), {"role": "user", "content": "hi"}]), + json!({}), + ); + assert!(body.get("instructions").is_none()); + assert_eq!(body["input"][0], image); + let body = request( + json!([{"role": "developer", "content": "d"}]), + json!({"instructions": ["x"]}), + ); + assert_eq!(body["instructions"], json!(["x"])); + assert_eq!( + body["input"], + json!([{"role": "system", "content": "d", "type": "message"}]) + ); + } +} diff --git a/litellm-rust/crates/llms/src/lib.rs b/litellm-rust/crates/llms/src/lib.rs index 6fd1c8817a9..d4c3344ab1e 100644 --- a/litellm-rust/crates/llms/src/lib.rs +++ b/litellm-rust/crates/llms/src/lib.rs @@ -9,6 +9,7 @@ pub mod cohere; pub mod deepseek; pub mod edenai; mod error; +pub mod fireworks_ai; pub mod hosted_vllm; pub mod minimax; pub mod mistral; diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs index a40dfbca926..38ca89f712b 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs @@ -47,7 +47,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py None => !model.contains('/'), Some(resolved) => match resolved.custom_llm_provider { "openai" => !resolved.model.contains('/'), - "azure" | "xai" | "hosted_vllm" => true, + "azure" | "xai" | "hosted_vllm" | "fireworks_ai" => true, _ => false, }, }; diff --git a/litellm/rust_bridge/host/routes/responses.py b/litellm/rust_bridge/host/routes/responses.py index 4a8f78833c0..1d354b3be9d 100644 --- a/litellm/rust_bridge/host/routes/responses.py +++ b/litellm/rust_bridge/host/routes/responses.py @@ -7,6 +7,8 @@ from typing import Final, cast # noqa: TID251 # narrows caller json the checke import litellm from litellm import get_llm_provider from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id +from litellm.llms.fireworks_ai.responses.transformation import FireworksAIResponsesAPIConfig from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig @@ -24,6 +26,7 @@ _CONFIGS: Final[Mapping[str, OpenAIResponsesAPIConfig]] = MappingProxyType( "azure": AzureOpenAIResponsesAPIConfig(), "xai": XAIResponsesAPIConfig(), "hosted_vllm": HostedVLLMResponsesAPIConfig(), + "fireworks_ai": FireworksAIResponsesAPIConfig(), } ) _AZURE_KEY_ENVIRONMENT: Final = ("AZURE_OPENAI_API_KEY", "AZURE_API_KEY") @@ -89,7 +92,7 @@ def decline_reason(request: Mapping[str, object]) -> str | None: def _provider_decline_reason(provider: str, model: str, request: Mapping[str, object]) -> str | None: if provider == "azure": return _azure_decline_reason(model, request) - if provider not in ("xai", "hosted_vllm"): + if provider not in ("xai", "hosted_vllm", "fireworks_ai"): return None provider_key, provider_base = provider_connection( str(request["model"]), @@ -101,6 +104,10 @@ def _provider_decline_reason(provider: str, model: str, request: Mapping[str, ob return "native xAI Responses does not implement OAuth" if provider == "hosted_vllm" and not provider_base: return "native hosted vLLM Responses needs an api_base" + if provider == "fireworks_ai" and not provider_key: + return "native Fireworks Responses needs an api key" + if provider == "fireworks_ai" and get_fireworks_session_id({"metadata": request.get("litellm_metadata")}): + return "native Fireworks Responses does not implement session affinity" return None @@ -126,6 +133,8 @@ def _model_rules_decline_reason( config: OpenAIResponsesAPIConfig, model: str, request: Mapping[str, object] ) -> str | None: """Request rewrites that read the model cost map or Python's regex engine stay on Python""" + if _has_file_search_tool(request.get("tools")) and not config.supports_native_file_search(): + return "native Responses does not implement emulated file_search" params: Final = {name: request[name] for name in PARAMETERS if request.get(name) is not None} try: config.map_openai_params(params, model, drop_params=False) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # the same rules the Python route applies @@ -159,6 +168,14 @@ def _repairs_tool_schemas( ) +def _has_file_search_tool(tools: object) -> bool: + return isinstance(tools, list) and any( + isinstance(tool, Mapping) + and cast(Mapping[str, object], tool).get("type") == "file_search" # cast-ok: caller json + for tool in cast(list[object], tools) # cast-ok: caller json + ) + + def _carries_cache_control(value: object) -> bool: pending: Final[list[object]] = [value] # mutable-ok: work stack for a depth-first walk of caller json while pending: diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py index 16b8696a019..f8db38f03bc 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py @@ -158,6 +158,15 @@ AFFINITY_REPLAY_INPUT: Final[list[JsonValue]] = [ AZURE_MODEL: Final = "azure/my-deployment" XAI_MODEL: Final = "xai/grok-4" +FIREWORKS_MODEL: Final = "fireworks_ai/qwen3-235b-a22b" +FIREWORKS_INSTRUCTION_INPUT: Final[list[JsonValue]] = [ + {"role": "system", "content": "Be brief"}, + {"role": "developer", "content": [{"type": "input_text", "text": "Answer in French"}]}, + {"role": "user", "content": "Capital of France?"}, + {"role": "developer", "content": "Mention the river"}, + {"role": "assistant", "content": "Paris"}, + {"role": "developer", "content": "Now be verbose"}, +] HOSTED_VLLM_MODEL: Final = "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct" XAI_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( @@ -355,6 +364,16 @@ CASES: Final = ( {"model": XAI_MODEL, "input": "hi", "tools": XAI_TOOLS, "metadata": {"team": "a"}, "max_output_tokens": 8}, upstream=XAI_RESPONSE, ), + Case( + "fireworks_folds_instruction_items", + {"model": FIREWORKS_MODEL, "input": FIREWORKS_INSTRUCTION_INPUT, "instructions": "You are helpful"}, + ), + Case("fireworks_router_resource_name", {"model": "fireworks_ai/glm-fast", "input": "hi"}), + Case( + "fireworks_session_affinity_stays_on_python", + {"model": FIREWORKS_MODEL, "input": "hi", "litellm_metadata": {"session_id": "sess-1"}}, + falls_back=True, + ), Case("hosted_vllm_placeholder_key", {"model": HOSTED_VLLM_MODEL, "input": "hi", "api_key": None}), Case( "hosted_vllm_keeps_replayed_reasoning_and_ids", diff --git a/tests/unit/rust_bridge/host/routes/test_responses.py b/tests/unit/rust_bridge/host/routes/test_responses.py index 04f9f5dc5b2..a096331a9a0 100644 --- a/tests/unit/rust_bridge/host/routes/test_responses.py +++ b/tests/unit/rust_bridge/host/routes/test_responses.py @@ -225,3 +225,38 @@ def test_provider_connection_matches_get_llm_provider(monkeypatch: pytest.Monkey "http://vllm:8000", ) assert provider_connection("gpt-4.1", None, None, None) == (None, None) + + +@pytest.mark.parametrize( + ("request_fields", "expected"), + [ + pytest.param({"api_key": "fw-key"}, None, id="api_key"), + pytest.param({}, "native Fireworks Responses needs an api key", id="no_key"), + pytest.param( + {"api_key": "fw-key", "litellm_metadata": {"session_id": "sess-1"}}, + "native Fireworks Responses does not implement session affinity", + id="session_affinity", + ), + pytest.param( + {"api_key": "fw-key", "tools": [{"type": "file_search", "vector_store_ids": ["vs_1"]}]}, + "native Responses does not implement emulated file_search", + id="emulated_file_search", + ), + ], +) +def test_fireworks_requests_outside_the_native_config_stay_on_python( + monkeypatch: pytest.MonkeyPatch, request_fields: dict[str, object], expected: str | None +) -> None: + for name in ("FIREWORKS_API_KEY", "FIREWORKS_AI_API_KEY", "FIREWORKSAI_API_KEY", "FIREWORKS_AI_TOKEN"): + monkeypatch.delenv(name, raising=False) + assert decline_reason({"model": "fireworks_ai/qwen3", "input": "hi", **request_fields}) == expected + + +def test_openai_file_search_stays_native() -> None: + request: Final = { + "model": "gpt-4.1", + "input": "hi", + "api_key": "sk-test", + "tools": [{"type": "file_search", "vector_store_ids": ["vs_1"]}], + } + assert decline_reason(request) is None