diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index ca80fd423..4640686a2 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -8227,6 +8227,7 @@ components: - reasoning - reasoning_effort - prompt_cache + - cache_control_breakpoints - sampling_params properties: tools: @@ -8243,6 +8244,12 @@ components: prompt_cache: type: boolean description: Whether the model endpoint supports prompt caching. + cache_control_breakpoints: + type: boolean + description: >- + Whether the endpoint only caches when the request marks the + cacheable prefix with Anthropic-style cache_control breakpoints + (e.g. Claude via OpenRouter). sampling_params: type: boolean description: Whether the model accepts classic sampling parameters (temperature, top_p). diff --git a/lib/crates/fabro-agent/src/cli.rs b/lib/crates/fabro-agent/src/cli.rs index 749f38b0a..f14041456 100644 --- a/lib/crates/fabro-agent/src/cli.rs +++ b/lib/crates/fabro-agent/src/cli.rs @@ -1073,12 +1073,13 @@ mod tests { max_output: None, }), features: Some(SettingsModelFeatures { - tools: Some(true), - vision: Some(false), - reasoning: Some(false), - reasoning_effort: None, - prompt_cache: None, - sampling_params: None, + tools: Some(true), + vision: Some(false), + reasoning: Some(false), + reasoning_effort: None, + prompt_cache: None, + cache_control_breakpoints: None, + sampling_params: None, }), ..ModelCatalogSettings::default() }); @@ -1158,12 +1159,13 @@ mod tests { max_output: None, }), features: Some(SettingsModelFeatures { - tools: Some(true), - vision: Some(false), - reasoning: Some(false), - reasoning_effort: None, - prompt_cache: None, - sampling_params: None, + tools: Some(true), + vision: Some(false), + reasoning: Some(false), + reasoning_effort: None, + prompt_cache: None, + cache_control_breakpoints: None, + sampling_params: None, }), ..ModelCatalogSettings::default() }); diff --git a/lib/crates/fabro-api/tests/model_features_round_trip.rs b/lib/crates/fabro-api/tests/model_features_round_trip.rs index 8e6f60aef..42f596665 100644 --- a/lib/crates/fabro-api/tests/model_features_round_trip.rs +++ b/lib/crates/fabro-api/tests/model_features_round_trip.rs @@ -11,12 +11,13 @@ fn model_features_reuses_canonical_type() { #[test] fn model_features_json_matches_openapi_shape() { let features = ModelFeatures { - tools: true, - vision: true, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: false, - sampling_params: true, + tools: true, + vision: true, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }; let json = serde_json::to_value(&features).unwrap(); @@ -25,6 +26,7 @@ fn model_features_json_matches_openapi_shape() { assert_eq!(json["reasoning"], true); assert_eq!(json["reasoning_effort"], "levels"); assert_eq!(json["prompt_cache"], false); + assert_eq!(json["cache_control_breakpoints"], false); assert_eq!(json["sampling_params"], true); let round_trip: ApiModelFeatures = serde_json::from_value(json).unwrap(); diff --git a/lib/crates/fabro-api/tests/model_round_trip.rs b/lib/crates/fabro-api/tests/model_round_trip.rs index 93d4e7692..57d3db557 100644 --- a/lib/crates/fabro-api/tests/model_round_trip.rs +++ b/lib/crates/fabro-api/tests/model_round_trip.rs @@ -24,12 +24,13 @@ fn model_json_matches_openapi_shape() { training: Some("2025-08-01".to_string()), knowledge_cutoff: Some("May 2025".to_string()), features: ModelFeatures { - tools: true, - vision: true, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: true, - sampling_params: true, + tools: true, + vision: true, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: true, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: Some(5.0), diff --git a/lib/crates/fabro-api/tests/provider_id_round_trip.rs b/lib/crates/fabro-api/tests/provider_id_round_trip.rs index e648b008f..970c7158f 100644 --- a/lib/crates/fabro-api/tests/provider_id_round_trip.rs +++ b/lib/crates/fabro-api/tests/provider_id_round_trip.rs @@ -34,12 +34,13 @@ fn provider_id_json_matches_openapi_shape_through_model() { training: None, knowledge_cutoff: None, features: ModelFeatures { - tools: false, - vision: false, - reasoning: false, - reasoning_effort: ReasoningEffortFeature::None, - prompt_cache: false, - sampling_params: true, + tools: false, + vision: false, + reasoning: false, + reasoning_effort: ReasoningEffortFeature::None, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: None, diff --git a/lib/crates/fabro-cli/src/commands/model.rs b/lib/crates/fabro-cli/src/commands/model.rs index 4e5bd6f39..f84bd44e2 100644 --- a/lib/crates/fabro-cli/src/commands/model.rs +++ b/lib/crates/fabro-cli/src/commands/model.rs @@ -538,12 +538,13 @@ mod tests { training: None, knowledge_cutoff: None, features: ModelFeatures { - tools: true, - vision: false, - reasoning: false, - reasoning_effort: ReasoningEffortFeature::None, - prompt_cache: false, - sampling_params: true, + tools: true, + vision: false, + reasoning: false, + reasoning_effort: ReasoningEffortFeature::None, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: Some(1.0), @@ -572,12 +573,13 @@ mod tests { training: None, knowledge_cutoff: None, features: ModelFeatures { - tools: true, - vision: false, - reasoning: false, - reasoning_effort: ReasoningEffortFeature::None, - prompt_cache: false, - sampling_params: true, + tools: true, + vision: false, + reasoning: false, + reasoning_effort: ReasoningEffortFeature::None, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: Some(1.0), diff --git a/lib/crates/fabro-config/src/builders.rs b/lib/crates/fabro-config/src/builders.rs index 1ffdc650d..a1e00907c 100644 --- a/lib/crates/fabro-config/src/builders.rs +++ b/lib/crates/fabro-config/src/builders.rs @@ -417,12 +417,13 @@ fn model_limits_to_catalog(limits: &LlmModelLimits) -> model_catalog::SettingsMo fn model_features_to_catalog(features: &LlmModelFeatures) -> model_catalog::SettingsModelFeatures { model_catalog::SettingsModelFeatures { - tools: features.tools, - vision: features.vision, - reasoning: features.reasoning, - reasoning_effort: features.reasoning_effort, - prompt_cache: features.prompt_cache, - sampling_params: features.sampling_params, + tools: features.tools, + vision: features.vision, + reasoning: features.reasoning, + reasoning_effort: features.reasoning_effort, + prompt_cache: features.prompt_cache, + cache_control_breakpoints: features.cache_control_breakpoints, + sampling_params: features.sampling_params, } } diff --git a/lib/crates/fabro-config/src/layers/llm.rs b/lib/crates/fabro-config/src/layers/llm.rs index ca29d2bbd..4f89c55fc 100644 --- a/lib/crates/fabro-config/src/layers/llm.rs +++ b/lib/crates/fabro-config/src/layers/llm.rs @@ -238,17 +238,19 @@ pub struct ModelLimits { #[serde(deny_unknown_fields)] pub struct ModelFeatures { #[serde(default, skip_serializing_if = "Option::is_none")] - pub tools: Option, + pub tools: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub vision: Option, + pub vision: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub reasoning: Option, + pub reasoning: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub reasoning_effort: Option, + pub reasoning_effort: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub prompt_cache: Option, + pub prompt_cache: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub sampling_params: Option, + pub cache_control_breakpoints: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sampling_params: Option, } /// User-facing allow-list for native control values Fabro accepts on this diff --git a/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs b/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs index 2803c221c..5ed999f87 100644 --- a/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs +++ b/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs @@ -8,7 +8,8 @@ use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use super::SYNTHETIC_TOOL_NAME; -use super::wire::{ApiMessage, ApiRequest, ApiToolDef, CacheControl, CountTokensRequest}; +use super::wire::{ApiMessage, ApiRequest, ApiToolDef, CountTokensRequest}; +use crate::codec::cache::{self, CacheControl}; use crate::codec::{AnthropicVersion, CodecCtx, EncodedRequest, extract_system_prompt}; use crate::types::{ ContentPart, Message, ReasoningEffort, ReasoningEffortFeature, Request, ResponseFormatType, @@ -48,7 +49,7 @@ pub(super) fn encode_count_tokens(ctx: &CodecCtx<'_>) -> EncodedRequest { /// hasn't opted out. fn auto_cache(ctx: &CodecCtx<'_>) -> bool { ctx.model.is_some_and(|m| m.features.prompt_cache) - && is_auto_cache_enabled(ctx.request.provider_options.as_ref()) + && cache::auto_cache_enabled(ctx.request.provider_options.as_ref(), "anthropic") } fn build_headers(ctx: &CodecCtx<'_>) -> Vec<(String, String)> { @@ -441,12 +442,6 @@ fn effort_to_budget_tokens(effort: ReasoningEffort, max_tokens: i64) -> i64 { budget.max(1024) } -fn is_auto_cache_enabled(provider_options: Option<&serde_json::Value>) -> bool { - anthropic_option(provider_options, "auto_cache") - .and_then(serde_json::Value::as_bool) - .unwrap_or(true) -} - fn system_with_cache_control(system: &str) -> serde_json::Value { serde_json::json!([{ "type": "text", @@ -462,18 +457,10 @@ fn apply_cache_control_to_last_tool(tools: &mut [ApiToolDef]) { } fn apply_cache_control_to_conversation_prefix(messages: &mut [ApiMessage]) { - let user_indices: Vec = messages - .iter() - .enumerate() - .filter(|(_, m)| m.role == "user") - .map(|(i, _)| i) - .collect(); - - if user_indices.len() < 2 { + let user_turns: Vec = messages.iter().map(|m| m.role == "user").collect(); + let Some(target_idx) = cache::conversation_breakpoint_index(&user_turns) else { return; - } - - let target_idx = user_indices[user_indices.len() - 2]; + }; if let Some(serde_json::Value::Object(map)) = messages[target_idx].content.last_mut() { map.insert( "cache_control".to_string(), @@ -651,37 +638,6 @@ reasoning = true .map(|(_, value)| value.as_str()) } - // --- auto_cache ---------------------------------------------------------- - - #[test] - fn auto_cache_enabled_by_default() { - assert!(is_auto_cache_enabled(None)); - } - - #[test] - fn auto_cache_enabled_when_true() { - let opts = serde_json::json!({"anthropic": {"auto_cache": true}}); - assert!(is_auto_cache_enabled(Some(&opts))); - } - - #[test] - fn auto_cache_disabled_when_false() { - let opts = serde_json::json!({"anthropic": {"auto_cache": false}}); - assert!(!is_auto_cache_enabled(Some(&opts))); - } - - #[test] - fn auto_cache_enabled_when_key_missing() { - let opts = serde_json::json!({"anthropic": {}}); - assert!(is_auto_cache_enabled(Some(&opts))); - } - - #[test] - fn auto_cache_enabled_when_anthropic_missing() { - let opts = serde_json::json!({"openai": {}}); - assert!(is_auto_cache_enabled(Some(&opts))); - } - // --- prompt-cache helpers ------------------------------------------------ #[test] diff --git a/lib/crates/fabro-llm/src/codec/anthropic_messages/wire.rs b/lib/crates/fabro-llm/src/codec/anthropic_messages/wire.rs index 80458dea4..70fb0ede6 100644 --- a/lib/crates/fabro-llm/src/codec/anthropic_messages/wire.rs +++ b/lib/crates/fabro-llm/src/codec/anthropic_messages/wire.rs @@ -1,5 +1,7 @@ //! Serde types mirroring the Anthropic Messages wire shapes. +use crate::codec::cache::CacheControl; + #[derive(serde::Serialize)] pub(super) struct ApiRequest { pub model: String, @@ -77,21 +79,6 @@ pub(super) struct ApiToolDef { pub cache_control: Option, } -/// Anthropic `cache_control` annotation. -#[derive(serde::Serialize, Clone)] -pub(super) struct CacheControl { - #[serde(rename = "type")] - pub kind: String, -} - -impl CacheControl { - pub(super) fn ephemeral() -> Self { - Self { - kind: "ephemeral".to_string(), - } - } -} - // --- Response types --- #[derive(serde::Deserialize)] diff --git a/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs b/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs index 54a38314e..39c9c5c10 100644 --- a/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs +++ b/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs @@ -342,7 +342,7 @@ fn apply_cache_point_to_conversation_prefix(messages: &mut [Value]) { /// codec). This is the passthrough for `additionalModelRequestFields`, /// `guardrailConfig`, `serviceTier`, and other Converse extensions. fn merge_provider_options(body: &mut Value, provider_options: Option<&Value>, provider_name: &str) { - merge_named_provider_options(body, provider_options, provider_name); + merge_named_provider_options(body, provider_options, provider_name, &[]); } #[cfg(test)] diff --git a/lib/crates/fabro-llm/src/codec/cache.rs b/lib/crates/fabro-llm/src/codec/cache.rs new file mode 100644 index 000000000..ed1afe8ab --- /dev/null +++ b/lib/crates/fabro-llm/src/codec/cache.rs @@ -0,0 +1,98 @@ +//! Shared prompt-cache policy: whether a request opts into explicit +//! Anthropic-style caching and where the conversation breakpoint lands. +//! Dialect codecs apply these decisions to their own wire shapes. + +/// Anthropic-style `cache_control` annotation. +#[derive(serde::Serialize, Clone)] +pub(crate) struct CacheControl { + #[serde(rename = "type")] + pub kind: String, +} + +impl CacheControl { + pub(crate) fn ephemeral() -> Self { + Self { + kind: "ephemeral".to_string(), + } + } +} + +/// Whether automatic prompt caching applies to this request: the +/// `provider_options..auto_cache` opt-out defaults to enabled. +pub(crate) fn auto_cache_enabled( + provider_options: Option<&serde_json::Value>, + namespace: &str, +) -> bool { + provider_options + .and_then(|opts| opts.get(namespace)) + .and_then(|ns| ns.get("auto_cache")) + .and_then(serde_json::Value::as_bool) + .unwrap_or(true) +} + +/// Index of the message carrying the conversation-prefix breakpoint: the +/// second-to-last user turn, so each iteration of an agent loop reuses the +/// prefix cached by the previous one. `user_turns[i]` is true when message +/// `i` advances the user side of the conversation (plain user messages, plus +/// tool results on dialects where they are separate messages). `None` until +/// the conversation has at least two user turns. +pub(crate) fn conversation_breakpoint_index(user_turns: &[bool]) -> Option { + let indices: Vec = user_turns + .iter() + .enumerate() + .filter_map(|(i, &is_user)| is_user.then_some(i)) + .collect(); + indices.len().checked_sub(2).map(|nth| indices[nth]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn auto_cache_enabled_by_default() { + assert!(auto_cache_enabled(None, "anthropic")); + } + + #[test] + fn auto_cache_enabled_when_true() { + let opts = serde_json::json!({"anthropic": {"auto_cache": true}}); + assert!(auto_cache_enabled(Some(&opts), "anthropic")); + } + + #[test] + fn auto_cache_disabled_when_false() { + let opts = serde_json::json!({"openrouter": {"auto_cache": false}}); + assert!(!auto_cache_enabled(Some(&opts), "openrouter")); + } + + #[test] + fn auto_cache_enabled_when_key_missing() { + let opts = serde_json::json!({"anthropic": {}}); + assert!(auto_cache_enabled(Some(&opts), "anthropic")); + } + + #[test] + fn auto_cache_reads_only_its_own_namespace() { + let opts = serde_json::json!({"openrouter": {"auto_cache": false}}); + assert!(auto_cache_enabled(Some(&opts), "anthropic")); + } + + #[test] + fn conversation_breakpoint_none_below_two_user_turns() { + assert_eq!(conversation_breakpoint_index(&[]), None); + assert_eq!(conversation_breakpoint_index(&[true]), None); + assert_eq!(conversation_breakpoint_index(&[true, false, false]), None); + } + + #[test] + fn conversation_breakpoint_with_exactly_two_user_turns() { + assert_eq!(conversation_breakpoint_index(&[true, false, true]), Some(0)); + } + + #[test] + fn conversation_breakpoint_targets_second_to_last_user_turn() { + let turns = [true, false, true, false, true]; + assert_eq!(conversation_breakpoint_index(&turns), Some(2)); + } +} diff --git a/lib/crates/fabro-llm/src/codec/mod.rs b/lib/crates/fabro-llm/src/codec/mod.rs index 9ddc1eb0a..5a7759b22 100644 --- a/lib/crates/fabro-llm/src/codec/mod.rs +++ b/lib/crates/fabro-llm/src/codec/mod.rs @@ -12,6 +12,7 @@ pub(crate) mod anthropic_messages; pub(crate) mod bedrock_converse; +pub(crate) mod cache; pub(crate) mod gemini_generate; pub(crate) mod openai_compatible; pub(crate) mod openai_responses; @@ -38,11 +39,14 @@ pub(crate) fn split_inclusive_token_total(total: i64, detail: i64) -> (i64, i64) /// Merge `provider_options.` fields into an encoded request /// body. Used by codecs whose provider-options namespace is adapter-name keyed -/// rather than a single fixed provider. +/// rather than a single fixed provider. `known_keys` are control keys the +/// codec consumed itself (e.g. `auto_cache`); they are not re-merged into the +/// body. pub(crate) fn merge_named_provider_options( body: &mut serde_json::Value, provider_options: Option<&serde_json::Value>, provider_name: &str, + known_keys: &[&str], ) { let Some(opts) = provider_options.and_then(|opts| opts.get(provider_name)) else { return; @@ -55,6 +59,9 @@ pub(crate) fn merge_named_provider_options( }; for (key, value) in opts_map { + if known_keys.contains(&key.as_str()) { + continue; + } body_map.insert(key.clone(), value.clone()); } } diff --git a/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs b/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs index 72920b34c..12be9e84b 100644 --- a/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs +++ b/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs @@ -1,10 +1,14 @@ //! Request encoding: canonical `Request` → Chat Completions body. use super::translate; -use super::wire::ApiRequest; -use crate::codec::{CodecCtx, EncodedRequest, merge_named_provider_options}; +use super::wire::{ApiRequest, ChatMessage}; +use crate::codec::{CodecCtx, EncodedRequest, cache, merge_named_provider_options}; use crate::error::Error; +/// Known `provider_options.` keys the codec consumes itself; +/// not re-merged into the body. +const KNOWN_OPTION_KEYS: &[&str] = &["auto_cache"]; + /// Build the Chat Completions request for `ctx.request`. `stream` toggles the /// `stream` body field. The body is assembled as a `serde_json::Value` so /// `provider_options.` fields can be merged in before sending. @@ -13,7 +17,10 @@ use crate::error::Error; /// the Chat Completions tool envelope cannot represent. pub(super) fn encode(ctx: &CodecCtx<'_>, stream: bool) -> Result { let request = ctx.request; - let chat_messages = translate::translate_messages(&request.messages); + let mut chat_messages = translate::translate_messages(&request.messages); + if explicit_cache_breakpoints(ctx) { + apply_cache_breakpoints(&mut chat_messages); + } let tools = request .tools .as_ref() @@ -64,6 +71,37 @@ pub(super) fn encode(ctx: &CodecCtx<'_>, stream: bool) -> Result) -> bool { + ctx.model + .is_some_and(|m| m.features.prompt_cache && m.features.cache_control_breakpoints) + && cache::auto_cache_enabled(ctx.request.provider_options.as_ref(), ctx.provider_name) +} + +/// Mark the cacheable prefix: the last system message (upstream, tools and +/// system precede the conversation, so this breakpoint covers them too) and +/// the second-to-last user turn. Tool results count as user turns — they ride +/// in user messages on the upstream Anthropic wire. +fn apply_cache_breakpoints(messages: &mut [ChatMessage]) { + if let Some(system) = messages.iter_mut().rev().find(|m| m.role == "system") { + if let Some(content) = system.content.as_mut() { + content.mark_cache_breakpoint(); + } + } + + let user_turns: Vec = messages + .iter() + .map(|m| m.role == "user" || m.role == "tool") + .collect(); + if let Some(idx) = cache::conversation_breakpoint_index(&user_turns) { + if let Some(content) = messages[idx].content.as_mut() { + content.mark_cache_breakpoint(); + } + } +} + /// Merge `provider_options.` fields into the serialized API /// request body. /// @@ -74,7 +112,7 @@ pub(super) fn merge_provider_options( provider_options: Option<&serde_json::Value>, provider_name: &str, ) { - merge_named_provider_options(body, provider_options, provider_name); + merge_named_provider_options(body, provider_options, provider_name, KNOWN_OPTION_KEYS); } #[cfg(test)] @@ -315,4 +353,68 @@ mod tests { merge_provider_options(&mut body, Some(&opts), "groq"); assert_eq!(body["model"], "test"); } + + #[test] + fn merge_provider_options_consumes_auto_cache_control_key() { + let mut body = serde_json::json!({"model": "test"}); + let opts = serde_json::json!({"groq": {"auto_cache": false, "top_k": 5}}); + merge_provider_options(&mut body, Some(&opts), "groq"); + assert!(body.get("auto_cache").is_none()); + assert_eq!(body["top_k"], 5); + } + + // --- apply_cache_breakpoints --------------------------------------------- + + fn chat_message(role: &str, text: &str) -> ChatMessage { + ChatMessage { + role: role.to_string(), + content: Some(super::super::wire::ChatContent::Text(text.to_string())), + reasoning_content: None, + tool_call_id: None, + tool_calls: None, + } + } + + fn marked(message: &ChatMessage) -> bool { + let json = serde_json::to_value(message).unwrap(); + json["content"].is_array() && json["content"][0]["cache_control"]["type"] == "ephemeral" + } + + #[test] + fn cache_breakpoints_on_first_turn_mark_only_the_system_prompt() { + let mut messages = vec![chat_message("system", "sys"), chat_message("user", "task")]; + apply_cache_breakpoints(&mut messages); + assert!(marked(&messages[0])); + assert!(!marked(&messages[1])); + } + + #[test] + fn cache_breakpoints_count_tool_results_as_user_turns() { + let mut messages = vec![ + chat_message("system", "sys"), + chat_message("user", "task"), + chat_message("assistant", "calling a tool"), + chat_message("tool", "tool output"), + chat_message("assistant", "one more"), + chat_message("tool", "more output"), + ]; + apply_cache_breakpoints(&mut messages); + assert!(marked(&messages[0])); + // Second-to-last user turn: the first tool result, not the user task. + assert!(!marked(&messages[1])); + assert!(marked(&messages[3])); + assert!(!marked(&messages[5])); + } + + #[test] + fn cache_breakpoints_without_system_mark_only_the_conversation() { + let mut messages = vec![ + chat_message("user", "task"), + chat_message("assistant", "answer"), + chat_message("user", "follow-up"), + ]; + apply_cache_breakpoints(&mut messages); + assert!(marked(&messages[0])); + assert!(!marked(&messages[2])); + } } diff --git a/lib/crates/fabro-llm/src/codec/openai_compatible/translate.rs b/lib/crates/fabro-llm/src/codec/openai_compatible/translate.rs index 127af26a2..dbef4831f 100644 --- a/lib/crates/fabro-llm/src/codec/openai_compatible/translate.rs +++ b/lib/crates/fabro-llm/src/codec/openai_compatible/translate.rs @@ -1,6 +1,6 @@ //! Pure mapping between canonical types and the Chat Completions wire shapes. -use super::wire::{ChatFunction, ChatMessage, ChatToolCall}; +use super::wire::{ChatContent, ChatFunction, ChatMessage, ChatToolCall}; use crate::error::Error; use crate::types::{ ContentPart, CostSource, FinishReason, Message, Request, ResponseFormat, ResponseFormatType, @@ -67,7 +67,7 @@ pub(super) fn translate_messages(messages: &[Message]) -> Vec { .map_or_else(|| tr.content.to_string(), str::to_string); Some(ChatMessage { role: "tool".to_string(), - content: Some(output), + content: Some(ChatContent::Text(output)), reasoning_content: None, tool_call_id: Some(tr.tool_call_id.clone()), tool_calls: None, @@ -109,7 +109,11 @@ pub(super) fn translate_messages(messages: &[Message]) -> Vec { } let text = content_text_with_fallbacks(&msg.content); - let content = if text.is_empty() { None } else { Some(text) }; + let content = if text.is_empty() { + None + } else { + Some(ChatContent::Text(text)) + }; let tool_calls = if tool_calls.is_empty() { None } else { @@ -276,7 +280,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("Let me check the weather") ); let tool_calls = translated[0].tool_calls.as_ref().unwrap(); @@ -318,7 +325,13 @@ mod tests { let msg = Message::user("Hello"); let translated = translate_messages(&[msg]); assert_eq!(translated[0].role, "user"); - assert_eq!(translated[0].content.as_deref(), Some("Hello")); + assert_eq!( + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), + Some("Hello") + ); assert!(translated[0].tool_calls.is_none()); } @@ -359,7 +372,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("[Audio content not supported by this provider]") ); } @@ -379,7 +395,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("[Document 'report.pdf': content type not supported by this provider]") ); } @@ -399,7 +418,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("[Document content not supported by this provider]") ); } @@ -421,7 +443,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("Check this: [Audio content not supported by this provider]") ); } diff --git a/lib/crates/fabro-llm/src/codec/openai_compatible/wire.rs b/lib/crates/fabro-llm/src/codec/openai_compatible/wire.rs index 396b74a72..5bc0a3b93 100644 --- a/lib/crates/fabro-llm/src/codec/openai_compatible/wire.rs +++ b/lib/crates/fabro-llm/src/codec/openai_compatible/wire.rs @@ -1,5 +1,6 @@ //! Serde types mirroring the OpenAI Chat Completions wire shapes. +use crate::codec::cache::CacheControl; use crate::codec::split_inclusive_token_total; use crate::types::{ReasoningEffort, TokenCounts}; @@ -31,7 +32,7 @@ pub(super) struct ApiRequest { pub(super) struct ChatMessage { pub role: String, #[serde(skip_serializing_if = "Option::is_none")] - pub content: Option, + pub content: Option, /// Reasoning/thinking content echoed back for providers that require it /// (Kimi). #[serde(skip_serializing_if = "Option::is_none")] @@ -42,6 +43,56 @@ pub(super) struct ChatMessage { pub tool_calls: Option>, } +/// Message content: plain text, or text parts when a part carries a +/// `cache_control` breakpoint (aggregators fronting Anthropic models forward +/// it upstream). Unmarked messages keep the plain-string form for maximum +/// compatibility with strict Chat Completions servers. +#[derive(serde::Serialize)] +#[serde(untagged)] +pub(super) enum ChatContent { + Text(String), + Parts(Vec), +} + +#[derive(serde::Serialize)] +pub(super) struct ChatTextPart { + #[serde(rename = "type")] + pub kind: String, + pub text: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +impl ChatContent { + /// Plain-text view for assertions. + #[cfg(test)] + pub(super) fn as_text(&self) -> Option<&str> { + match self { + Self::Text(text) => Some(text.as_str()), + Self::Parts(_) => None, + } + } + + /// Mark this content as a prompt-cache breakpoint, converting to parts + /// form so the annotation has somewhere to live. + pub(super) fn mark_cache_breakpoint(&mut self) { + match self { + Self::Text(text) => { + *self = Self::Parts(vec![ChatTextPart { + kind: "text".to_string(), + text: std::mem::take(text), + cache_control: Some(CacheControl::ephemeral()), + }]); + } + Self::Parts(parts) => { + if let Some(last) = parts.last_mut() { + last.cache_control = Some(CacheControl::ephemeral()); + } + } + } + } +} + #[derive(serde::Serialize)] pub(super) struct ChatToolCall { pub id: String, @@ -229,9 +280,53 @@ pub(super) struct AccumulatedToolCall { #[cfg(test)] mod tests { - use super::{ApiResponse, ApiUsage, StreamChunk}; + use super::{ApiResponse, ApiUsage, ChatContent, ChatTextPart, StreamChunk}; + use crate::codec::cache::CacheControl; use crate::types::TokenCounts; + #[test] + fn chat_content_text_serializes_as_plain_string() { + let content = ChatContent::Text("Hello".to_string()); + assert_eq!( + serde_json::to_value(&content).unwrap(), + serde_json::json!("Hello") + ); + } + + #[test] + fn mark_cache_breakpoint_converts_text_to_annotated_parts() { + let mut content = ChatContent::Text("Hello".to_string()); + content.mark_cache_breakpoint(); + assert_eq!( + serde_json::to_value(&content).unwrap(), + serde_json::json!([{ + "type": "text", + "text": "Hello", + "cache_control": {"type": "ephemeral"} + }]) + ); + } + + #[test] + fn mark_cache_breakpoint_annotates_last_existing_part() { + let mut content = ChatContent::Parts(vec![ + ChatTextPart { + kind: "text".to_string(), + text: "first".to_string(), + cache_control: None, + }, + ChatTextPart { + kind: "text".to_string(), + text: "second".to_string(), + cache_control: Some(CacheControl::ephemeral()), + }, + ]); + content.mark_cache_breakpoint(); + let json = serde_json::to_value(&content).unwrap(); + assert!(json[0].get("cache_control").is_none()); + assert_eq!(json[1]["cache_control"]["type"], "ephemeral"); + } + #[test] fn token_counts_bound_detail_to_parent_totals() { let usage: ApiUsage = serde_json::from_value(serde_json::json!({ diff --git a/lib/crates/fabro-llm/src/model_test.rs b/lib/crates/fabro-llm/src/model_test.rs index 3d8ea8eb0..aec0a83a7 100644 --- a/lib/crates/fabro-llm/src/model_test.rs +++ b/lib/crates/fabro-llm/src/model_test.rs @@ -223,12 +223,13 @@ mod tests { #[tokio::test] async fn run_model_test_deep_errors_when_model_lacks_tools() { let info = test_model_with(ModelFeatures { - tools: false, - vision: false, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: false, - sampling_params: true, + tools: false, + vision: false, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }); let outcome = run_model_test(&info, ModelTestMode::Deep, empty_test_client()).await; @@ -243,12 +244,13 @@ mod tests { #[test] fn deep_test_omits_effort_for_reasoning_without_effort_controls() { let info = test_model_with(ModelFeatures { - tools: true, - vision: false, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::None, - prompt_cache: true, - sampling_params: true, + tools: true, + vision: false, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::None, + prompt_cache: true, + cache_control_breakpoints: false, + sampling_params: true, }); let params = build_deep_test_params(&info, empty_test_client()) @@ -260,12 +262,13 @@ mod tests { #[test] fn deep_test_uses_high_effort_when_supported() { let info = test_model_with(ModelFeatures { - tools: true, - vision: false, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: true, - sampling_params: true, + tools: true, + vision: false, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: true, + cache_control_breakpoints: false, + sampling_params: true, }); let params = build_deep_test_params(&info, empty_test_client()) diff --git a/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs b/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs index b26416396..c8facd920 100644 --- a/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +++ b/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs @@ -10,7 +10,8 @@ use fabro_llm::types::{ Message, ReasoningEffort, Request, ResponseFormat, ResponseFormatType, ToolChoice, ToolDefinition, }; -use fabro_model::Catalog; +use fabro_model::catalog::LlmCatalogSettings; +use fabro_model::{Catalog, ProviderId}; use httpmock::prelude::*; use crate::support::{ @@ -256,6 +257,134 @@ async fn encode_kimi_k3_uses_catalog_reasoning_and_sampling_controls() { assert!(capture.body.get("top_p").is_none()); } +/// Counts JSON objects anywhere in `value` carrying a `cache_control` key. +fn count_cache_control_breakpoints(value: &serde_json::Value) -> usize { + match value { + serde_json::Value::Object(map) => { + usize::from(map.contains_key("cache_control")) + + map + .values() + .map(count_cache_control_breakpoints) + .sum::() + } + serde_json::Value::Array(items) => items.iter().map(count_cache_control_breakpoints).sum(), + _ => 0, + } +} + +/// Builtin catalog with the opt-in OpenRouter provider enabled. +fn openrouter_catalog() -> Arc { + let overrides: LlmCatalogSettings = toml::from_str("[providers.openrouter]\nenabled = true\n") + .expect("override TOML should parse"); + Arc::new( + Catalog::from_builtin_with_overrides(&overrides) + .expect("catalog with OpenRouter enabled should build"), + ) +} + +/// System + tools + two user turns against an OpenRouter model. +fn openrouter_multi_turn(model: &str) -> Request { + Request { + messages: vec![ + Message::system("You are a careful reviewer."), + Message::user("Review this."), + Message::assistant("Looking now."), + Message::user("Focus on the tests."), + ], + ..corpus_tools(model, None) + } +} + +/// OpenRouter serves Claude through this adapter, and Anthropic prompt +/// caching is opt-in per request: OpenRouter only forwards a cache write when +/// the body carries explicit ephemeral `cache_control` breakpoints (OpenAI +/// models cache implicitly; Anthropic models never do). The catalog row +/// declares `cache_control_breakpoints`, so the encoded request must mark the +/// cacheable prefix — otherwise every turn bills at the full uncached input +/// rate. +#[tokio::test] +async fn encode_openrouter_claude_marks_prompt_cache_breakpoints() { + let catalog = openrouter_catalog(); + let model = catalog + .get_on_provider(&ProviderId::new("openrouter"), "claude-fable-5") + .expect("OpenRouter Claude row should exist in the built-in catalog"); + assert!(model.features.prompt_cache); + assert!(model.features.cache_control_breakpoints); + + let request = openrouter_multi_turn("claude-fable-5"); + let capture = encode_capture_with(&request, move |adapter| { + adapter.with_name("openrouter").with_catalog(catalog) + }) + .await; + + assert_eq!(capture.body["model"], "anthropic/claude-fable-5"); + let messages = &capture.body["messages"]; + // The system prompt converts to parts form carrying a breakpoint; it + // covers the tool definitions too (tools precede system upstream). + assert_eq!(messages[0]["content"][0]["type"], "text"); + assert_eq!( + messages[0]["content"][0]["text"], + "You are a careful reviewer." + ); + assert_eq!( + messages[0]["content"][0]["cache_control"]["type"], + "ephemeral" + ); + // The second-to-last user turn carries the conversation breakpoint... + assert_eq!(messages[1]["content"][0]["text"], "Review this."); + assert_eq!( + messages[1]["content"][0]["cache_control"]["type"], + "ephemeral" + ); + // ...and the newest turn stays in plain-string form. + assert_eq!(messages[3]["content"], "Focus on the tests."); + assert_eq!(count_cache_control_breakpoints(&capture.body), 2); +} + +/// Models with implicit (server-side) caching must NOT get breakpoints even +/// though they support prompt caching — the annotation is an Anthropic-ism +/// the catalog row has to opt into. +#[tokio::test] +async fn encode_openrouter_implicit_cache_model_stays_plain() { + let catalog = openrouter_catalog(); + let model = catalog + .get_on_provider(&ProviderId::new("openrouter"), "gpt-5.6-luna") + .expect("OpenRouter GPT row should exist in the built-in catalog"); + assert!(model.features.prompt_cache); + assert!(!model.features.cache_control_breakpoints); + + let request = openrouter_multi_turn("gpt-5.6-luna"); + let capture = encode_capture_with(&request, move |adapter| { + adapter.with_name("openrouter").with_catalog(catalog) + }) + .await; + + assert_eq!(count_cache_control_breakpoints(&capture.body), 0); + assert_eq!( + capture.body["messages"][0]["content"], + "You are a careful reviewer." + ); +} + +/// `provider_options.openrouter.auto_cache = false` disables the breakpoints, +/// and the control key is consumed rather than merged into the body. +#[tokio::test] +async fn encode_openrouter_claude_auto_cache_opt_out() { + let request = Request { + provider_options: Some(serde_json::json!({"openrouter": {"auto_cache": false}})), + ..openrouter_multi_turn("claude-fable-5") + }; + let capture = encode_capture_with(&request, move |adapter| { + adapter + .with_name("openrouter") + .with_catalog(openrouter_catalog()) + }) + .await; + + assert_eq!(count_cache_control_breakpoints(&capture.body), 0); + assert!(capture.body.get("auto_cache").is_none()); +} + /// The provider_options namespace key is the runtime adapter NAME, not a /// static "openai_compatible" key (pinned in-module by /// `provider_options_uses_adapter_name`; this pins it from outside). diff --git a/lib/crates/fabro-model/src/catalog.rs b/lib/crates/fabro-model/src/catalog.rs index 69790f96d..cac9f878f 100644 --- a/lib/crates/fabro-model/src/catalog.rs +++ b/lib/crates/fabro-model/src/catalog.rs @@ -139,17 +139,19 @@ pub struct SettingsModelLimits { #[serde(deny_unknown_fields)] pub struct SettingsModelFeatures { #[serde(default)] - pub tools: Option, + pub tools: Option, #[serde(default)] - pub vision: Option, + pub vision: Option, #[serde(default)] - pub reasoning: Option, + pub reasoning: Option, #[serde(default)] - pub reasoning_effort: Option, + pub reasoning_effort: Option, #[serde(default)] - pub prompt_cache: Option, + pub prompt_cache: Option, #[serde(default)] - pub sampling_params: Option, + pub cache_control_breakpoints: Option, + #[serde(default)] + pub sampling_params: Option, } #[derive(Debug, Clone, Default, PartialEq, Deserialize)] @@ -574,6 +576,10 @@ pub enum CatalogBuildError { ReasoningEffortControlsWithoutReasoning { model: String }, #[error("model '{model}' declares reasoning_effort feature but features.reasoning is false")] ReasoningEffortWithoutReasoning { model: String }, + #[error( + "model '{model}' declares cache_control_breakpoints but features.prompt_cache is false" + )] + CacheControlBreakpointsWithoutPromptCache { model: String }, #[error( "model '{model}' must declare at least one reasoning_effort when features.reasoning_effort is levels or always_adaptive" )] @@ -1947,12 +1953,15 @@ fn merge_model_features_settings( fallback: &SettingsModelFeatures, ) -> SettingsModelFeatures { SettingsModelFeatures { - tools: higher.tools.or(fallback.tools), - vision: higher.vision.or(fallback.vision), - reasoning: higher.reasoning.or(fallback.reasoning), - reasoning_effort: higher.reasoning_effort.or(fallback.reasoning_effort), - prompt_cache: higher.prompt_cache.or(fallback.prompt_cache), - sampling_params: higher.sampling_params.or(fallback.sampling_params), + tools: higher.tools.or(fallback.tools), + vision: higher.vision.or(fallback.vision), + reasoning: higher.reasoning.or(fallback.reasoning), + reasoning_effort: higher.reasoning_effort.or(fallback.reasoning_effort), + prompt_cache: higher.prompt_cache.or(fallback.prompt_cache), + cache_control_breakpoints: higher + .cache_control_breakpoints + .or(fallback.cache_control_breakpoints), + sampling_params: higher.sampling_params.or(fallback.sampling_params), } } @@ -2261,6 +2270,15 @@ fn build_model_features( model: model_id.to_string(), }); } + let prompt_cache = features.prompt_cache.unwrap_or_default(); + let cache_control_breakpoints = features.cache_control_breakpoints.unwrap_or_default(); + if cache_control_breakpoints && !prompt_cache { + return Err( + CatalogBuildError::CacheControlBreakpointsWithoutPromptCache { + model: model_id.to_string(), + }, + ); + } Ok(ModelFeatures { tools: features @@ -2277,7 +2295,8 @@ fn build_model_features( })?, reasoning, reasoning_effort, - prompt_cache: features.prompt_cache.unwrap_or_default(), + prompt_cache, + cache_control_breakpoints, sampling_params: features.sampling_params.unwrap_or(true), }) } @@ -2977,6 +2996,7 @@ enabled = true 0.5, ReasoningEffortFeature::Levels, false, + false, BillingPolicy::OpenAi, ), ( @@ -2989,6 +3009,7 @@ enabled = true 0.25, ReasoningEffortFeature::Levels, false, + false, BillingPolicy::OpenAi, ), ( @@ -3001,6 +3022,7 @@ enabled = true 0.1, ReasoningEffortFeature::Levels, false, + false, BillingPolicy::OpenAi, ), ( @@ -3013,6 +3035,7 @@ enabled = true 0.5, ReasoningEffortFeature::Levels, false, + true, BillingPolicy::Anthropic, ), ( @@ -3025,6 +3048,7 @@ enabled = true 1.0, ReasoningEffortFeature::AlwaysAdaptive, false, + true, BillingPolicy::Anthropic, ), ]; @@ -3039,6 +3063,7 @@ enabled = true cache_input_cost, reasoning_effort, sampling_params, + cache_control_breakpoints, billing_policy, ) in expected { @@ -3055,6 +3080,10 @@ enabled = true assert!(model.features.prompt_cache, "{id}"); assert_eq!(model.features.reasoning_effort, reasoning_effort, "{id}"); assert_eq!(model.features.sampling_params, sampling_params, "{id}"); + assert_eq!( + model.features.cache_control_breakpoints, cache_control_breakpoints, + "{id}" + ); assert_eq!(model.costs.input_cost_per_mtok, Some(input_cost), "{id}"); assert_eq!(model.costs.output_cost_per_mtok, Some(output_cost), "{id}"); assert_eq!( @@ -3213,6 +3242,7 @@ enabled = true reasoning: true, reasoning_effort: Levels, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { @@ -3277,6 +3307,7 @@ enabled = true reasoning: true, reasoning_effort: AlwaysAdaptive, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: false, }, costs: ModelCosts { @@ -5683,6 +5714,39 @@ reasoning_effort = "levels" )); } + #[test] + fn catalog_from_settings_rejects_cache_control_breakpoints_without_prompt_cache() { + let settings = minimal_settings( + r#" +[providers.test] +display_name = "Test" +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "https://example.test/v1" + +[models.model] +provider = "test" +display_name = "Model" +family = "test" + +[models.model.limits] +context_window = 1000 + +[models.model.features] +tools = true +vision = false +reasoning = false +cache_control_breakpoints = true +"#, + ); + + assert!(matches!( + Catalog::from_settings(&settings).unwrap_err(), + CatalogBuildError::CacheControlBreakpointsWithoutPromptCache { model } + if model == "model" + )); + } + #[test] fn catalog_from_settings_rejects_always_adaptive_effort_without_reasoning() { let settings = minimal_settings( @@ -5843,6 +5907,7 @@ sampling_params = false reasoning: true, reasoning_effort: Levels, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { @@ -5899,6 +5964,7 @@ sampling_params = false reasoning: true, reasoning_effort: None, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: false, }, costs: ModelCosts { @@ -5947,6 +6013,7 @@ sampling_params = false reasoning: true, reasoning_effort: AlwaysAdaptive, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: false, }, costs: ModelCosts { @@ -6019,6 +6086,7 @@ sampling_params = false reasoning: true, reasoning_effort: Levels, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { @@ -6084,6 +6152,7 @@ sampling_params = false reasoning: true, reasoning_effort: Levels, prompt_cache: false, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { @@ -6140,6 +6209,7 @@ sampling_params = false reasoning: true, reasoning_effort: Levels, prompt_cache: false, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { diff --git a/lib/crates/fabro-model/src/catalog/providers/openrouter.toml b/lib/crates/fabro-model/src/catalog/providers/openrouter.toml index 4ef7d7c63..b83aa7662 100644 --- a/lib/crates/fabro-model/src/catalog/providers/openrouter.toml +++ b/lib/crates/fabro-model/src/catalog/providers/openrouter.toml @@ -50,6 +50,7 @@ vision = true reasoning = true reasoning_effort = "always_adaptive" prompt_cache = true +cache_control_breakpoints = true sampling_params = false [providers.openrouter.models."claude-fable-5".costs] @@ -76,6 +77,7 @@ vision = true reasoning = true reasoning_effort = "levels" prompt_cache = true +cache_control_breakpoints = true sampling_params = false [providers.openrouter.models."claude-opus-4-8".costs] @@ -98,6 +100,7 @@ tools = true vision = true reasoning = true prompt_cache = true +cache_control_breakpoints = true [providers.openrouter.models."claude-opus-4-7".costs] input_cost_per_mtok = 5.0 @@ -121,6 +124,7 @@ tools = true vision = true reasoning = true prompt_cache = true +cache_control_breakpoints = true [providers.openrouter.models."claude-sonnet-4-6".costs] input_cost_per_mtok = 3.0 @@ -144,6 +148,7 @@ tools = true vision = true reasoning = false prompt_cache = true +cache_control_breakpoints = true [providers.openrouter.models."claude-haiku-4-5".costs] input_cost_per_mtok = 1.0 diff --git a/lib/crates/fabro-model/src/types.rs b/lib/crates/fabro-model/src/types.rs index fb0164804..ae3804784 100644 --- a/lib/crates/fabro-model/src/types.rs +++ b/lib/crates/fabro-model/src/types.rs @@ -41,21 +41,28 @@ fn default_true() -> bool { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ModelFeatures { - pub tools: bool, - pub vision: bool, - pub reasoning: bool, + pub tools: bool, + pub vision: bool, + pub reasoning: bool, /// Whether this model endpoint supports a native reasoning-effort /// parameter. User-facing allowed effort values live in catalog controls. #[serde(default)] - pub reasoning_effort: ReasoningEffortFeature, + pub reasoning_effort: ReasoningEffortFeature, /// Whether this model endpoint supports prompt caching annotations. #[serde(default)] - pub prompt_cache: bool, + pub prompt_cache: bool, + /// Whether the endpoint only caches when the request marks the cacheable + /// prefix with Anthropic-style `cache_control` breakpoints. Set on + /// OpenAI-compatible routes fronting Anthropic models (e.g. Claude via + /// OpenRouter); dialects whose caching mechanism is implied (native + /// Anthropic, Bedrock) ignore it. + #[serde(default)] + pub cache_control_breakpoints: bool, /// Whether the model endpoint accepts classic sampling parameters /// (`temperature`, `top_p`). Models with always-on adaptive behavior /// reject them. #[serde(default = "default_true")] - pub sampling_params: bool, + pub sampling_params: bool, } impl ModelFeatures { @@ -221,12 +228,13 @@ mod tests { training: Some("training".to_string()), knowledge_cutoff: Some("knowledge-cutoff".to_string()), features: ModelFeatures { - tools: true, - vision: true, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: true, - sampling_params: true, + tools: true, + vision: true, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: true, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: Some(1.0), diff --git a/lib/packages/fabro-api-client/src/models/model-features.ts b/lib/packages/fabro-api-client/src/models/model-features.ts index 7eb314d8c..5f23381b3 100644 --- a/lib/packages/fabro-api-client/src/models/model-features.ts +++ b/lib/packages/fabro-api-client/src/models/model-features.ts @@ -38,6 +38,10 @@ export interface ModelFeatures { * Whether the model endpoint supports prompt caching. */ 'prompt_cache': boolean; + /** + * Whether the endpoint only caches when the request marks the cacheable prefix with Anthropic-style cache_control breakpoints (e.g. Claude via OpenRouter). + */ + 'cache_control_breakpoints': boolean; /** * Whether the model accepts classic sampling parameters (temperature, top_p). */