From e5f029022951a9ccfe104f8522e6e0356dbfd30e Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 23 Jul 2026 17:55:41 -0400 Subject: [PATCH] fix(llm): send cache_control breakpoints for Claude via OpenRouter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic prompt caching is opt-in per request: without explicit ephemeral cache_control breakpoints in the body, no cache writes or reads ever happen. The OpenAI-compatible codec never emitted them, so every run on openrouter Claude models billed the full conversation at the uncached input rate on every turn (0 cache tokens on the billing page, confirmed by OpenRouter's activity portal). - Add a `cache_control_breakpoints` model feature declaring that a route only caches when the request marks the cacheable prefix; set it on the builtin OpenRouter Claude rows. Catalog build rejects the flag without `prompt_cache`. - Teach the Chat Completions wire shape a parts-form content variant so a message can carry the annotation; unmarked messages keep the plain-string form for compatibility with strict servers. - Mark the last system message (covers tools + system upstream) and the second-to-last user turn, counting tool results as user turns — mirroring the anthropic codec's placement so agent loops get incremental cache hits. - Extract the shared placement/opt-out policy into codec::cache and refactor the anthropic codec onto it; anthropic wire snapshots are unchanged. - Honor `provider_options..auto_cache = false` as an opt-out and consume the control key instead of merging it into the body. - Mirror the new feature through settings (fabro-config), the OpenAPI schema, and the generated TypeScript client. Co-Authored-By: Claude Fable 5 --- docs/public/api-reference/fabro-api.yaml | 7 + lib/crates/fabro-agent/src/cli.rs | 26 ++-- .../tests/model_features_round_trip.rs | 14 +- .../fabro-api/tests/model_round_trip.rs | 13 +- .../fabro-api/tests/provider_id_round_trip.rs | 13 +- lib/crates/fabro-cli/src/commands/model.rs | 26 ++-- lib/crates/fabro-config/src/builders.rs | 13 +- lib/crates/fabro-config/src/layers/llm.rs | 14 +- .../src/codec/anthropic_messages/encode.rs | 56 +------- .../src/codec/anthropic_messages/wire.rs | 17 +-- .../src/codec/bedrock_converse/encode.rs | 2 +- lib/crates/fabro-llm/src/codec/cache.rs | 98 +++++++++++++ lib/crates/fabro-llm/src/codec/mod.rs | 9 +- .../src/codec/openai_compatible/request.rs | 110 ++++++++++++++- .../src/codec/openai_compatible/translate.rs | 43 ++++-- .../src/codec/openai_compatible/wire.rs | 99 ++++++++++++- lib/crates/fabro-llm/src/model_test.rs | 39 +++--- .../tests/it/wire/openai_compatible.rs | 131 +++++++++++++++++- lib/crates/fabro-model/src/catalog.rs | 96 +++++++++++-- .../src/catalog/providers/openrouter.toml | 5 + lib/crates/fabro-model/src/types.rs | 32 +++-- .../src/models/model-features.ts | 4 + 22 files changed, 687 insertions(+), 180 deletions(-) create mode 100644 lib/crates/fabro-llm/src/codec/cache.rs diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index ca80fd423..4640686a2 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -8227,6 +8227,7 @@ components: - reasoning - reasoning_effort - prompt_cache + - cache_control_breakpoints - sampling_params properties: tools: @@ -8243,6 +8244,12 @@ components: prompt_cache: type: boolean description: Whether the model endpoint supports prompt caching. + cache_control_breakpoints: + type: boolean + description: >- + Whether the endpoint only caches when the request marks the + cacheable prefix with Anthropic-style cache_control breakpoints + (e.g. Claude via OpenRouter). sampling_params: type: boolean description: Whether the model accepts classic sampling parameters (temperature, top_p). diff --git a/lib/crates/fabro-agent/src/cli.rs b/lib/crates/fabro-agent/src/cli.rs index 749f38b0a..f14041456 100644 --- a/lib/crates/fabro-agent/src/cli.rs +++ b/lib/crates/fabro-agent/src/cli.rs @@ -1073,12 +1073,13 @@ mod tests { max_output: None, }), features: Some(SettingsModelFeatures { - tools: Some(true), - vision: Some(false), - reasoning: Some(false), - reasoning_effort: None, - prompt_cache: None, - sampling_params: None, + tools: Some(true), + vision: Some(false), + reasoning: Some(false), + reasoning_effort: None, + prompt_cache: None, + cache_control_breakpoints: None, + sampling_params: None, }), ..ModelCatalogSettings::default() }); @@ -1158,12 +1159,13 @@ mod tests { max_output: None, }), features: Some(SettingsModelFeatures { - tools: Some(true), - vision: Some(false), - reasoning: Some(false), - reasoning_effort: None, - prompt_cache: None, - sampling_params: None, + tools: Some(true), + vision: Some(false), + reasoning: Some(false), + reasoning_effort: None, + prompt_cache: None, + cache_control_breakpoints: None, + sampling_params: None, }), ..ModelCatalogSettings::default() }); diff --git a/lib/crates/fabro-api/tests/model_features_round_trip.rs b/lib/crates/fabro-api/tests/model_features_round_trip.rs index 8e6f60aef..42f596665 100644 --- a/lib/crates/fabro-api/tests/model_features_round_trip.rs +++ b/lib/crates/fabro-api/tests/model_features_round_trip.rs @@ -11,12 +11,13 @@ fn model_features_reuses_canonical_type() { #[test] fn model_features_json_matches_openapi_shape() { let features = ModelFeatures { - tools: true, - vision: true, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: false, - sampling_params: true, + tools: true, + vision: true, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }; let json = serde_json::to_value(&features).unwrap(); @@ -25,6 +26,7 @@ fn model_features_json_matches_openapi_shape() { assert_eq!(json["reasoning"], true); assert_eq!(json["reasoning_effort"], "levels"); assert_eq!(json["prompt_cache"], false); + assert_eq!(json["cache_control_breakpoints"], false); assert_eq!(json["sampling_params"], true); let round_trip: ApiModelFeatures = serde_json::from_value(json).unwrap(); diff --git a/lib/crates/fabro-api/tests/model_round_trip.rs b/lib/crates/fabro-api/tests/model_round_trip.rs index 93d4e7692..57d3db557 100644 --- a/lib/crates/fabro-api/tests/model_round_trip.rs +++ b/lib/crates/fabro-api/tests/model_round_trip.rs @@ -24,12 +24,13 @@ fn model_json_matches_openapi_shape() { training: Some("2025-08-01".to_string()), knowledge_cutoff: Some("May 2025".to_string()), features: ModelFeatures { - tools: true, - vision: true, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: true, - sampling_params: true, + tools: true, + vision: true, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: true, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: Some(5.0), diff --git a/lib/crates/fabro-api/tests/provider_id_round_trip.rs b/lib/crates/fabro-api/tests/provider_id_round_trip.rs index e648b008f..970c7158f 100644 --- a/lib/crates/fabro-api/tests/provider_id_round_trip.rs +++ b/lib/crates/fabro-api/tests/provider_id_round_trip.rs @@ -34,12 +34,13 @@ fn provider_id_json_matches_openapi_shape_through_model() { training: None, knowledge_cutoff: None, features: ModelFeatures { - tools: false, - vision: false, - reasoning: false, - reasoning_effort: ReasoningEffortFeature::None, - prompt_cache: false, - sampling_params: true, + tools: false, + vision: false, + reasoning: false, + reasoning_effort: ReasoningEffortFeature::None, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: None, diff --git a/lib/crates/fabro-cli/src/commands/model.rs b/lib/crates/fabro-cli/src/commands/model.rs index 4e5bd6f39..f84bd44e2 100644 --- a/lib/crates/fabro-cli/src/commands/model.rs +++ b/lib/crates/fabro-cli/src/commands/model.rs @@ -538,12 +538,13 @@ mod tests { training: None, knowledge_cutoff: None, features: ModelFeatures { - tools: true, - vision: false, - reasoning: false, - reasoning_effort: ReasoningEffortFeature::None, - prompt_cache: false, - sampling_params: true, + tools: true, + vision: false, + reasoning: false, + reasoning_effort: ReasoningEffortFeature::None, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: Some(1.0), @@ -572,12 +573,13 @@ mod tests { training: None, knowledge_cutoff: None, features: ModelFeatures { - tools: true, - vision: false, - reasoning: false, - reasoning_effort: ReasoningEffortFeature::None, - prompt_cache: false, - sampling_params: true, + tools: true, + vision: false, + reasoning: false, + reasoning_effort: ReasoningEffortFeature::None, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: Some(1.0), diff --git a/lib/crates/fabro-config/src/builders.rs b/lib/crates/fabro-config/src/builders.rs index 1ffdc650d..a1e00907c 100644 --- a/lib/crates/fabro-config/src/builders.rs +++ b/lib/crates/fabro-config/src/builders.rs @@ -417,12 +417,13 @@ fn model_limits_to_catalog(limits: &LlmModelLimits) -> model_catalog::SettingsMo fn model_features_to_catalog(features: &LlmModelFeatures) -> model_catalog::SettingsModelFeatures { model_catalog::SettingsModelFeatures { - tools: features.tools, - vision: features.vision, - reasoning: features.reasoning, - reasoning_effort: features.reasoning_effort, - prompt_cache: features.prompt_cache, - sampling_params: features.sampling_params, + tools: features.tools, + vision: features.vision, + reasoning: features.reasoning, + reasoning_effort: features.reasoning_effort, + prompt_cache: features.prompt_cache, + cache_control_breakpoints: features.cache_control_breakpoints, + sampling_params: features.sampling_params, } } diff --git a/lib/crates/fabro-config/src/layers/llm.rs b/lib/crates/fabro-config/src/layers/llm.rs index ca29d2bbd..4f89c55fc 100644 --- a/lib/crates/fabro-config/src/layers/llm.rs +++ b/lib/crates/fabro-config/src/layers/llm.rs @@ -238,17 +238,19 @@ pub struct ModelLimits { #[serde(deny_unknown_fields)] pub struct ModelFeatures { #[serde(default, skip_serializing_if = "Option::is_none")] - pub tools: Option, + pub tools: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub vision: Option, + pub vision: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub reasoning: Option, + pub reasoning: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub reasoning_effort: Option, + pub reasoning_effort: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub prompt_cache: Option, + pub prompt_cache: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub sampling_params: Option, + pub cache_control_breakpoints: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sampling_params: Option, } /// User-facing allow-list for native control values Fabro accepts on this diff --git a/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs b/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs index 2803c221c..5ed999f87 100644 --- a/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs +++ b/lib/crates/fabro-llm/src/codec/anthropic_messages/encode.rs @@ -8,7 +8,8 @@ use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use super::SYNTHETIC_TOOL_NAME; -use super::wire::{ApiMessage, ApiRequest, ApiToolDef, CacheControl, CountTokensRequest}; +use super::wire::{ApiMessage, ApiRequest, ApiToolDef, CountTokensRequest}; +use crate::codec::cache::{self, CacheControl}; use crate::codec::{AnthropicVersion, CodecCtx, EncodedRequest, extract_system_prompt}; use crate::types::{ ContentPart, Message, ReasoningEffort, ReasoningEffortFeature, Request, ResponseFormatType, @@ -48,7 +49,7 @@ pub(super) fn encode_count_tokens(ctx: &CodecCtx<'_>) -> EncodedRequest { /// hasn't opted out. fn auto_cache(ctx: &CodecCtx<'_>) -> bool { ctx.model.is_some_and(|m| m.features.prompt_cache) - && is_auto_cache_enabled(ctx.request.provider_options.as_ref()) + && cache::auto_cache_enabled(ctx.request.provider_options.as_ref(), "anthropic") } fn build_headers(ctx: &CodecCtx<'_>) -> Vec<(String, String)> { @@ -441,12 +442,6 @@ fn effort_to_budget_tokens(effort: ReasoningEffort, max_tokens: i64) -> i64 { budget.max(1024) } -fn is_auto_cache_enabled(provider_options: Option<&serde_json::Value>) -> bool { - anthropic_option(provider_options, "auto_cache") - .and_then(serde_json::Value::as_bool) - .unwrap_or(true) -} - fn system_with_cache_control(system: &str) -> serde_json::Value { serde_json::json!([{ "type": "text", @@ -462,18 +457,10 @@ fn apply_cache_control_to_last_tool(tools: &mut [ApiToolDef]) { } fn apply_cache_control_to_conversation_prefix(messages: &mut [ApiMessage]) { - let user_indices: Vec = messages - .iter() - .enumerate() - .filter(|(_, m)| m.role == "user") - .map(|(i, _)| i) - .collect(); - - if user_indices.len() < 2 { + let user_turns: Vec = messages.iter().map(|m| m.role == "user").collect(); + let Some(target_idx) = cache::conversation_breakpoint_index(&user_turns) else { return; - } - - let target_idx = user_indices[user_indices.len() - 2]; + }; if let Some(serde_json::Value::Object(map)) = messages[target_idx].content.last_mut() { map.insert( "cache_control".to_string(), @@ -651,37 +638,6 @@ reasoning = true .map(|(_, value)| value.as_str()) } - // --- auto_cache ---------------------------------------------------------- - - #[test] - fn auto_cache_enabled_by_default() { - assert!(is_auto_cache_enabled(None)); - } - - #[test] - fn auto_cache_enabled_when_true() { - let opts = serde_json::json!({"anthropic": {"auto_cache": true}}); - assert!(is_auto_cache_enabled(Some(&opts))); - } - - #[test] - fn auto_cache_disabled_when_false() { - let opts = serde_json::json!({"anthropic": {"auto_cache": false}}); - assert!(!is_auto_cache_enabled(Some(&opts))); - } - - #[test] - fn auto_cache_enabled_when_key_missing() { - let opts = serde_json::json!({"anthropic": {}}); - assert!(is_auto_cache_enabled(Some(&opts))); - } - - #[test] - fn auto_cache_enabled_when_anthropic_missing() { - let opts = serde_json::json!({"openai": {}}); - assert!(is_auto_cache_enabled(Some(&opts))); - } - // --- prompt-cache helpers ------------------------------------------------ #[test] diff --git a/lib/crates/fabro-llm/src/codec/anthropic_messages/wire.rs b/lib/crates/fabro-llm/src/codec/anthropic_messages/wire.rs index 80458dea4..70fb0ede6 100644 --- a/lib/crates/fabro-llm/src/codec/anthropic_messages/wire.rs +++ b/lib/crates/fabro-llm/src/codec/anthropic_messages/wire.rs @@ -1,5 +1,7 @@ //! Serde types mirroring the Anthropic Messages wire shapes. +use crate::codec::cache::CacheControl; + #[derive(serde::Serialize)] pub(super) struct ApiRequest { pub model: String, @@ -77,21 +79,6 @@ pub(super) struct ApiToolDef { pub cache_control: Option, } -/// Anthropic `cache_control` annotation. -#[derive(serde::Serialize, Clone)] -pub(super) struct CacheControl { - #[serde(rename = "type")] - pub kind: String, -} - -impl CacheControl { - pub(super) fn ephemeral() -> Self { - Self { - kind: "ephemeral".to_string(), - } - } -} - // --- Response types --- #[derive(serde::Deserialize)] diff --git a/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs b/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs index 54a38314e..39c9c5c10 100644 --- a/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs +++ b/lib/crates/fabro-llm/src/codec/bedrock_converse/encode.rs @@ -342,7 +342,7 @@ fn apply_cache_point_to_conversation_prefix(messages: &mut [Value]) { /// codec). This is the passthrough for `additionalModelRequestFields`, /// `guardrailConfig`, `serviceTier`, and other Converse extensions. fn merge_provider_options(body: &mut Value, provider_options: Option<&Value>, provider_name: &str) { - merge_named_provider_options(body, provider_options, provider_name); + merge_named_provider_options(body, provider_options, provider_name, &[]); } #[cfg(test)] diff --git a/lib/crates/fabro-llm/src/codec/cache.rs b/lib/crates/fabro-llm/src/codec/cache.rs new file mode 100644 index 000000000..ed1afe8ab --- /dev/null +++ b/lib/crates/fabro-llm/src/codec/cache.rs @@ -0,0 +1,98 @@ +//! Shared prompt-cache policy: whether a request opts into explicit +//! Anthropic-style caching and where the conversation breakpoint lands. +//! Dialect codecs apply these decisions to their own wire shapes. + +/// Anthropic-style `cache_control` annotation. +#[derive(serde::Serialize, Clone)] +pub(crate) struct CacheControl { + #[serde(rename = "type")] + pub kind: String, +} + +impl CacheControl { + pub(crate) fn ephemeral() -> Self { + Self { + kind: "ephemeral".to_string(), + } + } +} + +/// Whether automatic prompt caching applies to this request: the +/// `provider_options..auto_cache` opt-out defaults to enabled. +pub(crate) fn auto_cache_enabled( + provider_options: Option<&serde_json::Value>, + namespace: &str, +) -> bool { + provider_options + .and_then(|opts| opts.get(namespace)) + .and_then(|ns| ns.get("auto_cache")) + .and_then(serde_json::Value::as_bool) + .unwrap_or(true) +} + +/// Index of the message carrying the conversation-prefix breakpoint: the +/// second-to-last user turn, so each iteration of an agent loop reuses the +/// prefix cached by the previous one. `user_turns[i]` is true when message +/// `i` advances the user side of the conversation (plain user messages, plus +/// tool results on dialects where they are separate messages). `None` until +/// the conversation has at least two user turns. +pub(crate) fn conversation_breakpoint_index(user_turns: &[bool]) -> Option { + let indices: Vec = user_turns + .iter() + .enumerate() + .filter_map(|(i, &is_user)| is_user.then_some(i)) + .collect(); + indices.len().checked_sub(2).map(|nth| indices[nth]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn auto_cache_enabled_by_default() { + assert!(auto_cache_enabled(None, "anthropic")); + } + + #[test] + fn auto_cache_enabled_when_true() { + let opts = serde_json::json!({"anthropic": {"auto_cache": true}}); + assert!(auto_cache_enabled(Some(&opts), "anthropic")); + } + + #[test] + fn auto_cache_disabled_when_false() { + let opts = serde_json::json!({"openrouter": {"auto_cache": false}}); + assert!(!auto_cache_enabled(Some(&opts), "openrouter")); + } + + #[test] + fn auto_cache_enabled_when_key_missing() { + let opts = serde_json::json!({"anthropic": {}}); + assert!(auto_cache_enabled(Some(&opts), "anthropic")); + } + + #[test] + fn auto_cache_reads_only_its_own_namespace() { + let opts = serde_json::json!({"openrouter": {"auto_cache": false}}); + assert!(auto_cache_enabled(Some(&opts), "anthropic")); + } + + #[test] + fn conversation_breakpoint_none_below_two_user_turns() { + assert_eq!(conversation_breakpoint_index(&[]), None); + assert_eq!(conversation_breakpoint_index(&[true]), None); + assert_eq!(conversation_breakpoint_index(&[true, false, false]), None); + } + + #[test] + fn conversation_breakpoint_with_exactly_two_user_turns() { + assert_eq!(conversation_breakpoint_index(&[true, false, true]), Some(0)); + } + + #[test] + fn conversation_breakpoint_targets_second_to_last_user_turn() { + let turns = [true, false, true, false, true]; + assert_eq!(conversation_breakpoint_index(&turns), Some(2)); + } +} diff --git a/lib/crates/fabro-llm/src/codec/mod.rs b/lib/crates/fabro-llm/src/codec/mod.rs index 9ddc1eb0a..5a7759b22 100644 --- a/lib/crates/fabro-llm/src/codec/mod.rs +++ b/lib/crates/fabro-llm/src/codec/mod.rs @@ -12,6 +12,7 @@ pub(crate) mod anthropic_messages; pub(crate) mod bedrock_converse; +pub(crate) mod cache; pub(crate) mod gemini_generate; pub(crate) mod openai_compatible; pub(crate) mod openai_responses; @@ -38,11 +39,14 @@ pub(crate) fn split_inclusive_token_total(total: i64, detail: i64) -> (i64, i64) /// Merge `provider_options.` fields into an encoded request /// body. Used by codecs whose provider-options namespace is adapter-name keyed -/// rather than a single fixed provider. +/// rather than a single fixed provider. `known_keys` are control keys the +/// codec consumed itself (e.g. `auto_cache`); they are not re-merged into the +/// body. pub(crate) fn merge_named_provider_options( body: &mut serde_json::Value, provider_options: Option<&serde_json::Value>, provider_name: &str, + known_keys: &[&str], ) { let Some(opts) = provider_options.and_then(|opts| opts.get(provider_name)) else { return; @@ -55,6 +59,9 @@ pub(crate) fn merge_named_provider_options( }; for (key, value) in opts_map { + if known_keys.contains(&key.as_str()) { + continue; + } body_map.insert(key.clone(), value.clone()); } } diff --git a/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs b/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs index 72920b34c..12be9e84b 100644 --- a/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs +++ b/lib/crates/fabro-llm/src/codec/openai_compatible/request.rs @@ -1,10 +1,14 @@ //! Request encoding: canonical `Request` → Chat Completions body. use super::translate; -use super::wire::ApiRequest; -use crate::codec::{CodecCtx, EncodedRequest, merge_named_provider_options}; +use super::wire::{ApiRequest, ChatMessage}; +use crate::codec::{CodecCtx, EncodedRequest, cache, merge_named_provider_options}; use crate::error::Error; +/// Known `provider_options.` keys the codec consumes itself; +/// not re-merged into the body. +const KNOWN_OPTION_KEYS: &[&str] = &["auto_cache"]; + /// Build the Chat Completions request for `ctx.request`. `stream` toggles the /// `stream` body field. The body is assembled as a `serde_json::Value` so /// `provider_options.` fields can be merged in before sending. @@ -13,7 +17,10 @@ use crate::error::Error; /// the Chat Completions tool envelope cannot represent. pub(super) fn encode(ctx: &CodecCtx<'_>, stream: bool) -> Result { let request = ctx.request; - let chat_messages = translate::translate_messages(&request.messages); + let mut chat_messages = translate::translate_messages(&request.messages); + if explicit_cache_breakpoints(ctx) { + apply_cache_breakpoints(&mut chat_messages); + } let tools = request .tools .as_ref() @@ -64,6 +71,37 @@ pub(super) fn encode(ctx: &CodecCtx<'_>, stream: bool) -> Result) -> bool { + ctx.model + .is_some_and(|m| m.features.prompt_cache && m.features.cache_control_breakpoints) + && cache::auto_cache_enabled(ctx.request.provider_options.as_ref(), ctx.provider_name) +} + +/// Mark the cacheable prefix: the last system message (upstream, tools and +/// system precede the conversation, so this breakpoint covers them too) and +/// the second-to-last user turn. Tool results count as user turns — they ride +/// in user messages on the upstream Anthropic wire. +fn apply_cache_breakpoints(messages: &mut [ChatMessage]) { + if let Some(system) = messages.iter_mut().rev().find(|m| m.role == "system") { + if let Some(content) = system.content.as_mut() { + content.mark_cache_breakpoint(); + } + } + + let user_turns: Vec = messages + .iter() + .map(|m| m.role == "user" || m.role == "tool") + .collect(); + if let Some(idx) = cache::conversation_breakpoint_index(&user_turns) { + if let Some(content) = messages[idx].content.as_mut() { + content.mark_cache_breakpoint(); + } + } +} + /// Merge `provider_options.` fields into the serialized API /// request body. /// @@ -74,7 +112,7 @@ pub(super) fn merge_provider_options( provider_options: Option<&serde_json::Value>, provider_name: &str, ) { - merge_named_provider_options(body, provider_options, provider_name); + merge_named_provider_options(body, provider_options, provider_name, KNOWN_OPTION_KEYS); } #[cfg(test)] @@ -315,4 +353,68 @@ mod tests { merge_provider_options(&mut body, Some(&opts), "groq"); assert_eq!(body["model"], "test"); } + + #[test] + fn merge_provider_options_consumes_auto_cache_control_key() { + let mut body = serde_json::json!({"model": "test"}); + let opts = serde_json::json!({"groq": {"auto_cache": false, "top_k": 5}}); + merge_provider_options(&mut body, Some(&opts), "groq"); + assert!(body.get("auto_cache").is_none()); + assert_eq!(body["top_k"], 5); + } + + // --- apply_cache_breakpoints --------------------------------------------- + + fn chat_message(role: &str, text: &str) -> ChatMessage { + ChatMessage { + role: role.to_string(), + content: Some(super::super::wire::ChatContent::Text(text.to_string())), + reasoning_content: None, + tool_call_id: None, + tool_calls: None, + } + } + + fn marked(message: &ChatMessage) -> bool { + let json = serde_json::to_value(message).unwrap(); + json["content"].is_array() && json["content"][0]["cache_control"]["type"] == "ephemeral" + } + + #[test] + fn cache_breakpoints_on_first_turn_mark_only_the_system_prompt() { + let mut messages = vec![chat_message("system", "sys"), chat_message("user", "task")]; + apply_cache_breakpoints(&mut messages); + assert!(marked(&messages[0])); + assert!(!marked(&messages[1])); + } + + #[test] + fn cache_breakpoints_count_tool_results_as_user_turns() { + let mut messages = vec![ + chat_message("system", "sys"), + chat_message("user", "task"), + chat_message("assistant", "calling a tool"), + chat_message("tool", "tool output"), + chat_message("assistant", "one more"), + chat_message("tool", "more output"), + ]; + apply_cache_breakpoints(&mut messages); + assert!(marked(&messages[0])); + // Second-to-last user turn: the first tool result, not the user task. + assert!(!marked(&messages[1])); + assert!(marked(&messages[3])); + assert!(!marked(&messages[5])); + } + + #[test] + fn cache_breakpoints_without_system_mark_only_the_conversation() { + let mut messages = vec![ + chat_message("user", "task"), + chat_message("assistant", "answer"), + chat_message("user", "follow-up"), + ]; + apply_cache_breakpoints(&mut messages); + assert!(marked(&messages[0])); + assert!(!marked(&messages[2])); + } } diff --git a/lib/crates/fabro-llm/src/codec/openai_compatible/translate.rs b/lib/crates/fabro-llm/src/codec/openai_compatible/translate.rs index 127af26a2..dbef4831f 100644 --- a/lib/crates/fabro-llm/src/codec/openai_compatible/translate.rs +++ b/lib/crates/fabro-llm/src/codec/openai_compatible/translate.rs @@ -1,6 +1,6 @@ //! Pure mapping between canonical types and the Chat Completions wire shapes. -use super::wire::{ChatFunction, ChatMessage, ChatToolCall}; +use super::wire::{ChatContent, ChatFunction, ChatMessage, ChatToolCall}; use crate::error::Error; use crate::types::{ ContentPart, CostSource, FinishReason, Message, Request, ResponseFormat, ResponseFormatType, @@ -67,7 +67,7 @@ pub(super) fn translate_messages(messages: &[Message]) -> Vec { .map_or_else(|| tr.content.to_string(), str::to_string); Some(ChatMessage { role: "tool".to_string(), - content: Some(output), + content: Some(ChatContent::Text(output)), reasoning_content: None, tool_call_id: Some(tr.tool_call_id.clone()), tool_calls: None, @@ -109,7 +109,11 @@ pub(super) fn translate_messages(messages: &[Message]) -> Vec { } let text = content_text_with_fallbacks(&msg.content); - let content = if text.is_empty() { None } else { Some(text) }; + let content = if text.is_empty() { + None + } else { + Some(ChatContent::Text(text)) + }; let tool_calls = if tool_calls.is_empty() { None } else { @@ -276,7 +280,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("Let me check the weather") ); let tool_calls = translated[0].tool_calls.as_ref().unwrap(); @@ -318,7 +325,13 @@ mod tests { let msg = Message::user("Hello"); let translated = translate_messages(&[msg]); assert_eq!(translated[0].role, "user"); - assert_eq!(translated[0].content.as_deref(), Some("Hello")); + assert_eq!( + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), + Some("Hello") + ); assert!(translated[0].tool_calls.is_none()); } @@ -359,7 +372,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("[Audio content not supported by this provider]") ); } @@ -379,7 +395,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("[Document 'report.pdf': content type not supported by this provider]") ); } @@ -399,7 +418,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("[Document content not supported by this provider]") ); } @@ -421,7 +443,10 @@ mod tests { }; let translated = translate_messages(&[msg]); assert_eq!( - translated[0].content.as_deref(), + translated[0] + .content + .as_ref() + .and_then(ChatContent::as_text), Some("Check this: [Audio content not supported by this provider]") ); } diff --git a/lib/crates/fabro-llm/src/codec/openai_compatible/wire.rs b/lib/crates/fabro-llm/src/codec/openai_compatible/wire.rs index 396b74a72..5bc0a3b93 100644 --- a/lib/crates/fabro-llm/src/codec/openai_compatible/wire.rs +++ b/lib/crates/fabro-llm/src/codec/openai_compatible/wire.rs @@ -1,5 +1,6 @@ //! Serde types mirroring the OpenAI Chat Completions wire shapes. +use crate::codec::cache::CacheControl; use crate::codec::split_inclusive_token_total; use crate::types::{ReasoningEffort, TokenCounts}; @@ -31,7 +32,7 @@ pub(super) struct ApiRequest { pub(super) struct ChatMessage { pub role: String, #[serde(skip_serializing_if = "Option::is_none")] - pub content: Option, + pub content: Option, /// Reasoning/thinking content echoed back for providers that require it /// (Kimi). #[serde(skip_serializing_if = "Option::is_none")] @@ -42,6 +43,56 @@ pub(super) struct ChatMessage { pub tool_calls: Option>, } +/// Message content: plain text, or text parts when a part carries a +/// `cache_control` breakpoint (aggregators fronting Anthropic models forward +/// it upstream). Unmarked messages keep the plain-string form for maximum +/// compatibility with strict Chat Completions servers. +#[derive(serde::Serialize)] +#[serde(untagged)] +pub(super) enum ChatContent { + Text(String), + Parts(Vec), +} + +#[derive(serde::Serialize)] +pub(super) struct ChatTextPart { + #[serde(rename = "type")] + pub kind: String, + pub text: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +impl ChatContent { + /// Plain-text view for assertions. + #[cfg(test)] + pub(super) fn as_text(&self) -> Option<&str> { + match self { + Self::Text(text) => Some(text.as_str()), + Self::Parts(_) => None, + } + } + + /// Mark this content as a prompt-cache breakpoint, converting to parts + /// form so the annotation has somewhere to live. + pub(super) fn mark_cache_breakpoint(&mut self) { + match self { + Self::Text(text) => { + *self = Self::Parts(vec![ChatTextPart { + kind: "text".to_string(), + text: std::mem::take(text), + cache_control: Some(CacheControl::ephemeral()), + }]); + } + Self::Parts(parts) => { + if let Some(last) = parts.last_mut() { + last.cache_control = Some(CacheControl::ephemeral()); + } + } + } + } +} + #[derive(serde::Serialize)] pub(super) struct ChatToolCall { pub id: String, @@ -229,9 +280,53 @@ pub(super) struct AccumulatedToolCall { #[cfg(test)] mod tests { - use super::{ApiResponse, ApiUsage, StreamChunk}; + use super::{ApiResponse, ApiUsage, ChatContent, ChatTextPart, StreamChunk}; + use crate::codec::cache::CacheControl; use crate::types::TokenCounts; + #[test] + fn chat_content_text_serializes_as_plain_string() { + let content = ChatContent::Text("Hello".to_string()); + assert_eq!( + serde_json::to_value(&content).unwrap(), + serde_json::json!("Hello") + ); + } + + #[test] + fn mark_cache_breakpoint_converts_text_to_annotated_parts() { + let mut content = ChatContent::Text("Hello".to_string()); + content.mark_cache_breakpoint(); + assert_eq!( + serde_json::to_value(&content).unwrap(), + serde_json::json!([{ + "type": "text", + "text": "Hello", + "cache_control": {"type": "ephemeral"} + }]) + ); + } + + #[test] + fn mark_cache_breakpoint_annotates_last_existing_part() { + let mut content = ChatContent::Parts(vec![ + ChatTextPart { + kind: "text".to_string(), + text: "first".to_string(), + cache_control: None, + }, + ChatTextPart { + kind: "text".to_string(), + text: "second".to_string(), + cache_control: Some(CacheControl::ephemeral()), + }, + ]); + content.mark_cache_breakpoint(); + let json = serde_json::to_value(&content).unwrap(); + assert!(json[0].get("cache_control").is_none()); + assert_eq!(json[1]["cache_control"]["type"], "ephemeral"); + } + #[test] fn token_counts_bound_detail_to_parent_totals() { let usage: ApiUsage = serde_json::from_value(serde_json::json!({ diff --git a/lib/crates/fabro-llm/src/model_test.rs b/lib/crates/fabro-llm/src/model_test.rs index 3d8ea8eb0..aec0a83a7 100644 --- a/lib/crates/fabro-llm/src/model_test.rs +++ b/lib/crates/fabro-llm/src/model_test.rs @@ -223,12 +223,13 @@ mod tests { #[tokio::test] async fn run_model_test_deep_errors_when_model_lacks_tools() { let info = test_model_with(ModelFeatures { - tools: false, - vision: false, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: false, - sampling_params: true, + tools: false, + vision: false, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: false, + cache_control_breakpoints: false, + sampling_params: true, }); let outcome = run_model_test(&info, ModelTestMode::Deep, empty_test_client()).await; @@ -243,12 +244,13 @@ mod tests { #[test] fn deep_test_omits_effort_for_reasoning_without_effort_controls() { let info = test_model_with(ModelFeatures { - tools: true, - vision: false, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::None, - prompt_cache: true, - sampling_params: true, + tools: true, + vision: false, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::None, + prompt_cache: true, + cache_control_breakpoints: false, + sampling_params: true, }); let params = build_deep_test_params(&info, empty_test_client()) @@ -260,12 +262,13 @@ mod tests { #[test] fn deep_test_uses_high_effort_when_supported() { let info = test_model_with(ModelFeatures { - tools: true, - vision: false, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: true, - sampling_params: true, + tools: true, + vision: false, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: true, + cache_control_breakpoints: false, + sampling_params: true, }); let params = build_deep_test_params(&info, empty_test_client()) diff --git a/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs b/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs index b26416396..c8facd920 100644 --- a/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +++ b/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs @@ -10,7 +10,8 @@ use fabro_llm::types::{ Message, ReasoningEffort, Request, ResponseFormat, ResponseFormatType, ToolChoice, ToolDefinition, }; -use fabro_model::Catalog; +use fabro_model::catalog::LlmCatalogSettings; +use fabro_model::{Catalog, ProviderId}; use httpmock::prelude::*; use crate::support::{ @@ -256,6 +257,134 @@ async fn encode_kimi_k3_uses_catalog_reasoning_and_sampling_controls() { assert!(capture.body.get("top_p").is_none()); } +/// Counts JSON objects anywhere in `value` carrying a `cache_control` key. +fn count_cache_control_breakpoints(value: &serde_json::Value) -> usize { + match value { + serde_json::Value::Object(map) => { + usize::from(map.contains_key("cache_control")) + + map + .values() + .map(count_cache_control_breakpoints) + .sum::() + } + serde_json::Value::Array(items) => items.iter().map(count_cache_control_breakpoints).sum(), + _ => 0, + } +} + +/// Builtin catalog with the opt-in OpenRouter provider enabled. +fn openrouter_catalog() -> Arc { + let overrides: LlmCatalogSettings = toml::from_str("[providers.openrouter]\nenabled = true\n") + .expect("override TOML should parse"); + Arc::new( + Catalog::from_builtin_with_overrides(&overrides) + .expect("catalog with OpenRouter enabled should build"), + ) +} + +/// System + tools + two user turns against an OpenRouter model. +fn openrouter_multi_turn(model: &str) -> Request { + Request { + messages: vec![ + Message::system("You are a careful reviewer."), + Message::user("Review this."), + Message::assistant("Looking now."), + Message::user("Focus on the tests."), + ], + ..corpus_tools(model, None) + } +} + +/// OpenRouter serves Claude through this adapter, and Anthropic prompt +/// caching is opt-in per request: OpenRouter only forwards a cache write when +/// the body carries explicit ephemeral `cache_control` breakpoints (OpenAI +/// models cache implicitly; Anthropic models never do). The catalog row +/// declares `cache_control_breakpoints`, so the encoded request must mark the +/// cacheable prefix — otherwise every turn bills at the full uncached input +/// rate. +#[tokio::test] +async fn encode_openrouter_claude_marks_prompt_cache_breakpoints() { + let catalog = openrouter_catalog(); + let model = catalog + .get_on_provider(&ProviderId::new("openrouter"), "claude-fable-5") + .expect("OpenRouter Claude row should exist in the built-in catalog"); + assert!(model.features.prompt_cache); + assert!(model.features.cache_control_breakpoints); + + let request = openrouter_multi_turn("claude-fable-5"); + let capture = encode_capture_with(&request, move |adapter| { + adapter.with_name("openrouter").with_catalog(catalog) + }) + .await; + + assert_eq!(capture.body["model"], "anthropic/claude-fable-5"); + let messages = &capture.body["messages"]; + // The system prompt converts to parts form carrying a breakpoint; it + // covers the tool definitions too (tools precede system upstream). + assert_eq!(messages[0]["content"][0]["type"], "text"); + assert_eq!( + messages[0]["content"][0]["text"], + "You are a careful reviewer." + ); + assert_eq!( + messages[0]["content"][0]["cache_control"]["type"], + "ephemeral" + ); + // The second-to-last user turn carries the conversation breakpoint... + assert_eq!(messages[1]["content"][0]["text"], "Review this."); + assert_eq!( + messages[1]["content"][0]["cache_control"]["type"], + "ephemeral" + ); + // ...and the newest turn stays in plain-string form. + assert_eq!(messages[3]["content"], "Focus on the tests."); + assert_eq!(count_cache_control_breakpoints(&capture.body), 2); +} + +/// Models with implicit (server-side) caching must NOT get breakpoints even +/// though they support prompt caching — the annotation is an Anthropic-ism +/// the catalog row has to opt into. +#[tokio::test] +async fn encode_openrouter_implicit_cache_model_stays_plain() { + let catalog = openrouter_catalog(); + let model = catalog + .get_on_provider(&ProviderId::new("openrouter"), "gpt-5.6-luna") + .expect("OpenRouter GPT row should exist in the built-in catalog"); + assert!(model.features.prompt_cache); + assert!(!model.features.cache_control_breakpoints); + + let request = openrouter_multi_turn("gpt-5.6-luna"); + let capture = encode_capture_with(&request, move |adapter| { + adapter.with_name("openrouter").with_catalog(catalog) + }) + .await; + + assert_eq!(count_cache_control_breakpoints(&capture.body), 0); + assert_eq!( + capture.body["messages"][0]["content"], + "You are a careful reviewer." + ); +} + +/// `provider_options.openrouter.auto_cache = false` disables the breakpoints, +/// and the control key is consumed rather than merged into the body. +#[tokio::test] +async fn encode_openrouter_claude_auto_cache_opt_out() { + let request = Request { + provider_options: Some(serde_json::json!({"openrouter": {"auto_cache": false}})), + ..openrouter_multi_turn("claude-fable-5") + }; + let capture = encode_capture_with(&request, move |adapter| { + adapter + .with_name("openrouter") + .with_catalog(openrouter_catalog()) + }) + .await; + + assert_eq!(count_cache_control_breakpoints(&capture.body), 0); + assert!(capture.body.get("auto_cache").is_none()); +} + /// The provider_options namespace key is the runtime adapter NAME, not a /// static "openai_compatible" key (pinned in-module by /// `provider_options_uses_adapter_name`; this pins it from outside). diff --git a/lib/crates/fabro-model/src/catalog.rs b/lib/crates/fabro-model/src/catalog.rs index 69790f96d..cac9f878f 100644 --- a/lib/crates/fabro-model/src/catalog.rs +++ b/lib/crates/fabro-model/src/catalog.rs @@ -139,17 +139,19 @@ pub struct SettingsModelLimits { #[serde(deny_unknown_fields)] pub struct SettingsModelFeatures { #[serde(default)] - pub tools: Option, + pub tools: Option, #[serde(default)] - pub vision: Option, + pub vision: Option, #[serde(default)] - pub reasoning: Option, + pub reasoning: Option, #[serde(default)] - pub reasoning_effort: Option, + pub reasoning_effort: Option, #[serde(default)] - pub prompt_cache: Option, + pub prompt_cache: Option, #[serde(default)] - pub sampling_params: Option, + pub cache_control_breakpoints: Option, + #[serde(default)] + pub sampling_params: Option, } #[derive(Debug, Clone, Default, PartialEq, Deserialize)] @@ -574,6 +576,10 @@ pub enum CatalogBuildError { ReasoningEffortControlsWithoutReasoning { model: String }, #[error("model '{model}' declares reasoning_effort feature but features.reasoning is false")] ReasoningEffortWithoutReasoning { model: String }, + #[error( + "model '{model}' declares cache_control_breakpoints but features.prompt_cache is false" + )] + CacheControlBreakpointsWithoutPromptCache { model: String }, #[error( "model '{model}' must declare at least one reasoning_effort when features.reasoning_effort is levels or always_adaptive" )] @@ -1947,12 +1953,15 @@ fn merge_model_features_settings( fallback: &SettingsModelFeatures, ) -> SettingsModelFeatures { SettingsModelFeatures { - tools: higher.tools.or(fallback.tools), - vision: higher.vision.or(fallback.vision), - reasoning: higher.reasoning.or(fallback.reasoning), - reasoning_effort: higher.reasoning_effort.or(fallback.reasoning_effort), - prompt_cache: higher.prompt_cache.or(fallback.prompt_cache), - sampling_params: higher.sampling_params.or(fallback.sampling_params), + tools: higher.tools.or(fallback.tools), + vision: higher.vision.or(fallback.vision), + reasoning: higher.reasoning.or(fallback.reasoning), + reasoning_effort: higher.reasoning_effort.or(fallback.reasoning_effort), + prompt_cache: higher.prompt_cache.or(fallback.prompt_cache), + cache_control_breakpoints: higher + .cache_control_breakpoints + .or(fallback.cache_control_breakpoints), + sampling_params: higher.sampling_params.or(fallback.sampling_params), } } @@ -2261,6 +2270,15 @@ fn build_model_features( model: model_id.to_string(), }); } + let prompt_cache = features.prompt_cache.unwrap_or_default(); + let cache_control_breakpoints = features.cache_control_breakpoints.unwrap_or_default(); + if cache_control_breakpoints && !prompt_cache { + return Err( + CatalogBuildError::CacheControlBreakpointsWithoutPromptCache { + model: model_id.to_string(), + }, + ); + } Ok(ModelFeatures { tools: features @@ -2277,7 +2295,8 @@ fn build_model_features( })?, reasoning, reasoning_effort, - prompt_cache: features.prompt_cache.unwrap_or_default(), + prompt_cache, + cache_control_breakpoints, sampling_params: features.sampling_params.unwrap_or(true), }) } @@ -2977,6 +2996,7 @@ enabled = true 0.5, ReasoningEffortFeature::Levels, false, + false, BillingPolicy::OpenAi, ), ( @@ -2989,6 +3009,7 @@ enabled = true 0.25, ReasoningEffortFeature::Levels, false, + false, BillingPolicy::OpenAi, ), ( @@ -3001,6 +3022,7 @@ enabled = true 0.1, ReasoningEffortFeature::Levels, false, + false, BillingPolicy::OpenAi, ), ( @@ -3013,6 +3035,7 @@ enabled = true 0.5, ReasoningEffortFeature::Levels, false, + true, BillingPolicy::Anthropic, ), ( @@ -3025,6 +3048,7 @@ enabled = true 1.0, ReasoningEffortFeature::AlwaysAdaptive, false, + true, BillingPolicy::Anthropic, ), ]; @@ -3039,6 +3063,7 @@ enabled = true cache_input_cost, reasoning_effort, sampling_params, + cache_control_breakpoints, billing_policy, ) in expected { @@ -3055,6 +3080,10 @@ enabled = true assert!(model.features.prompt_cache, "{id}"); assert_eq!(model.features.reasoning_effort, reasoning_effort, "{id}"); assert_eq!(model.features.sampling_params, sampling_params, "{id}"); + assert_eq!( + model.features.cache_control_breakpoints, cache_control_breakpoints, + "{id}" + ); assert_eq!(model.costs.input_cost_per_mtok, Some(input_cost), "{id}"); assert_eq!(model.costs.output_cost_per_mtok, Some(output_cost), "{id}"); assert_eq!( @@ -3213,6 +3242,7 @@ enabled = true reasoning: true, reasoning_effort: Levels, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { @@ -3277,6 +3307,7 @@ enabled = true reasoning: true, reasoning_effort: AlwaysAdaptive, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: false, }, costs: ModelCosts { @@ -5683,6 +5714,39 @@ reasoning_effort = "levels" )); } + #[test] + fn catalog_from_settings_rejects_cache_control_breakpoints_without_prompt_cache() { + let settings = minimal_settings( + r#" +[providers.test] +display_name = "Test" +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "https://example.test/v1" + +[models.model] +provider = "test" +display_name = "Model" +family = "test" + +[models.model.limits] +context_window = 1000 + +[models.model.features] +tools = true +vision = false +reasoning = false +cache_control_breakpoints = true +"#, + ); + + assert!(matches!( + Catalog::from_settings(&settings).unwrap_err(), + CatalogBuildError::CacheControlBreakpointsWithoutPromptCache { model } + if model == "model" + )); + } + #[test] fn catalog_from_settings_rejects_always_adaptive_effort_without_reasoning() { let settings = minimal_settings( @@ -5843,6 +5907,7 @@ sampling_params = false reasoning: true, reasoning_effort: Levels, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { @@ -5899,6 +5964,7 @@ sampling_params = false reasoning: true, reasoning_effort: None, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: false, }, costs: ModelCosts { @@ -5947,6 +6013,7 @@ sampling_params = false reasoning: true, reasoning_effort: AlwaysAdaptive, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: false, }, costs: ModelCosts { @@ -6019,6 +6086,7 @@ sampling_params = false reasoning: true, reasoning_effort: Levels, prompt_cache: true, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { @@ -6084,6 +6152,7 @@ sampling_params = false reasoning: true, reasoning_effort: Levels, prompt_cache: false, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { @@ -6140,6 +6209,7 @@ sampling_params = false reasoning: true, reasoning_effort: Levels, prompt_cache: false, + cache_control_breakpoints: false, sampling_params: true, }, costs: ModelCosts { diff --git a/lib/crates/fabro-model/src/catalog/providers/openrouter.toml b/lib/crates/fabro-model/src/catalog/providers/openrouter.toml index 4ef7d7c63..b83aa7662 100644 --- a/lib/crates/fabro-model/src/catalog/providers/openrouter.toml +++ b/lib/crates/fabro-model/src/catalog/providers/openrouter.toml @@ -50,6 +50,7 @@ vision = true reasoning = true reasoning_effort = "always_adaptive" prompt_cache = true +cache_control_breakpoints = true sampling_params = false [providers.openrouter.models."claude-fable-5".costs] @@ -76,6 +77,7 @@ vision = true reasoning = true reasoning_effort = "levels" prompt_cache = true +cache_control_breakpoints = true sampling_params = false [providers.openrouter.models."claude-opus-4-8".costs] @@ -98,6 +100,7 @@ tools = true vision = true reasoning = true prompt_cache = true +cache_control_breakpoints = true [providers.openrouter.models."claude-opus-4-7".costs] input_cost_per_mtok = 5.0 @@ -121,6 +124,7 @@ tools = true vision = true reasoning = true prompt_cache = true +cache_control_breakpoints = true [providers.openrouter.models."claude-sonnet-4-6".costs] input_cost_per_mtok = 3.0 @@ -144,6 +148,7 @@ tools = true vision = true reasoning = false prompt_cache = true +cache_control_breakpoints = true [providers.openrouter.models."claude-haiku-4-5".costs] input_cost_per_mtok = 1.0 diff --git a/lib/crates/fabro-model/src/types.rs b/lib/crates/fabro-model/src/types.rs index fb0164804..ae3804784 100644 --- a/lib/crates/fabro-model/src/types.rs +++ b/lib/crates/fabro-model/src/types.rs @@ -41,21 +41,28 @@ fn default_true() -> bool { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ModelFeatures { - pub tools: bool, - pub vision: bool, - pub reasoning: bool, + pub tools: bool, + pub vision: bool, + pub reasoning: bool, /// Whether this model endpoint supports a native reasoning-effort /// parameter. User-facing allowed effort values live in catalog controls. #[serde(default)] - pub reasoning_effort: ReasoningEffortFeature, + pub reasoning_effort: ReasoningEffortFeature, /// Whether this model endpoint supports prompt caching annotations. #[serde(default)] - pub prompt_cache: bool, + pub prompt_cache: bool, + /// Whether the endpoint only caches when the request marks the cacheable + /// prefix with Anthropic-style `cache_control` breakpoints. Set on + /// OpenAI-compatible routes fronting Anthropic models (e.g. Claude via + /// OpenRouter); dialects whose caching mechanism is implied (native + /// Anthropic, Bedrock) ignore it. + #[serde(default)] + pub cache_control_breakpoints: bool, /// Whether the model endpoint accepts classic sampling parameters /// (`temperature`, `top_p`). Models with always-on adaptive behavior /// reject them. #[serde(default = "default_true")] - pub sampling_params: bool, + pub sampling_params: bool, } impl ModelFeatures { @@ -221,12 +228,13 @@ mod tests { training: Some("training".to_string()), knowledge_cutoff: Some("knowledge-cutoff".to_string()), features: ModelFeatures { - tools: true, - vision: true, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::Levels, - prompt_cache: true, - sampling_params: true, + tools: true, + vision: true, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: true, + cache_control_breakpoints: false, + sampling_params: true, }, costs: ModelCosts { input_cost_per_mtok: Some(1.0), diff --git a/lib/packages/fabro-api-client/src/models/model-features.ts b/lib/packages/fabro-api-client/src/models/model-features.ts index 7eb314d8c..5f23381b3 100644 --- a/lib/packages/fabro-api-client/src/models/model-features.ts +++ b/lib/packages/fabro-api-client/src/models/model-features.ts @@ -38,6 +38,10 @@ export interface ModelFeatures { * Whether the model endpoint supports prompt caching. */ 'prompt_cache': boolean; + /** + * Whether the endpoint only caches when the request marks the cacheable prefix with Anthropic-style cache_control breakpoints (e.g. Claude via OpenRouter). + */ + 'cache_control_breakpoints': boolean; /** * Whether the model accepts classic sampling parameters (temperature, top_p). */