diff --git a/strix/core/inputs.py b/strix/core/inputs.py index f383261e..9be6e457 100644 --- a/strix/core/inputs.py +++ b/strix/core/inputs.py @@ -26,6 +26,13 @@ if TYPE_CHECKING: from strix.config.settings import ReasoningEffort +_CACHE_LOCATION_MESSAGE = "message" +_CACHE_LOCATION_TOOL_CONFIG = "tool_config" +_CACHE_ROLE_SYSTEM = "system" +_CACHE_INDEX_PREVIOUS_MESSAGE = -2 +_CACHE_INDEX_LAST_MESSAGE = -1 + + def _accepts_required_tool_choice(model_name: str | None) -> bool: name = (model_name or "").strip().lower() for prefix in ("litellm/", "any-llm/"): @@ -312,7 +319,7 @@ def _reasoning_settings( def _prompt_cache_extra_args(model_name: str) -> dict[str, Any] | None: """LiteLLM ``cache_control_injection_points`` for Claude prompt caching. - System prompt + rolling last-message breakpoint everywhere; ``tool_config`` + System prompt + rolling recent-message breakpoints everywhere; ``tool_config`` only on Bedrock Converse (the only route whose LiteLLM transform consumes it — elsewhere it leaks onto the wire and native Anthropic 400s). Unmapped Bedrock models get no points at all: Bedrock rejects the passed-through @@ -323,10 +330,17 @@ def _prompt_cache_extra_args(model_name: str) -> dict[str, Any] | None: if is_bedrock_route(model_name) and not bedrock_route_supports_prompt_caching(model_name): return None - points: list[dict[str, Any]] = [{"location": "message", "role": "system"}] + points: list[dict[str, Any]] = [ + {"location": _CACHE_LOCATION_MESSAGE, "role": _CACHE_ROLE_SYSTEM} + ] if is_bedrock_route(model_name): - points.append({"location": "tool_config"}) - points.append({"location": "message", "index": -1}) + points.append({"location": _CACHE_LOCATION_TOOL_CONFIG}) + points.extend( + [ + {"location": _CACHE_LOCATION_MESSAGE, "index": _CACHE_INDEX_PREVIOUS_MESSAGE}, + {"location": _CACHE_LOCATION_MESSAGE, "index": _CACHE_INDEX_LAST_MESSAGE}, + ] + ) return {"cache_control_injection_points": points} diff --git a/tests/test_inputs.py b/tests/test_inputs.py index e12c56c5..0a064a39 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -71,6 +71,7 @@ def test_make_model_settings_enables_prompt_cache_for_bedrock_claude() -> None: assert _cache_points("bedrock/global.anthropic.claude-opus-4-8") == [ {"location": "message", "role": "system"}, {"location": "tool_config"}, + {"location": "message", "index": -2}, {"location": "message", "index": -1}, ] @@ -86,6 +87,7 @@ def test_make_model_settings_enables_prompt_cache_for_bedrock_claude() -> None: def test_make_model_settings_enables_prompt_cache_for_non_bedrock_claude(model_name: str) -> None: assert _cache_points(model_name) == [ {"location": "message", "role": "system"}, + {"location": "message", "index": -2}, {"location": "message", "index": -1}, ] @@ -131,6 +133,7 @@ def test_prompt_cache_kept_for_non_bedrock_claude_even_if_unmapped(monkeypatch: for model in ("anthropic/claude-brand-new-9", "openrouter/anthropic/claude-brand-new"): assert _cache_points(model) == [ {"location": "message", "role": "system"}, + {"location": "message", "index": -2}, {"location": "message", "index": -1}, ]