fix(cache_control): stamp the OpenAI dialect for a bare deployment name served by a flagged provider

A deployment written as `model: openai.gpt-5.6-sol` with `custom_llm_provider: bedrock_mantle` has no cost-map row
of its own and no openai row of the same name, so the dialect stamp's cheap gate (`supports_openai_prompt_cache_breakpoint`)
returned early and the configured point was never stamped. On `/v1/responses` and on the chat seeding path the hook
sees no provider, so it fell back to the Anthropic `cache_control` marker, which Bedrock Mantle strips.

The gate now also passes a model whose serving provider is already known and whose provider-keyed row carries the
flag, which is the same row the dialect resolution reads and costs no provider lookup. A bare name without a provider
is still left alone, so models below GPT-5.6 keep their points untouched and resolve nothing.
This commit is contained in:
mateo-berri 2026-10-07 20:52:53 -07:00
parent 72a912c8d5
commit edc40a731c
2 changed files with 70 additions and 1 deletions

View file

@ -770,7 +770,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
api_base: object,
prompt_cache_options: object,
) -> Sequence[Mapping[str, object]]:
if not supports_openai_prompt_cache_breakpoint(model):
if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider):
return points
return AnthropicCacheControlHook._stamped(
points,
@ -780,6 +780,20 @@ class AnthropicCacheControlHook(CustomPromptManagement):
),
)
@staticmethod
def _may_target_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool:
"""Cheap gate before the dialect is resolved: the model's own row or version, or, when the
serving provider is already known, a row keyed for that provider (``openai.gpt-5.6-sol``
served by ``bedrock_mantle``), which costs no provider lookup."""
if supports_openai_prompt_cache_breakpoint(model):
return True
if custom_llm_provider is None:
return False
return (
_hosted_openai_dialect_flag(model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider)
is not None
)
@staticmethod
def _stamped(points: Sequence[Mapping[str, object]], key: str, value: object) -> Sequence[Mapping[str, object]]:
return [{**point, key: value} for point in points]

View file

@ -3994,6 +3994,61 @@ class TestBedrockMantleGptShipsTheOpenAIDialect:
assert messages[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}]
assert params["prompt_cache_options"] == {"mode": "implicit"}
BARE_MODEL = "openai.gpt-5.6-sol"
POINTS = [{"location": "message", "role": "system"}]
MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}]
MARKED_SYSTEM = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}]
def test_a_bare_deployment_name_with_its_provider_is_stamped_on_the_openai_dialect(self):
"""A deployment written as ``model: openai.gpt-5.6-sol`` plus ``custom_llm_provider: bedrock_mantle``
has no row of its own and no openai row of the same name, so only the provider-keyed row can
answer; the stamp must read it the way the dialect resolution does."""
stamped = AnthropicCacheControlHook._stamped_with_dialect(
copy.deepcopy(self.POINTS), self.BARE_MODEL, "bedrock_mantle", None, None
)
assert stamped[0]["_litellm_openai_dialect"] is True
def test_a_bare_deployment_name_without_its_provider_keeps_its_points_and_costs_no_lookup(self):
points = copy.deepcopy(self.POINTS)
with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve:
assert AnthropicCacheControlHook._stamped_with_dialect(points, self.BARE_MODEL, None, None, None) is points
resolve.assert_not_called()
def test_the_chat_seed_carries_the_resolved_provider_for_a_bare_deployment_name(self):
params: dict = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
AnthropicCacheControlHook.maybe_seed_default_injection_points(
non_default_params=params,
messages=copy.deepcopy(self.MESSAGES),
model=self.BARE_MODEL,
custom_llm_provider="bedrock_mantle",
)
_, messages, out = AnthropicCacheControlHook().get_chat_completion_prompt(
model=self.BARE_MODEL,
messages=copy.deepcopy(self.MESSAGES),
non_default_params=params,
prompt_id=None,
prompt_variables=None,
dynamic_callback_params={},
)
assert messages[0]["content"] == self.MARKED_SYSTEM
assert out["prompt_cache_options"] == {"mode": "implicit"}
def test_the_responses_stamp_carries_the_resolved_provider_for_a_bare_deployment_name(self):
from litellm.responses.main import _stamp_injection_points_with_dialect
kwargs: dict = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
_stamp_injection_points_with_dialect(kwargs, self.BARE_MODEL, "bedrock_mantle")
_, messages, out = AnthropicCacheControlHook().get_chat_completion_prompt(
model=self.BARE_MODEL,
messages=copy.deepcopy(self.MESSAGES),
non_default_params=kwargs,
prompt_id=None,
prompt_variables=None,
dynamic_callback_params={},
)
assert messages[0]["content"] == self.MARKED_SYSTEM
assert out["prompt_cache_options"] == {"mode": "implicit"}
class TestRecordGatewayInjection:
"""The injection marker spend accounting gates prompt-caching savings on."""