mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-30 06:43:27 +00:00
fix(responses): drop a malformed prompt_cache_breakpoint under drop_params on the direct /v1/responses path (#45330)
* fix(responses): drop a malformed prompt_cache_breakpoint under drop_params on the direct /v1/responses path * refactor(responses): reuse the shared prompt_cache_breakpoint_for_wire helper in the chat bridge * fix(responses): read a non-flag deployment drop_params as unset in the marker walk, like the param mapper * fix(responses): drop every prompt_cache_breakpoint under drop_params when the model has no breakpoints Also drop a marker set on the message item itself, sanitize before the cache_control conversion so a client typo no longer erases that breakpoint, and skip items that carry no marker so unmarked history costs no extra walk * fix(responses): keep a valid prompt_cache_breakpoint unless the model is known to reject it A deployment alias the cost map does not know (an Azure deployment name, a fine-tune, a litellm_proxy alias) kept its valid markers before this PR and must keep them under drop_params. base_model is read from litellm_params first, then model_info, as chat completions does. * fix(responses): drop a valid marker on known non-breakpoint models and keep the tool schema gate unchanged * fix(responses): judge breakpoint support by model name so cost-map aliases keep valid markers * fix(responses): keep the caller's request stripped of cache_control as before when a marker is copied * fix(responses): keep valid prompt cache markers in place and let cache_control fill a dropped one Under drop_params the marker step now hands back the caller's own objects when every marker is already wire-safe, so a request with valid markers is no longer copied and walked a second time. A deployment that turns cache_control into markers keeps the caller's own markers too, whatever its base_model says, so one request never gets two answers about breakpoint support. On /v1/messages a malformed raw marker next to cache_control is dropped before the conversion, so the block gets the cache hint cache_control alone would give, as on /v1/responses. * test(responses): keep a null marker and the original key order on a model that reads breakpoints --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
310c96162b
commit
b93d26bc85
9 changed files with 605 additions and 56 deletions
|
|
@ -36,6 +36,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
|
|||
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
|
|
@ -61,7 +62,6 @@ from litellm.types.llms.openai import (
|
|||
ChatCompletionToolCallChunk,
|
||||
ChatCompletionToolCallFunctionChunk,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
PromptCacheBreakpoint,
|
||||
Reasoning,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
ResponsesAPIResponse,
|
||||
|
|
@ -631,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li
|
|||
return "length"
|
||||
|
||||
|
||||
_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
|
||||
|
||||
|
||||
def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
|
||||
if marker is None or not drop_params:
|
||||
return marker
|
||||
try:
|
||||
return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
|
||||
except ValidationError:
|
||||
verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
|
||||
return None
|
||||
|
||||
|
||||
def _input_file_from_file_value(file_value: object) -> dict[str, object]:
|
||||
if not isinstance(file_value, dict):
|
||||
return {"type": "input_file"}
|
||||
|
|
@ -1679,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
if original_type == "text":
|
||||
converted = with_prompt_cache_breakpoint(
|
||||
self._convert_content_str_to_input_text(item.get("text", ""), role),
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: text -> %s", converted)
|
||||
|
|
@ -1692,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
cast(ChatCompletionImageObject, item), role
|
||||
),
|
||||
),
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: image_url -> %s", converted)
|
||||
elif original_type == "input_audio":
|
||||
converted = with_prompt_cache_breakpoint(
|
||||
{"type": "input_audio", "input_audio": item.get("input_audio")},
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: input_audio -> %s", converted)
|
||||
|
|
@ -1715,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
_input_file_from_file_value(
|
||||
cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked
|
||||
),
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: file -> %s", converted)
|
||||
|
|
@ -1739,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
# Default to input_text for unknown types
|
||||
converted = with_prompt_cache_breakpoint(
|
||||
self._convert_content_str_to_input_text(str(item.get("text", item)), role),
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted)
|
||||
|
|
|
|||
|
|
@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
|
|||
CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
|
||||
OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
|
||||
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
|
||||
_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
|
||||
_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
|
||||
OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
|
||||
{"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"}
|
||||
)
|
||||
|
|
@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None:
|
|||
|
||||
|
||||
def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
|
||||
|
||||
|
||||
def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
|
||||
hosted_flag: Final = (
|
||||
None
|
||||
if custom_llm_provider is None
|
||||
|
|
@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str
|
|||
model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
|
||||
if model_map_flag is not None:
|
||||
return model_map_flag
|
||||
version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower())
|
||||
if version_match is None:
|
||||
fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
|
||||
if fine_tuned_base is not None:
|
||||
return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None
|
||||
return _named_model_breakpoint_support(model)
|
||||
|
||||
|
||||
def _named_model_breakpoint_support(model: str) -> bool | None:
|
||||
bare_model: Final = model.rsplit("/", 1)[-1].lower()
|
||||
if _O_SERIES_PATTERN.match(bare_model) is not None:
|
||||
return False
|
||||
version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
|
||||
if version_match is None:
|
||||
return None
|
||||
version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
|
||||
return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from pathlib import Path
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -2073,6 +2073,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
|
|||
return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key
|
||||
|
||||
|
||||
_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
|
||||
|
||||
|
||||
def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
|
||||
if marker is None or not drop_params:
|
||||
return marker
|
||||
try:
|
||||
return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
|
||||
except ValidationError:
|
||||
verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
|
||||
return None
|
||||
|
||||
|
||||
def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
|
||||
"""Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block."""
|
||||
if block.get("cache_control") is None:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,11 @@ from litellm.integrations.anthropic_cache_control_hook import (
|
|||
supports_openai_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.responses.utils import auto_reasoning_summary
|
||||
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -90,14 +94,28 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str
|
|||
return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
|
||||
|
||||
|
||||
def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]:
|
||||
if "prompt_cache_breakpoint" not in block:
|
||||
return cache_control_as_prompt_cache_breakpoint(block)
|
||||
wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True)
|
||||
return cache_control_as_prompt_cache_breakpoint(
|
||||
with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker)
|
||||
)
|
||||
|
||||
|
||||
def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None:
|
||||
"""Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints
|
||||
drops raw ones under drop_params."""
|
||||
drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params"))
|
||||
if _reads_prompt_cache_breakpoint(model, forwarded_kwargs):
|
||||
return cache_control_as_prompt_cache_breakpoint
|
||||
return (
|
||||
_cache_control_as_wire_safe_prompt_cache_breakpoint
|
||||
if drops_params
|
||||
else cache_control_as_prompt_cache_breakpoint
|
||||
)
|
||||
if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs):
|
||||
return None
|
||||
if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
|
||||
if drops_params:
|
||||
return _without_prompt_cache_breakpoint
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Callable, Collection, Mapping, Sequence
|
||||
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints
|
||||
|
|
@ -10,7 +10,11 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
openai_prompt_cache_breakpoint_support,
|
||||
targets_openai_api,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
|
|
@ -20,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
cache_control_as_prompt_cache_breakpoint,
|
||||
drop_non_python_regex_patterns,
|
||||
flatten_combinators_and_drop_non_python_regex_patterns,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
|
@ -66,6 +71,7 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE,
|
|||
_PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS
|
||||
_ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"})
|
||||
_PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"})
|
||||
_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"})
|
||||
_COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"})
|
||||
_COMPACT_REJECTED_PARAMS: Final = frozenset(
|
||||
{
|
||||
|
|
@ -93,6 +99,13 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str,
|
|||
_CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]])
|
||||
|
||||
|
||||
class _DeclaredModel(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
|
||||
|
||||
base_model: object = None
|
||||
model_info: object = None
|
||||
|
||||
|
||||
class _DeploymentHost(BaseModel):
|
||||
"""`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`."""
|
||||
|
||||
|
|
@ -216,6 +229,63 @@ def _with_breakpoint_on_last_reading_part(
|
|||
return (*parts[:last], marked_last, *parts[last + 1 :])
|
||||
|
||||
|
||||
def _json_object_or_none(value: object) -> Mapping[str, object] | None:
|
||||
return value if isinstance(value, Mapping) else None
|
||||
|
||||
|
||||
def _json_array_or_empty(value: object) -> Sequence[object]:
|
||||
return value if isinstance(value, (list, tuple)) else ()
|
||||
|
||||
|
||||
def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object:
|
||||
fields: Final = _json_object_or_none(item)
|
||||
if fields is None or not _carries_prompt_cache_breakpoint(fields):
|
||||
return item
|
||||
wire_safe_fields: Final = {
|
||||
key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers)
|
||||
for key, value in fields.items()
|
||||
if key != "prompt_cache_breakpoint"
|
||||
}
|
||||
unchanged: Final = "prompt_cache_breakpoint" not in fields and _same_objects(
|
||||
wire_safe_fields.values(), fields.values()
|
||||
)
|
||||
return item if unchanged else wire_safe_fields
|
||||
|
||||
|
||||
def _same_objects(rebuilt: Iterable[object], originals: Iterable[object]) -> bool:
|
||||
return all(new is old for new, old in zip(rebuilt, originals, strict=True))
|
||||
|
||||
|
||||
def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool:
|
||||
return "prompt_cache_breakpoint" in fields or any(
|
||||
_parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS
|
||||
)
|
||||
|
||||
|
||||
def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool:
|
||||
return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts))
|
||||
|
||||
|
||||
def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object:
|
||||
if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value):
|
||||
return value
|
||||
parts: Final = _json_array_or_empty(value)
|
||||
wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts)
|
||||
return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts)
|
||||
|
||||
|
||||
def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object:
|
||||
fields: Final = _json_object_or_none(part)
|
||||
if fields is None or "prompt_cache_breakpoint" not in fields:
|
||||
return part
|
||||
marker: Final = fields["prompt_cache_breakpoint"]
|
||||
wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None
|
||||
if keeps_markers and wire_marker == marker:
|
||||
return part
|
||||
unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
|
||||
return with_prompt_cache_breakpoint(unmarked, wire_marker)
|
||||
|
||||
|
||||
class _ReasoningSupportEntry(LiteLLMBaseModel):
|
||||
litellm_provider: str | None = None
|
||||
supports_reasoning: bool | None = None
|
||||
|
|
@ -544,12 +614,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
validated_input: Final = self._validate_input_param(
|
||||
self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params)
|
||||
)
|
||||
marked_input: Final = (
|
||||
self._input_with_prompt_cache_breakpoints(validated_input)
|
||||
if self._reads_prompt_cache_breakpoints(
|
||||
model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options
|
||||
)
|
||||
else validated_input
|
||||
reads_breakpoints: Final = self._reads_prompt_cache_breakpoints(
|
||||
model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options
|
||||
)
|
||||
marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints(
|
||||
model=model, input=validated_input, litellm_params=litellm_params, reads_breakpoints=reads_breakpoints
|
||||
)
|
||||
maps_anthropic_tool_search: Final = self._targets_openai_api(litellm_params) and any(
|
||||
is_anthropic_tool_search_chat_tool(tool) for tool in tools or ()
|
||||
|
|
@ -559,20 +628,71 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
if maps_anthropic_tool_search
|
||||
else tools
|
||||
)
|
||||
replayable_input: Final = (
|
||||
input_without_unanswered_anthropic_tool_search_calls(marked_input)
|
||||
if maps_anthropic_tool_search
|
||||
else marked_input
|
||||
)
|
||||
stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools(
|
||||
model=model, input=replayable_input, tools=mapped_tools
|
||||
model=model,
|
||||
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
|
||||
tools=mapped_tools,
|
||||
)
|
||||
if marker_safe_input is not validated_input:
|
||||
self._strip_caller_input_as_if_unmarked(
|
||||
model=model,
|
||||
caller_input=validated_input,
|
||||
reads_breakpoints=reads_breakpoints,
|
||||
maps_anthropic_tool_search=maps_anthropic_tool_search,
|
||||
)
|
||||
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
|
||||
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
|
||||
model=model, tools=object_schema_tools, litellm_params=litellm_params
|
||||
)
|
||||
return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools
|
||||
|
||||
def _replayable_input(
|
||||
self, input: str | ResponseInputParam, reads_breakpoints: bool, maps_anthropic_tool_search: bool
|
||||
) -> str | ResponseInputParam:
|
||||
marked_input: Final = self._input_with_prompt_cache_breakpoints(input) if reads_breakpoints else input
|
||||
if not maps_anthropic_tool_search:
|
||||
return marked_input
|
||||
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
|
||||
|
||||
def _strip_caller_input_as_if_unmarked(
|
||||
self,
|
||||
model: str,
|
||||
caller_input: str | ResponseInputParam,
|
||||
reads_breakpoints: bool,
|
||||
maps_anthropic_tool_search: bool,
|
||||
) -> None:
|
||||
self.remove_cache_control_flag_from_input_and_tools(
|
||||
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
|
||||
)
|
||||
|
||||
def _input_with_wire_safe_prompt_cache_breakpoints(
|
||||
self,
|
||||
model: str,
|
||||
input: str | ResponseInputParam,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
reads_breakpoints: bool,
|
||||
) -> str | ResponseInputParam:
|
||||
if not isinstance(input, list) or not drops_params(litellm_params):
|
||||
return input
|
||||
keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints(
|
||||
model=model, litellm_params=litellm_params
|
||||
)
|
||||
wire_safe_items: Final = tuple(
|
||||
_item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input
|
||||
)
|
||||
if _same_objects(wire_safe_items, input):
|
||||
return input
|
||||
wire_safe_input: Final = list(wire_safe_items)
|
||||
return cast("ResponseInputParam", wire_safe_input) # cast-ok: items keep their shape, minus a rejected marker
|
||||
|
||||
def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool:
|
||||
provider: Final = _provider_name(self.custom_llm_provider)
|
||||
base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params)
|
||||
known_support: Final = (
|
||||
openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model))
|
||||
)
|
||||
return next((support for support in known_support if support is not None), None) is not False
|
||||
|
||||
def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool:
|
||||
host: Final = _DeploymentHost.model_validate(litellm_params)
|
||||
return self.custom_llm_provider == LlmProviders.OPENAI and targets_openai_api(host.api_base or host.base_url)
|
||||
|
|
@ -688,14 +808,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
|
||||
@staticmethod
|
||||
def _without_foreign_tool_call_item_id(item: object) -> object:
|
||||
if not isinstance(item, dict):
|
||||
fields: Final = _json_object_or_none(item)
|
||||
if fields is None:
|
||||
return item
|
||||
item_type: Final = item.get("type")
|
||||
item_id: Final = item.get("id")
|
||||
item_type: Final = fields.get("type")
|
||||
item_id: Final = fields.get("id")
|
||||
genuine_prefix: Final = TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE.get(item_type) if isinstance(item_type, str) else None
|
||||
if genuine_prefix is None or not isinstance(item_id, str) or item_id.startswith(genuine_prefix):
|
||||
return item
|
||||
return {key: value for key, value in item.items() if key != "id"}
|
||||
return {key: value for key, value in fields.items() if key != "id"}
|
||||
|
||||
def _sanitized_tool_schemas_for_openai(
|
||||
self,
|
||||
|
|
@ -739,6 +860,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None
|
||||
return base_model if isinstance(base_model, str) and base_model else model
|
||||
|
||||
@staticmethod
|
||||
def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str:
|
||||
declared: Final = _DeclaredModel.model_validate(litellm_params)
|
||||
model_info: Final = _json_object_or_none(declared.model_info)
|
||||
candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model"))
|
||||
return next((name for name in candidates if isinstance(name, str) and name), model)
|
||||
|
||||
@staticmethod
|
||||
def _sanitized_tool_entry(
|
||||
entry: Mapping[str, object],
|
||||
|
|
|
|||
|
|
@ -39,14 +39,6 @@ _CASES: Final[tuple[tuple[str, Mode, JsonValue, JsonValue], ...]] = (
|
|||
)
|
||||
_CASE_IDS: Final = tuple(case[0] for case in _CASES)
|
||||
_CASE_VALUES: Final = tuple(case[1:] for case in _CASES)
|
||||
_ADAPTER_CASES: Final[tuple[tuple[str, Mode, JsonValue, JsonValue], ...]] = (
|
||||
("valid-on", "on", pcb.EXPLICIT, pcb.EXPLICIT),
|
||||
("valid-off", "off", pcb.EXPLICIT, pcb.EXPLICIT),
|
||||
("malformed-on", "on", "yes", "yes"),
|
||||
("malformed-off", "off", "yes", "yes"),
|
||||
)
|
||||
_ADAPTER_CASE_IDS: Final = tuple(case[0] for case in _ADAPTER_CASES)
|
||||
_ADAPTER_CASE_VALUES: Final = tuple(case[1:] for case in _ADAPTER_CASES)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -331,7 +323,7 @@ def _anthropic_text(message: anthropic.types.Message, marker: str) -> None:
|
|||
assert content.type == "text" and content.text == rv.answer(marker), message
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _ADAPTER_CASE_VALUES, ids=_ADAPTER_CASE_IDS)
|
||||
@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _CASE_VALUES, ids=_CASE_IDS)
|
||||
def test_anthropic_sdk_marker_on_user_text(
|
||||
bridge: _Bridge, mode: Mode, breakpoint: JsonValue, expected: JsonValue
|
||||
) -> None:
|
||||
|
|
@ -540,14 +532,17 @@ def test_anthropic_sdk_cache_control_on_an_assistant_turn_is_dropped(bridge: _Br
|
|||
], "OpenAI accepts a breakpoint on output_text but writes no cache for it (live gpt-5.6, 2026-10-12)"
|
||||
|
||||
|
||||
def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> None:
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "expected"), (("on", pcb.EXPLICIT), ("off", _UNKNOWN_KEY)), ids=("normalized-on", "verbatim-off")
|
||||
)
|
||||
def test_native_responses_marker_with_an_unknown_key(bridge: _Bridge, mode: Mode, expected: JsonValue) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)}
|
||||
response: Final = bridge.gateway.request(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": bridge.on,
|
||||
"model": bridge.model(mode),
|
||||
"input": [{"type": "message", "role": "user", "content": [pcb.marked(block, _UNKNOWN_KEY)]}],
|
||||
**pcb.NO_CACHE,
|
||||
},
|
||||
|
|
@ -559,8 +554,31 @@ def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> No
|
|||
request: Final = pcb.posted(bridge.wire, marker)
|
||||
_wire_body(request)
|
||||
on_wire: Final = pcb.single_block(pcb.input_items(request), "user")
|
||||
assert on_wire == pcb.marked(block, _UNKNOWN_KEY), on_wire
|
||||
bridge.spend.landed(bridge.on, response.headers["x-litellm-call-id"], marker)
|
||||
assert on_wire == pcb.marked(block, expected), on_wire
|
||||
bridge.spend.landed(bridge.model(mode), response.headers["x-litellm-call-id"], marker)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _CASE_VALUES, ids=_CASE_IDS)
|
||||
def test_openai_sdk_native_responses_marker_on_user_text(
|
||||
bridge: _Bridge, mode: Mode, breakpoint: JsonValue, expected: JsonValue
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)}
|
||||
with openai.OpenAI(api_key=bridge.gateway.key, base_url=_v1(bridge.gateway), max_retries=0) as client:
|
||||
raw: Final = client.responses.with_raw_response.create(
|
||||
model=bridge.model(mode),
|
||||
input=[{"type": "message", "role": "user", "content": [pcb.marked(block, breakpoint)]}],
|
||||
extra_body=dict(pcb.NO_CACHE),
|
||||
)
|
||||
response: Final = raw.parse()
|
||||
assert pcb.answers(response.id, marker), response
|
||||
assert response.output_text == rv.answer(marker), response
|
||||
request: Final = pcb.posted(bridge.wire, marker)
|
||||
_wire_body(request)
|
||||
on_wire: Final = pcb.single_block(pcb.input_items(request), "user")
|
||||
assert on_wire["type"] == "input_text" and on_wire["text"] == pcb.prompt(marker), on_wire
|
||||
pcb.assert_marker(on_wire, expected)
|
||||
bridge.spend.landed(bridge.model(mode), raw.headers["x-litellm-call-id"], marker)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("breakpoint", _MALFORMED_VALUES, ids=_MALFORMED_IDS)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
merge_consecutive_system_messages,
|
||||
native_redacted_thinking_data,
|
||||
parse_tool_call_arguments,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
salvage_concatenated_tool_arguments,
|
||||
schema_grammar_branch_count,
|
||||
|
|
@ -2328,6 +2329,52 @@ def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_w
|
|||
assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages)
|
||||
|
||||
|
||||
_MALFORMED_PROMPT_CACHE_BREAKPOINTS: Final = (
|
||||
"yes",
|
||||
1,
|
||||
["explicit"],
|
||||
"",
|
||||
{},
|
||||
{"mode": "bogus"},
|
||||
{"mode": "explicit", "ttl": "1h"},
|
||||
)
|
||||
_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS: Final = (
|
||||
"string",
|
||||
"int",
|
||||
"list",
|
||||
"empty-string",
|
||||
"empty-object",
|
||||
"bogus-mode",
|
||||
"bad-ttl",
|
||||
)
|
||||
_VALID_PROMPT_CACHE_BREAKPOINTS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep"))
|
||||
def test_prompt_cache_breakpoint_for_wire_passes_a_missing_marker_through(drop_params: bool) -> None:
|
||||
assert prompt_cache_breakpoint_for_wire(None, drop_params) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS)
|
||||
def test_prompt_cache_breakpoint_for_wire_drops_a_malformed_marker_under_drop_params(marker: object) -> None:
|
||||
assert prompt_cache_breakpoint_for_wire(marker, True) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS)
|
||||
def test_prompt_cache_breakpoint_for_wire_forwards_a_malformed_marker_without_drop_params(marker: object) -> None:
|
||||
assert prompt_cache_breakpoint_for_wire(marker, False) is marker
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep"))
|
||||
@pytest.mark.parametrize("marker", _VALID_PROMPT_CACHE_BREAKPOINTS, ids=("explicit", "explicit-30m"))
|
||||
def test_prompt_cache_breakpoint_for_wire_keeps_a_valid_marker(marker: dict[str, str], drop_params: bool) -> None:
|
||||
assert prompt_cache_breakpoint_for_wire(marker, drop_params) == marker
|
||||
|
||||
|
||||
def test_prompt_cache_breakpoint_for_wire_strips_an_unknown_key_under_drop_params() -> None:
|
||||
assert prompt_cache_breakpoint_for_wire({"mode": "explicit", "note": "kept"}, True) == {"mode": "explicit"}
|
||||
|
||||
|
||||
def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None:
|
||||
marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}}
|
||||
|
||||
|
|
|
|||
|
|
@ -556,6 +556,28 @@ def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drop
|
|||
]
|
||||
|
||||
|
||||
def _gpt_5_6_input_for_a_cached_block(drop_params: bool, raw_mark: Mapping[str, object]) -> object:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs={**OPENAI_API, "drop_params": drop_params},
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": [{"type": "text", "text": "look", "cache_control": EPHEMERAL, **raw_mark}]}],
|
||||
)
|
||||
return responses_kwargs["input"]
|
||||
|
||||
|
||||
def test_a_malformed_raw_marker_beside_cache_control_gives_way_to_it_only_under_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
malformed: Final = {"prompt_cache_breakpoint": "yes"}
|
||||
|
||||
assert _gpt_5_6_input_for_a_cached_block(True, malformed) == _gpt_5_6_input_for_a_cached_block(True, {})
|
||||
assert _gpt_5_6_input_for_a_cached_block(False, malformed) == [
|
||||
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "look", **malformed}]}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "extra_kwargs"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.types.llms.openai import (
|
|||
OutputTextDeltaEvent,
|
||||
ResponseAPIUsage,
|
||||
ResponseCompletedEvent,
|
||||
ResponseInputParam,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
|
|
@ -4246,6 +4247,305 @@ async def test_extra_body_merges_with_request_data(extra_body_mock_response_data
|
|||
assert request_body["custom_field"] == "custom_value"
|
||||
|
||||
|
||||
_MALFORMED_MARKERS: Final = ("yes", 1, {"mode": "bogus"}, {"mode": "explicit", "ttl": "1h"})
|
||||
_MALFORMED_MARKER_IDS: Final = ("string", "int", "bogus-mode", "bad-ttl")
|
||||
_VALID_MARKERS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"})
|
||||
_VALID_MARKER_IDS: Final = ("explicit", "explicit-30m")
|
||||
_PARTS: Final[Mapping[str, Mapping[str, object]]] = {
|
||||
"input_text": {"type": "input_text", "text": "Say ok"},
|
||||
"input_image": {"type": "input_image", "image_url": "https://example.com/a.png"},
|
||||
"input_file": {"type": "input_file", "file_id": "file-abc"},
|
||||
}
|
||||
_CONFIGS: Final = (OpenAIResponsesAPIConfig(), AzureOpenAIResponsesAPIConfig())
|
||||
_CONFIG_IDS: Final = ("openai", "azure")
|
||||
|
||||
|
||||
def _marked_message(part_type: str, marker: object) -> list[dict[str, object]]:
|
||||
return [{"type": "message", "role": "user", "content": [{**_PARTS[part_type], "prompt_cache_breakpoint": marker}]}]
|
||||
|
||||
|
||||
def _transformed_input(
|
||||
config: BaseResponsesAPIConfig, input: str | list[dict[str, object]], litellm_params: GenericLiteLLMParams
|
||||
) -> object:
|
||||
request: Final = config.transform_responses_api_request(
|
||||
model="gpt-6.1-sol",
|
||||
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
|
||||
response_api_optional_request_params={},
|
||||
litellm_params=litellm_params,
|
||||
headers={},
|
||||
)
|
||||
return request["input"]
|
||||
|
||||
|
||||
def _single_part(transformed_input: object) -> Mapping[str, object]:
|
||||
(item,) = cast(list[Mapping[str, object]], transformed_input)
|
||||
(part,) = cast(list[Mapping[str, object]], item["content"])
|
||||
return part
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS)
|
||||
@pytest.mark.parametrize("part_type", tuple(_PARTS))
|
||||
@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS)
|
||||
def test_malformed_prompt_cache_breakpoint_is_dropped_under_deployment_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig, part_type: str, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
transformed: Final = _transformed_input(
|
||||
config, _marked_message(part_type, marker), GenericLiteLLMParams(drop_params=True)
|
||||
)
|
||||
assert _single_part(transformed) == _PARTS[part_type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS)
|
||||
def test_malformed_prompt_cache_breakpoint_is_dropped_under_global_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
transformed: Final = _transformed_input(_CONFIGS[0], _marked_message("input_text", marker), GenericLiteLLMParams())
|
||||
assert _single_part(transformed) == _PARTS["input_text"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", (False, None), ids=("false", "null"))
|
||||
@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS)
|
||||
def test_malformed_prompt_cache_breakpoint_is_forwarded_verbatim_without_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch, marker: object, drop_params: bool | None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
input: Final = _marked_message("input_text", marker)
|
||||
assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=drop_params)) == input
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS)
|
||||
def test_a_non_flag_deployment_drop_params_keeps_the_marker_like_the_param_mapper_does(
|
||||
monkeypatch: pytest.MonkeyPatch, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
input: Final = _marked_message("input_text", marker)
|
||||
assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params="maybe")) == input
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep"))
|
||||
@pytest.mark.parametrize("marker", _VALID_MARKERS, ids=_VALID_MARKER_IDS)
|
||||
def test_valid_prompt_cache_breakpoint_reaches_the_wire_under_both_settings(
|
||||
monkeypatch: pytest.MonkeyPatch, marker: Mapping[str, str], drop_params: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
input: Final = _marked_message("input_text", marker)
|
||||
assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=drop_params)) == input
|
||||
|
||||
|
||||
def test_prompt_cache_breakpoint_unknown_key_is_trimmed_under_drop_params(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
input: Final = _marked_message("input_text", {"mode": "explicit", "note": "kept"})
|
||||
dropped: Final = _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True))
|
||||
assert _single_part(dropped)["prompt_cache_breakpoint"] == {"mode": "explicit"}
|
||||
assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=False)) == input
|
||||
|
||||
|
||||
def test_malformed_prompt_cache_breakpoint_on_a_tool_output_part_is_dropped_under_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
marked_output: Final = [{"type": "input_text", "text": "42", "prompt_cache_breakpoint": "yes"}]
|
||||
input: Final[list[dict[str, object]]] = [{"type": "function_call_output", "call_id": "call_1", "output": marked_output}]
|
||||
(item,) = cast(
|
||||
list[Mapping[str, object]], _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True))
|
||||
)
|
||||
assert item["output"] == [{"type": "input_text", "text": "42"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("input", ("hi", [{"type": "message", "role": "user", "content": "hi"}]), ids=("string", "unmarked"))
|
||||
def test_input_without_a_marker_is_untouched_under_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch, input: str | list[dict[str, object]]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) == input
|
||||
|
||||
|
||||
def _transformed_input_for(
|
||||
model: str,
|
||||
input: list[dict[str, object]],
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
config: BaseResponsesAPIConfig = _CONFIGS[0],
|
||||
) -> object:
|
||||
request: Final = config.transform_responses_api_request(
|
||||
model=model,
|
||||
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
|
||||
response_api_optional_request_params={},
|
||||
litellm_params=litellm_params,
|
||||
headers={},
|
||||
)
|
||||
return request["input"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"marker", (*_VALID_MARKERS, *_MALFORMED_MARKERS), ids=(*_VALID_MARKER_IDS, *_MALFORMED_MARKER_IDS)
|
||||
)
|
||||
def test_every_prompt_cache_breakpoint_is_dropped_under_drop_params_for_a_model_without_breakpoints(
|
||||
monkeypatch: pytest.MonkeyPatch, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
input: Final = _marked_message("input_text", marker)
|
||||
dropped: Final = _transformed_input_for("gpt-5.4-mini", input, GenericLiteLLMParams(drop_params=True))
|
||||
assert _single_part(dropped) == _PARTS["input_text"]
|
||||
assert _transformed_input_for("gpt-5.4-mini", input, GenericLiteLLMParams(drop_params=False)) == input
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
("o4-mini", "ft:gpt-4o-mini-2024-07-18:my-org::abc123", "ft:o4-mini-2025-04-16:my-org::abc123"),
|
||||
ids=("o-series", "fine-tune-of-older-gpt", "fine-tune-of-o-series"),
|
||||
)
|
||||
def test_a_valid_marker_is_dropped_under_drop_params_on_a_model_known_to_lack_breakpoints(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
input: Final = _marked_message("input_text", {"mode": "explicit"})
|
||||
dropped: Final = _transformed_input_for(model, input, GenericLiteLLMParams(drop_params=True))
|
||||
assert _single_part(dropped) == _PARTS["input_text"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS)
|
||||
def test_a_cost_map_entry_registered_under_the_deployment_name_keeps_a_valid_marker_under_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
registered: Final = {"id": "litellm-qa-sol-alias", "supports_vision": True, "input_cost_per_token": 1e-6}
|
||||
monkeypatch.setitem(litellm.model_cost, "litellm-qa-sol-alias", registered)
|
||||
valid: Final = _marked_message("input_text", {"mode": "explicit"})
|
||||
assert _transformed_input_for("litellm-qa-sol-alias", valid, GenericLiteLLMParams(drop_params=True), config) == valid
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS)
|
||||
def test_a_valid_marker_survives_drop_params_on_a_deployment_name_with_unknown_breakpoint_support(
|
||||
monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
litellm_params: Final = GenericLiteLLMParams(drop_params=True)
|
||||
valid: Final = _marked_message("input_text", {"mode": "explicit"})
|
||||
malformed: Final = _marked_message("input_text", {"mode": "explicit", "ttl": "1h"})
|
||||
assert _transformed_input_for("litellm-qa-sol-alias", valid, litellm_params, config) == valid
|
||||
dropped: Final = _transformed_input_for("litellm-qa-sol-alias", malformed, litellm_params, config)
|
||||
assert _single_part(dropped) == _PARTS["input_text"]
|
||||
|
||||
|
||||
def _declaring_base_model(where: str, base_model: str) -> GenericLiteLLMParams:
|
||||
if where == "litellm_params":
|
||||
return GenericLiteLLMParams(drop_params=True, base_model=base_model)
|
||||
return GenericLiteLLMParams(drop_params=True, model_info={"base_model": base_model})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("where", ("litellm_params", "model_info"))
|
||||
def test_a_declared_base_model_decides_whether_a_valid_marker_survives_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch, where: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
input: Final = _marked_message("input_text", {"mode": "explicit"})
|
||||
kept: Final = _transformed_input_for("gpt-5.4-mini-prod", input, _declaring_base_model(where, "gpt-6.1-sol"))
|
||||
assert kept == input
|
||||
dropped: Final = _transformed_input_for("prod-deployment", input, _declaring_base_model(where, "gpt-5.4-mini"))
|
||||
assert _single_part(dropped) == _PARTS["input_text"]
|
||||
|
||||
|
||||
def test_a_deployment_that_marks_cache_control_keeps_the_callers_own_marker_whatever_its_base_model_says(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
cached: Final = {"type": "input_text", "text": "a", "cache_control": {"type": "ephemeral"}}
|
||||
marked: Final = {"type": "input_text", "text": "b", "prompt_cache_breakpoint": {"mode": "explicit"}}
|
||||
input: Final = [{"type": "message", "role": "user", "content": [cached, marked]}]
|
||||
transformed: Final = _transformed_input_for("gpt-6.1-sol", input, _declaring_base_model("model_info", "gpt-5.5"))
|
||||
(item,) = cast(list[Mapping[str, object]], transformed)
|
||||
parts: Final = cast(list[Mapping[str, object]], item["content"])
|
||||
assert [part.get("prompt_cache_breakpoint") for part in parts] == [{"mode": "explicit"}, {"mode": "explicit"}]
|
||||
|
||||
|
||||
def test_a_model_that_takes_markers_sends_kept_parts_byte_for_byte_and_drops_only_the_rejected_marker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
kept: Final = [
|
||||
{"type": "input_text", "text": "a", "prompt_cache_breakpoint": None},
|
||||
{"prompt_cache_breakpoint": {"mode": "explicit"}, "type": "input_text", "text": "b"},
|
||||
]
|
||||
rejected: Final = {"type": "input_text", "text": "c"}
|
||||
bad_ttl: Final = {"prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}
|
||||
input: Final = [
|
||||
{"type": "message", "role": "user", "content": kept},
|
||||
{"type": "message", "role": "user", "content": [{**bad_ttl, **rejected}]},
|
||||
]
|
||||
transformed: Final = _transformed_input_for("gpt-6.1-sol", input, GenericLiteLLMParams(drop_params=True))
|
||||
assert json.dumps(transformed) == json.dumps(
|
||||
[{"type": "message", "role": "user", "content": kept}, {"type": "message", "role": "user", "content": [rejected]}]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("content", ("Say ok", [{"type": "input_text", "text": "Say ok"}]), ids=("string", "parts"))
|
||||
@pytest.mark.parametrize("marker", (_VALID_MARKERS[0], "yes"), ids=("explicit", "string"))
|
||||
def test_a_prompt_cache_breakpoint_on_the_message_item_is_dropped_under_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch, content: object, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
unmarked: Final[dict[str, object]] = {"type": "message", "role": "user", "content": content}
|
||||
input: Final = [{**unmarked, "prompt_cache_breakpoint": marker}]
|
||||
assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) == [unmarked]
|
||||
assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=False)) == input
|
||||
|
||||
|
||||
def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
part: Final = {"type": "input_text", "text": "Say ok", "cache_control": {"type": "ephemeral"}}
|
||||
input: Final = [{"type": "message", "role": "user", "content": [{**part, "prompt_cache_breakpoint": "yes"}]}]
|
||||
transformed: Final = _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True))
|
||||
assert _single_part(transformed) == {
|
||||
"type": "input_text",
|
||||
"text": "Say ok",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
}
|
||||
|
||||
|
||||
def _caller_input_after_transform(model: str, marker: object) -> str:
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker}
|
||||
input: Final = [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Say ok", "cache_control": hint, **marked},
|
||||
{"type": "input_text", "text": "Thanks", "cache_control": hint},
|
||||
],
|
||||
"cache_control": hint,
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}],
|
||||
},
|
||||
]
|
||||
_CONFIGS[0].transform_responses_api_request(
|
||||
model=model,
|
||||
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
|
||||
response_api_optional_request_params={},
|
||||
litellm_params=GenericLiteLLMParams(drop_params=True),
|
||||
headers={},
|
||||
)
|
||||
return json.dumps(input)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol"))
|
||||
@pytest.mark.parametrize("marker", (_VALID_MARKERS[0], "yes"), ids=("explicit", "malformed"))
|
||||
def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_unmarked_one_would(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
marker_json: Final = f', "prompt_cache_breakpoint": {json.dumps(marker)}'
|
||||
unmarked: Final = _caller_input_after_transform(model, None)
|
||||
assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures(httpx_transport.__name__)
|
||||
async def test_aresponses_forwards_previous_response_id_to_openai() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue