mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-11 22:51:28 +00:00
fix(responses): extract cache_control_injection_points from extra_body
When cache_control_injection_points is passed via extra_body in the Responses API, the prompt management hook system cannot find it because it looks for top-level kwargs, not params nested inside extra_body. Extract recognised DynamicPromptManagementParam keys from extra_body into kwargs before the hook dispatch runs, in both the sync and async Responses API paths. Fixes #25335
This commit is contained in:
parent
97f722f558
commit
a9cd368629
2 changed files with 70 additions and 0 deletions
|
|
@ -80,6 +80,25 @@ def _has_file_search_tool(tools: Optional[Any]) -> bool:
|
|||
return any(isinstance(t, dict) and t.get("type") == "file_search" for t in tools)
|
||||
|
||||
|
||||
def _extract_dynamic_params_from_extra_body(
|
||||
extra_body: Dict[str, Any], kwargs: Dict[str, Any]
|
||||
) -> None:
|
||||
"""Extract dynamic prompt management params from extra_body into kwargs.
|
||||
|
||||
The Responses API receives params like ``cache_control_injection_points``
|
||||
inside ``extra_body``, but the prompt management hook system expects them
|
||||
as top-level kwargs. Move recognised keys so the hooks can find them.
|
||||
"""
|
||||
if not isinstance(extra_body, dict):
|
||||
return
|
||||
|
||||
from litellm.types.utils import DynamicPromptManagementParamLiteral
|
||||
|
||||
for param in DynamicPromptManagementParamLiteral.list_all_params():
|
||||
if param in extra_body and param not in kwargs:
|
||||
kwargs[param] = extra_body.pop(param)
|
||||
|
||||
|
||||
def mock_responses_api_response(
|
||||
mock_response: str = "In a peaceful grove beneath a silver moon, a unicorn named Lumina discovered a hidden pool that reflected the stars. As she dipped her horn into the water, the pool began to shimmer, revealing a pathway to a magical realm of endless night skies. Filled with wonder, Lumina whispered a wish for all who dream to find their own hidden magic, and as she glanced back, her hoofprints sparkled like stardust.",
|
||||
):
|
||||
|
|
@ -484,6 +503,10 @@ async def aresponses(
|
|||
prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None))
|
||||
original_model = model
|
||||
|
||||
# Extract dynamic prompt management params from extra_body
|
||||
if extra_body:
|
||||
_extract_dynamic_params_from_extra_body(extra_body, kwargs)
|
||||
|
||||
if isinstance(
|
||||
litellm_logging_obj, LiteLLMLoggingObj
|
||||
) and litellm_logging_obj.should_run_prompt_management_hooks(
|
||||
|
|
@ -783,6 +806,11 @@ def responses(
|
|||
local_vars=local_vars,
|
||||
)
|
||||
|
||||
# Extract dynamic prompt management params (e.g. cache_control_injection_points)
|
||||
# from extra_body into kwargs so the hook system can find them.
|
||||
if extra_body:
|
||||
_extract_dynamic_params_from_extra_body(extra_body, kwargs)
|
||||
|
||||
#########################################################
|
||||
# PROMPT MANAGEMENT
|
||||
# If aresponses() already ran the async hook, it pops prompt_id and
|
||||
|
|
|
|||
|
|
@ -1073,3 +1073,45 @@ async def test_anthropic_cache_control_hook_string_negative_index():
|
|||
f"Expected cachePoint in last message content, got: {last_message_content}. "
|
||||
"String index '-1' was not parsed correctly (str.isdigit() returns False for negative strings)."
|
||||
)
|
||||
|
||||
|
||||
def test_extract_dynamic_params_from_extra_body():
|
||||
"""
|
||||
Verify that cache_control_injection_points inside extra_body gets
|
||||
extracted into kwargs so the prompt-management hook system can find it.
|
||||
|
||||
Regression test for https://github.com/BerriAI/litellm/issues/25335
|
||||
"""
|
||||
from litellm.responses.main import _extract_dynamic_params_from_extra_body
|
||||
|
||||
injection_points = [
|
||||
{"location": "message", "role": "system", "index": 0},
|
||||
{"location": "message", "role": "user", "index": -1},
|
||||
]
|
||||
|
||||
extra_body: dict = {"cache_control_injection_points": injection_points, "other_key": "keep"}
|
||||
kwargs: dict = {}
|
||||
|
||||
_extract_dynamic_params_from_extra_body(extra_body, kwargs)
|
||||
|
||||
# cache_control_injection_points should be moved to kwargs
|
||||
assert kwargs["cache_control_injection_points"] == injection_points
|
||||
# should be removed from extra_body
|
||||
assert "cache_control_injection_points" not in extra_body
|
||||
# other keys should remain
|
||||
assert extra_body["other_key"] == "keep"
|
||||
|
||||
|
||||
def test_extract_dynamic_params_does_not_overwrite_existing_kwargs():
|
||||
"""If kwargs already has the key, extra_body should not overwrite it."""
|
||||
from litellm.responses.main import _extract_dynamic_params_from_extra_body
|
||||
|
||||
extra_body: dict = {"cache_control_injection_points": [{"from": "extra_body"}]}
|
||||
kwargs: dict = {"cache_control_injection_points": [{"from": "kwargs"}]}
|
||||
|
||||
_extract_dynamic_params_from_extra_body(extra_body, kwargs)
|
||||
|
||||
# kwargs value should remain unchanged
|
||||
assert kwargs["cache_control_injection_points"] == [{"from": "kwargs"}]
|
||||
# extra_body should still have it since it wasn't consumed
|
||||
assert "cache_control_injection_points" in extra_body
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue