diff --git a/litellm/router_utils/prompt_caching_cache.py b/litellm/router_utils/prompt_caching_cache.py index 4ec0bc62e7f..f78b5982186 100644 --- a/litellm/router_utils/prompt_caching_cache.py +++ b/litellm/router_utils/prompt_caching_cache.py @@ -94,8 +94,10 @@ def _is_ephemeral_mark(mark: object) -> bool: return mark_map.get("type") == "ephemeral" -def _tool_carries_ephemeral_mark(tool: ChatCompletionToolParam) -> bool: - tool_map: Final = cast( # cast-ok: TypedDict.get rejects keys the type does not declare +def _tool_carries_ephemeral_mark(tool: object) -> bool: + if not isinstance(tool, dict): + return False + tool_map: Final = cast( # cast-ok: tools are plain JSON dicts Mapping[str, object], tool ) if _is_ephemeral_mark(tool_map.get("cache_control")): diff --git a/tests/unit/router_utils/test_prompt_caching_cache.py b/tests/unit/router_utils/test_prompt_caching_cache.py index 46af2454dea..ce8f04b8c54 100644 --- a/tests/unit/router_utils/test_prompt_caching_cache.py +++ b/tests/unit/router_utils/test_prompt_caching_cache.py @@ -608,6 +608,23 @@ def test_unmarked_tools_change_the_key_marked_tools_do_not_share(): assert key_a != key_b +def test_a_non_dict_tool_entry_still_derives_a_key_instead_of_raising(): + """ + A client can send `tools: ["not-a-tool"]`. The provider rejects that with its own 400, so the + routing check has to key on what it was given and let the request through, never raise a 500. + """ + messages = cast( + list[AllMessageValues], + [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}], + ) + tools: Final = cast(list[ChatCompletionToolParam], ["not-a-tool", _marked_tool("read", {"type": "ephemeral"})]) + + positions = PromptCachingCache.prefix_positions(messages, tools) + + assert [position.position for position in positions] == [0, 1] + assert PromptCachingCache.get_prompt_caching_cache_key(messages, tools) == positions[-1].cache_key + + def test_empty_and_missing_tools_seed_the_same_key(): """`tools=[]` and no tools at all are the same request to the provider, so they must key alike.""" messages = cast(