mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
14b4e9c0c3
Derive the router affinity cache TTL from the cacheable prompt prefix so one-hour ephemeral cache hints stay routable for their advertised lifetime instead of expiring after the previous five-minute default. Related: BerriAI/litellm#28427 Tested: python -m pytest tests/router_unit_tests/test_router_prompt_caching.py -q; python -m pytest tests/test_litellm/test_prompt_caching_cache.py -q; python -m ruff check litellm/router_utils/prompt_caching_cache.py tests/router_unit_tests/test_router_prompt_caching.py tests/test_litellm/test_prompt_caching_cache.py; python -m black litellm/router_utils/prompt_caching_cache.py tests/router_unit_tests/test_router_prompt_caching.py tests/test_litellm/test_prompt_caching_cache.py Co-authored-by: OmX <omx@oh-my-codex.dev>
19 lines
625 B
Python
19 lines
625 B
Python
from litellm.router_utils.prompt_caching_cache import PromptCachingCache
|
|
|
|
|
|
def test_prompt_caching_affinity_ttl_respects_one_hour_cache_control():
|
|
messages = [
|
|
{
|
|
"role": "system",
|
|
"content": [
|
|
{
|
|
"type": "text",
|
|
"text": "Long-lived prompt cache prefix",
|
|
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
|
}
|
|
],
|
|
},
|
|
{"role": "user", "content": "This is outside the cacheable prefix"},
|
|
]
|
|
|
|
assert PromptCachingCache.get_prompt_caching_ttl_seconds(messages) == 3600
|