Merge pull request #33637 from BerriAI/litellm_lit4520_cache_min_tokens

fix(router): resolve prompt cache minimum per model instead of a flat 1024

litellm/constants.py

@@ -2,7 +2,7 @@ import os
 import sys
 from typing import List, Literal, Optional
 
-from litellm.litellm_core_utils.env_utils import get_env_int
+from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_none
 
 DEFAULT_HEALTH_CHECK_PROMPT = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
 AZURE_DEFAULT_RESPONSES_API_VERSION = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
@@ -269,9 +269,18 @@ TOOL_POLICY_CACHE_TTL_SECONDS = int(os.getenv("TOOL_POLICY_CACHE_TTL_SECONDS", 6
 MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8)))
 MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int(os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000))
 ###############################################################################################
-MINIMUM_PROMPT_CACHE_TOKEN_COUNT = int(
-    os.getenv("MINIMUM_PROMPT_CACHE_TOKEN_COUNT", 1024)
-)  # minimum number of tokens to cache a prompt by Anthropic
+# Providers will not cache a prefix below a minimum size. That minimum is per-model, not global:
+# Anthropic's ranges from 512 to 4096 depending on the model, and can differ per platform for the
+# same model. The real minimum is resolved from `prompt_cache_min_tokens` in the model cost map;
+# this value is only the fallback for models the cost map has no entry for, and doubles as a global
+# escape hatch when `MINIMUM_PROMPT_CACHE_TOKEN_COUNT` is explicitly set.
+MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE: int | None = get_env_int_or_none("MINIMUM_PROMPT_CACHE_TOKEN_COUNT")
+DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT = 1024
+MINIMUM_PROMPT_CACHE_TOKEN_COUNT = (
+    MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE
+    if MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE is not None
+    else DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT
+)
 DEFAULT_TRIM_RATIO = float(
     os.getenv("DEFAULT_TRIM_RATIO", 0.75)
 )  # default ratio of tokens to trim from the end of a prompt

litellm/litellm_core_utils/env_utils.py

@@ -19,3 +19,19 @@ def get_env_int(env_var: str, default: int) -> int:
         return int(raw)
     except (ValueError, TypeError):
         return default
+
+
+def get_env_int_or_none(env_var: str) -> int | None:
+    """Parse an environment variable as an integer, returning None when it is unset or unusable.
+
+    Use this instead of `get_env_int` when callers must distinguish "explicitly configured"
+    from "left at the default", for example when an override should take precedence over a
+    value resolved from somewhere else.
+    """
+    raw = os.getenv(env_var)
+    if raw is None:
+        return None
+    try:
+        return int(raw.strip())
+    except (ValueError, TypeError):
+        return None

litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py

@@ -8,14 +8,40 @@ from typing import List, Optional, cast
 
 from litellm import verbose_logger
 from litellm.caching.dual_cache import DualCache
+from litellm.constants import DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT
 from litellm.integrations.custom_logger import CustomLogger, Span
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import CallTypes, StandardLoggingPayload
-from litellm.utils import is_prompt_caching_valid_prompt
+from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt
 
 from ..prompt_caching_cache import PromptCachingCache
 
 
+def _get_min_token_count_for_deployments(healthy_deployments: list[dict]) -> int:
+    """
+    Returns the lowest minimum cacheable prefix across a model group.
+
+    This gate only decides whether the cache lookup is worth doing. It cannot cause a wrong pin,
+    because a deployment is only pinned when the cache already holds an entry for the prefix, and
+    entries are written by `async_log_success_event` against the deployment's real model. A model
+    that will not cache a prefix never records one, so there is nothing to pin it to.
+
+    That makes the lowest minimum in the group the correct threshold rather than the highest.
+    `model` here is the model-group alias the operator chose, not a model name, so the threshold
+    has to come from the deployments themselves, and a group may mix models whose minimums differ.
+    Taking the highest would skip the lookup for a prefix a lower-minimum member genuinely cached,
+    losing a cache hit it had earned. The lowest can only cost a lookup that finds nothing.
+    """
+    return min(
+        (
+            get_prompt_cache_min_tokens(model=deployment["litellm_params"]["model"])
+            for deployment in healthy_deployments
+            if deployment.get("litellm_params", {}).get("model")
+        ),
+        default=DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT,
+    )
+
+
 class PromptCachingDeploymentCheck(CustomLogger):
     def __init__(self, cache: DualCache):
         self.cache = cache
@@ -31,7 +57,8 @@ class PromptCachingDeploymentCheck(CustomLogger):
         if messages is not None and is_prompt_caching_valid_prompt(
             messages=messages,
             model=model,
-        ):  # prompt > 1024 tokens
+            min_token_count=_get_min_token_count_for_deployments(healthy_deployments),
+        ):
             prompt_cache = PromptCachingCache(
                 cache=self.cache,
             )

litellm/types/utils.py

@@ -197,6 +197,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
     cache_read_input_token_cost_above_272k_tokens: Optional[float]
     cache_read_input_token_cost_above_272k_tokens_priority: Optional[float]
     cache_read_input_token_cost_above_512k_tokens: Optional[float]
+    # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses.
+    # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT.
+    prompt_cache_min_tokens: Optional[int]
     input_cost_per_character: Optional[float]  # only for vertex ai models
     input_cost_per_audio_token: Optional[float]
     input_cost_per_token_above_128k_tokens: Optional[float]  # only for vertex ai models

litellm/utils.py

@@ -73,7 +73,8 @@ from litellm.constants import (
     JITTER,
     MAX_RETRY_DELAY,
     MAX_TOKEN_TRIMMING_ATTEMPTS,
-    MINIMUM_PROMPT_CACHE_TOKEN_COUNT,
+    DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT,
+    MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
     OPENAI_EMBEDDING_PARAMS,
     TOOL_CHOICE_OBJECT_TOKEN_COUNT,
 )
@@ -5402,6 +5403,7 @@ def _get_model_info_helper(
                     "cache_creation_input_token_cost_above_200k_tokens", None
                 ),
                 cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
+                prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
                 cache_read_input_token_cost_above_200k_tokens=_model_info.get(
                     "cache_read_input_token_cost_above_200k_tokens", None
                 ),
@@ -9039,16 +9041,46 @@ def should_use_cohere_v1_client(api_base: Optional[str], present_version_params:
     return api_base.endswith("/v1/rerank") or (uses_v1_params and not api_base.endswith("/v2/rerank"))
 
 
+def get_prompt_cache_min_tokens(model: str) -> int:
+    """
+    Returns the smallest prefix `model` will actually cache.
+
+    Resolution order is an explicitly configured `MINIMUM_PROMPT_CACHE_TOKEN_COUNT`, then the
+    model's `prompt_cache_min_tokens` in the cost map, then the provider-agnostic default. The
+    cost map is the source of truth because the real minimum is per-model and per-platform:
+    Anthropic's ranges from 512 to 4096 and moves in both directions across releases, and the
+    same model can differ by platform.
+
+    Never raises. An unresolvable model falls back to the default rather than propagating, so a
+    caller cannot mistake "no entry for this model" for "this prompt is not cacheable".
+    """
+    if MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE is not None:
+        return MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE
+    try:
+        min_tokens = get_model_info(model=model).get("prompt_cache_min_tokens")
+    except Exception:
+        return DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT
+    if min_tokens is None:
+        return DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT
+    return min_tokens
+
+
 def is_prompt_caching_valid_prompt(
     model: str,
     messages: Optional[List[AllMessageValues]],
     tools: Optional[List[ChatCompletionToolParam]] = None,
     custom_llm_provider: Optional[str] = None,
+    min_token_count: int | None = None,
 ) -> bool:
     """
     Returns true if the prompt is valid for prompt caching.
 
-    OpenAI + Anthropic providers have a minimum token count of 1024 for prompt caching.
+    The minimum cacheable prefix is per-model, so it is resolved from `model` unless the caller
+    passes `min_token_count`. Callers that only hold a model-group alias (the router's deployment
+    checks) must resolve the threshold themselves and pass it, because an alias resolves to
+    nothing here and would silently fall back to the default.
+
+    OpenAI's minimum is a flat 1024 across models, which the default already covers.
     """
     try:
         if messages is None and tools is None:
@@ -9061,7 +9093,9 @@ def is_prompt_caching_valid_prompt(
             model=model,
             use_default_image_token_count=True,
         )
-        return token_count >= MINIMUM_PROMPT_CACHE_TOKEN_COUNT
+        if min_token_count is None:
+            min_token_count = get_prompt_cache_min_tokens(model=model)
+        return token_count >= min_token_count
     except Exception as e:
         verbose_logger.error(f"Error in is_prompt_caching_valid_prompt: {e}")
         return False

tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py

@@ -0,0 +1,207 @@
+import os
+import sys
+from typing import List, cast
+
+import pytest
+
+sys.path.insert(0, os.path.abspath("../.."))
+
+import litellm
+from litellm.caching.dual_cache import DualCache
+from litellm.constants import DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT
+from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import (
+    PromptCachingDeploymentCheck,
+    _get_min_token_count_for_deployments,
+)
+from litellm.router_utils.prompt_caching_cache import PromptCachingCache
+from litellm.types.llms.openai import AllMessageValues
+from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt, token_counter
+
+MODEL_GROUP_ALIAS = "my-claude-group"
+OPUS_4_6_MIN_TOKENS = 4096
+
+
+@pytest.fixture(autouse=True)
+def local_model_cost_map(monkeypatch):
+    """
+    The remote cost map does not carry `prompt_cache_min_tokens` yet, so a test that reads the
+    default map would pass here and flake in CI. Force the in-repo map.
+
+    `get_model_info` is lru_cached, so swapping `model_cost` is not enough on its own: an earlier
+    test that resolved these models against the remote map leaves entries with no
+    `prompt_cache_min_tokens`, and the stale hit resolves to the default. Clear on the way out too,
+    so the entries these tests warm against the local map do not leak into later tests.
+    """
+    original_model_cost = litellm.model_cost
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+    litellm.get_model_info.cache_clear()
+    try:
+        yield
+    finally:
+        litellm.model_cost = original_model_cost
+        litellm.get_model_info.cache_clear()
+
+
+def _deployments(*models: str) -> List[dict]:
+    return [
+        {
+            "model_name": MODEL_GROUP_ALIAS,
+            "litellm_params": {"model": model},
+            "model_info": {"id": f"dep-{index}"},
+        }
+        for index, model in enumerate(models, start=1)
+    ]
+
+
+def _messages(word_count: int) -> List[AllMessageValues]:
+    return cast(
+        List[AllMessageValues],
+        [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "word " * word_count,
+                        "cache_control": {"type": "ephemeral"},
+                    }
+                ],
+            }
+        ],
+    )
+
+
+def test_get_min_token_count_for_deployments_takes_min_across_mixed_group():
+    """
+    A group may legally mix models whose real minimums differ, and one gate decides for every
+    member. The threshold must be the lowest minimum in the group. This gate only decides whether
+    the cache lookup happens, so taking the highest would skip the lookup for a prefix the Sonnet
+    4.5 deployment genuinely cached and lose a hit it had earned.
+    """
+    assert get_prompt_cache_min_tokens(model="anthropic/claude-opus-4-5") == 4096
+    assert get_prompt_cache_min_tokens(model="anthropic/claude-sonnet-4-5") == 1024
+
+    deployments = _deployments("anthropic/claude-opus-4-5", "anthropic/claude-sonnet-4-5")
+
+    assert _get_min_token_count_for_deployments(deployments) == 1024
+
+
+def test_write_gate_is_what_prevents_a_pin_below_the_model_minimum():
+    """
+    The invariant the read gate relies on. A deployment can only be pinned when the cache already
+    holds an entry for the prefix, and `async_log_success_event` writes entries against the real
+    deployment model. Opus 4.5 never records an entry for a prefix it will not cache, so no read
+    threshold is what keeps it from being pinned.
+    """
+    messages = _messages(word_count=1400)
+
+    token_count = token_counter(messages=messages, model="anthropic/claude-opus-4-5", use_default_image_token_count=True)
+    assert 1024 < token_count < 4096
+
+    assert is_prompt_caching_valid_prompt(model="anthropic/claude-opus-4-5", messages=messages) is False
+    assert is_prompt_caching_valid_prompt(model="anthropic/claude-sonnet-4-5", messages=messages) is True
+
+
+def test_get_min_token_count_for_deployments_falls_back_to_default_for_empty_group():
+    """An empty group has no member minimum to read, so it must fall back rather than crash."""
+    assert _get_min_token_count_for_deployments([]) == DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT
+
+
+@pytest.mark.asyncio
+async def test_async_filter_deployments_does_not_narrow_prompt_below_model_minimum():
+    """
+    The regression. Opus 4.6 will not cache a prefix under 4096 tokens, so a ~1400-token prompt is
+    not cacheable and routing must stay free across the whole group. Previously the check resolved
+    its threshold from `model`, which is the operator's group alias and matches nothing in the cost
+    map, silently fell back to 1024, judged this prompt cacheable, and pinned every request to one
+    deployment for a cache hit the provider was never going to serve.
+    """
+    cache = DualCache()
+    check = PromptCachingDeploymentCheck(cache=cache)
+    deployments = _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6")
+    messages = _messages(word_count=1400)
+
+    token_count = token_counter(messages=messages, model="anthropic/claude-opus-4-6", use_default_image_token_count=True)
+    assert DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT < token_count < OPUS_4_6_MIN_TOKENS
+
+    await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None)
+
+    filtered = await check.async_filter_deployments(
+        model=MODEL_GROUP_ALIAS,
+        healthy_deployments=deployments,
+        messages=messages,
+    )
+
+    assert filtered == deployments
+
+
+@pytest.mark.asyncio
+async def test_async_filter_deployments_narrows_prompt_above_model_minimum():
+    """
+    The positive control for the regression above: once the same group's prompt clears Opus 4.6's
+    real 4096-token minimum the prefix is genuinely cacheable, so the check must still pin the
+    deployment that served it. Proves the fix tightened the gate rather than disabling the feature.
+    """
+    cache = DualCache()
+    check = PromptCachingDeploymentCheck(cache=cache)
+    deployments = _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6")
+    messages = _messages(word_count=5000)
+
+    token_count = token_counter(messages=messages, model="anthropic/claude-opus-4-6", use_default_image_token_count=True)
+    assert token_count > OPUS_4_6_MIN_TOKENS
+
+    await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None)
+
+    filtered = await check.async_filter_deployments(
+        model=MODEL_GROUP_ALIAS,
+        healthy_deployments=deployments,
+        messages=messages,
+    )
+
+    assert filtered == [deployments[1]]
+
+
+@pytest.mark.asyncio
+async def test_async_filter_deployments_narrows_for_group_whose_model_minimum_is_lower():
+    """
+    Same ~1400-token prompt that must not pin an Opus 4.6 group, on an Opus 4.8 group whose real
+    minimum is 1024. Here the prefix is cacheable and the check must pin. Proves the threshold is
+    resolved per-model from the deployments rather than tightened for everyone.
+    """
+    cache = DualCache()
+    check = PromptCachingDeploymentCheck(cache=cache)
+    deployments = _deployments("anthropic/claude-opus-4-8", "anthropic/claude-opus-4-8")
+    messages = _messages(word_count=1400)
+
+    assert get_prompt_cache_min_tokens(model="anthropic/claude-opus-4-8") == DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT
+
+    await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None)
+
+    filtered = await check.async_filter_deployments(
+        model=MODEL_GROUP_ALIAS,
+        healthy_deployments=deployments,
+        messages=messages,
+    )
+
+    assert filtered == [deployments[1]]
+
+
+@pytest.mark.asyncio
+async def test_wildcard_route_resolves_underlying_model_minimum(local_model_cost_map):
+    from litellm import Router
+
+    router = Router(
+        model_list=[
+            {
+                "model_name": "anthropic/*",
+                "litellm_params": {"model": "anthropic/*", "api_key": "sk-fake"},
+                "model_info": {"id": "wild-1"},
+            }
+        ]
+    )
+
+    deployments = await router.async_get_healthy_deployments(model="anthropic/claude-opus-4-6", request_kwargs={})
+
+    assert deployments[0]["litellm_params"]["model"] == "anthropic/claude-opus-4-6"
+    assert _get_min_token_count_for_deployments(deployments) == 4096

tests/test_litellm/test_utils.py

@@ -26,7 +26,9 @@ from litellm.utils import (
     _is_streaming_request,
     get_llm_provider,
     get_optional_params_image_gen,
+    get_prompt_cache_min_tokens,
     is_cached_message,
+    is_prompt_caching_valid_prompt,
 )
 
 # Adds the parent directory to the system path
@@ -842,6 +844,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
                 "supports_parallel_function_calling": {"type": "boolean"},
                 "supports_parallel_tool_use_config": {"type": "boolean"},
                 "supports_pdf_input": {"type": "boolean"},
+                "prompt_cache_min_tokens": {"type": "number"},
                 "supports_prompt_caching": {"type": "boolean"},
                 "supports_response_schema": {"type": "boolean"},
                 "supports_system_messages": {"type": "boolean"},
@@ -4741,3 +4744,73 @@ def test_gemini_image_models_do_not_support_reasoning(
         f"{model} incorrectly classified as reasoning-capable. "
         "Add 'supports_reasoning: false' to its model_cost entry."
     )
+
+
+PROMPT_CACHE_MESSAGES = [{"role": "user", "content": "the quick brown fox jumps over the lazy dog " * 155}]
+
+
+@pytest.mark.parametrize(
+    "model, expected_min_tokens",
+    [
+        ("claude-opus-4-6", 4096),
+        ("claude-opus-4-7", 2048),
+        ("claude-opus-4-8", 1024),
+        ("claude-fable-5", 512),
+    ],
+)
+def test_get_prompt_cache_min_tokens_resolves_per_model(
+    model: str, expected_min_tokens: int, local_model_cost_map: None
+) -> None:
+    """The smallest cacheable prefix is a per-model property, read from the cost map's
+    prompt_cache_min_tokens. Anthropic's minimum spans 512..4096 across models and moves in both
+    directions across releases, so a single global constant is wrong for every model but one."""
+    assert get_prompt_cache_min_tokens(model=model) == expected_min_tokens
+
+
+def test_get_prompt_cache_min_tokens_differs_per_platform_for_same_model(local_model_cost_map: None) -> None:
+    """The same model can carry a different minimum per platform, so the threshold must come from
+    the platform's own cost-map entry rather than being derived from the model family name."""
+    assert get_prompt_cache_min_tokens(model="claude-fable-5") == 512
+    assert get_prompt_cache_min_tokens(model="anthropic.claude-fable-5") == 1024
+    assert get_prompt_cache_min_tokens(model="claude-fable-5") != get_prompt_cache_min_tokens(
+        model="anthropic.claude-fable-5"
+    )
+
+
+def test_get_prompt_cache_min_tokens_unmapped_model_falls_back_to_default(local_model_cost_map: None) -> None:
+    """get_model_info raises for a model it has no entry for. The resolver must swallow that and
+    fall back to the default, otherwise the raise reaches callers that would read it as
+    "not cacheable" -- turning an unknown model into a silently uncacheable one."""
+    assert get_prompt_cache_min_tokens(model="totally-unknown-model-xyz") == 1024
+
+
+def test_is_prompt_caching_valid_prompt_uses_per_model_minimum(local_model_cost_map: None) -> None:
+    """Regression: a prompt between two models' minimums is cacheable on one and not the other.
+    A 1403-token prompt clears claude-opus-4-8's 1024 minimum but not claude-opus-4-6's 4096, so
+    the flat-1024 check reported claude-opus-4-6 as cacheable and the cache write was rejected
+    upstream. Both assertions must live together: is_prompt_caching_valid_prompt returns False on
+    any internal error, so the True case is what proves the False case isn't a swallowed exception."""
+    token_count = litellm.token_counter(
+        model="claude-opus-4-6", messages=PROMPT_CACHE_MESSAGES, use_default_image_token_count=True
+    )
+    assert 1024 <= token_count < 4096, (
+        f"prompt drifted to {token_count} tokens; it must sit between claude-opus-4-8's 1024 minimum "
+        "and claude-opus-4-6's 4096 minimum for this test to distinguish them"
+    )
+
+    assert is_prompt_caching_valid_prompt(model="claude-opus-4-6", messages=PROMPT_CACHE_MESSAGES) is False
+    assert is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES) is True
+
+
+def test_is_prompt_caching_valid_prompt_explicit_min_token_count_overrides_model(local_model_cost_map: None) -> None:
+    """An explicit min_token_count wins over the model-resolved value in both directions. Callers
+    holding only a model-group alias resolve the threshold themselves and pass it, because an alias
+    resolves to nothing here and would silently fall back to the default."""
+    assert (
+        is_prompt_caching_valid_prompt(model="claude-opus-4-6", messages=PROMPT_CACHE_MESSAGES, min_token_count=512)
+        is True
+    )
+    assert (
+        is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES, min_token_count=8192)
+        is False
+    )