From fe0eee64512ff1832b227dde5a1cd0649ce4a5bd Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 17 Sep 2026 03:02:42 +0000 Subject: [PATCH] fix(anthropic): keep prompt cache prediction supported for queue-bounded deployments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/anthropic/prompt_cache_prediction.py | 1 + .../anthropic/test_anthropic_prompt_cache_prediction.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index e69a02bd93a..a0ce5bf0360 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -50,6 +50,7 @@ _DEPLOYMENT_OPTIONS: Final = frozenset( "max_retries", "num_retries", "max_parallel_requests", + "max_parallel_requests_queue_size", "input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost", diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py index 2b36866a1a0..c13217a0d46 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -181,6 +181,15 @@ async def test_environment_credential_matches_native_count_and_observed_scope( assert observed.scope == cache_scope(_CALLER, _DEPLOYMENT, target.api_key, target.model) +def test_deployment_concurrency_knobs_keep_native_prediction_supported() -> None: + target: Final = resolve_prediction_target(LiteLLM_Params( + model=f"anthropic/{_MODEL}", api_key=_KEY, api_base="https://api.anthropic.com", + max_parallel_requests=1, max_parallel_requests_queue_size=0, + )) + assert isinstance(target, NativePredictionTarget) + assert (target.model, target.api_key) == (_MODEL, _KEY) + + @pytest.mark.parametrize("inline_key", [None, _KEY]) @pytest.mark.asyncio async def test_named_credential_is_explicitly_unsupported_before_count(