From 811fa6e52b4731b13802cdca6b763da9bf215d66 Mon Sep 17 00:00:00 2001 From: 16niraj <1niraj607@gmail.com> Date: Tue, 20 Oct 2026 18:22:25 +0530 Subject: [PATCH 1/3] feat(caching): remove embeddings from default supported_call_types to prevent Redis exhaustion Fixes #46520. Embeddings cache per-input by default, which fills up shared Redis instances with unique strings and evicts critical rate-limit and spend-tracking keys. This removes them from the default list while allowing users to opt-in via cache_params. --- litellm/caching/caching.py | 2 ++ litellm/types/caching.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 5f5b8210096..c7a0372ac85 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -108,6 +108,8 @@ class Cache: default_in_memory_ttl: float | None = None, default_in_redis_ttl: float | None = None, similarity_threshold: float | None = None, + # Note: embeddings are excluded by default to prevent per-input caching from exhausting shared Redis/memory limits. + # Users can opt-in by passing `supported_call_types=["embedding", "aembedding"]` via cache_params. supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES), # s3 Bucket, boto3 configuration azure_account_url: str | None = None, diff --git a/litellm/types/caching.py b/litellm/types/caching.py index b9bcbbb93dd..ee729d75926 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -44,8 +44,6 @@ CachingSupportedCallTypes = Literal[ DEFAULT_CACHING_SUPPORTED_CALL_TYPES: tuple[CachingSupportedCallTypes, ...] = ( "completion", "acompletion", - "embedding", - "aembedding", "atranscription", "transcription", "atext_completion", From 26ac93887d3c08acf1e5406a125c5f958ec0518c Mon Sep 17 00:00:00 2001 From: 16niraj <1niraj607@gmail.com> Date: Sun, 25 Oct 2026 09:36:47 +0530 Subject: [PATCH 2/3] feat(caching): support excluded_call_types and call_type_ttl in Cache (#46520) --- litellm/caching/caching.py | 44 ++++++++++++++++----- litellm/types/caching.py | 2 + tests/unit/caching/test_caching.py | 63 ++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 9 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index c7a0372ac85..4288fab75a3 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -108,9 +108,9 @@ class Cache: default_in_memory_ttl: float | None = None, default_in_redis_ttl: float | None = None, similarity_threshold: float | None = None, - # Note: embeddings are excluded by default to prevent per-input caching from exhausting shared Redis/memory limits. - # Users can opt-in by passing `supported_call_types=["embedding", "aembedding"]` via cache_params. supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES), + excluded_call_types: list[CachingSupportedCallTypes] | None = None, + call_type_ttl: dict[str, float] | None = None, # s3 Bucket, boto3 configuration azure_account_url: str | None = None, azure_blob_container: str | None = None, @@ -316,9 +316,17 @@ class Cache: litellm.logging_callback_manager.add_litellm_success_callback("cache") if "cache" not in litellm._async_success_callback: litellm.logging_callback_manager.add_litellm_async_success_callback("cache") - self.supported_call_types = ( - supported_call_types # default to ["completion", "acompletion", "embedding", "aembedding"] - ) + if excluded_call_types and supported_call_types is not None: + excluded_set: Final = set(excluded_call_types) + self.supported_call_types: list[CachingSupportedCallTypes] | None = [ + ct for ct in supported_call_types if ct not in excluded_set + ] + else: + self.supported_call_types = ( + supported_call_types # default to ["completion", "acompletion", "embedding", "aembedding"] + ) + self.excluded_call_types: list[CachingSupportedCallTypes] | None = excluded_call_types + self.call_type_ttl: dict[str, float] | None = call_type_ttl self.type = type self.namespace = namespace self.redis_flush_size = redis_flush_size @@ -709,6 +717,20 @@ class Cache: print_verbose(f"An exception occurred: {traceback.format_exc()}") return None + def _get_ttl_for_call_type(self, kwargs: Mapping[str, object]) -> float | None: + if self.call_type_ttl: + call_type: Final = kwargs.get("call_type") or ( + "embedding" if kwargs.get("embedding") is True else None + ) + if isinstance(call_type, str): + if call_type in self.call_type_ttl: + return self.call_type_ttl[call_type] + if call_type in ("embedding", "aembedding"): + emb_ttl: Final = self.call_type_ttl.get("embedding") or self.call_type_ttl.get("aembedding") + if emb_ttl is not None: + return emb_ttl + return self.ttl + def _add_cache_logic(self, result, **kwargs): """ Common implementation across sync + async add_cache functions @@ -723,8 +745,9 @@ class Cache: result = result.model_dump_json(exclude_unset=True) ## DEFAULT TTL ## - if self.ttl is not None: - kwargs["ttl"] = self.ttl + resolved_ttl: Final = self._get_ttl_for_call_type(kwargs) + if resolved_ttl is not None: + kwargs["ttl"] = resolved_ttl ## Get Cache-Controls ## _cache_kwargs: Final = kwargs.get("cache", None) if isinstance(_cache_kwargs, dict): @@ -913,6 +936,8 @@ class Cache: prompt_tokens_details=prompt_tokens_details, ) + if "call_type" not in kwargs and "embedding" not in kwargs: + kwargs["embedding"] = True cache_key, cached_data, kwargs = self._add_cache_logic( result=embedding_dict, **kwargs, @@ -940,8 +965,9 @@ class Cache: return # set default ttl if not set - if self.ttl is not None: - kwargs["ttl"] = self.ttl + pipeline_ttl: Final = self._get_ttl_for_call_type({**kwargs, "embedding": True}) + if pipeline_ttl is not None: + kwargs["ttl"] = pipeline_ttl cache_list: Final = [] if isinstance(kwargs["input"], list): diff --git a/litellm/types/caching.py b/litellm/types/caching.py index ee729d75926..b9bcbbb93dd 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -44,6 +44,8 @@ CachingSupportedCallTypes = Literal[ DEFAULT_CACHING_SUPPORTED_CALL_TYPES: tuple[CachingSupportedCallTypes, ...] = ( "completion", "acompletion", + "embedding", + "aembedding", "atranscription", "transcription", "atext_completion", diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 91083921130..628245ce2dd 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -1566,3 +1566,66 @@ def test_cache_key_separates_model_ids_unless_a_model_group_owns_the_key() -> No assert key_a != bare assert key_a == key_a_again assert grouped_a == grouped_b + + +def test_cache_excluded_call_types_filters_defaults() -> None: + cache: Final = Cache( + type=LiteLLMCacheType.LOCAL, + excluded_call_types=["embedding", "aembedding"], + ) + assert cache.supported_call_types is not None + assert "embedding" not in cache.supported_call_types + assert "aembedding" not in cache.supported_call_types + assert "completion" in cache.supported_call_types + assert "acompletion" in cache.supported_call_types + + +@pytest.mark.asyncio +async def test_cache_call_type_ttl_overrides_default_ttl_for_embeddings() -> None: + from litellm.types.utils import EmbeddingResponse + + cache: Final = Cache( + type=LiteLLMCacheType.LOCAL, + ttl=3600, + call_type_ttl={"embedding": 60}, + ) + response: Final = EmbeddingResponse( + model="text-embedding-3-small", + data=[{"embedding": [0.1, 0.2], "index": 0, "object": "embedding"}], + ) + _, _, updated_kwargs = cache.add_embedding_response_to_cache( + result=response, + input="hello world", + kwargs={"model": "text-embedding-3-small"}, + idx_in_result_data=0, + ) + assert updated_kwargs["ttl"] == 60 + + # Explicit per-request cache={"ttl": ...} still overrides call_type_ttl + _, _, override_kwargs = cache.add_embedding_response_to_cache( + result=response, + input="hello world", + kwargs={"model": "text-embedding-3-small", "cache": {"ttl": 15}}, + idx_in_result_data=0, + ) + assert override_kwargs["ttl"] == 15 + + # Pipeline bulk write also forwards call_type_ttl + mock_backend: Final = AsyncMock() + await cache.async_add_cache_pipeline( + result=response, + dynamic_cache_object=mock_backend, + model="text-embedding-3-small", + input=["hello world"], + ) + mock_backend.async_set_cache_pipeline.assert_awaited_once() + assert mock_backend.async_set_cache_pipeline.call_args.kwargs["ttl"] == 60 + + # Non-embedding calls still use default ttl (3600) + _, _, completion_kwargs = cache._add_cache_logic( + result="cached-completion", + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + call_type="completion", + ) + assert completion_kwargs["ttl"] == 3600 From b3af5f8c0fe7e6110fc2e3f23792abd7bcbab88b Mon Sep 17 00:00:00 2001 From: 16niraj <1niraj607@gmail.com> Date: Tue, 27 Oct 2026 03:03:53 +0530 Subject: [PATCH 3/3] Fix Ruff formatting --- litellm/caching/caching.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 4288fab75a3..1a1de3eeab3 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -719,9 +719,7 @@ class Cache: def _get_ttl_for_call_type(self, kwargs: Mapping[str, object]) -> float | None: if self.call_type_ttl: - call_type: Final = kwargs.get("call_type") or ( - "embedding" if kwargs.get("embedding") is True else None - ) + call_type: Final = kwargs.get("call_type") or ("embedding" if kwargs.get("embedding") is True else None) if isinstance(call_type, str): if call_type in self.call_type_ttl: return self.call_type_ttl[call_type]