mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
feat(anthropic): allow enabling prompt caching via environment variables
Both enable_anthropic_prompt_caching and anthropic_prompt_caching_ttl are now read from LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING and LITELLM_ANTHROPIC_PROMPT_CACHING_TTL at import, so the flag can be turned on without a config file. An unsupported ttl falls back to the provider default rather than reaching the provider verbatim
This commit is contained in:
parent
04afc962b1
commit
f7a3e22b22
2 changed files with 57 additions and 2 deletions
|
|
@ -315,8 +315,11 @@ disable_token_counter: bool = False
|
|||
disable_add_transform_inline_image_block: bool = False
|
||||
disable_add_user_agent_to_request_tags: bool = False
|
||||
disable_anthropic_gemini_context_caching_transform: bool = False
|
||||
enable_anthropic_prompt_caching: bool = False
|
||||
anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = None
|
||||
enable_anthropic_prompt_caching: bool = os.getenv("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", "false").lower() == "true"
|
||||
_anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL")
|
||||
anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
|
||||
"1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None
|
||||
)
|
||||
disable_vertex_batch_output_transformation: bool = False
|
||||
extra_spend_tag_headers: Optional[List[str]] = None
|
||||
in_memory_llm_clients_cache: "LLMClientCache"
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@ import copy
|
|||
import datetime
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import unittest
|
||||
from typing import List, Optional, Tuple
|
||||
from unittest.mock import ANY, MagicMock, Mock, patch
|
||||
|
|
@ -1669,3 +1671,53 @@ class TestEnableAnthropicPromptCaching:
|
|||
|
||||
assert result_sys == "sys"
|
||||
assert result_msgs == messages
|
||||
|
||||
|
||||
class TestAnthropicPromptCachingEnvVars:
|
||||
"""Both settings are read from the environment at import, so an admin can enable
|
||||
auto-caching without a config file. Each case re-imports litellm in a subprocess
|
||||
so the env is read fresh without contaminating this process's module graph.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _import_litellm_with_env(env_override: dict) -> Tuple[bool, Optional[str]]:
|
||||
env = os.environ.copy()
|
||||
env.pop("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", None)
|
||||
env.pop("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL", None)
|
||||
env.update(env_override)
|
||||
script = textwrap.dedent(
|
||||
"""
|
||||
import json, litellm
|
||||
print(json.dumps([litellm.enable_anthropic_prompt_caching, litellm.anthropic_prompt_caching_ttl]))
|
||||
"""
|
||||
)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script], capture_output=True, text=True, env=env, timeout=300
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
enabled, ttl = json.loads(result.stdout.strip().splitlines()[-1])
|
||||
return enabled, ttl
|
||||
|
||||
def test_unset_env_leaves_auto_caching_off(self):
|
||||
assert self._import_litellm_with_env({}) == (False, None)
|
||||
|
||||
@pytest.mark.parametrize("value", ["true", "True", "TRUE"])
|
||||
def test_env_enables_auto_caching_case_insensitively(self, value):
|
||||
enabled, _ = self._import_litellm_with_env({"LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING": value})
|
||||
assert enabled is True
|
||||
|
||||
@pytest.mark.parametrize("value", ["false", "0", "yes", ""])
|
||||
def test_env_only_enables_on_true(self, value):
|
||||
enabled, _ = self._import_litellm_with_env({"LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING": value})
|
||||
assert enabled is False
|
||||
|
||||
@pytest.mark.parametrize("value", ["5m", "1h"])
|
||||
def test_ttl_env_is_applied(self, value):
|
||||
_, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value})
|
||||
assert ttl == value
|
||||
|
||||
@pytest.mark.parametrize("value", ["10m", "1H", "3600", "ephemeral"])
|
||||
def test_unsupported_ttl_env_falls_back_to_provider_default(self, value):
|
||||
"""An unparseable TTL must fall back to Anthropic's 5m default, never reach the provider verbatim."""
|
||||
_, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value})
|
||||
assert ttl is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue