From 861e15daba1b69db8d6d8a8fe49fafd2d022ea2f Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Tue, 11 Aug 2026 01:25:46 +0530 Subject: [PATCH 1/2] feat(bedrock): make the IAM credential cache bounds env configurable BEDROCK_IAM_CACHE_MAX_ENTRIES and BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES were compile-time literals, so an operator whose live credential-key cardinality exceeds 1000 had no way to raise the bound short of patching the package. The cache then evicts and re-assumes, restoring the sts:AssumeRole volume the cache exists to remove. Both now use the os.getenv pattern the rest of constants.py already uses, with the current values as defaults. --- litellm/constants.py | 4 ++-- tests/test_litellm/test_constants.py | 33 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index f2ac96162eb..1b4ed46c468 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -374,10 +374,10 @@ MAX_STRING_LENGTH_PROMPT_IN_DB: Final = int(os.getenv("MAX_STRING_LENGTH_PROMPT_ BEDROCK_MAX_POLICY_SIZE: Final = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75)) # One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed # identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session. -BEDROCK_IAM_CACHE_MAX_ENTRIES: Final = 1000 +BEDROCK_IAM_CACHE_MAX_ENTRIES: Final = int(os.getenv("BEDROCK_IAM_CACHE_MAX_ENTRIES", 1000)) # Single-flight lock stripes over that cache. Only keys landing on the same stripe wait for each # other, so a burst of distinct identities still resolves its credentials in parallel. -BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES: Final = 64 +BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES: Final = int(os.getenv("BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES", 64)) # Retire a cached STS credential this many seconds before AWS expires it, so a request that reads it # still has a usable credential for the whole call. STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 diff --git a/tests/test_litellm/test_constants.py b/tests/test_litellm/test_constants.py index b3c13c6e26e..5e212f56df4 100644 --- a/tests/test_litellm/test_constants.py +++ b/tests/test_litellm/test_constants.py @@ -71,3 +71,36 @@ def _build_constant_env_var_map() -> dict[str, str]: env_var_map[constant_name] = env_var_name return env_var_map + + +@pytest.mark.parametrize( + "constant_name, override", + [ + ("BEDROCK_IAM_CACHE_MAX_ENTRIES", 4096), + ("BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES", 128), + ], +) +def test_bedrock_iam_cache_bounds_are_env_overridable(constant_name: str, override: int) -> None: + try: + with mock.patch.dict(os.environ, {constant_name: str(override)}): + reloaded = importlib.reload(constants) + assert getattr(reloaded, constant_name) == override + finally: + importlib.reload(constants) + + +@pytest.mark.parametrize( + "constant_name, expected_default", + [ + ("BEDROCK_IAM_CACHE_MAX_ENTRIES", 1000), + ("BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES", 64), + ], +) +def test_bedrock_iam_cache_bounds_keep_their_defaults(constant_name: str, expected_default: int) -> None: + try: + with mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop(constant_name, None) + reloaded = importlib.reload(constants) + assert getattr(reloaded, constant_name) == expected_default + finally: + importlib.reload(constants) From 6fd3e8973184f1f672acfdc911c3ff3703d1b19a Mon Sep 17 00:00:00 2001 From: Aryan Pardeshi Date: Tue, 11 Aug 2026 15:57:49 +0530 Subject: [PATCH 2/2] fix(lint): use string envvar defaults for the bedrock cache bounds PLW1508 flags os.getenv(name, ): the default now matches the pattern the rest of constants.py already uses. --- litellm/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 1b4ed46c468..809485eb9ed 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -374,10 +374,10 @@ MAX_STRING_LENGTH_PROMPT_IN_DB: Final = int(os.getenv("MAX_STRING_LENGTH_PROMPT_ BEDROCK_MAX_POLICY_SIZE: Final = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75)) # One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed # identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session. -BEDROCK_IAM_CACHE_MAX_ENTRIES: Final = int(os.getenv("BEDROCK_IAM_CACHE_MAX_ENTRIES", 1000)) +BEDROCK_IAM_CACHE_MAX_ENTRIES: Final = int(os.getenv("BEDROCK_IAM_CACHE_MAX_ENTRIES", "1000")) # Single-flight lock stripes over that cache. Only keys landing on the same stripe wait for each # other, so a burst of distinct identities still resolves its credentials in parallel. -BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES: Final = int(os.getenv("BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES", 64)) +BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES: Final = int(os.getenv("BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES", "64")) # Retire a cached STS credential this many seconds before AWS expires it, so a request that reads it # still has a usable credential for the whole call. STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60