diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index a4d37810f82..435506831dc 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -65,6 +65,7 @@ from litellm.types.llms.openai import ( OpenAIMessageContentListBlock, ) from litellm.types.utils import ( + CacheCreationTokenDetails, ChatCompletionMessageToolCall, CompletionTokensDetailsWrapper, Function, @@ -1807,6 +1808,26 @@ class AmazonConverseConfig(BaseConfig): thinking_blocks_list.append(_redacted_block) return thinking_blocks_list + @staticmethod + def _parse_cache_details(usage: ConverseTokenUsageBlock) -> "CacheCreationTokenDetails | None": + """Split ``cacheDetails`` into 5m/1h buckets, or ``None`` unless the split fully + accounts for ``cacheWriteInputTokens``, since a partial or unrecognized-ttl + breakdown would understate the cache-write cost. + + https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_CacheDetail.html + """ + cache_details: Final = usage.get("cacheDetails") + if not cache_details: + return None + tokens_5m: Final = sum(d["inputTokens"] for d in cache_details if d.get("ttl") == "5m") + tokens_1h: Final = sum(d["inputTokens"] for d in cache_details if d.get("ttl") == "1h") + if tokens_5m + tokens_1h != usage.get("cacheWriteInputTokens", 0): + return None + return CacheCreationTokenDetails( + ephemeral_5m_input_tokens=tokens_5m, + ephemeral_1h_input_tokens=tokens_1h, + ) + @staticmethod def thinking_tokens_from_additional_fields(additional_fields: object) -> int | None: """Converse omits thinking tokens from its usage block; they only arrive under @@ -1878,6 +1899,7 @@ class AmazonConverseConfig(BaseConfig): prompt_tokens_details: Final = PromptTokensDetailsWrapper( cached_tokens=cache_read_input_tokens, cache_creation_tokens=cache_creation_input_tokens, + cache_creation_token_details=self._parse_cache_details(usage), text_tokens=raw_input_tokens, ) estimated_reasoning_tokens: Final = ( diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index ed23db597ae..bed0ba3dc08 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -221,14 +221,22 @@ class ConverseResponseOutputBlock(TypedDict): message: MessageBlock | None -class ConverseTokenUsageBlock(TypedDict): - inputTokens: int - outputTokens: int - totalTokens: int - cacheReadInputTokenCount: int - cacheReadInputTokens: int - cacheWriteInputTokenCount: int - cacheWriteInputTokens: int +class CacheDetailBlock(TypedDict): + """Per-TTL cache-write breakdown, read-only AWS response data. https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_CacheDetail.html""" + + inputTokens: ReadOnly[int] + ttl: ReadOnly[Literal["5m", "1h"]] + + +class ConverseTokenUsageBlock(TypedDict, total=False): + inputTokens: Required[ReadOnly[int]] + outputTokens: Required[ReadOnly[int]] + totalTokens: Required[ReadOnly[int]] + cacheReadInputTokenCount: ReadOnly[int] + cacheReadInputTokens: ReadOnly[int] + cacheWriteInputTokenCount: ReadOnly[int] + cacheWriteInputTokens: ReadOnly[int] + cacheDetails: ReadOnly[list[CacheDetailBlock]] # mutable-ok: AWS response array, never mutated after parsing class ServiceTierBlock(TypedDict): diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index c5ab6027f45..f7f569ec14e 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -47,6 +47,97 @@ def test_transform_usage(): assert openai_usage.completion_tokens_details.text_tokens == usage["outputTokens"] +def test_transform_usage_with_cache_details(): + """cacheDetails should split cacheWriteInputTokens into the 5m/1h TTL breakdown + so cost calc can bill the 1h portion at its own (higher) rate instead of + defaulting the whole write to the 5m rate. See issue #36760.""" + usage = ConverseTokenUsageBlock( + **{ + "inputTokens": 76, + "outputTokens": 259, + "totalTokens": 335, + "cacheWriteInputTokens": 362, + "cacheDetails": [ + {"inputTokens": 74, "ttl": "1h"}, + {"inputTokens": 288, "ttl": "5m"}, + ], + } + ) + config = AmazonConverseConfig() + openai_usage = config.transform_usage(usage) + details = openai_usage.prompt_tokens_details.cache_creation_token_details + assert details is not None + assert details.ephemeral_1h_input_tokens == 74 + assert details.ephemeral_5m_input_tokens == 288 + + +def test_transform_usage_with_mismatched_cache_details_falls_back(): + """An unrecognized ttl or partial breakdown must not silently understate + cache-write cost, so the split is only used when it fully accounts for + cacheWriteInputTokens.""" + usage = ConverseTokenUsageBlock( + **{ + "inputTokens": 76, + "outputTokens": 259, + "totalTokens": 335, + "cacheWriteInputTokens": 362, + "cacheDetails": [{"inputTokens": 74, "ttl": "1h"}], # missing the 5m entry + } + ) + config = AmazonConverseConfig() + openai_usage = config.transform_usage(usage) + assert ( + getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) + is None + ) + + +def test_transform_usage_without_cache_details_stays_none(): + """No cacheDetails in the response (older models/regions) should leave + cache_creation_token_details unset, same as before this field existed.""" + usage = ConverseTokenUsageBlock( + **{ + "inputTokens": 3, + "outputTokens": 401, + "totalTokens": 2193, + "cacheWriteInputTokens": 1789, + } + ) + config = AmazonConverseConfig() + openai_usage = config.transform_usage(usage) + assert ( + getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) + is None + ) + + +def test_bedrock_converse_1h_cache_write_billed_at_1h_rate(monkeypatch): + """Regression for issue #36760: without the cacheDetails split, the whole + write is billed at the (cheaper) 5m rate.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + usage = ConverseTokenUsageBlock( + **{ + "inputTokens": 16, + "outputTokens": 4, + "totalTokens": 11652, + "cacheReadInputTokens": 0, + "cacheWriteInputTokens": 11632, + "cacheDetails": [{"inputTokens": 11632, "ttl": "1h"}], + } + ) + openai_usage = AmazonConverseConfig().transform_usage(usage) + model = "bedrock/converse/global.anthropic.claude-opus-4-8" + prompt_cost, completion_cost = litellm.cost_calculator.cost_per_token(model=model, usage_object=openai_usage) + model_info = litellm.get_model_info(model=model) + expected_prompt_cost = ( + 16 * model_info["input_cost_per_token"] + 11632 * model_info["cache_creation_input_token_cost_above_1hr"] + ) + assert prompt_cost == pytest.approx(expected_prompt_cost) + assert prompt_cost > 16 * model_info["input_cost_per_token"] + 11632 * model_info["cache_creation_input_token_cost"] + assert completion_cost == pytest.approx(4 * model_info["output_cost_per_token"]) + + def test_transform_usage_with_reasoning_content(): """Test that completion_tokens_details correctly tracks reasoning vs text tokens.""" usage = ConverseTokenUsageBlock(