diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index ff2c3491cb4..c6fcb28d194 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -193,6 +193,11 @@ def _usage_of(kwargs: Mapping[str, object], response_obj: object) -> Usage | Non return socket_usage if socket_usage is not None else get_usage_object(response_obj) +def _cache_read_tokens(usage: Usage) -> int: + details: Final = usage.prompt_tokens_details + return (details.cached_tokens or 0) if details is not None else 0 + + def _settled_tokens(usage: Usage | None, rate_limit_type: str) -> int: if usage is None: return 0 @@ -200,9 +205,9 @@ def _settled_tokens(usage: Usage | None, rate_limit_type: str) -> int: case "output": return usage.completion_tokens case "input": - return usage.prompt_tokens + return max(0, usage.prompt_tokens - _cache_read_tokens(usage)) case _: - return usage.total_tokens + return max(0, usage.total_tokens - _cache_read_tokens(usage)) def model_limit_at_saturation(kind: BorrowerKind) -> str: diff --git a/litellm/rust/crates/callbacks-parallel-request-limiter/src/logged.rs b/litellm/rust/crates/callbacks-parallel-request-limiter/src/logged.rs index 4116563b5cf..464664dabeb 100644 --- a/litellm/rust/crates/callbacks-parallel-request-limiter/src/logged.rs +++ b/litellm/rust/crates/callbacks-parallel-request-limiter/src/logged.rs @@ -232,6 +232,8 @@ pub fn aggregate_only(usage: Option<&UsageObject>) -> i64 { /// `resolve_reconciled_usage` of a usage object: its prompt, completion and cache-read counts, /// cache reads still inside the prompt, or None when it reports neither prompt nor completion. +/// A dict in Anthropic's own shape reports `input_tokens` without its cache reads and cache +/// writes, so its prompt is the sum of the three. pub fn reconciled(usage: Option<&UsageObject>) -> Option { let (prompt_tokens, completion_tokens, cached_tokens) = match usage? { UsageObject::Chat { members } => ( @@ -252,17 +254,33 @@ pub fn reconciled(usage: Option<&UsageObject>) -> Option { Some(Value::Object(details)) => details.get("cached_tokens"), _ => None, }; - ( - count(either( - members.get("prompt_tokens"), - members.get("input_tokens"), - )), - count(either( - members.get("completion_tokens"), - members.get("output_tokens"), - )), - count(either(detailed, members.get("cache_read_input_tokens"))), - ) + let completion_tokens = count(either( + members.get("completion_tokens"), + members.get("output_tokens"), + )); + let cache_reads = count(members.get("cache_read_input_tokens")); + let cache_writes = count(members.get("cache_creation_input_tokens")); + let anthropic_shaped = !truthy(members.get("prompt_tokens")) + && detailed.is_none() + && (cache_reads != 0 || cache_writes != 0); + if anthropic_shaped { + ( + count(members.get("input_tokens")) + .saturating_add(cache_writes) + .saturating_add(cache_reads), + completion_tokens, + cache_reads, + ) + } else { + ( + count(either( + members.get("prompt_tokens"), + members.get("input_tokens"), + )), + completion_tokens, + count(either(detailed, members.get("cache_read_input_tokens"))), + ) + } } UsageObject::Other => return None, }; @@ -573,6 +591,48 @@ mod tests { ); } + #[rstest] + #[case::anthropic_shape_adds_cache_writes_and_reads_to_the_prompt( + json!({ + "input_tokens": 40, "output_tokens": 25, + "cache_creation_input_tokens": 60, "cache_read_input_tokens": 900, + }), + (1000, 25, 900, 100) + )] + #[case::anthropic_shape_with_cache_writes_only( + json!({"input_tokens": 40, "output_tokens": 25, "cache_creation_input_tokens": 60}), + (100, 25, 0, 100) + )] + #[case::prompt_tokens_already_hold_the_cache_reads( + json!({"prompt_tokens": 100, "completion_tokens": 25, "cache_read_input_tokens": 30}), + (100, 25, 30, 70) + )] + #[case::input_tokens_details_already_hold_the_cache_reads( + json!({ + "input_tokens": 100, "output_tokens": 25, + "input_tokens_details": {"cached_tokens": 30}, "cache_creation_input_tokens": 60, + }), + (100, 25, 30, 70) + )] + fn a_plain_dict_reconciles_its_billable_input( + #[case] usage: Value, + #[case] expected: (i64, i64, i64, i64), + ) { + let resolved = reconciled(Some(&UsageObject::Mapping { + members: members(usage), + nested: None, + })); + assert_eq!( + resolved.map(|usage| ( + usage.prompt_tokens, + usage.completion_tokens, + usage.cached_tokens, + usage.billable_input_tokens(), + )), + Some(expected) + ); + } + #[test] fn a_rerank_reconciles_its_billed_units() { let response = LoggedObject::Rerank { diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py index 0da186e2b8d..c9b4b690d18 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1356,10 +1356,7 @@ async def test_async_log_success_event_increments_by_actual_tokens(monkeypatch): # Create mock response with 50 completion tokens mock_response = MagicMock(spec=ModelResponse) - mock_response.usage = MagicMock(spec=Usage) - mock_response.usage.prompt_tokens = 10 - mock_response.usage.completion_tokens = 50 - mock_response.usage.total_tokens = 60 + mock_response.usage = Usage(prompt_tokens=10, completion_tokens=50, total_tokens=60) # Create kwargs with priority in user_api_key_auth_metadata kwargs = { @@ -1518,10 +1515,7 @@ async def test_async_log_success_event_uses_team_priority_from_auth_metadata(mon # Create mock response mock_response = MagicMock(spec=ModelResponse) - mock_response.usage = MagicMock(spec=Usage) - mock_response.usage.prompt_tokens = 10 - mock_response.usage.completion_tokens = 20 - mock_response.usage.total_tokens = 30 + mock_response.usage = Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) # Simulate team metadata inheritance: priority is in user_api_key_auth_metadata # This is how the proxy passes team metadata to the callback @@ -2548,6 +2542,27 @@ async def test_an_input_token_limit_leaves_the_reply_length_alone(monkeypatch: p assert "max_tokens" not in prompt +@pytest.mark.parametrize(("rate_limit_type", "counted"), [("input", 20), ("total", 25), ("output", 5)]) +@pytest.mark.asyncio +async def test_cache_read_tokens_are_left_out_of_the_tokens_a_response_settles( + monkeypatch: pytest.MonkeyPatch, rate_limit_type: str, counted: int +): + from litellm.proxy import proxy_server + from litellm.types.utils import PromptTokensDetailsWrapper + + monkeypatch.setitem(proxy_server.general_settings, "token_rate_limit_type", rate_limit_type) + handler = _borrowing_handler(monkeypatch, borrows=True) + usage: Final = litellm.Usage( + prompt_tokens=50, + completion_tokens=5, + total_tokens=55, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=30), + ) + await _check(handler, _team_key(), saturation=0.5) + await _settle_tokens(handler, 0, response=litellm.ModelResponse(usage=usage)) + assert await _team_tokens_remaining(handler) == 100 - counted + + @pytest.mark.asyncio async def test_a_late_failure_from_a_fallback_that_did_not_serve_is_charged_to_that_target( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index 4ba884ff31f..039f6cd46d9 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -735,7 +735,28 @@ async def test_anthropic_messages_usage_reconciles_split_project_quota(rate_limi handler, cache, "proj-anthropic", - {"usage": {"input_tokens": 100, "output_tokens": 25, "cache_read_input_tokens": 30}}, + { + "usage": { + "input_tokens": 40, + "output_tokens": 25, + "cache_creation_input_tokens": 60, + "cache_read_input_tokens": 900, + } + }, + ) + + assert settled == (100, 25), "uncached input plus cache writes, with cache reads left out" + + +@pytest.mark.asyncio +async def test_usage_dict_with_prompt_tokens_keeps_cache_reads_inside_the_prompt_count(rate_limiter): + handler, cache = rate_limiter + + _, settled = await _settle_project_io( + handler, + cache, + "proj-prompt-tokens", + {"usage": {"prompt_tokens": 100, "completion_tokens": 25, "cache_read_input_tokens": 30}}, ) assert settled == (70, 25)