fix(proxy): leave cache-read tokens out of the priority limiter's token count (#52209)

* fix(proxy): leave cache-read tokens out of the priority limiter's token count

The priority limiter counted cache-read tokens that the TPM limiter leaves out, so mostly cached prompts were throttled early. Usage dicts in Anthropic's own shape now count uncached input plus cache writes.

Fixes #51970

Co-authored-by: yifei tang <fl2326151118@163.com>

* fix(parallel_request_limiter): count cache writes and cache reads in the prompt of an Anthropic-shaped usage dict

A usage dict in Anthropic's own shape reports input_tokens without its cache reads and
cache writes. The Rust reconciliation read input_tokens as the full prompt and subtracted
cache reads from it, so billable input dropped the cache writes and clamped to 0.

Fixes #51970

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yifei tang <fl2326151118@163.com>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kenneth-berri 2026-11-17 11:32:25 -08:00 • committed by GitHub
parent b5e719392e
commit 52412090b9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 123 additions and 22 deletions

View file

@ -193,6 +193,11 @@ def _usage_of(kwargs: Mapping[str, object], response_obj: object) -> Usage | Non
return socket_usage if socket_usage is not None else get_usage_object(response_obj)
def _cache_read_tokens(usage: Usage) -> int:
details: Final = usage.prompt_tokens_details
return (details.cached_tokens or 0) if details is not None else 0
def _settled_tokens(usage: Usage | None, rate_limit_type: str) -> int:
if usage is None:
return 0
@ -200,9 +205,9 @@ def _settled_tokens(usage: Usage | None, rate_limit_type: str) -> int:
case "output":
return usage.completion_tokens
case "input":
return usage.prompt_tokens
return max(0, usage.prompt_tokens - _cache_read_tokens(usage))
case _:
return usage.total_tokens
return max(0, usage.total_tokens - _cache_read_tokens(usage))
def model_limit_at_saturation(kind: BorrowerKind) -> str:

View file

@ -232,6 +232,8 @@ pub fn aggregate_only(usage: Option<&UsageObject>) -> i64 {
/// `resolve_reconciled_usage` of a usage object: its prompt, completion and cache-read counts,
/// cache reads still inside the prompt, or None when it reports neither prompt nor completion.
/// A dict in Anthropic's own shape reports `input_tokens` without its cache reads and cache
/// writes, so its prompt is the sum of the three.
pub fn reconciled(usage: Option<&UsageObject>) -> Option<Usage> {
let (prompt_tokens, completion_tokens, cached_tokens) = match usage? {
UsageObject::Chat { members } => (
@ -252,17 +254,33 @@ pub fn reconciled(usage: Option<&UsageObject>) -> Option<Usage> {
Some(Value::Object(details)) => details.get("cached_tokens"),
_ => None,
};
(
count(either(
members.get("prompt_tokens"),
members.get("input_tokens"),
)),
count(either(
members.get("completion_tokens"),
members.get("output_tokens"),
)),
count(either(detailed, members.get("cache_read_input_tokens"))),
)
let completion_tokens = count(either(
members.get("completion_tokens"),
members.get("output_tokens"),
));
let cache_reads = count(members.get("cache_read_input_tokens"));
let cache_writes = count(members.get("cache_creation_input_tokens"));
let anthropic_shaped = !truthy(members.get("prompt_tokens"))
&& detailed.is_none()
&& (cache_reads != 0 || cache_writes != 0);
if anthropic_shaped {
(
count(members.get("input_tokens"))
.saturating_add(cache_writes)
.saturating_add(cache_reads),
completion_tokens,
cache_reads,
)
} else {
(
count(either(
members.get("prompt_tokens"),
members.get("input_tokens"),
)),
completion_tokens,
count(either(detailed, members.get("cache_read_input_tokens"))),
)
}
}
UsageObject::Other => return None,
};
@ -573,6 +591,48 @@ mod tests {
);
}
#[rstest]
#[case::anthropic_shape_adds_cache_writes_and_reads_to_the_prompt(
json!({
"input_tokens": 40, "output_tokens": 25,
"cache_creation_input_tokens": 60, "cache_read_input_tokens": 900,
}),
(1000, 25, 900, 100)
)]
#[case::anthropic_shape_with_cache_writes_only(
json!({"input_tokens": 40, "output_tokens": 25, "cache_creation_input_tokens": 60}),
(100, 25, 0, 100)
)]
#[case::prompt_tokens_already_hold_the_cache_reads(
json!({"prompt_tokens": 100, "completion_tokens": 25, "cache_read_input_tokens": 30}),
(100, 25, 30, 70)
)]
#[case::input_tokens_details_already_hold_the_cache_reads(
json!({
"input_tokens": 100, "output_tokens": 25,
"input_tokens_details": {"cached_tokens": 30}, "cache_creation_input_tokens": 60,
}),
(100, 25, 30, 70)
)]
fn a_plain_dict_reconciles_its_billable_input(
#[case] usage: Value,
#[case] expected: (i64, i64, i64, i64),
) {
let resolved = reconciled(Some(&UsageObject::Mapping {
members: members(usage),
nested: None,
}));
assert_eq!(
resolved.map(|usage| (
usage.prompt_tokens,
usage.completion_tokens,
usage.cached_tokens,
usage.billable_input_tokens(),
)),
Some(expected)
);
}
#[test]
fn a_rerank_reconciles_its_billed_units() {
let response = LoggedObject::Rerank {

View file

@ -1356,10 +1356,7 @@ async def test_async_log_success_event_increments_by_actual_tokens(monkeypatch):
# Create mock response with 50 completion tokens
mock_response = MagicMock(spec=ModelResponse)
mock_response.usage = MagicMock(spec=Usage)
mock_response.usage.prompt_tokens = 10
mock_response.usage.completion_tokens = 50
mock_response.usage.total_tokens = 60
mock_response.usage = Usage(prompt_tokens=10, completion_tokens=50, total_tokens=60)
# Create kwargs with priority in user_api_key_auth_metadata
kwargs = {
@ -1518,10 +1515,7 @@ async def test_async_log_success_event_uses_team_priority_from_auth_metadata(mon
# Create mock response
mock_response = MagicMock(spec=ModelResponse)
mock_response.usage = MagicMock(spec=Usage)
mock_response.usage.prompt_tokens = 10
mock_response.usage.completion_tokens = 20
mock_response.usage.total_tokens = 30
mock_response.usage = Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30)
# Simulate team metadata inheritance: priority is in user_api_key_auth_metadata
# This is how the proxy passes team metadata to the callback
@ -2548,6 +2542,27 @@ async def test_an_input_token_limit_leaves_the_reply_length_alone(monkeypatch: p
assert "max_tokens" not in prompt
@pytest.mark.parametrize(("rate_limit_type", "counted"), [("input", 20), ("total", 25), ("output", 5)])
@pytest.mark.asyncio
async def test_cache_read_tokens_are_left_out_of_the_tokens_a_response_settles(
monkeypatch: pytest.MonkeyPatch, rate_limit_type: str, counted: int
):
from litellm.proxy import proxy_server
from litellm.types.utils import PromptTokensDetailsWrapper
monkeypatch.setitem(proxy_server.general_settings, "token_rate_limit_type", rate_limit_type)
handler = _borrowing_handler(monkeypatch, borrows=True)
usage: Final = litellm.Usage(
prompt_tokens=50,
completion_tokens=5,
total_tokens=55,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=30),
)
await _check(handler, _team_key(), saturation=0.5)
await _settle_tokens(handler, 0, response=litellm.ModelResponse(usage=usage))
assert await _team_tokens_remaining(handler) == 100 - counted
@pytest.mark.asyncio
async def test_a_late_failure_from_a_fallback_that_did_not_serve_is_charged_to_that_target(
monkeypatch: pytest.MonkeyPatch,

View file

@ -735,7 +735,28 @@ async def test_anthropic_messages_usage_reconciles_split_project_quota(rate_limi
handler,
cache,
"proj-anthropic",
{"usage": {"input_tokens": 100, "output_tokens": 25, "cache_read_input_tokens": 30}},
{
"usage": {
"input_tokens": 40,
"output_tokens": 25,
"cache_creation_input_tokens": 60,
"cache_read_input_tokens": 900,
}
},
)
assert settled == (100, 25), "uncached input plus cache writes, with cache reads left out"
@pytest.mark.asyncio
async def test_usage_dict_with_prompt_tokens_keeps_cache_reads_inside_the_prompt_count(rate_limiter):
handler, cache = rate_limiter
_, settled = await _settle_project_io(
handler,
cache,
"proj-prompt-tokens",
{"usage": {"prompt_tokens": 100, "completion_tokens": 25, "cache_read_input_tokens": 30}},
)
assert settled == (70, 25)