mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): leave cache-read tokens out of the priority limiter's token count (#52209)
* fix(proxy): leave cache-read tokens out of the priority limiter's token count The priority limiter counted cache-read tokens that the TPM limiter leaves out, so mostly cached prompts were throttled early. Usage dicts in Anthropic's own shape now count uncached input plus cache writes. Fixes #51970 Co-authored-by: yifei tang <fl2326151118@163.com> * fix(parallel_request_limiter): count cache writes and cache reads in the prompt of an Anthropic-shaped usage dict A usage dict in Anthropic's own shape reports input_tokens without its cache reads and cache writes. The Rust reconciliation read input_tokens as the full prompt and subtracted cache reads from it, so billable input dropped the cache writes and clamped to 0. Fixes #51970 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yifei tang <fl2326151118@163.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
b5e719392e
commit
52412090b9
4 changed files with 123 additions and 22 deletions
|
|
@ -193,6 +193,11 @@ def _usage_of(kwargs: Mapping[str, object], response_obj: object) -> Usage | Non
|
|||
return socket_usage if socket_usage is not None else get_usage_object(response_obj)
|
||||
|
||||
|
||||
def _cache_read_tokens(usage: Usage) -> int:
|
||||
details: Final = usage.prompt_tokens_details
|
||||
return (details.cached_tokens or 0) if details is not None else 0
|
||||
|
||||
|
||||
def _settled_tokens(usage: Usage | None, rate_limit_type: str) -> int:
|
||||
if usage is None:
|
||||
return 0
|
||||
|
|
@ -200,9 +205,9 @@ def _settled_tokens(usage: Usage | None, rate_limit_type: str) -> int:
|
|||
case "output":
|
||||
return usage.completion_tokens
|
||||
case "input":
|
||||
return usage.prompt_tokens
|
||||
return max(0, usage.prompt_tokens - _cache_read_tokens(usage))
|
||||
case _:
|
||||
return usage.total_tokens
|
||||
return max(0, usage.total_tokens - _cache_read_tokens(usage))
|
||||
|
||||
|
||||
def model_limit_at_saturation(kind: BorrowerKind) -> str:
|
||||
|
|
|
|||
|
|
@ -232,6 +232,8 @@ pub fn aggregate_only(usage: Option<&UsageObject>) -> i64 {
|
|||
|
||||
/// `resolve_reconciled_usage` of a usage object: its prompt, completion and cache-read counts,
|
||||
/// cache reads still inside the prompt, or None when it reports neither prompt nor completion.
|
||||
/// A dict in Anthropic's own shape reports `input_tokens` without its cache reads and cache
|
||||
/// writes, so its prompt is the sum of the three.
|
||||
pub fn reconciled(usage: Option<&UsageObject>) -> Option<Usage> {
|
||||
let (prompt_tokens, completion_tokens, cached_tokens) = match usage? {
|
||||
UsageObject::Chat { members } => (
|
||||
|
|
@ -252,17 +254,33 @@ pub fn reconciled(usage: Option<&UsageObject>) -> Option<Usage> {
|
|||
Some(Value::Object(details)) => details.get("cached_tokens"),
|
||||
_ => None,
|
||||
};
|
||||
(
|
||||
count(either(
|
||||
members.get("prompt_tokens"),
|
||||
members.get("input_tokens"),
|
||||
)),
|
||||
count(either(
|
||||
members.get("completion_tokens"),
|
||||
members.get("output_tokens"),
|
||||
)),
|
||||
count(either(detailed, members.get("cache_read_input_tokens"))),
|
||||
)
|
||||
let completion_tokens = count(either(
|
||||
members.get("completion_tokens"),
|
||||
members.get("output_tokens"),
|
||||
));
|
||||
let cache_reads = count(members.get("cache_read_input_tokens"));
|
||||
let cache_writes = count(members.get("cache_creation_input_tokens"));
|
||||
let anthropic_shaped = !truthy(members.get("prompt_tokens"))
|
||||
&& detailed.is_none()
|
||||
&& (cache_reads != 0 || cache_writes != 0);
|
||||
if anthropic_shaped {
|
||||
(
|
||||
count(members.get("input_tokens"))
|
||||
.saturating_add(cache_writes)
|
||||
.saturating_add(cache_reads),
|
||||
completion_tokens,
|
||||
cache_reads,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
count(either(
|
||||
members.get("prompt_tokens"),
|
||||
members.get("input_tokens"),
|
||||
)),
|
||||
completion_tokens,
|
||||
count(either(detailed, members.get("cache_read_input_tokens"))),
|
||||
)
|
||||
}
|
||||
}
|
||||
UsageObject::Other => return None,
|
||||
};
|
||||
|
|
@ -573,6 +591,48 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::anthropic_shape_adds_cache_writes_and_reads_to_the_prompt(
|
||||
json!({
|
||||
"input_tokens": 40, "output_tokens": 25,
|
||||
"cache_creation_input_tokens": 60, "cache_read_input_tokens": 900,
|
||||
}),
|
||||
(1000, 25, 900, 100)
|
||||
)]
|
||||
#[case::anthropic_shape_with_cache_writes_only(
|
||||
json!({"input_tokens": 40, "output_tokens": 25, "cache_creation_input_tokens": 60}),
|
||||
(100, 25, 0, 100)
|
||||
)]
|
||||
#[case::prompt_tokens_already_hold_the_cache_reads(
|
||||
json!({"prompt_tokens": 100, "completion_tokens": 25, "cache_read_input_tokens": 30}),
|
||||
(100, 25, 30, 70)
|
||||
)]
|
||||
#[case::input_tokens_details_already_hold_the_cache_reads(
|
||||
json!({
|
||||
"input_tokens": 100, "output_tokens": 25,
|
||||
"input_tokens_details": {"cached_tokens": 30}, "cache_creation_input_tokens": 60,
|
||||
}),
|
||||
(100, 25, 30, 70)
|
||||
)]
|
||||
fn a_plain_dict_reconciles_its_billable_input(
|
||||
#[case] usage: Value,
|
||||
#[case] expected: (i64, i64, i64, i64),
|
||||
) {
|
||||
let resolved = reconciled(Some(&UsageObject::Mapping {
|
||||
members: members(usage),
|
||||
nested: None,
|
||||
}));
|
||||
assert_eq!(
|
||||
resolved.map(|usage| (
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens,
|
||||
usage.cached_tokens,
|
||||
usage.billable_input_tokens(),
|
||||
)),
|
||||
Some(expected)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rerank_reconciles_its_billed_units() {
|
||||
let response = LoggedObject::Rerank {
|
||||
|
|
|
|||
|
|
@ -1356,10 +1356,7 @@ async def test_async_log_success_event_increments_by_actual_tokens(monkeypatch):
|
|||
|
||||
# Create mock response with 50 completion tokens
|
||||
mock_response = MagicMock(spec=ModelResponse)
|
||||
mock_response.usage = MagicMock(spec=Usage)
|
||||
mock_response.usage.prompt_tokens = 10
|
||||
mock_response.usage.completion_tokens = 50
|
||||
mock_response.usage.total_tokens = 60
|
||||
mock_response.usage = Usage(prompt_tokens=10, completion_tokens=50, total_tokens=60)
|
||||
|
||||
# Create kwargs with priority in user_api_key_auth_metadata
|
||||
kwargs = {
|
||||
|
|
@ -1518,10 +1515,7 @@ async def test_async_log_success_event_uses_team_priority_from_auth_metadata(mon
|
|||
|
||||
# Create mock response
|
||||
mock_response = MagicMock(spec=ModelResponse)
|
||||
mock_response.usage = MagicMock(spec=Usage)
|
||||
mock_response.usage.prompt_tokens = 10
|
||||
mock_response.usage.completion_tokens = 20
|
||||
mock_response.usage.total_tokens = 30
|
||||
mock_response.usage = Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30)
|
||||
|
||||
# Simulate team metadata inheritance: priority is in user_api_key_auth_metadata
|
||||
# This is how the proxy passes team metadata to the callback
|
||||
|
|
@ -2548,6 +2542,27 @@ async def test_an_input_token_limit_leaves_the_reply_length_alone(monkeypatch: p
|
|||
assert "max_tokens" not in prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("rate_limit_type", "counted"), [("input", 20), ("total", 25), ("output", 5)])
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_read_tokens_are_left_out_of_the_tokens_a_response_settles(
|
||||
monkeypatch: pytest.MonkeyPatch, rate_limit_type: str, counted: int
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper
|
||||
|
||||
monkeypatch.setitem(proxy_server.general_settings, "token_rate_limit_type", rate_limit_type)
|
||||
handler = _borrowing_handler(monkeypatch, borrows=True)
|
||||
usage: Final = litellm.Usage(
|
||||
prompt_tokens=50,
|
||||
completion_tokens=5,
|
||||
total_tokens=55,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=30),
|
||||
)
|
||||
await _check(handler, _team_key(), saturation=0.5)
|
||||
await _settle_tokens(handler, 0, response=litellm.ModelResponse(usage=usage))
|
||||
assert await _team_tokens_remaining(handler) == 100 - counted
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_late_failure_from_a_fallback_that_did_not_serve_is_charged_to_that_target(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
|
|
@ -735,7 +735,28 @@ async def test_anthropic_messages_usage_reconciles_split_project_quota(rate_limi
|
|||
handler,
|
||||
cache,
|
||||
"proj-anthropic",
|
||||
{"usage": {"input_tokens": 100, "output_tokens": 25, "cache_read_input_tokens": 30}},
|
||||
{
|
||||
"usage": {
|
||||
"input_tokens": 40,
|
||||
"output_tokens": 25,
|
||||
"cache_creation_input_tokens": 60,
|
||||
"cache_read_input_tokens": 900,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert settled == (100, 25), "uncached input plus cache writes, with cache reads left out"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_usage_dict_with_prompt_tokens_keeps_cache_reads_inside_the_prompt_count(rate_limiter):
|
||||
handler, cache = rate_limiter
|
||||
|
||||
_, settled = await _settle_project_io(
|
||||
handler,
|
||||
cache,
|
||||
"proj-prompt-tokens",
|
||||
{"usage": {"prompt_tokens": 100, "completion_tokens": 25, "cache_read_input_tokens": 30}},
|
||||
)
|
||||
|
||||
assert settled == (70, 25)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue