mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
75613bf22f
* test: add regression coverage for twelve closed issues Adds targeted regression tests for behavior that was fixed but left ungated, so the fixes cannot silently regress: - #33772 openai cache_write_tokens cost - #34309 Responses API cache cost_breakdown - #35363 /v1/responses batch spend - #36619 auto-router api_base/api_key leak on a shared model name - #35359 batch fallbacks within the owning model group - #36523 passthrough streamed Responses spend log - #36646 passthrough embeddings spend log - #37147 non-object metadata on create_batch is a 400 - #35362 unscoped list files reads the managed-file store - #33221 gpt-5.6 bridges to Responses on function tools alone - #34487 LLM complexity classifier runs for every caller metadata shape - #35124 streamed /v1/messages emits success logging on both bridges Cost assertions read rates from litellm.model_cost rather than hardcoding dollar amounts, so they do not drift on repricing. * fix: stop the new regression tests polluting and tripping over shared global state Two shard failures, both from global state the new tests share with their neighbours rather than from the behaviour under test. test_main.py's local_cost_map pinned litellm.model_cost but left the get_model_info lru_cache warm, so completion_cost billed at whatever prices were cached earlier in the process while the assertions read the pinned map. Clear the cache on both sides of the fixture, matching the local_model_cost_map fixture in tests/test_litellm/conftest.py. The anthropic messages streaming tests called GLOBAL_LOGGING_WORKER.flush() on whatever queue happened to be around. A queue left non-empty by an earlier test is still bound to that test's loop, so join() either hangs or raises "bound to a different event loop". Rebind to the running loop before the call and wait for the captured payload instead of a fixed sleep.
119 lines
4.4 KiB
Python
119 lines
4.4 KiB
Python
"""Token and spend reconciliation for /v1/responses batches.
|
|
|
|
Regression guard for https://github.com/BerriAI/litellm/issues/35363: a batch
|
|
output line built by the Responses API reports ``input_tokens`` /
|
|
``output_tokens`` where a chat line reports ``prompt_tokens`` /
|
|
``completion_tokens``. The usage object was constructed straight from the raw
|
|
dict, which accepts the unrecognized names without raising and yields zeros, so
|
|
a completed Responses batch reconciled to 0 tokens and $0.00 spend with no
|
|
error, and per-key budgets were never charged for it.
|
|
|
|
Line shape decides the parse, not the batch's declared endpoint, so an output
|
|
file mixing Responses-shaped and chat-shaped lines sums across both.
|
|
"""
|
|
|
|
from typing import Literal, get_args, get_type_hints
|
|
|
|
import pytest
|
|
|
|
import litellm
|
|
import litellm.batches.batch_utils as bu
|
|
from litellm.types.llms.openai import CreateBatchRequest
|
|
|
|
MODEL = "gpt-5.6"
|
|
|
|
|
|
def _responses_line(input_tokens: int, output_tokens: int) -> dict:
|
|
return {
|
|
"response": {
|
|
"status_code": 200,
|
|
"body": {
|
|
"model": MODEL,
|
|
"usage": {
|
|
"input_tokens": input_tokens,
|
|
"output_tokens": output_tokens,
|
|
"total_tokens": input_tokens + output_tokens,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
|
|
def _chat_line(prompt_tokens: int, completion_tokens: int) -> dict:
|
|
return {
|
|
"response": {
|
|
"status_code": 200,
|
|
"body": {
|
|
"model": MODEL,
|
|
"usage": {
|
|
"prompt_tokens": prompt_tokens,
|
|
"completion_tokens": completion_tokens,
|
|
"total_tokens": prompt_tokens + completion_tokens,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
|
|
def test_responses_shaped_usage_maps_onto_prompt_and_completion_tokens():
|
|
"""The Responses names land on the chat-shaped counters instead of being
|
|
dropped for unrecognized keys."""
|
|
usage = bu._get_batch_job_usage_from_response_body(
|
|
{"usage": {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150}}
|
|
)
|
|
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (100, 50, 150)
|
|
|
|
|
|
async def test_responses_batch_reconciles_to_real_tokens_and_spend(local_model_cost_map):
|
|
"""A completed Responses batch records the provider's token counts and a
|
|
non-zero spend at the model's batch rates."""
|
|
model_info = litellm.get_model_info(model=MODEL, custom_llm_provider="openai")
|
|
input_tokens = 33
|
|
output_tokens = 57
|
|
|
|
cost, usage, models = await bu.calculate_batch_cost_and_usage(
|
|
file_content_dictionary=[_responses_line(input_tokens, output_tokens)],
|
|
custom_llm_provider="openai",
|
|
model_name=MODEL,
|
|
model_info=model_info,
|
|
)
|
|
|
|
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (
|
|
input_tokens,
|
|
output_tokens,
|
|
input_tokens + output_tokens,
|
|
)
|
|
assert models == [MODEL]
|
|
assert cost == pytest.approx(
|
|
input_tokens * model_info["input_cost_per_token_batches"]
|
|
+ output_tokens * model_info["output_cost_per_token_batches"]
|
|
)
|
|
assert cost > 0.0
|
|
|
|
|
|
async def test_mixed_shape_batch_output_sums_across_both_line_shapes(local_model_cost_map):
|
|
"""An output file carrying both line shapes sums both. A fix keyed off the
|
|
batch's declared endpoint rather than each line's shape would miss this."""
|
|
model_info = litellm.get_model_info(model=MODEL, custom_llm_provider="openai")
|
|
|
|
cost, usage, _ = await bu.calculate_batch_cost_and_usage(
|
|
file_content_dictionary=[_responses_line(100, 50), _chat_line(33, 57)],
|
|
custom_llm_provider="openai",
|
|
model_name=MODEL,
|
|
model_info=model_info,
|
|
)
|
|
|
|
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (133, 107, 240)
|
|
assert cost == pytest.approx(
|
|
133 * model_info["input_cost_per_token_batches"] + 107 * model_info["output_cost_per_token_batches"]
|
|
)
|
|
|
|
|
|
def test_create_batch_endpoint_accepts_v1_responses():
|
|
"""A type-checked caller can pass endpoint="/v1/responses", which the runtime
|
|
already forwarded correctly."""
|
|
endpoint_annotation = get_type_hints(CreateBatchRequest)["endpoint"]
|
|
assert "/v1/responses" in get_args(endpoint_annotation)
|
|
|
|
for create_fn in (litellm.create_batch, litellm.acreate_batch):
|
|
assert "/v1/responses" in get_args(get_type_hints(create_fn)["endpoint"])
|