mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(batches): log no cost for a batch retrieve that is not final yet
A retrieve now reads OpenAI's batch usage into the chat counts, so a retrieve that saw the batch completed before its output file arrived priced those counts for the logging callbacks, and the final retrieve priced the whole batch again. A batch that is not final now logs no cost, as it did before. Batch usage dumps also pass exclude_unset and exclude_defaults through to the usage keys.
This commit is contained in:
parent
b126dae208
commit
d740675074
4 changed files with 51 additions and 14 deletions
|
|
@ -2071,6 +2071,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
|
||||
router_model_id = hidden_params["model_id"]
|
||||
|
||||
if isinstance(priced_result, LiteLLMBatch) and not batch_cost_is_final(priced_result):
|
||||
return 0.0
|
||||
|
||||
# Fallback: extract router_model_id from litellm_params when not available
|
||||
# from the result object. ResponsesAPIResponse objects (used by /v1/responses
|
||||
# streaming) don't carry _hidden_params["model_id"] like ModelResponse does.
|
||||
|
|
|
|||
|
|
@ -4872,10 +4872,19 @@ class LiteLLMBatch(Batch):
|
|||
return usage
|
||||
return {
|
||||
**usage.model_dump(
|
||||
mode=info.mode, include=info.include, exclude=info.exclude, exclude_none=info.exclude_none
|
||||
mode=info.mode,
|
||||
include=info.include,
|
||||
exclude=info.exclude,
|
||||
exclude_unset=info.exclude_unset,
|
||||
exclude_defaults=info.exclude_defaults,
|
||||
exclude_none=info.exclude_none,
|
||||
),
|
||||
**_batch_usage_from_chat_usage(usage).model_dump(
|
||||
mode=info.mode, include=info.include, exclude=info.exclude
|
||||
mode=info.mode,
|
||||
include=info.include,
|
||||
exclude=info.exclude,
|
||||
exclude_unset=info.exclude_unset,
|
||||
exclude_defaults=info.exclude_defaults,
|
||||
),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1519,22 +1519,33 @@ class TestRetrieveBatchPricesOnlyFinalBatches:
|
|||
litellm_call_id="batch-call-2",
|
||||
function_id="f",
|
||||
)
|
||||
obj.custom_llm_provider = "openai"
|
||||
obj.update_environment_variables(
|
||||
model="gpt-5.6-luna", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai"
|
||||
)
|
||||
return obj
|
||||
|
||||
@staticmethod
|
||||
def _batch(status: str, output_file_id: str | None):
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
return LiteLLMBatch(
|
||||
id="batch_6a9c99e185588190877d391f8b9d7f8a",
|
||||
completion_window="24h",
|
||||
created_at=1,
|
||||
endpoint="/v1/chat/completions",
|
||||
input_file_id="file-in",
|
||||
object="batch",
|
||||
status="validating",
|
||||
output_file_id=output_file_id,
|
||||
return LiteLLMBatch.model_validate(
|
||||
{
|
||||
"id": "batch_6a9c99e185588190877d391f8b9d7f8a",
|
||||
"completion_window": "24h",
|
||||
"created_at": 1,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"input_file_id": "file-in",
|
||||
"object": "batch",
|
||||
"status": "validating",
|
||||
"output_file_id": output_file_id,
|
||||
"usage": {
|
||||
"input_tokens": 29,
|
||||
"input_tokens_details": {"cached_tokens": 0},
|
||||
"output_tokens": 48,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 77,
|
||||
},
|
||||
}
|
||||
).model_copy(update={"status": status})
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1548,11 +1559,12 @@ class TestRetrieveBatchPricesOnlyFinalBatches:
|
|||
handle_completed_batch = AsyncMock()
|
||||
monkeypatch.setattr(logging_module, "handle_completed_batch", handle_completed_batch)
|
||||
batch = self._batch(status, output_file_id)
|
||||
logging_obj = self._logging_obj()
|
||||
|
||||
await self._logging_obj()._async_success_handler_body(result=batch, start_time=None, end_time=None)
|
||||
await logging_obj._async_success_handler_body(result=batch, start_time=None, end_time=None)
|
||||
|
||||
handle_completed_batch.assert_not_awaited()
|
||||
assert "response_cost" not in batch._hidden_params
|
||||
assert logging_obj.model_call_details["response_cost"] == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_batch_with_output_is_priced(self, monkeypatch) -> None:
|
||||
|
|
|
|||
|
|
@ -1154,3 +1154,16 @@ def test_batch_dump_applies_nested_include_and_exclude_to_both_usage_shapes() ->
|
|||
assert {"prompt_tokens_details", "input_tokens_details"}.isdisjoint(excluded)
|
||||
assert (excluded["prompt_tokens"], excluded["input_tokens"]) == (21, 21)
|
||||
assert included == {"prompt_tokens": 21, "input_tokens": 21}
|
||||
|
||||
|
||||
def test_batch_dump_applies_exclude_defaults_to_the_chat_usage_keys() -> None:
|
||||
batch: Final = LiteLLMBatch.model_validate(
|
||||
{**_BATCH_FIELDS, "usage": Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)}
|
||||
)
|
||||
|
||||
python_usage: Final = batch.model_dump(exclude_defaults=True)["usage"]
|
||||
json_usage: Final = json.loads(batch.model_dump_json(exclude_defaults=True, warnings=False))["usage"]
|
||||
|
||||
assert python_usage == json_usage
|
||||
assert {"prompt_tokens_details", "completion_tokens_details"}.isdisjoint(python_usage)
|
||||
assert (python_usage["prompt_tokens"], python_usage["input_tokens"]) == (10, 10)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue