chore(router): add is_batch_retrieve_call_type for the batch charge-once backport

Prerequisite for the #47625 backport to stable/1.103.x. Ports only the BATCH_RETRIEVE_CALL_TYPES / is_batch_retrieve_call_type helper from main 9acf09f60d, which #47625's charge-once helpers call. The rest of that commit (keeping batch retrieves out of router tpm/rpm counters) is not taken: its test passes on this line's tip, since batch retrieves are not stamped with a model group here.
This commit is contained in:
mateo-berri 2026-10-20 17:03:41 -07:00 • committed by Mrinal Chanshetty
parent a83924f8c8
commit a8b8b3bd60

View file

@ -5,6 +5,7 @@ from typing import Final
from litellm._logging import verbose_logger
from litellm.types.llms.openai import FileTypes, OpenAIFilesPurpose
from litellm.types.utils import CallTypes
class InMemoryFile(io.BytesIO):
@ -170,3 +171,20 @@ def _get_router_metadata_variable_name(function_name: str | None) -> str:
return "litellm_metadata"
else:
return "metadata"
BATCH_RETRIEVE_CALL_TYPES: Final = frozenset(
{
CallTypes.aretrieve_batch.value,
CallTypes.retrieve_batch.value,
}
)
def is_batch_retrieve_call_type(call_type: object) -> bool:
"""
A batch retrieve reports the whole job's token usage, which the provider spent
asynchronously over the life of the batch, and reports it again on every poll of the
finished batch. Per-minute usage counters must not be fed from it.
"""
return isinstance(call_type, str) and call_type in BATCH_RETRIEVE_CALL_TYPES