mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
chore(router): add is_batch_retrieve_call_type for the batch charge-once backport
Prerequisite for the #47625 backport to stable/1.103.x. Ports only the BATCH_RETRIEVE_CALL_TYPES / is_batch_retrieve_call_type helper from main 9acf09f60d, which #47625's charge-once helpers call. The rest of that commit (keeping batch retrieves out of router tpm/rpm counters) is not taken: its test passes on this line's tip, since batch retrieves are not stamped with a model group here.
This commit is contained in:
parent
a83924f8c8
commit
a8b8b3bd60
1 changed files with 18 additions and 0 deletions
|
|
@ -5,6 +5,7 @@ from typing import Final
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.llms.openai import FileTypes, OpenAIFilesPurpose
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
|
||||
class InMemoryFile(io.BytesIO):
|
||||
|
|
@ -170,3 +171,20 @@ def _get_router_metadata_variable_name(function_name: str | None) -> str:
|
|||
return "litellm_metadata"
|
||||
else:
|
||||
return "metadata"
|
||||
|
||||
|
||||
BATCH_RETRIEVE_CALL_TYPES: Final = frozenset(
|
||||
{
|
||||
CallTypes.aretrieve_batch.value,
|
||||
CallTypes.retrieve_batch.value,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def is_batch_retrieve_call_type(call_type: object) -> bool:
|
||||
"""
|
||||
A batch retrieve reports the whole job's token usage, which the provider spent
|
||||
asynchronously over the life of the batch, and reports it again on every poll of the
|
||||
finished batch. Per-minute usage counters must not be fed from it.
|
||||
"""
|
||||
return isinstance(call_type, str) and call_type in BATCH_RETRIEVE_CALL_TYPES
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue