mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): count a Responses WebSocket turn's tokens before the next turn is admitted
This commit is contained in:
parent
5eb1042bbf
commit
103efe969b
11 changed files with 447 additions and 58 deletions
|
|
@ -1666,6 +1666,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
|
|||
BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
|
||||
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_usage_based_routing_counted_tokens"
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
|
||||
|
|
|
|||
|
|
@ -244,7 +244,7 @@ if TYPE_CHECKING:
|
|||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
|
||||
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnUsageRecorder
|
||||
from litellm.types.google_genai.main import GenerateContentResponse
|
||||
from litellm.types.llms.openai_evals import (
|
||||
CancelEvalResponse,
|
||||
|
|
@ -7143,6 +7143,7 @@ class BaseLLMHTTPHandler:
|
|||
first_message: str | None = None,
|
||||
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
|
||||
turn_admission: Callable[[], Awaitable[None]] | None = None,
|
||||
turn_usage_recorder: "ResponsesWsTurnUsageRecorder | None" = None,
|
||||
frame_guard: "ResponsesWsFrameGuard | None" = None,
|
||||
**kwargs: object,
|
||||
) -> Exception | None:
|
||||
|
|
@ -7303,6 +7304,7 @@ class BaseLLMHTTPHandler:
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
request_defaults=request_defaults,
|
||||
turn_admission=turn_admission,
|
||||
turn_usage_recorder=turn_usage_recorder,
|
||||
frame_guard=frame_guard,
|
||||
)
|
||||
return await streaming.bidirectional_forward()
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from litellm.constants import (
|
|||
SESSION_ID_GENERATED_METADATA_KEY,
|
||||
SESSION_ID_OMITTED_METADATA_KEY,
|
||||
TYPESAFE_DANGER_ZONE_HEADER_PREFIX,
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
|
||||
VECTOR_STORE_CALLER_METADATA_KWARG,
|
||||
X_LITELLM_DISABLE_CALLBACKS,
|
||||
)
|
||||
|
|
@ -518,6 +519,7 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
|
|||
"request_retry_count",
|
||||
CLIENT_OUTPUT_CEILING_METADATA_KEY,
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
|
||||
}
|
||||
)
|
||||
_ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override"
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ from litellm.types.llms.openai import (
|
|||
|
||||
# Handle ResponseText import with fallback
|
||||
if TYPE_CHECKING:
|
||||
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
|
||||
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnUsageRecorder
|
||||
from litellm.types.llms.openai import ResponseText
|
||||
else:
|
||||
ResponseText = str # Fallback for ResponseText import
|
||||
|
|
@ -2781,6 +2781,7 @@ async def _aresponses_websocket(
|
|||
api_key: str | None = None,
|
||||
timeout: float | None = None,
|
||||
turn_admission: Callable[[], Awaitable[None]] | None = None,
|
||||
turn_usage_recorder: "ResponsesWsTurnUsageRecorder | None" = None,
|
||||
responses_ws_frame_guard: "ResponsesWsFrameGuard | None" = None,
|
||||
**kwargs,
|
||||
) -> Exception | None:
|
||||
|
|
@ -2875,6 +2876,7 @@ async def _aresponses_websocket(
|
|||
custom_llm_provider=_custom_llm_provider,
|
||||
request_defaults=_build_responses_websocket_request_defaults(deployment_kwargs),
|
||||
turn_admission=turn_admission,
|
||||
turn_usage_recorder=turn_usage_recorder,
|
||||
frame_guard=responses_ws_frame_guard,
|
||||
**remaining_kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2245,6 +2245,7 @@ class GuardedResponsesWsFrame:
|
|||
|
||||
|
||||
ResponsesWsFrameGuard: TypeAlias = Callable[[str], Awaitable[GuardedResponsesWsFrame]]
|
||||
ResponsesWsTurnUsageRecorder: TypeAlias = Callable[[Mapping[str, object], int], Awaitable[None]]
|
||||
|
||||
|
||||
async def _reject_guarded_turn(
|
||||
|
|
@ -2444,6 +2445,7 @@ class ResponsesWebSocketStreaming:
|
|||
custom_llm_provider: str | None = None,
|
||||
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
|
||||
turn_admission: Callable[[], Awaitable[None]] | None = None,
|
||||
turn_usage_recorder: ResponsesWsTurnUsageRecorder | None = None,
|
||||
frame_guard: ResponsesWsFrameGuard | None = None,
|
||||
):
|
||||
self.websocket = websocket
|
||||
|
|
@ -2466,6 +2468,7 @@ class ResponsesWebSocketStreaming:
|
|||
self.authorized_model: str | None = authorized_model
|
||||
self.request_defaults: ResponsesWebSocketRequestDefaults | None = request_defaults
|
||||
self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission
|
||||
self.turn_usage_recorder: ResponsesWsTurnUsageRecorder | None = turn_usage_recorder
|
||||
|
||||
def _start_turn(
|
||||
self, client_frame: str | None, guardrail_information: tuple[Mapping[str, object], ...] = ()
|
||||
|
|
@ -2537,7 +2540,7 @@ class ResponsesWebSocketStreaming:
|
|||
"pii_tokens": {**(known if _is_str_mapping(known) else {}), **pii_tokens},
|
||||
}
|
||||
|
||||
def _record_backend_event(self, event: str) -> None:
|
||||
async def _record_backend_event(self, event: str) -> None:
|
||||
event_obj: Final = _json_object_or_none(event)
|
||||
if event_obj is None or event_obj.get("type") not in RESPONSES_WS_LOGGED_EVENT_TYPES:
|
||||
return
|
||||
|
|
@ -2546,10 +2549,25 @@ class ResponsesWebSocketStreaming:
|
|||
turn: Final = self._open_turns[0].with_event(event_obj)
|
||||
if event_obj.get("type") in _RESPONSES_WS_TERMINAL_EVENT_TYPES:
|
||||
self._open_turns = self._open_turns[1:]
|
||||
if event_obj.get("type") in _HISTORY_TERMINAL_EVENT_TYPES:
|
||||
await self._record_turn_usage(turn)
|
||||
self._log_turn(turn)
|
||||
return
|
||||
self._open_turns = (turn, *self._open_turns[1:])
|
||||
|
||||
async def _record_turn_usage(self, turn: _ResponsesWsTurn) -> None:
|
||||
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
|
||||
|
||||
turn_litellm_params: Final = _json_object_or_none(
|
||||
turn.logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
|
||||
)
|
||||
if self.turn_usage_recorder is None or turn_litellm_params is None:
|
||||
return
|
||||
usage: Final = ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(
|
||||
turn.events
|
||||
)
|
||||
await self.turn_usage_recorder(turn_litellm_params, usage.total_tokens)
|
||||
|
||||
def _log_open_turns(self) -> None:
|
||||
open_turns: Final = self._open_turns
|
||||
self._open_turns = ()
|
||||
|
|
@ -2634,7 +2652,7 @@ class ResponsesWebSocketStreaming:
|
|||
|
||||
# Log the output-masked form so PII redacted by apply_to_output
|
||||
# guardrails does not appear in success logs.
|
||||
self._record_backend_event(wrapped_str)
|
||||
await self._record_backend_event(wrapped_str)
|
||||
|
||||
await self.websocket.send_text(wrapped_str)
|
||||
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ from litellm.constants import (
|
|||
ROUTING_REQUEST_TAGS_METADATA_KEY,
|
||||
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
|
||||
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -869,6 +870,30 @@ def _request_model_group(kwargs: Mapping[str, object]) -> str | None:
|
|||
return model_group if isinstance(model_group, str) else None
|
||||
|
||||
|
||||
def _routed_deployment_id(deployment_metadata: Mapping[str, object]) -> str | None:
|
||||
model_info: Final = deployment_metadata.get("model_info")
|
||||
deployment_id: Final = (
|
||||
_MODEL_INFO_ADAPTER.validate_python(model_info).get("id") if isinstance(model_info, dict) else None
|
||||
)
|
||||
return None if deployment_id is None else str(deployment_id)
|
||||
|
||||
|
||||
async def _count_before_callbacks(
|
||||
deployment_metadata: dict[str, object], # mutable-ok: the success callbacks read the marker from it
|
||||
marker: str,
|
||||
tokens: int,
|
||||
count: Callable[[], Awaitable[object]] | None,
|
||||
) -> None:
|
||||
if count is None:
|
||||
return
|
||||
deployment_metadata[marker] = tokens # rebind-ok: the success callbacks read the marker off this call's metadata
|
||||
try:
|
||||
await count()
|
||||
except Exception:
|
||||
deployment_metadata.pop(marker, None)
|
||||
raise
|
||||
|
||||
|
||||
def _mid_stream_error_before_output(error: "MidStreamFallbackError") -> bool:
|
||||
return not error.generated_content or error.is_pre_first_chunk
|
||||
|
||||
|
|
@ -6423,7 +6448,10 @@ class Router:
|
|||
"model": model_name,
|
||||
**_with_router_resolved_session_model(kwargs.get("session"), model_name),
|
||||
**(
|
||||
{"turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs)}
|
||||
{
|
||||
"turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs),
|
||||
"turn_usage_recorder": partial(self.async_record_deployment_turn_usage, model, kwargs),
|
||||
}
|
||||
if generic_function_name == "_aresponses_websocket"
|
||||
else {}
|
||||
),
|
||||
|
|
@ -9565,30 +9593,114 @@ class Router:
|
|||
model_group: Final = request_kwargs.get("model")
|
||||
if deployment_metadata is None or not isinstance(model_group, str):
|
||||
return
|
||||
model_info: Final = deployment_metadata["model_info"]
|
||||
deployment_id: Final = model_info.get("id") if isinstance(model_info, dict) else None
|
||||
if deployment_id is None:
|
||||
return
|
||||
total_tokens: Final = response_total_token_count(response)
|
||||
deployment_name: Final = deployment_metadata.get("deployment")
|
||||
deployment_metadata[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] = total_tokens
|
||||
try:
|
||||
await self._increment_deployment_usage(
|
||||
deployment_id=str(deployment_id),
|
||||
deployment_name=deployment_name if isinstance(deployment_name, str) else None,
|
||||
await _count_before_callbacks(
|
||||
deployment_metadata,
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
|
||||
total_tokens,
|
||||
self._deployment_usage_count(
|
||||
deployment_metadata=deployment_metadata,
|
||||
model_group=model_group,
|
||||
total_tokens=total_tokens,
|
||||
rpm_increment=1,
|
||||
parent_otel_span=get_parent_otel_span_from_kwargs(request_kwargs),
|
||||
)
|
||||
except Exception:
|
||||
deployment_metadata.pop(ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, None)
|
||||
raise
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.Router::increment_deployment_usage_for_response(): Exception occured - %s", e
|
||||
)
|
||||
|
||||
async def async_record_deployment_turn_usage(
|
||||
self,
|
||||
model_group: str,
|
||||
request_kwargs: Mapping[str, object],
|
||||
turn_litellm_params: Mapping[str, object],
|
||||
total_tokens: int,
|
||||
) -> None:
|
||||
try:
|
||||
deployment_metadata: Final = find_deployment_metadata(turn_litellm_params)
|
||||
if deployment_metadata is None:
|
||||
return
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(request_kwargs)
|
||||
await _count_before_callbacks(
|
||||
deployment_metadata,
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
|
||||
total_tokens,
|
||||
self._deployment_usage_count(
|
||||
deployment_metadata=deployment_metadata,
|
||||
model_group=model_group,
|
||||
total_tokens=total_tokens,
|
||||
parent_otel_span=parent_otel_span,
|
||||
),
|
||||
)
|
||||
await _count_before_callbacks(
|
||||
deployment_metadata,
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
|
||||
total_tokens,
|
||||
self._routing_strategy_usage_count(
|
||||
deployment_metadata=deployment_metadata,
|
||||
model_group=model_group,
|
||||
request_kwargs=request_kwargs,
|
||||
total_tokens=total_tokens,
|
||||
parent_otel_span=parent_otel_span,
|
||||
),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # a failed early count must not break the turn; its logging counts it
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.Router::async_record_deployment_turn_usage(): Exception occured - %s", e
|
||||
)
|
||||
|
||||
def _deployment_usage_count(
|
||||
self,
|
||||
*,
|
||||
deployment_metadata: Mapping[str, object],
|
||||
model_group: str,
|
||||
total_tokens: int,
|
||||
parent_otel_span: Span | None,
|
||||
) -> Callable[[], Awaitable[object]] | None:
|
||||
deployment_id: Final = _routed_deployment_id(deployment_metadata)
|
||||
if deployment_id is None:
|
||||
return None
|
||||
deployment_name: Final = deployment_metadata.get("deployment")
|
||||
return partial(
|
||||
self._increment_deployment_usage,
|
||||
deployment_id=deployment_id,
|
||||
deployment_name=deployment_name if isinstance(deployment_name, str) else None,
|
||||
model_group=model_group,
|
||||
total_tokens=total_tokens,
|
||||
rpm_increment=1,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
||||
def _routing_strategy_usage_count(
|
||||
self,
|
||||
*,
|
||||
deployment_metadata: Mapping[str, object],
|
||||
model_group: str,
|
||||
request_kwargs: Mapping[str, object],
|
||||
total_tokens: int,
|
||||
parent_otel_span: Span | None,
|
||||
) -> Callable[[], Awaitable[object]] | None:
|
||||
routing_kwargs: Final = {key: value for key, value in request_kwargs.items() if key != "litellm_logging_obj"}
|
||||
_, selector = self._get_routing_context( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped request dict
|
||||
model=model_group, request_kwargs=routing_kwargs
|
||||
)
|
||||
deployment_id: Final = _routed_deployment_id(deployment_metadata)
|
||||
deployment_name: Final = deployment_metadata.get("deployment")
|
||||
if (
|
||||
not isinstance(selector, LowestTPMLoggingHandler_v2)
|
||||
or deployment_id is None
|
||||
or not isinstance(deployment_name, str)
|
||||
):
|
||||
return None
|
||||
return partial(
|
||||
selector.async_increment_tpm,
|
||||
deployment_id=deployment_id,
|
||||
deployment_name=deployment_name,
|
||||
tokens=total_tokens,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
||||
async def _increment_deployment_usage(
|
||||
self,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -8,15 +8,19 @@ from dataclasses import dataclass
|
|||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm import token_counter
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_logger, verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.constants import USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import get_counted_usage_tokens
|
||||
from litellm.types.router import RouterErrors
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload
|
||||
from litellm.utils import get_utc_datetime, print_verbose
|
||||
|
|
@ -35,6 +39,13 @@ class RoutingArgs(LiteLLMPydanticObjectBase):
|
|||
ttl: int = 1 * 60 # 1min (RPM/TPM expire key)
|
||||
|
||||
|
||||
class _SuccessKwargs(TypedDict, total=False):
|
||||
litellm_params: ReadOnly[Mapping[str, object] | None]
|
||||
|
||||
|
||||
_SUCCESS_KWARGS_ADAPTER: Final = TypeAdapter(_SuccessKwargs)
|
||||
|
||||
|
||||
_active_prefetched_usage: Final[ContextVar["PrefetchedUsage | None"]] = ContextVar("prefetched_usage", default=None)
|
||||
|
||||
|
||||
|
|
@ -296,7 +307,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
|
|||
)
|
||||
|
||||
@with_service_target("router_usage")
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
|
||||
if is_batch_retrieve_call_type(kwargs.get("call_type")):
|
||||
return
|
||||
try:
|
||||
|
|
@ -308,30 +319,19 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
|
|||
raise ValueError("standard_logging_object not passed in.")
|
||||
model_group: Final = standard_logging_object.get("model_group")
|
||||
model: Final = standard_logging_object["hidden_params"]["litellm_model_name"]
|
||||
id = standard_logging_object.get("model_id")
|
||||
if model_group is None or id is None:
|
||||
return
|
||||
elif isinstance(id, int):
|
||||
id = str(id)
|
||||
model_id: Final = standard_logging_object.get("model_id")
|
||||
total_tokens: Final = standard_logging_object.get("total_tokens")
|
||||
# ------------
|
||||
# Setup values
|
||||
# ------------
|
||||
dt: Final = get_utc_datetime()
|
||||
current_minute: Final = dt.strftime("%H-%M") # use the same timezone regardless of system clock
|
||||
|
||||
tpm_key: Final = f"{id}:{model}:tpm:{current_minute}"
|
||||
# ------------
|
||||
# Update usage
|
||||
# ------------
|
||||
# update cache
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
## TPM
|
||||
await self.router_cache.async_increment_cache_post_call(
|
||||
key=tpm_key,
|
||||
value=total_tokens,
|
||||
ttl=self.routing_args.ttl,
|
||||
parent_otel_span=parent_otel_span,
|
||||
if model_group is None or not isinstance(model_id, (str, int)) or not isinstance(total_tokens, int):
|
||||
return
|
||||
counted_tokens: Final = get_counted_usage_tokens(
|
||||
_SUCCESS_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_params") or {},
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
|
||||
)
|
||||
await self.async_increment_tpm(
|
||||
deployment_id=str(model_id),
|
||||
deployment_name=model if isinstance(model, str) else None,
|
||||
tokens=total_tokens if counted_tokens is None else max(0, total_tokens - counted_tokens),
|
||||
parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
|
||||
)
|
||||
|
||||
### TESTING ###
|
||||
|
|
@ -342,6 +342,18 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
|
|||
"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - %s", e
|
||||
)
|
||||
|
||||
@with_service_target("router_usage")
|
||||
async def async_increment_tpm(
|
||||
self, *, deployment_id: str, deployment_name: str | None, tokens: int, parent_otel_span: Span | None
|
||||
) -> None:
|
||||
current_minute: Final = get_utc_datetime().strftime("%H-%M")
|
||||
await self.router_cache.async_increment_cache_post_call(
|
||||
key=f"{deployment_id}:{deployment_name}:tpm:{current_minute}",
|
||||
value=tokens,
|
||||
ttl=self.routing_args.ttl,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
||||
def _return_potential_deployments(
|
||||
self,
|
||||
healthy_deployments: list[dict],
|
||||
|
|
|
|||
|
|
@ -29,14 +29,12 @@ def find_deployment_metadata(kwargs: Mapping[str, object]) -> dict[str, object]
|
|||
return next((bucket for bucket in buckets if isinstance(bucket, dict) and "model_info" in bucket), None)
|
||||
|
||||
|
||||
def get_counted_usage_tokens(litellm_params: Mapping[str, object]) -> int | None:
|
||||
def get_counted_usage_tokens(
|
||||
litellm_params: Mapping[str, object], marker: str = ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
|
||||
) -> int | None:
|
||||
buckets: Final = (litellm_params.get(channel) for channel in _METADATA_CHANNELS)
|
||||
counted: Final = next(
|
||||
(
|
||||
bucket[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY]
|
||||
for bucket in buckets
|
||||
if isinstance(bucket, dict) and ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY in bucket
|
||||
),
|
||||
(bucket[marker] for bucket in buckets if isinstance(bucket, dict) and marker in bucket),
|
||||
None,
|
||||
)
|
||||
return counted if isinstance(counted, int) and not isinstance(counted, bool) else None
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.constants import (
|
|||
SERVER_STREAMING_CLASSIFICATION_MARKER,
|
||||
SESSION_ID_GENERATED_METADATA_KEY,
|
||||
SESSION_ID_OMITTED_METADATA_KEY,
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import (
|
||||
|
|
@ -8759,6 +8760,7 @@ _PLANTED_STAMPS = {
|
|||
"previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}],
|
||||
"_client_output_ceiling": {"api_base": "https://attacker.example"},
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9,
|
||||
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY: 10**9,
|
||||
"client_key": "client_value",
|
||||
}
|
||||
|
||||
|
|
@ -8793,6 +8795,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo
|
|||
assert "request_retry_count" not in updated["metadata"]
|
||||
assert "previous_models" not in updated["metadata"]
|
||||
assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"]
|
||||
assert USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"]
|
||||
assert updated["metadata"]["client_key"] == "client_value"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1582,6 +1582,9 @@ class TestWebSocketProjectQuotaEnforcement:
|
|||
quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once()
|
||||
|
||||
|
||||
_TURN_USAGE: Final = {"input_tokens": 20, "output_tokens": 15, "total_tokens": 35}
|
||||
|
||||
|
||||
def _turn_frames() -> tuple[str, str, str]:
|
||||
return tuple(json.dumps({"type": "response.create", "input": word}) for word in ("one", "two", "three"))
|
||||
|
||||
|
|
@ -1662,6 +1665,66 @@ def _errors(websocket: _ClientSocket) -> list[tuple[object, object]]:
|
|||
class TestWebSocketTurnAdmission:
|
||||
"""A turn after the connection's first is admitted by the deployment again before it reaches the provider."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("terminal_event", "counted_before_it_reaches_the_client"),
|
||||
[
|
||||
({"type": "response.completed", "response": {"usage": _TURN_USAGE}}, [("ws-dep", 35)]),
|
||||
({"type": "response.incomplete", "response": {"usage": _TURN_USAGE}}, [("ws-dep", 35)]),
|
||||
({"type": "response.failed", "response": {"usage": _TURN_USAGE, "error": {"code": "server_error"}}}, []),
|
||||
],
|
||||
)
|
||||
async def test_native_relay_counts_a_finished_turn_before_forwarding_its_terminal_event(
|
||||
self, terminal_event: dict[str, object], counted_before_it_reaches_the_client: list[tuple[object, int]]
|
||||
):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import websockets.exceptions
|
||||
|
||||
counted: Final[list[tuple[object, int]]] = []
|
||||
|
||||
async def record_turn_usage(turn_litellm_params: Mapping[str, object], total_tokens: int) -> None:
|
||||
await asyncio.sleep(0)
|
||||
metadata: Final = turn_litellm_params["litellm_metadata"]
|
||||
assert isinstance(metadata, dict)
|
||||
counted.append((metadata["model_info"]["id"], total_tokens))
|
||||
|
||||
class Backend:
|
||||
def __init__(self, *events: dict[str, object]) -> None:
|
||||
self.events: Final = list(events)
|
||||
|
||||
async def recv(self, decode: bool = False) -> str:
|
||||
if not self.events:
|
||||
raise websockets.exceptions.ConnectionClosed(None, None)
|
||||
return json.dumps(self.events.pop(0))
|
||||
|
||||
class Client:
|
||||
def __init__(self) -> None:
|
||||
self.seen: Final[list[tuple[object, list[tuple[object, int]]]]] = []
|
||||
|
||||
async def send_text(self, data: str) -> None:
|
||||
self.seen.append((json.loads(data)["type"], list(counted)))
|
||||
|
||||
logging_obj: Final = MagicMock()
|
||||
logging_obj.litellm_params = {"litellm_metadata": {"model_info": {"id": "ws-dep"}}}
|
||||
logging_obj.copy_for_new_call.return_value = logging_obj
|
||||
logging_obj.dispatch_success_handlers = AsyncMock()
|
||||
logging_obj.dispatch_failure_handlers = AsyncMock()
|
||||
client: Final = Client()
|
||||
handler: Final = _make_streaming(
|
||||
websocket=client,
|
||||
backend_ws=Backend({"type": "response.created", "response": {}}, terminal_event),
|
||||
logging_obj=logging_obj,
|
||||
turn_usage_recorder=record_turn_usage,
|
||||
)
|
||||
|
||||
await handler.backend_to_client()
|
||||
|
||||
assert client.seen == [
|
||||
("response.created", []),
|
||||
(terminal_event["type"], counted_before_it_reaches_the_client),
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_relay_drops_a_later_turn_the_deployment_refuses(self):
|
||||
from unittest.mock import AsyncMock
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import gc
|
|||
import json
|
||||
import os
|
||||
import weakref
|
||||
from collections.abc import Iterator, Mapping
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from types import MethodType
|
||||
from typing import Final
|
||||
|
|
@ -19,8 +19,11 @@ from litellm.caching.in_memory_cache import InMemoryCache
|
|||
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.litellm_params import RoutingStrategyName
|
||||
from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -545,31 +548,203 @@ async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> d
|
|||
routed.update(kwargs)
|
||||
|
||||
call.__name__ = function_name
|
||||
await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call)
|
||||
await router._ageneric_api_call_with_fallbacks_helper(
|
||||
model=model, original_generic_function=call, litellm_metadata={"model_group": model}
|
||||
)
|
||||
return routed
|
||||
|
||||
|
||||
async def _finish_35_token_websocket_turn(routed: Mapping[str, object], call_id: str) -> None:
|
||||
def _open_websocket_turn(routed: Mapping[str, object], call_id: str) -> Logging:
|
||||
import time
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
turn: Final = Logging(
|
||||
connection: Final = Logging(
|
||||
model="gpt-5-mini",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type=CallTypes.aresponses_websocket.value,
|
||||
start_time=time.time(),
|
||||
litellm_call_id=call_id,
|
||||
litellm_call_id=f"{call_id}-connection",
|
||||
function_id=call_id,
|
||||
)
|
||||
turn.update_environment_variables(litellm_params={"litellm_metadata": routed["litellm_metadata"]}, optional_params={})
|
||||
connection.update_environment_variables(litellm_params=get_litellm_params(**routed), optional_params={})
|
||||
return connection.copy_for_new_call(litellm_call_id=call_id, start_time=datetime.now())
|
||||
|
||||
|
||||
async def _complete_35_token_turn(turn: Logging) -> None:
|
||||
await turn.dispatch_success_handlers(
|
||||
[{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15, "total_tokens": 35}}}],
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
|
||||
|
||||
async def _finish_35_token_websocket_turn(routed: Mapping[str, object], call_id: str) -> None:
|
||||
await _complete_35_token_turn(_open_websocket_turn(routed, call_id))
|
||||
|
||||
|
||||
def _with_success_callbacks_of(monkeypatch: pytest.MonkeyPatch, build_router: Callable[[], Router]) -> Router:
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
monkeypatch.setattr(litellm, "success_callback", [])
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [])
|
||||
router: Final = build_router()
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, *litellm.callbacks])
|
||||
return router
|
||||
|
||||
|
||||
def _usage_based_routing_router(monkeypatch: pytest.MonkeyPatch, model_name: str, tpm: int) -> Router:
|
||||
return _with_success_callbacks_of(
|
||||
monkeypatch,
|
||||
lambda: Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model_name,
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "tpm": tpm},
|
||||
"model_info": {"id": f"{model_name}-dep"},
|
||||
}
|
||||
],
|
||||
routing_strategy="usage-based-routing-v2",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
_TurnAdmission = Callable[[], Awaitable[None]]
|
||||
_TurnUsageRecorder = Callable[[Mapping[str, object], int], Awaitable[None]]
|
||||
|
||||
|
||||
async def _websocket_turn_hooks(
|
||||
router: Router, model_name: str
|
||||
) -> tuple[dict[str, object], _TurnAdmission, _TurnUsageRecorder]:
|
||||
routed: Final = await _kwargs_routed_to(router, model_name, "_aresponses_websocket")
|
||||
admit_turn: Final = routed["turn_admission"]
|
||||
record_turn: Final = routed["turn_usage_recorder"]
|
||||
assert callable(admit_turn)
|
||||
assert callable(record_turn)
|
||||
return routed, admit_turn, record_turn
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("router_minute_pinned")
|
||||
async def test_usage_based_routing_refuses_the_next_websocket_turn_before_the_finished_turn_s_logging_runs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
router: Final = _usage_based_routing_router(monkeypatch, "ws-tpm-v2-early", tpm=30)
|
||||
routed, admit_turn, record_turn = await _websocket_turn_hooks(router, "ws-tpm-v2-early")
|
||||
await admit_turn()
|
||||
|
||||
await record_turn(_open_websocket_turn(routed, "ws-tpm-v2-early-turn").litellm_params, 35)
|
||||
|
||||
with pytest.raises(litellm.RateLimitError, match="ws-tpm-v2-early"):
|
||||
await admit_turn()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("router_minute_pinned")
|
||||
async def test_websocket_turn_counted_before_its_logging_is_not_counted_again_when_the_logging_runs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
router: Final = _usage_based_routing_router(monkeypatch, "ws-tpm-v2-once", tpm=69)
|
||||
routed, admit_turn, record_turn = await _websocket_turn_hooks(router, "ws-tpm-v2-once")
|
||||
await admit_turn()
|
||||
first_turn: Final = _open_websocket_turn(routed, "ws-tpm-v2-once-first")
|
||||
|
||||
await record_turn(first_turn.litellm_params, 35)
|
||||
assert await router.get_model_group_usage("ws-tpm-v2-once") == (35, 1)
|
||||
await _complete_35_token_turn(first_turn)
|
||||
|
||||
assert await router.get_model_group_usage("ws-tpm-v2-once") == (35, 1)
|
||||
await admit_turn()
|
||||
await record_turn(_open_websocket_turn(routed, "ws-tpm-v2-once-second").litellm_params, 35)
|
||||
with pytest.raises(litellm.RateLimitError, match="ws-tpm-v2-once"):
|
||||
await admit_turn()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("router_minute_pinned")
|
||||
async def test_usage_based_routing_counts_an_http_call_once(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
router: Final = _usage_based_routing_router(monkeypatch, "http-tpm-v2", tpm=1000)
|
||||
|
||||
response: Final = await router.acompletion(
|
||||
model="http-tpm-v2", messages=[{"role": "user", "content": "hi"}], mock_response="pong"
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
await GLOBAL_LOGGING_WORKER.flush()
|
||||
|
||||
tpm_keys, _ = router.lowesttpm_logger_v2.usage_counter_keys(router.get_model_list(model_name="http-tpm-v2"))
|
||||
assert await router.cache.async_get_cache(key=tpm_keys[0]) == response.usage.total_tokens
|
||||
|
||||
|
||||
class _FirstIncrementFailingCache(DualCache):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(in_memory_cache=InMemoryCache())
|
||||
self.failed: bool = False
|
||||
|
||||
async def async_increment_cache_pipeline(
|
||||
self, increment_list: list[RedisPipelineIncrementOperation], **kwargs: object
|
||||
) -> list[float] | None:
|
||||
if self.failed:
|
||||
return await super().async_increment_cache_pipeline(increment_list, **kwargs)
|
||||
self.failed = True
|
||||
raise RuntimeError("cache unavailable")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("router_minute_pinned")
|
||||
async def test_websocket_turn_whose_early_count_fails_is_counted_in_full_by_its_logging(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
router: Final = _with_success_callbacks_of(monkeypatch, lambda: _rpm_tpm_router("ws-early-count-fails"))
|
||||
router.cache = _FirstIncrementFailingCache()
|
||||
routed, _, record_turn = await _websocket_turn_hooks(router, "gpt-5-mini")
|
||||
turn: Final = _open_websocket_turn(routed, "ws-early-count-fails-turn")
|
||||
|
||||
await record_turn(turn.litellm_params, 35)
|
||||
await _complete_35_token_turn(turn)
|
||||
|
||||
assert await router.get_model_group_usage("gpt-5-mini") == (35, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("router_minute_pinned")
|
||||
async def test_websocket_turn_on_a_router_without_usage_based_routing_is_counted_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
router: Final = _with_success_callbacks_of(monkeypatch, lambda: _rpm_tpm_router("ws-default-routing"))
|
||||
routed, _, record_turn = await _websocket_turn_hooks(router, "gpt-5-mini")
|
||||
turn: Final = _open_websocket_turn(routed, "ws-default-routing-turn")
|
||||
|
||||
await record_turn(turn.litellm_params, 35)
|
||||
assert await router.get_model_group_usage("gpt-5-mini") == (35, 1)
|
||||
await _complete_35_token_turn(turn)
|
||||
|
||||
assert await router.get_model_group_usage("gpt-5-mini") == (35, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("router_minute_pinned")
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_metadata",
|
||||
[{}, {"model_info": {"id": None}}],
|
||||
ids=["no_deployment", "deployment_without_id"],
|
||||
)
|
||||
async def test_websocket_turn_not_attributed_to_a_deployment_does_not_throttle_the_next_turn(
|
||||
monkeypatch: pytest.MonkeyPatch, deployment_metadata: dict[str, object]
|
||||
):
|
||||
router: Final = _usage_based_routing_router(monkeypatch, "ws-tpm-v2-unrouted", tpm=30)
|
||||
routed, admit_turn, record_turn = await _websocket_turn_hooks(router, "ws-tpm-v2-unrouted")
|
||||
await admit_turn()
|
||||
turn: Final = _open_websocket_turn(
|
||||
{**routed, "litellm_metadata": {"model_group": "ws-tpm-v2-unrouted", **deployment_metadata}},
|
||||
"ws-tpm-v2-unrouted-turn",
|
||||
)
|
||||
|
||||
await record_turn(turn.litellm_params, 35)
|
||||
await _complete_35_token_turn(turn)
|
||||
|
||||
await admit_turn()
|
||||
assert await router.get_model_group_usage("ws-tpm-v2-unrouted") == (None, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_budget():
|
||||
router = Router(
|
||||
|
|
@ -1198,6 +1373,7 @@ def _rpm_tpm_router(model_id: str) -> Router:
|
|||
def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime:
|
||||
pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc)
|
||||
monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned)
|
||||
monkeypatch.setattr("litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime", lambda: pinned)
|
||||
return pinned
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue