Merge remote-tracking branch 'origin/main' into lit8011-pr41622
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

This commit is contained in:
yucheng-berri 2026-10-20 19:47:34 -07:00
commit 7ee528e294
64 changed files with 2990 additions and 1635 deletions

View file

@ -33,7 +33,7 @@ while IFS= read -r file || [ -n "$file" ]; do
*) has_backend=true ;;
esac
case "$file" in
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json)
has_cost_map=true ;;
tests/unit/proxy/*) : ;;
*) outside_cost_map_set=true ;;

View file

@ -4,8 +4,7 @@ Every pull request whose diff against its merge base touches one of the two cost
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
get the file checks and additionally may only touch those two files or the legacy schema
path and may only add or update models.
get the file checks and additionally may only touch those two files and may only add or update models.
"""
from __future__ import annotations
@ -21,7 +20,6 @@ from typing import Final
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
@ -110,7 +108,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
*(
f"bot PRs may only change the cost map files, not {path}"
for path in changed_files
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
if path not in GUARDED_PATHS
),
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),

View file

@ -37,6 +37,7 @@ from litellm.harness.context import GatewayTarget
from litellm.harness.errors import HarnessError, HarnessInstallFailed
from litellm.harness.types import Harness, Usage
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
@ -690,18 +691,21 @@ class ModelEndpoint:
async def _invoke_sdk(
self,
route: str,
kwargs: dict[str, Any], # mutable-ok: injects stream_options into the SDK kwargs
kwargs: dict[str, Any], # mutable-ok: SDK call kwargs built by _sdk_kwargs
) -> object:
if route == ROUTE_MESSAGES:
return await litellm.anthropic.messages.acreate(**kwargs)
if route == ROUTE_CHAT:
if kwargs.get("stream"):
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
kwargs["stream_options"] = { # mutable-ok: JSON field sent to litellm.acompletion
"include_usage": True,
**stream_options,
if not kwargs.get("stream"):
return await litellm.acompletion(**kwargs)
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
return await litellm.acompletion(
**{
**kwargs,
"stream_options": {"include_usage": True, **stream_options},
STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options,
}
return await litellm.acompletion(**kwargs)
)
return await litellm.aresponses(**kwargs)
async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response:

View file

@ -138,6 +138,7 @@ from litellm.types.interactions import (
InteractionsAPIResponse,
InteractionsAPIStreamingResponse,
)
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.openai import (
AllMessageValues,
Batch,
@ -595,6 +596,7 @@ class Logging(LiteLLMLoggingBaseClass):
prometheusLogger, \
slack_app
custom_pricing: bool = False
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
streamed_anthropic_message_id: str | None = None
@ -649,11 +651,11 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_call_id: str,
function_id: str,
litellm_trace_id: str | None = None,
dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
applied_guardrails: list[str] | None = None,
kwargs: dict | None = None,
log_raw_request_response: bool = False,
@ -698,6 +700,7 @@ class Logging(LiteLLMLoggingBaseClass):
self._pre_call_session_id: str = session_id_var.get()
_sid: Final = kwargs.get("litellm_session_id") if kwargs else None
self.litellm_session_id: str = str(_sid) if _sid else ""
self.stream_usage_injected: Final = kwargs is not None and kwargs.get(STREAM_USAGE_INJECTED_FIELD) is True
# supports_correlation_logging is False for calls originating from the
# sync client entry point (wrapper() in utils.py): a plain OS thread
# has no per-call context isolation the way an asyncio Task does, and
@ -726,15 +729,17 @@ class Logging(LiteLLMLoggingBaseClass):
self.raw_request_only = raw_request_only
# Initialize dynamic callbacks
self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks
self.dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = dynamic_input_callbacks
self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_success_callbacks
)
self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_async_success_callbacks
)
self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks
self.dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = (
self.dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_failure_callbacks
)
self.dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
dynamic_async_failure_callbacks
)
@ -851,10 +856,32 @@ class Logging(LiteLLMLoggingBaseClass):
self.dynamic_async_failure_callbacks, callback
)
def replace_global_callback(self, callback: CustomLogger) -> None:
self.add_dynamic_callback(callback)
self.replaced_global_callback_keys = self.replaced_global_callback_keys | {
litellm.logging_callback_manager.get_custom_logger_key(callback)
}
def add_dynamic_function_callbacks(
self,
*,
async_success: Callable[..., object],
failure: Callable[..., object],
async_failure: Callable[..., object],
) -> None:
self.dynamic_async_success_callbacks = self._with_dynamic_callback(
self.dynamic_async_success_callbacks, async_success
)
self.dynamic_failure_callbacks = self._with_dynamic_callback(self.dynamic_failure_callbacks, failure)
self.dynamic_async_failure_callbacks = self._with_dynamic_callback(
self.dynamic_async_failure_callbacks, async_failure
)
@staticmethod
def _with_dynamic_callback(
callbacks: Sequence[str | Callable | CustomLogger] | None, callback: CustomLogger
) -> list[str | Callable | CustomLogger]:
callbacks: Sequence[str | Callable[..., object] | CustomLogger] | None,
callback: Callable[..., object] | CustomLogger,
) -> list[str | Callable[..., object] | CustomLogger]:
existing: Final = tuple(callbacks or ())
return [*existing, *(() if callback in existing else (callback,))]
@ -891,9 +918,9 @@ class Logging(LiteLLMLoggingBaseClass):
def _process_dynamic_callback_list(
self,
callback_list: list[str | Callable | CustomLogger] | None,
callback_list: list[str | Callable[..., object] | CustomLogger] | None,
dynamic_callbacks_type: Literal["input", "success", "failure", "async_success", "async_failure"],
) -> list[str | Callable | CustomLogger] | None:
) -> list[str | Callable[..., object] | CustomLogger] | None:
"""
Helper function to initialize CustomLogger compatible callbacks in self.dynamic_* callbacks
@ -905,7 +932,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback_list is None:
return None
processed_list: Final[list[str | Callable | CustomLogger]] = []
processed_list: Final[list[str | Callable[..., object] | CustomLogger]] = []
for callback in callback_list:
if isinstance(callback, str) and callback in litellm._known_custom_logger_compatible_callbacks:
for callback_instance in self._resolve_dynamic_callback_string(callback):
@ -1566,7 +1593,10 @@ class Logging(LiteLLMLoggingBaseClass):
"api_call_start_time"
]
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
callbacks: Final = (
*self._without_replaced_global_callbacks(litellm.input_callback),
*(self.dynamic_input_callbacks or ()),
)
for callback in callbacks:
try:
if callback == "supabase" and supabaseClient is not None:
@ -1739,7 +1769,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
callbacks: Final = (
*self._without_replaced_global_callbacks(litellm.input_callback),
*(self.dynamic_input_callbacks or ()),
)
for callback in callbacks:
try:
if callback == "sentry" and add_breadcrumb:
@ -4516,14 +4549,26 @@ class Logging(LiteLLMLoggingBaseClass):
_filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks)
return len(_filtered_failure_callbacks) > 0
def _without_replaced_global_callbacks(
self, global_callbacks: Iterable[str | Callable[..., object] | CustomLogger]
) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
if not self.replaced_global_callback_keys:
return tuple(global_callbacks)
return tuple(
litellm.logging_callback_manager.without_custom_logger_keys(
global_callbacks, self.replaced_global_callback_keys
)
)
def get_combined_callback_list(
self,
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None,
global_callbacks: list[str | Callable[..., object] | CustomLogger],
) -> list[str | Callable[..., object] | CustomLogger]:
kept_global_callbacks: Final = self._without_replaced_global_callbacks(global_callbacks)
if dynamic_success_callbacks is None:
return list(global_callbacks)
return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
return list(kept_global_callbacks)
return list(dict.fromkeys((*dynamic_success_callbacks, *kept_global_callbacks)))
def _remove_internal_litellm_callbacks(self, callbacks: list) -> list:
"""

View file

@ -1,4 +1,6 @@
from collections.abc import Callable
import threading
from collections.abc import Callable, Iterable, Iterator
from types import MethodType
from typing import TYPE_CHECKING, Final
import litellm
@ -15,6 +17,7 @@ else:
_custom_logger_compatible_callbacks_literal = str
_generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
_callback_lists_lock: Final = threading.RLock()
class LoggingCallbackManager:
@ -43,16 +46,18 @@ class LoggingCallbackManager:
Add a input callback to litellm.input_callback.
Auto-routes async callbacks to litellm._async_input_callback.
"""
if not isinstance(callback, str) and self._is_async_callable(callback):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback)
with _callback_lists_lock:
if not isinstance(callback, str) and self._is_async_callable(callback):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback)
def add_litellm_service_callback(self, callback: CustomLogger | str | Callable[..., object]):
"""
Add a service callback to litellm.service_callback
"""
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback)
with _callback_lists_lock:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback)
def add_litellm_callback(self, callback: CustomLogger | str | Callable[..., object]):
"""
@ -60,10 +65,11 @@ class LoggingCallbackManager:
Ensures no duplicates are added.
"""
self._safe_add_callback_to_list(
callback=callback,
parent_list=litellm.callbacks,
)
with _callback_lists_lock:
self._safe_add_callback_to_list(
callback=callback,
parent_list=litellm.callbacks,
)
def add_litellm_success_callback(self, callback: CustomLogger | str | Callable[..., object]):
"""
@ -71,37 +77,41 @@ class LoggingCallbackManager:
Auto-routes async callbacks to litellm._async_success_callback.
Special-cases 'dynamodb' and 'openmeter' as async callbacks.
"""
if (
isinstance(callback, str)
and callback in ("dynamodb", "openmeter")
or not isinstance(callback, str)
and self._is_async_callable(callback)
):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback)
with _callback_lists_lock:
if (
isinstance(callback, str)
and callback in ("dynamodb", "openmeter")
or not isinstance(callback, str)
and self._is_async_callable(callback)
):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback)
def add_litellm_failure_callback(self, callback: CustomLogger | str | Callable[..., object]):
"""
Add a failure callback to `litellm.failure_callback`.
Auto-routes async callbacks to litellm._async_failure_callback.
"""
if not isinstance(callback, str) and self._is_async_callable(callback):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback)
with _callback_lists_lock:
if not isinstance(callback, str) and self._is_async_callable(callback):
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
else:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback)
def add_litellm_async_success_callback(self, callback: CustomLogger | Callable[..., object] | str):
"""
Add a success callback to litellm._async_success_callback
"""
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
with _callback_lists_lock:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
def add_litellm_async_failure_callback(self, callback: CustomLogger | Callable[..., object] | str):
"""
Add a failure callback to litellm._async_failure_callback
"""
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
with _callback_lists_lock:
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
def remove_callback_from_list_by_object(self, callback_list, obj, require_self=True):
"""
@ -110,13 +120,14 @@ class LoggingCallbackManager:
if not isinstance(callback_list, list): # Not list -> do nothing
return
if require_self:
remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj]
else:
remove_list = [c for c in callback_list if c == obj]
with _callback_lists_lock:
if require_self:
remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj]
else:
remove_list = [c for c in callback_list if c == obj]
for c in remove_list:
callback_list.remove(c)
for c in remove_list:
callback_list.remove(c)
def remove_callbacks_by_type(self, callback_list, callback_type):
"""
@ -134,10 +145,11 @@ class LoggingCallbackManager:
if not isinstance(callback_list, list):
return
remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)]
with _callback_lists_lock:
remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)]
for c in remove_list:
callback_list.remove(c)
for c in remove_list:
callback_list.remove(c)
def _add_string_callback_to_list(
self, callback: str, parent_list: list[CustomLogger | Callable[..., object] | str]
@ -336,6 +348,55 @@ class LoggingCallbackManager:
return "-".join(key_parts)
get_custom_logger_key = _get_custom_logger_key
callback_lists_lock: Final = _callback_lists_lock
def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]:
dropped_ids: Final = frozenset(map(id, callbacks))
owner_ids: Final = frozenset(map(id, method_owners))
def keep(callback: object) -> bool:
if id(callback) in dropped_ids:
return False
return not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids)
with _callback_lists_lock:
vacated_keys: Final = frozenset(
self._get_custom_logger_key(callback)
for callback in litellm.callbacks
if isinstance(callback, CustomLogger) and id(callback) in dropped_ids
)
litellm.callbacks = [callback for callback in litellm.callbacks if keep(callback)]
litellm.input_callback = [callback for callback in litellm.input_callback if keep(callback)]
litellm.success_callback = [callback for callback in litellm.success_callback if keep(callback)]
litellm.failure_callback = [callback for callback in litellm.failure_callback if keep(callback)]
litellm.service_callback = [callback for callback in litellm.service_callback if keep(callback)]
litellm._async_success_callback = [
callback for callback in litellm._async_success_callback if keep(callback)
]
litellm._async_failure_callback = [
callback for callback in litellm._async_failure_callback if keep(callback)
]
return vacated_keys
def with_custom_logger_keys(
self, candidates: Iterable[CustomLogger], keys: frozenset[str]
) -> tuple[CustomLogger, ...]:
if not keys:
return ()
class_names: Final = frozenset(key.split("-", 1)[0] for key in keys)
same_class: Final = (candidate for candidate in candidates if type(candidate).__name__ in class_names)
return tuple(candidate for candidate in same_class if self._get_custom_logger_key(candidate) in keys)
def without_custom_logger_keys(
self, callbacks: Iterable[str | Callable[..., object] | CustomLogger], keys: frozenset[str]
) -> Iterator[str | Callable[..., object] | CustomLogger]:
return (
callback
for callback in callbacks
if not (isinstance(callback, CustomLogger) and self._get_custom_logger_key(callback) in keys)
)
def _reset_all_callbacks(self):
"""
Reset all callbacks to an empty list
@ -363,19 +424,21 @@ class LoggingCallbackManager:
_get_all_callbacks = get_all_callbacks
def remove_callback_from_all_lists(self, obj, require_self=False) -> None:
def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None:
"""
Remove a callback object from every callback list it may have been
promoted into, so a re-initialized callback leaves no stale instance behind.
"""
for callback_list in (
litellm.callbacks,
litellm.success_callback,
litellm.failure_callback,
litellm._async_success_callback,
litellm._async_failure_callback,
):
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
with _callback_lists_lock:
for callback_list in (
litellm.callbacks,
litellm.input_callback,
litellm.success_callback,
litellm.failure_callback,
litellm._async_success_callback,
litellm._async_failure_callback,
):
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
def get_active_additional_logging_utils_from_custom_logger(
self,

View file

@ -342,6 +342,10 @@ class CustomStreamWrapper:
self.messages = getattr(logging_obj, "messages", None)
self.sent_stream_usage = False
self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False
self.caller_requested_stream_usage: Final = (
self.check_send_stream_usage(getattr(logging_obj, "stream_options", None))
and getattr(logging_obj, "stream_usage_injected", False) is not True
)
self.tool_call = False
# keep track of the returned chunks - used for calculating the input/output tokens for stream options
self.chunks: list[ModelResponseStream] = []
@ -2070,7 +2074,7 @@ class CustomStreamWrapper:
except StopIteration:
if self.sent_last_chunk is True:
complete_streaming_response: Final = self._build_complete_streaming_response()
response = self.model_response_creator()
response = self._new_usage_chunk()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
@ -2222,27 +2226,10 @@ class CustomStreamWrapper:
)
if _has_usage:
# Store a copy ONLY when usage stripping below will mutate
# the chunk. For non-usage chunks (vast majority), store
# directly to avoid expensive model_copy() per chunk.
self.chunks.append(processed_chunk.model_copy())
# Strip usage from the outgoing chunk so it's not sent twice
# (once in the chunk, once in _hidden_params).
obj_dict = processed_chunk.model_dump()
if "usage" in obj_dict:
del obj_dict["usage"]
processed_chunk = self.model_response_creator(
chunk=obj_dict,
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
Mapping[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR)
),
)
is_empty = is_model_response_stream_empty(
model_response=cast(ModelResponseStream, processed_chunk)
)
if is_empty:
usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk)
if usage_free_chunk is None:
continue
processed_chunk = usage_free_chunk
else:
# No usage data — safe to store directly without copying
self.chunks.append(processed_chunk)
@ -2273,13 +2260,18 @@ class CustomStreamWrapper:
if processed_chunk is None:
continue
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self._accumulate_streamed_delta(choice.delta)
if processed_chunk.choices:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# RETURN RESULT
self.chunks.append(processed_chunk)
return processed_chunk
if not self.send_stream_usage or getattr(processed_chunk, "usage", None) is None:
self.chunks.append(processed_chunk)
return processed_chunk
usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk)
if usage_free_chunk is None:
continue
return usage_free_chunk
except (StopAsyncIteration, StopIteration):
return await self._finalize_completed_stream(cache_hit=cache_hit)
except httpx.TimeoutException as e: # if httpx read timeout error occues
@ -2306,7 +2298,7 @@ class CustomStreamWrapper:
async def _finalize_completed_stream(self, cache_hit: bool) -> "ModelResponseStream":
if self.sent_last_chunk is True:
complete_streaming_response: Final = await self._async_build_complete_streaming_response()
response: Final = self.model_response_creator()
response: Final = self._new_usage_chunk()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
@ -2463,6 +2455,22 @@ class CustomStreamWrapper:
return False
return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
def _new_usage_chunk(self) -> ModelResponseStream:
usage_chunk: Final = self.model_response_creator()
if self.caller_requested_stream_usage:
usage_chunk.choices = []
return usage_chunk
def _record_and_strip_usage_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
self.chunks.append(chunk.model_copy())
usage_free_chunk: Final = self.model_response_creator(
chunk={key: value for key, value in chunk.model_dump().items() if key != "usage"},
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
Mapping[str, object], getattr(chunk, HIDDEN_PARAMS_ATTR)
),
)
return None if is_model_response_stream_empty(model_response=usage_free_chunk) else usage_free_chunk
def _accumulate_streamed_delta(self, delta: object) -> None:
# Shared by the sync, async, and non-aiohttp iteration sites so answer
# text and the disqualifying-content latch stay in step across all three.

View file

@ -2,17 +2,20 @@ import json
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from itertools import accumulate, chain
from typing import Final, TypeAlias, TypeVar, cast # noqa: TID251 # sum_usage_metadata casts computed keys
from typing import Final, Literal, TypeAlias, TypeVar, cast # noqa: TID251 # sum_usage_metadata casts computed keys
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.exceptions import Timeout
from litellm.litellm_core_utils.env_utils import get_env_int_or_none
from litellm.types.llms.vertex_ai import (
Candidates,
Citation,
CitationMetadata,
GenerateContentResponseBody,
GenerationConfig,
GroundingMetadata,
GroundingSegment,
GroundingSupport,
@ -48,8 +51,14 @@ HOP_FAILURES: Final = (VertexAIError, httpx.HTTPError, Timeout)
HOP_END_FINISH_REASON: Final = "CONTINUATION"
CUT_OFF_FINISH_REASONS: Final = frozenset({"MAX_TOKENS", HOP_END_FINISH_REASON})
DEFAULT_OUTPUT_BUDGET_ENV: Final = "LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS"
_OUTPUT_LIMIT_KEYS: Final = frozenset({"max_output_tokens", "maxOutputTokens"})
_REQUEST_KEYS_THAT_SKIP_THE_DEFAULT_BUDGET: Final = frozenset({"generation_config", "continuationToken"})
_TEXT_ONLY_MODALITIES: Final = TypeAdapter(list[Literal["TEXT"]])
_VERTEX_V1_PATH: Final = "/v1/projects/"
_VERTEX_PATH_ACCEPTING_CONTINUATION_TOKEN: Final = "/v1beta1/projects/"
_CANDIDATE_COUNT_KEYS: Final = ("candidate_count", "candidateCount")
LongDecodeHops: TypeAlias = tuple[GenerateContentResponseBody, ...]
AsyncHopPost: TypeAlias = Callable[[RequestBody], Awaitable[httpx.Response]]
@ -76,15 +85,53 @@ def _int_or_none(value: object) -> int | None:
return value if isinstance(value, int) else None
def _asks_for_one_candidate(generation_config: Mapping[str, object]) -> bool:
return all(
_int_or_none(generation_config[key] or 1) == 1 for key in _CANDIDATE_COUNT_KEYS if key in generation_config
)
def continuation_cap_of(request_body: RequestBody) -> int | None:
generation_config: Final = request_body.get("generationConfig")
if generation_config is None or "max_output_tokens" not in generation_config:
return None
if "candidate_count" in generation_config and _int_or_none(generation_config["candidate_count"]) != 1:
if not _asks_for_one_candidate(generation_config):
return None
return _int_or_none(generation_config["max_output_tokens"])
def _asks_for_text_only(generation_config: Mapping[str, object]) -> bool:
if "response_modalities" in generation_config:
return False
try:
_ = _TEXT_ONLY_MODALITIES.validate_python(generation_config.get("responseModalities", []))
except ValidationError:
return False
return True
def with_default_output_budget(
request_body: RequestBody, model_max_output_tokens: Callable[[], int | None]
) -> RequestBody:
generation_config: Final = request_body.get("generationConfig", GenerationConfig())
request_json: Final[Mapping[str, object]] = request_body
budget: Final = get_env_int_or_none(DEFAULT_OUTPUT_BUDGET_ENV)
if budget is None or budget <= 0:
return request_body
if not _REQUEST_KEYS_THAT_SKIP_THE_DEFAULT_BUDGET.isdisjoint(request_body):
return request_body
if not isinstance(request_json.get("generationConfig", {}), dict):
return request_body
if not _OUTPUT_LIMIT_KEYS.isdisjoint(generation_config):
return request_body
if not _asks_for_text_only(generation_config):
return request_body
model_max: Final = model_max_output_tokens()
if model_max is None:
return request_body
return {**request_body, "generationConfig": {**generation_config, "max_output_tokens": min(budget, model_max)}}
def client_continuation_token(candidate: Candidates) -> str | None:
if candidate.get("finishReason") not in CUT_OFF_FINISH_REASONS:
return None

View file

@ -7,16 +7,19 @@ Why separate file? Make it easy to see how transformation works
import os
import re
from collections.abc import Callable, Mapping
from functools import partial
from typing import TYPE_CHECKING, Any, Final, Literal, cast
from urllib.parse import quote
import httpx
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel, PositiveInt, TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_image_mime_type_from_url,
)
@ -65,6 +68,7 @@ from ..common_utils import (
get_supports_response_schema,
get_supports_system_message,
)
from .long_decode import with_default_output_budget
# VertexBase is imported only for type checking to avoid introducing a
# module-load-time cyclic import to vertex_llm_base. The instance is lazily
@ -1304,13 +1308,75 @@ def sync_transform_request_body(
vertex_auth_header=vertex_auth_header,
)
return transform_request_body(
messages=messages,
model=model,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
cached_content=cached_content,
optional_params=optional_params,
return _with_default_output_budget(
transform_request_body(
messages=messages,
model=model,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
cached_content=cached_content,
optional_params=optional_params,
),
model,
custom_llm_provider,
litellm_params.get("model_info"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is an untyped dict
)
class _CostMapOutputModalities(TypedDict):
supported_output_modalities: NotRequired[ReadOnly[tuple[str, ...]]]
_COST_MAP_OUTPUT_MODALITIES: Final = TypeAdapter(_CostMapOutputModalities)
def _answers_text(cost_map_key: str) -> bool:
try:
row: Final = _COST_MAP_OUTPUT_MODALITIES.validate_python(litellm.model_cost.get(cost_map_key)) # pyright: ignore[reportUnknownMemberType] # cost map rows are untyped JSON
except ValidationError:
return False
return "text" in row.get("supported_output_modalities", ("text",))
class _OutputLimit(TypedDict):
max_output_tokens: NotRequired[ReadOnly[PositiveInt | None]]
_OUTPUT_LIMIT: Final = TypeAdapter(_OutputLimit)
def _output_limit(fields: object) -> int | None:
try:
return _OUTPUT_LIMIT.validate_python(fields).get("max_output_tokens")
except ValidationError:
return None
def _catalog_capped_output_limit(cost_map_key: str, override: object) -> int | None:
limits: Final = (
_output_limit(GetModelCostMap.loaded_model_cost_map().get(cost_map_key)),
_output_limit(override),
)
return min((limit for limit in limits if limit is not None), default=None)
def _text_chat_model_max_output_tokens(
model: str, custom_llm_provider: str, deployment_model_info: object
) -> int | None:
info: Final = litellm.utils.lookup_model_info_for_registration(model, custom_llm_provider)
if isinstance(info, litellm.utils.ModelInfoLookupMiss) or info.get("mode") != "chat":
return None
if not _answers_text(info["key"]):
return None
return _catalog_capped_output_limit(info["key"], deployment_model_info or info)
def _with_default_output_budget(
request_body: RequestBody, model: str, custom_llm_provider: str, deployment_model_info: object
) -> RequestBody:
return with_default_output_budget(
request_body,
partial(_text_chat_model_max_output_tokens, model, custom_llm_provider, deployment_model_info),
)
@ -1380,21 +1446,8 @@ async def async_transform_request_body(
messages, should_inline=should_inline_remote_media(custom_llm_provider)
)
if _openai_messages_may_need_sync_gcs_metadata_fetch(inlined_messages):
# _transform_request_body may issue a sync httpx.get (up to 5s timeout)
# via _get_gcs_object_content_type to fetch GCS object metadata. Run the
# whole sync transformation on a worker thread so it does not block the
# async event loop.
return await asyncify(transform_request_body)(
messages=inlined_messages,
model=model,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
cached_content=cached_content,
optional_params=optional_params,
)
return transform_request_body(
transform: Final = partial(
transform_request_body,
messages=inlined_messages,
model=model,
custom_llm_provider=custom_llm_provider,
@ -1402,6 +1455,21 @@ async def async_transform_request_body(
cached_content=cached_content,
optional_params=optional_params,
)
# _transform_request_body may issue a sync httpx.get (up to 5s timeout)
# via _get_gcs_object_content_type to fetch GCS object metadata. Run the
# whole sync transformation on a worker thread so it does not block the
# async event loop.
request_body: Final[RequestBody] = (
await asyncify(transform)()
if _openai_messages_may_need_sync_gcs_metadata_fetch(inlined_messages)
else transform()
)
return _with_default_output_budget(
request_body,
model,
custom_llm_provider,
litellm_params.get("model_info"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is an untyped dict
)
def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage:

View file

@ -268,6 +268,7 @@ from litellm.proxy.litellm_pre_call_utils import (
sanitize_for_log,
)
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.utils import (
CallTypesLiteral,
ModelResponse,
@ -555,22 +556,29 @@ def model_deployments_support_stream_options(
return all(_litellm_model_supports_stream_options(m) for m in candidate_models)
_STREAM_USAGE_FLAGS: Final = ("_litellm_strip_stream_usage", STREAM_USAGE_INJECTED_FIELD)
def stream_usage_tracking_updates(
data: Mapping[str, object],
general_settings: Mapping[str, object],
route_type: str,
supports_stream_options: Callable[[], bool],
) -> Mapping[str, object]:
scrub: Final = {"_litellm_strip_stream_usage": False} if "_litellm_strip_stream_usage" in data else {}
scrub: Final = {flag: False for flag in _STREAM_USAGE_FLAGS if flag in data}
if data.get("stream", False) is not True:
return scrub
always_include: Final = general_settings.get("always_include_stream_usage")
stream_options: Final = data.get("stream_options")
if always_include is True:
if "stream_options" not in data:
return {**scrub, "stream_options": {"include_usage": True}}
return {**scrub, "stream_options": {"include_usage": True}, STREAM_USAGE_INJECTED_FIELD: True}
if isinstance(stream_options, dict) and "include_usage" not in stream_options:
return {**scrub, "stream_options": {**stream_options, "include_usage": True}}
return {
**scrub,
"stream_options": {**stream_options, "include_usage": True},
STREAM_USAGE_INJECTED_FIELD: True,
}
return scrub
if always_include is False and route_type == "agenerate_content_stream" and "stream_options" not in data:
return {**scrub, "stream_options": {"include_usage": False}}
@ -584,6 +592,7 @@ def stream_usage_tracking_updates(
return {
"stream_options": {**merged_stream_options, "include_usage": True},
"_litellm_strip_stream_usage": True,
STREAM_USAGE_INJECTED_FIELD: True,
}

View file

@ -494,7 +494,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail):
sse: Final = AnthropicSSERestorer(step)
events: Final = ResponsesStreamRestorer(step, rehydrate)
carries: Final[CarryWindows] = {}
last_chunk = None # rebind-ok: tracks the most recent chunk for the final flush.
last_chunk = None # rebind-ok: tracks the most recent chunk with choices for the final flush.
async for chunk in response:
if isinstance(chunk, (bytes, str)):
@ -506,7 +506,8 @@ class LLMShieldProxyGuardrail(CustomGuardrail):
yield event
continue
restored_chunk = detached(chunk)
last_chunk = restored_chunk
if getattr(restored_chunk, "choices", None):
last_chunk = restored_chunk
for choice in attribute_items(restored_chunk, "choices"):
await self._restore_choice(choice, carries, session_id)
yield restored_chunk

View file

@ -374,6 +374,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
self._redis_increment_queue_lock = asyncio.Lock()
self._redis_increment_flush_lock = asyncio.Lock()
self._detached_increment_operations = None
self._retired = False
self._flush_after_retire = None
self.deployment_budget_config = None
@with_service_target("model_budgets")

View file

@ -268,7 +268,7 @@ def sanitize_for_log(value: object) -> str:
from litellm.router import Router
from litellm.secret_managers.main import get_secret_bool
from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES
from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES, STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS
from litellm.types.router import BaggageMappedTagsStamp, holds_api_key_secret_reference
from litellm.types.services import ServiceTypes
@ -2370,7 +2370,15 @@ def refresh_proxy_server_request_body_snapshot(
if not isinstance(proxy_server_request, dict):
return
_body_snapshot_exclude: Final = (
frozenset({"secret_fields", "proxy_server_request", "litellm_logging_obj", VECTOR_STORE_CALLER_METADATA_KWARG})
frozenset(
{
"secret_fields",
"proxy_server_request",
"litellm_logging_obj",
VECTOR_STORE_CALLER_METADATA_KWARG,
STREAM_USAGE_INJECTED_FIELD,
}
)
| _TRANSPORT_ONLY_CREDENTIAL_KEYS
| _CALLBACK_CREDENTIAL_KEYS
)

View file

@ -880,8 +880,8 @@ def _build_passthrough_failure_request_payload(
@dataclass(frozen=True, slots=True)
class _TeamCallbackWiring:
success_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg
failure_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg
success_callbacks: "list[str | Callable[..., object] | CustomLogger] | None" = None # mutable-ok: Logging init arg
failure_callbacks: "list[str | Callable[..., object] | CustomLogger] | None" = None # mutable-ok: Logging init arg
logging_kwargs: dict[str, str | dict[str, str]] | None = None # mutable-ok: Logging.__init__ arg

View file

@ -976,6 +976,7 @@ from litellm.secret_managers.main import (
)
from litellm.tracing.config import is_lens_tracing_enabled
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.anthropic import (
AnthropicMessagesRequest,
AnthropicResponse,
@ -18219,6 +18220,7 @@ async def async_queue_request(
try:
data = await request.json()
data.pop("_litellm_strip_stream_usage", None)
data.pop(STREAM_USAGE_INJECTED_FIELD, None)
# Include original request and headers in the data
data["proxy_server_request"] = {

View file

@ -1,9 +1,14 @@
import asyncio
import hashlib
import json
from collections import OrderedDict
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from datetime import datetime
from enum import Enum, auto
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, TypeVar
from uuid import uuid4
import httpx
from fastapi import HTTPException, status
@ -12,6 +17,7 @@ from typing_extensions import Never
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.router_utils.common_utils import is_proxy_admin_request
@ -58,7 +64,34 @@ _ROUTER_ONLY_SETTINGS: Final = frozenset(
)
def _route_user_config_request(data: dict, route_type: str):
class UserConfigRouters:
def __init__(self, max_size: int) -> None:
self._max_size: Final = max_size
self._routers: Final[OrderedDict[str, LitellmRouter]] = OrderedDict() # mutable-ok: LRU evicts in place
def get(self, config: Mapping[str, object], build: Callable[[], LitellmRouter]) -> LitellmRouter:
key: Final = hashlib.sha256(json.dumps(config, sort_keys=True, default=str).encode()).hexdigest()
cached: Final = self._routers.get(key)
if cached is not None:
self._routers.move_to_end(key)
return cached
router: Final = build()
router.serve_through_request_logging()
self._routers[key] = router
if len(self._routers) > self._max_size:
self._routers.popitem(last=False)[1].discard()
return router
def clear(self) -> None:
for router in self._routers.values():
router.discard()
self._routers.clear()
user_config_routers: Final = UserConfigRouters(max_size=32)
async def _route_user_config_request(data: dict, route_type: str):
"""Route a request using the user-provided router config."""
router_config: Final = data.pop("user_config")
@ -67,10 +100,24 @@ def _route_user_config_request(data: dict, route_type: str):
valid_args: Final = litellm.Router.get_valid_args()
filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
user_router: Final = litellm.Router(**filtered_config)
ret_val: Final = getattr(user_router, f"{route_type}")(**data)
user_router.discard()
return ret_val
logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)
cached_router: Final = user_config_routers.get(filtered_config, lambda: litellm.Router(**filtered_config))
cached_router.bind_to_request_logging(logging_obj)
return await getattr(cached_router, f"{route_type}")(**{**data, "litellm_logging_obj": logging_obj})
def _request_logging_obj(data: Mapping[str, object], route_type: str) -> LiteLLMLoggingObj:
arrived_with: Final = data.get("litellm_logging_obj")
if isinstance(arrived_with, LiteLLMLoggingObj):
return arrived_with
logging_obj, _ = litellm.utils.function_setup(
original_function=route_type,
rules_obj=litellm.utils.Rules(),
start_time=datetime.now(), # noqa: DTZ005 # Logging subtracts it from naive end times
is_async_call=True,
**{"litellm_call_id": str(uuid4()), **data},
)
return logging_obj
def _is_a2a_agent_model(model_name: object) -> bool:

View file

@ -38,6 +38,7 @@ from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime, timezone
from functools import lru_cache, partial
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Generic, Literal, NoReturn, Optional, TypeAlias, TypeVar, Union, cast
@ -361,10 +362,11 @@ from litellm.router_utils.routing_groups import (
validate_routing_strategy,
)
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch
from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage
from litellm.scheduler import FlowItem, Scheduler
from litellm.secret_managers.main import str_to_bool
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.litellm_params import RoutingStrategyName
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD, RoutingStrategyName
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionToolParam,
@ -552,6 +554,7 @@ _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
_CallbackT = TypeVar("_CallbackT")
_DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object])
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset(
{"model", "api_base", "api_key", "api_version", "responses_api_version"}
)
@ -1046,6 +1049,8 @@ class RoutingArgs(enum.Enum):
# as the per-request one built from a caller-supplied user_config, drops out on its
# own rather than leaving entries behind that nothing can withdraw.
_live_routers: Final["weakref.WeakSet[Router]"] = weakref.WeakSet()
_routers_holding_global_callbacks: Final["weakref.WeakSet[Router]"] = weakref.WeakSet()
_routers_holding_global_callbacks_lock: Final = threading.Lock()
def _replay_live_router_model_cost() -> None:
@ -1578,22 +1583,18 @@ class Router:
self._init_routing_groups(self._routing_groups_input)
self._override_selectors: dict[str, RouterStrategySelector | None] = {}
self._override_selectors_lock = threading.Lock()
self._request_scoped_selectors: tuple[CustomLogger, ...] = ()
self._request_scoped_policies: tuple[CustomLogger, ...] = ()
self.access_groups = None
with _routers_holding_global_callbacks_lock:
_routers_holding_global_callbacks.add(self)
## USAGE TRACKING ##
if isinstance(litellm._async_success_callback, list):
litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success)
else:
litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success)
litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success)
if isinstance(litellm.success_callback, list):
litellm.logging_callback_manager.add_litellm_success_callback(self.sync_deployment_callback_on_success)
else:
litellm.success_callback = [self.sync_deployment_callback_on_success]
if isinstance(litellm._async_failure_callback, list):
litellm.logging_callback_manager.add_litellm_async_failure_callback(
self.async_deployment_callback_on_failure
)
else:
litellm._async_failure_callback = [self.async_deployment_callback_on_failure]
litellm.logging_callback_manager.add_litellm_async_failure_callback(self.async_deployment_callback_on_failure)
## COOLDOWNS ##
if isinstance(litellm.failure_callback, list):
litellm.logging_callback_manager.add_litellm_failure_callback(self.deployment_callback_on_failure)
@ -1689,20 +1690,116 @@ class Router:
# Stop contributing to cost-map rebuilds straight away rather than waiting
# for this router to be collected.
_live_routers.discard(self)
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm._async_success_callback, self)
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.success_callback, self)
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm._async_failure_callback, self)
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.failure_callback, self)
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.input_callback, self)
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.service_callback, self)
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, self)
self.retire_routing_state()
self._release_global_callbacks()
# Remove ForwardClientSideHeadersByModelGroup if it exists
if self.optional_callbacks is not None:
for callback in self.optional_callbacks:
litellm.logging_callback_manager.remove_callback_from_list_by_object(
litellm.callbacks, callback, require_self=False
)
def _release_global_callbacks(self) -> None:
with _routers_holding_global_callbacks_lock:
_routers_holding_global_callbacks.discard(self)
slack_alerting_loggers: Final = self._slack_alerting_loggers()
adaptive_router_hooks: Final = self._adaptive_router_hooks()
vacated_keys: Final = litellm.logging_callback_manager.drop_callbacks(
(
*self._callback_selectors(),
*self._override_selectors.values(),
*(self.optional_callbacks or ()),
*slack_alerting_loggers,
*adaptive_router_hooks,
),
method_owners=(self, *slack_alerting_loggers),
)
with _routers_holding_global_callbacks_lock:
survivors: Final = tuple(_routers_holding_global_callbacks)
candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors)
for candidate in litellm.logging_callback_manager.with_custom_logger_keys(candidates, vacated_keys):
self._register_router_selector(candidate)
adaptive_keys: Final = frozenset(
map(litellm.logging_callback_manager.get_custom_logger_key, adaptive_router_hooks)
)
adaptive_survivor: Final = next((router for router in survivors if router.adaptive_routers), None)
if adaptive_survivor is not None and vacated_keys & adaptive_keys:
adaptive_survivor._sync_adaptive_router_hooks()
def serve_through_request_logging(self) -> None:
self._request_scoped_selectors = (*self._callback_selectors(), *self._request_adaptive_router_hooks())
self._request_scoped_policies = (*self._optional_loggers(), *self._slack_alerting_loggers())
self._release_global_callbacks()
def bind_to_request_logging(self, logging_obj: LiteLLMLogging) -> None:
logging_obj.add_dynamic_function_callbacks(
async_success=self.deployment_callback_on_success,
failure=self.deployment_callback_on_failure,
async_failure=self.async_deployment_callback_on_failure,
)
for selector in self._request_scoped_selectors:
logging_obj.replace_global_callback(selector)
for policy in self._request_policies_not_set_globally():
logging_obj.add_dynamic_callback(policy)
def _request_policies_not_set_globally(self) -> tuple[CustomLogger, ...]:
global_keys: Final = frozenset(
map(
litellm.logging_callback_manager.get_custom_logger_key,
(callback for callback in litellm.callbacks if isinstance(callback, CustomLogger)),
)
)
return tuple(
policy
for policy in self._request_scoped_policies
if litellm.logging_callback_manager.get_custom_logger_key(policy) not in global_keys
)
def _request_adaptive_router_hooks(self) -> tuple[CustomLogger, ...]:
from litellm.router_strategy.adaptive_router.hooks import (
AdaptiveRouterPostCallHook,
)
return tuple(
AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)
for tagged in chain.from_iterable(self.adaptive_routers.values())
)
def _routing_callbacks(self) -> Iterable[object]:
if not (self._request_scoped_selectors or self._request_scoped_policies):
return litellm.callbacks
replaced_keys: Final = frozenset(
map(litellm.logging_callback_manager.get_custom_logger_key, self._request_scoped_selectors)
)
return chain(
litellm.logging_callback_manager.without_custom_logger_keys(litellm.callbacks, replaced_keys),
self._request_scoped_selectors,
self._request_policies_not_set_globally(),
)
def retire_routing_state(self) -> None:
for strategy in (*self._global_callback_candidates(), *self._override_selectors.values()):
if isinstance(strategy, BaseRoutingStrategy | RouterBudgetLimiting):
strategy.retire()
def _slack_alerting_loggers(self) -> tuple[CustomLogger, ...]:
slack_alerting_logger: Final = getattr(self, "slack_alerting_logger", None)
return (slack_alerting_logger,) if isinstance(slack_alerting_logger, CustomLogger) else ()
def _adaptive_router_hooks(self) -> tuple[CustomLogger, ...]:
from litellm.router_strategy.adaptive_router.hooks import (
AdaptiveRouterPostCallHook,
)
own_strategy_ids: Final = frozenset(
id(tagged.strategy) for tagged in chain.from_iterable(self.adaptive_routers.values())
)
hooks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook)
return tuple(
hook
for hook in hooks
if isinstance(hook, AdaptiveRouterPostCallHook) and id(hook.adaptive_router) in own_strategy_ids
)
def _optional_loggers(self) -> tuple[CustomLogger, ...]:
return tuple(callback for callback in self.optional_callbacks or () if isinstance(callback, CustomLogger))
def _global_callback_candidates(self) -> tuple[CustomLogger, ...]:
return (*self._callback_selectors(), *self._optional_loggers(), *self._slack_alerting_loggers())
@staticmethod
def _create_redis_cache(
@ -1808,7 +1905,7 @@ class Router:
return selector
@staticmethod
def _register_router_selector(selector: RouterStrategySelector) -> None:
def _register_router_selector(selector: CustomLogger) -> None:
if isinstance(selector, LeastBusyLoggingHandler):
if isinstance(litellm.input_callback, list):
litellm.logging_callback_manager.add_litellm_input_callback(selector)
@ -1817,23 +1914,28 @@ class Router:
if isinstance(litellm.callbacks, list):
litellm.logging_callback_manager.add_litellm_callback(selector)
def _callback_selectors(self) -> tuple[RouterStrategySelector, ...]:
return tuple(
selector
for selector in (
*(getattr(self, attr, None) for attr in self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.values()),
*chain.from_iterable(selectors.values() for selectors in self._group_selectors.values()),
)
if selector is not None
)
def _unregister_router_selectors(self, selectors: Sequence[object]) -> None:
"""
Drop router-owned strategy selectors from litellm's global callback
lists by identity. Used before re-init (`routing_strategy_init` /
`_init_routing_groups`) so repeated `update_settings` calls don't
accumulate dead selectors that keep receiving callback events.
lists. Used before re-init (`routing_strategy_init` /
`_init_routing_groups`) and by `discard` so dead selectors stop
receiving callback events. `function_setup` copies `litellm.callbacks`
into the per-event lists on every request, so those are purged too.
"""
for selector in selectors:
if isinstance(selector, BaseRoutingStrategy):
selector.retire()
selector_ids: Final = {id(s) for s in selectors if s is not None}
if not selector_ids:
return
if isinstance(litellm.callbacks, list):
litellm.callbacks = [c for c in litellm.callbacks if id(c) not in selector_ids]
if isinstance(litellm.input_callback, list):
litellm.input_callback = [c for c in litellm.input_callback if id(c) not in selector_ids]
litellm.logging_callback_manager.drop_callbacks(selectors)
def _apply_updated_routing_strategy_args(self) -> None:
"""
@ -4673,7 +4775,8 @@ class Router:
deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
):
kwargs.pop(key, None)
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=self.default_litellm_params)
defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params)
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults)
model_info = deployment.get("model_info", {}).copy()
deployment_litellm_model_name = deployment["litellm_params"]["model"]
@ -4817,6 +4920,19 @@ class Router:
data=_DEPLOYMENT_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]),
)
deployment_litellm_params: Final = _DEPLOYMENT_PARAMS_ADAPTER.validate_python(deployment["litellm_params"])
if router_config_requests_stream_usage(
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
(
_LITELLM_PARAMS_ADAPTER.validate_python(
deployment_litellm_params
if isinstance(deployment_litellm_params, Mapping)
else deployment_litellm_params.model_dump(exclude_none=True)
),
defaults,
),
):
kwargs[STREAM_USAGE_INJECTED_FIELD] = True
self._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name)
hop_kwargs: Final = _FALLBACK_HOP_ADAPTER.validate_python(kwargs)
if _is_fallback_hop(hop_kwargs):
@ -9789,7 +9905,7 @@ class Router:
Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
"""
for _callback in litellm.callbacks:
for _callback in self._routing_callbacks():
if isinstance(_callback, CustomLogger):
_callback.pre_call_check(deployment)
@ -9810,7 +9926,7 @@ class Router:
Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
"""
for _callback in litellm.callbacks:
for _callback in self._routing_callbacks():
if isinstance(_callback, CustomLogger):
try:
await _callback.async_pre_call_check(deployment, parent_otel_span)
@ -9947,7 +10063,7 @@ class Router:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
"""
returned_healthy_deployments = healthy_deployments
for _callback in litellm.callbacks:
for _callback in self._routing_callbacks():
if isinstance(_callback, CustomLogger):
try:
returned_healthy_deployments = await _callback.async_filter_deployments(
@ -10602,6 +10718,7 @@ class Router:
changes, otherwise a released router keeps recording turns through its hook."""
from litellm.router_strategy.adaptive_router.hooks import (
AdaptiveRouterPostCallHook,
AdaptiveRouterResponseHeaderHook,
)
for callback in litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook):
@ -10611,6 +10728,8 @@ class Router:
litellm.logging_callback_manager.add_litellm_callback(
AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)
)
if self.adaptive_routers:
litellm.logging_callback_manager.add_litellm_callback(AdaptiveRouterResponseHeaderHook())
def init_adaptive_router_deployment(self, deployment: Deployment) -> None:
"""

View file

@ -1,6 +1,9 @@
"""Adaptive router strategy. See README.md for design overview."""
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
from litellm.router_strategy.adaptive_router.hooks import (
AdaptiveRouterPostCallHook,
AdaptiveRouterResponseHeaderHook,
)
__all__ = ["AdaptiveRouter", "AdaptiveRouterPostCallHook"]
__all__ = ["AdaptiveRouter", "AdaptiveRouterPostCallHook", "AdaptiveRouterResponseHeaderHook"]

View file

@ -170,12 +170,7 @@ def _assistant_content_and_tool_calls(response_obj: Any) -> tuple[object, Sequen
return content, tool_calls
class AdaptiveRouterPostCallHook(CustomLogger):
"""One hook instance per AdaptiveRouter. Registered into litellm.callbacks."""
def __init__(self, adaptive_router: AdaptiveRouter) -> None:
self.adaptive_router = adaptive_router
class AdaptiveRouterResponseHeaderHook(CustomLogger):
async def async_post_call_response_headers_hook(
self,
data: Mapping[str, object],
@ -200,6 +195,13 @@ class AdaptiveRouterPostCallHook(CustomLogger):
return None
return {ADAPTIVE_ROUTER_RESPONSE_HEADER: chosen}
class AdaptiveRouterPostCallHook(CustomLogger):
"""One hook instance per AdaptiveRouter. Registered into litellm.callbacks."""
def __init__(self, adaptive_router: AdaptiveRouter) -> None:
self.adaptive_router = adaptive_router
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
await self._record(kwargs, response_obj, response_status=200)

View file

@ -24,6 +24,8 @@ class BaseRoutingStrategy(ABC):
self.dual_cache = dual_cache
self.redis_increment_operation_queue: list[RedisPipelineIncrementOperation] = []
self._sync_task: asyncio.Task[None] | None = None
self._retired = False
self._flush_after_retire: asyncio.Task[None] | None = None
if should_batch_redis_writes:
self.setup_sync_task(default_sync_interval)
@ -47,13 +49,25 @@ class BaseRoutingStrategy(ABC):
def retire(self) -> None:
self.cancel_sync_task()
if not self.redis_increment_operation_queue:
self._retired = True
self._schedule_flush_after_retire()
def _schedule_flush_after_retire(self) -> None:
if self._flush_after_retire is not None and not self._flush_after_retire.done():
return
if not self.redis_increment_operation_queue or self.dual_cache.redis_cache is None:
return
try:
loop: Final = asyncio.get_running_loop()
except RuntimeError:
return
loop.create_task(self._push_in_memory_increments_to_redis())
self._flush_after_retire = loop.create_task(self._drain_increments_after_retire())
async def _drain_increments_after_retire(self) -> None:
if not self.redis_increment_operation_queue:
return
await self._push_in_memory_increments_to_redis()
await self._drain_increments_after_retire()
async def cleanup(self):
"""Cleanup method to be called when shutting down"""
@ -98,6 +112,8 @@ class BaseRoutingStrategy(ABC):
self.redis_increment_operation_queue.append(increment_op)
self.add_to_in_memory_keys_to_update(key=key)
if self._retired:
self._schedule_flush_after_retire()
return result
@with_service_target("router_usage")

View file

@ -128,7 +128,9 @@ class RouterBudgetLimiting(CustomLogger):
self._redis_increment_queue_lock = asyncio.Lock()
self._redis_increment_flush_lock = asyncio.Lock()
self._detached_increment_operations: tuple[RedisPipelineIncrementOperation, ...] | None = None
asyncio.create_task(self.periodic_sync_in_memory_spend_with_redis())
self._sync_task: asyncio.Task[None] = asyncio.create_task(self.periodic_sync_in_memory_spend_with_redis())
self._retired = False
self._flush_after_retire: asyncio.Task[None] | None = None
self.provider_budget_config: GenericBudgetConfigType | None = provider_budget_config
self.deployment_budget_config: GenericBudgetConfigType | None = None
self.tag_budget_config: GenericBudgetConfigType | None = None
@ -140,6 +142,30 @@ class RouterBudgetLimiting(CustomLogger):
if isinstance(litellm.callbacks, list):
litellm.logging_callback_manager.add_litellm_callback(self)
def retire(self) -> None:
self._sync_task.cancel()
self._retired = True
self._schedule_flush_after_retire()
def _schedule_flush_after_retire(self) -> None:
if self._flush_after_retire is not None and not self._flush_after_retire.done():
return
if not self._has_unwritten_increments():
return
try:
loop: Final = asyncio.get_running_loop()
except RuntimeError:
return
self._flush_after_retire = loop.create_task(self._drain_increments_after_retire())
def _has_unwritten_increments(self) -> bool:
return bool(self.redis_increment_operation_queue) or self._detached_increment_operations is not None
async def _drain_increments_after_retire(self) -> None:
while self._has_unwritten_increments():
if not await self._push_in_memory_increments_to_redis():
await asyncio.sleep(DEFAULT_REDIS_SYNC_INTERVAL)
@with_service_target("router_budgets")
async def async_filter_deployments(
self,
@ -427,6 +453,8 @@ class RouterBudgetLimiting(CustomLogger):
if self.dual_cache.redis_cache is None:
return
self.redis_increment_operation_queue.append(increment_op)
if self._retired:
self._schedule_flush_after_retire()
def _get_redis_increment_queue_lock(self) -> asyncio.Lock:
return self._redis_increment_queue_lock

View file

@ -0,0 +1,17 @@
from collections.abc import Mapping, Sequence
def router_config_requests_stream_usage(
request_kwargs: Mapping[str, object], configured_params: Sequence[Mapping[str, object]]
) -> bool:
if request_kwargs.get("stream") is not True or "stream_options" in request_kwargs:
return False
return any(_includes_usage(params.get("stream_options")) for params in configured_params)
def _includes_usage(stream_options: object) -> bool:
match stream_options:
case {"include_usage": True}:
return True
case _:
return False

View file

@ -54,6 +54,7 @@ RoutingStrategyName: TypeAlias = Literal[
TRUSTED_CALLBACK_VARS_FIELD: Final = "litellm_trusted_callback_vars"
ADDRESSED_RESPONSE_ID_FIELD: Final = "_litellm_addressed_response_id"
STREAM_USAGE_INJECTED_FIELD: Final = "_litellm_stream_usage_injected"
WIRE_NAME: Final = "wire_name"
@ -397,6 +398,7 @@ class ProxyRequestState:
trusted_callback_vars: Mapping[str, str] | None = field(default=None, metadata=wire(TRUSTED_CALLBACK_VARS_FIELD))
addressed_response_id: str | None = field(default=None, metadata=wire(ADDRESSED_RESPONSE_ID_FIELD))
strip_stream_usage: bool | None = field(default=None, metadata=wire("_litellm_strip_stream_usage"))
stream_usage_injected: bool | None = field(default=None, metadata=wire(STREAM_USAGE_INJECTED_FIELD))
client_side_timeout: bool | None = None
model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None

View file

@ -1126,83 +1126,90 @@ def function_setup(
dynamic_callbacks: Final[list[str | Callable | CustomLogger] | None] = kwargs.pop("callbacks", None)
all_callbacks: Final = get_dynamic_callbacks(dynamic_callbacks=dynamic_callbacks)
if len(all_callbacks) > 0:
for callback in all_callbacks:
# check if callback is a string - e.g. "lago", "openmeter"
if isinstance(callback, str):
callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(
callback,
internal_usage_cache=None,
llm_router=None,
)
if callback is None or any(
type(cb) is type(callback) for cb in litellm._async_success_callback
): # don't double add a callback
continue
if callback not in litellm.input_callback:
litellm.input_callback.append(callback)
if callback not in litellm.success_callback:
litellm.logging_callback_manager.add_litellm_success_callback(callback)
if callback not in litellm.failure_callback:
litellm.logging_callback_manager.add_litellm_failure_callback(callback)
if callback not in litellm._async_success_callback:
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
if callback not in litellm._async_failure_callback:
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
print_verbose(f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}")
with litellm.logging_callback_manager.callback_lists_lock:
if len(all_callbacks) > 0:
for callback in all_callbacks:
# check if callback is a string - e.g. "lago", "openmeter"
if isinstance(callback, str):
callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(
callback,
internal_usage_cache=None,
llm_router=None,
)
if callback is None or any(
type(cb) is type(callback) for cb in litellm._async_success_callback
): # don't double add a callback
continue
if callback not in litellm.input_callback:
litellm.input_callback.append(callback)
if callback not in litellm.success_callback:
litellm.logging_callback_manager.add_litellm_success_callback(callback)
if callback not in litellm.failure_callback:
litellm.logging_callback_manager.add_litellm_failure_callback(callback)
if callback not in litellm._async_success_callback:
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
if callback not in litellm._async_failure_callback:
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
print_verbose(
f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}"
)
if (
len(litellm.input_callback) > 0 or len(litellm.success_callback) > 0 or len(litellm.failure_callback) > 0
) and len(callback_list) == 0:
callback_list = list(set(litellm.input_callback + litellm.success_callback + litellm.failure_callback))
callbacks_to_set_up: Final = (
list(set(litellm.input_callback + litellm.success_callback + litellm.failure_callback))
if len(callback_list) == 0
else []
)
if callbacks_to_set_up:
callback_list = callbacks_to_set_up
## ASYNC CALLBACKS - safety net for callbacks added via direct append
if len(litellm.input_callback) > 0:
removed_async_items = []
for index, callback in enumerate(litellm.input_callback):
if coroutine_checker.is_async_callable(callback):
litellm._async_input_callback.append(callback)
removed_async_items.append(index)
# Pop the async items from input_callback in reverse order to avoid index issues
for index in reversed(removed_async_items):
litellm.input_callback.pop(index)
if len(litellm.success_callback) > 0:
removed_async_items = []
for index, callback in enumerate(litellm.success_callback):
if coroutine_checker.is_async_callable(callback):
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
removed_async_items.append(index)
elif callback == "dynamodb" or callback == "openmeter":
# dynamo is an async callback, it's used for the proxy and needs to be async
# we only support async dynamo db logging for acompletion/aembedding since that's used on proxy
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
removed_async_items.append(index)
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
add_custom_logger_callback_to_specific_event(callback, "success")
# Pop the async items from success_callback in reverse order to avoid index issues
for index in reversed(removed_async_items):
litellm.success_callback.pop(index)
if len(litellm.failure_callback) > 0:
removed_async_items = []
for index, callback in enumerate(litellm.failure_callback):
if coroutine_checker.is_async_callable(callback):
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
removed_async_items.append(index)
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
add_custom_logger_callback_to_specific_event(callback, "failure")
# Pop the async items from failure_callback in reverse order to avoid index issues
for index in reversed(removed_async_items):
litellm.failure_callback.pop(index)
if callbacks_to_set_up:
get_set_callbacks: Final[Callable[[], Callable[..., None]]] = litellm_utils.get_set_callbacks
get_set_callbacks()(callback_list=callback_list, function_id=function_id)
## ASYNC CALLBACKS - safety net for callbacks added via direct append
if len(litellm.input_callback) > 0:
removed_async_items = []
for index, callback in enumerate(litellm.input_callback):
if coroutine_checker.is_async_callable(callback):
litellm._async_input_callback.append(callback)
removed_async_items.append(index)
# Pop the async items from input_callback in reverse order to avoid index issues
for index in reversed(removed_async_items):
litellm.input_callback.pop(index)
if len(litellm.success_callback) > 0:
removed_async_items = []
for index, callback in enumerate(litellm.success_callback):
if coroutine_checker.is_async_callable(callback):
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
removed_async_items.append(index)
elif callback == "dynamodb" or callback == "openmeter":
# dynamo is an async callback, it's used for the proxy and needs to be async
# we only support async dynamo db logging for acompletion/aembedding since that's used on proxy
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
removed_async_items.append(index)
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
add_custom_logger_callback_to_specific_event(callback, "success")
# Pop the async items from success_callback in reverse order to avoid index issues
for index in reversed(removed_async_items):
litellm.success_callback.pop(index)
if len(litellm.failure_callback) > 0:
removed_async_items = []
for index, callback in enumerate(litellm.failure_callback):
if coroutine_checker.is_async_callable(callback):
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
removed_async_items.append(index)
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
add_custom_logger_callback_to_specific_event(callback, "failure")
# Pop the async items from failure_callback in reverse order to avoid index issues
for index in reversed(removed_async_items):
litellm.failure_callback.pop(index)
get_set_callbacks()(callback_list=callbacks_to_set_up, function_id=function_id)
### DYNAMIC CALLBACKS ###
dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = None
dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = None
dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None
dynamic_async_failure_callbacks: Final[list[str | Callable | CustomLogger] | None] = None
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None
dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None
dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None
dynamic_async_failure_callbacks: Final[list[str | Callable[..., object] | CustomLogger] | None] = None
if kwargs.get("success_callback", None) is not None and isinstance(kwargs["success_callback"], list):
removed_async_items = []
for index, callback in enumerate(kwargs["success_callback"]):
@ -7962,12 +7969,16 @@ class TextCompletionStreamWrapper:
text_choices: Final = TextChoices()
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
raise Exception
delta: Final = chunk["choices"][0]["delta"]
text_choices["text"] = delta["content"]
text_choices["reasoning_content"] = delta.get("reasoning_content")
text_choices["index"] = chunk["choices"][0]["index"]
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
response["choices"] = [text_choices]
if chunk["choices"]:
delta: Final = chunk["choices"][0]["delta"]
text_choices["text"] = delta["content"]
text_choices["reasoning_content"] = delta.get("reasoning_content")
text_choices["index"] = chunk["choices"][0]["index"]
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
response["choices"] = [text_choices]
else:
response["choices"] = []
# only pass usage when stream_options["include_usage"] is True
if self.stream_options and self.stream_options.get("include_usage", False) is True:

File diff suppressed because it is too large Load diff

View file

@ -146,6 +146,18 @@ ignored_function_names = [
"async_admit_deployment_turn", # Bound as the WebSocket turn_admission partial, tested through it in test_router_helper_utils.py
"_deployment_as_configured_now", # Read by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py
"_admit_turn_through_routing_strategy", # Run by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py
"_callback_selectors", # Run by discard, tested through the discard tests in test_router_helper_utils.py
"bind_to_request_logging", # Run by the proxy user_config route, tested through the user_config tests in test_route_llm_request.py
"serve_through_request_logging", # Run by the proxy user_config route, tested through the user_config tests in test_route_llm_request.py
"_request_adaptive_router_hooks", # Run by serve_through_request_logging, tested through the user_config adaptive test
"_release_global_callbacks", # Run by discard, tested through the discard tests in test_router_helper_utils.py
"_routing_callbacks", # Read by the pre-call checks, tested through the user_config limit tests in test_route_llm_request.py
"_request_policies_not_set_globally", # Read by bind_to_request_logging, tested through the user_config budget test in test_route_llm_request.py
"_optional_loggers", # Read by discard and serve_through_request_logging, tested through the budget limiter discard test
"retire_routing_state", # Run by discard, tested through the budget limiter discard test
"_slack_alerting_loggers", # Run by discard, tested through the Slack alerting discard test in test_router_helper_utils.py
"_adaptive_router_hooks", # Run by discard, tested through the adaptive router discard tests in test_router_helper_utils.py
"_global_callback_candidates", # Run by discard, tested through the survivor discard tests in test_router_helper_utils.py
"without_deployments_that_cannot_batch", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
"batch_create_routing", # Tested through the mixed-group acreate_batch tests in test_router.py
"is_batch_create_routing", # Tested through the chat-after-batch-create test in test_router.py

View file

@ -46,6 +46,11 @@ model_list:
model: openai/gpt-6-luna
api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
api_key: perf-mock-not-checked
- model_name: gpt-6-luna-responses
litellm_params:
model: openai/responses/gpt-6-luna
api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
api_key: perf-mock-not-checked
- model_name: claude-sonnet-5
litellm_params:
model: anthropic/claude-sonnet-5
@ -58,6 +63,13 @@ model_list:
aws_region_name: us-east-1
aws_access_key_id: perf-mock-not-checked
aws_secret_access_key: perf-mock-not-checked
- model_name: claude-sonnet-5-bedrock-converse
litellm_params:
model: bedrock/converse/anthropic.claude-sonnet-5
aws_bedrock_runtime_endpoint: os.environ/E2E_PERF_MOCK_BEDROCK_URL
aws_region_name: us-east-1
aws_access_key_id: perf-mock-not-checked
aws_secret_access_key: perf-mock-not-checked
guardrails:
- guardrail_name: perf-pre
litellm_params:

View file

@ -0,0 +1,15 @@
"""`/v1/chat/completions` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's chat completions
route, `gpt-6-luna-responses` (`openai/responses/gpt-6-luna`) is bridged to its Responses route."""
from __future__ import annotations
from typing import Final
from .direct_vs_bridge import direct_vs_bridge
CASE: Final = direct_vs_bridge(
"chat_completions_2rps",
"chat_completions",
direct="gpt-6-luna",
bridge="gpt-6-luna-responses",
)

View file

@ -0,0 +1,31 @@
"""The mixed case's traffic on one public API, half of it on a deployment the gateway serves on
that provider's own API (direct) and half on a deployment it has to translate to another provider
API (bridge). Same 2 rps, request count, shapes, stream share, callers, guardrails and limits as
`mixed_size_requests_2rps`, so a gap between the two halves and the mixed baseline is the
translation's cost."""
from __future__ import annotations
from dataclasses import replace
from setup.performance_test_case import Endpoint, PerformanceTestCase, VirtualKey
from .mixed_size_requests_2rps import CASE as MIXED
def _scoped(key: VirtualKey, direct: str) -> VirtualKey:
return replace(key, models=(direct,)) if key.models else key
def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) -> PerformanceTestCase:
return replace(
MIXED,
name=name,
teams=(
replace(MIXED.team("team-a"), models=(direct, bridge)),
replace(MIXED.team("team-b"), models=(bridge,)),
),
virtual_keys=tuple(_scoped(key, direct) for key in MIXED.virtual_keys),
model_groups=(direct, bridge),
traffic=replace(MIXED.traffic, endpoints={endpoint: 1.0}),
)

View file

@ -8,8 +8,8 @@ new hook that deep-copies the request inline all land here as a new row. Runs on
`PYTHONASYNCIODEBUG=1` and `LITELLM_EVENT_LOOP=asyncio` (uvloop reports nothing and crashed under debug locally),
and the runner `E2E_PERF_EVENT_LOOP=1`, so the nightly build, which sets neither, skips it. The allowlist counts
are twice the highest count any of ten back-to-back local runs showed on that row (rows that showed nothing in
those ten but did in the earlier before/after legs for 46925 and 46967 get 2), so each run sits at or under half
of every cap and a listed location doubling in frequency is a failure, not a flake. The
those ten but did in the earlier before/after legs for 46925 and 46967, or in the first scheduled-lane run, get 2),
so each run sits at or under half of every cap and a listed location doubling in frequency is a failure, not a flake. The
`garbage collection` bucket is the gen-2 passes the collector logs in debug mode (0.6 to 0.7 s each, one every
minute or two of this traffic, plus the ramp while the heap fills on a local whole-file run), which asyncio would
otherwise charge to a random task; its count is the signal for a change that makes much more garbage."""
@ -44,6 +44,7 @@ CASE: Final = replace(
KnownStall("run_coroutine_job", max_count=2),
KnownStall("user_api_key_auth.py", max_count=2),
KnownStall("route_streaming_logging_to_handler", max_count=2),
KnownStall("TaskHandle._run_coro", max_count=2),
),
),
)

View file

@ -0,0 +1,16 @@
"""`/v1/messages` at 2 rps: `claude-sonnet-5` is served by the Anthropic mock's Messages route,
`claude-sonnet-5-bedrock-converse` (`bedrock/converse/anthropic.claude-sonnet-5`) is bridged to the
Bedrock mock's Converse route."""
from __future__ import annotations
from typing import Final
from .direct_vs_bridge import direct_vs_bridge
CASE: Final = direct_vs_bridge(
"messages_2rps",
"messages",
direct="claude-sonnet-5",
bridge="claude-sonnet-5-bedrock-converse",
)

View file

@ -0,0 +1,15 @@
"""`/v1/responses` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's Responses route,
`claude-sonnet-5` is bridged to the Anthropic mock's Messages route."""
from __future__ import annotations
from typing import Final
from .direct_vs_bridge import direct_vs_bridge
CASE: Final = direct_vs_bridge(
"responses_2rps",
"responses",
direct="gpt-6-luna",
bridge="claude-sonnet-5",
)

View file

@ -565,6 +565,7 @@ def _diagnostics(observation: Observation) -> list[str]:
return [
" diagnostics (not limits):",
f" bodies off reply {observation.bodies_off_reply()}",
f" mock routes {_mock_routes(observation)}",
f" guardrail calls per request {len(observation.guardrail_calls) / count:.3f}",
f" guardrail missed {observation.guardrail_missed() if observation.guardrail_calls else 'n/a'}",
f" retried share {len(observation.retried_markers) / count:.4f}",
@ -575,6 +576,12 @@ def _diagnostics(observation: Observation) -> list[str]:
]
def _mock_routes(observation: Observation) -> str:
"""Which provider route each request landed on, so a direct-vs-bridge case shows its split."""
counts: Final = Counter(f"{seen.provider}/{seen.route}" for seen in observation.mock_requests)
return ", ".join(f"{route} {count}" for route, count in sorted(counts.items())) or "n/a"
def _datastore_lines(observation: Observation) -> list[str]:
"""What the per-request Postgres and Redis counts are made of, so a red run names the statement or
command that grew. `plumbing` (BEGIN, COMMIT, session settings) and `unclassified` count in neither limit."""

View file

@ -459,7 +459,6 @@ async def test_openai_clients_parse_text_completion_streams_and_choices(gateway:
def test_text_completion_usage_chunk_has_empty_choices(gateway: Gateway) -> None:
pytest.skip("BUG: the /v1/completions include_usage chunk carries choices [{index: 0}] instead of []")
identity: Final = f"cmpl-usage-{uuid.uuid4().hex}"
prompt: Final = f"raw-stream {identity}"
with wire_server(_text_provider(identity, {prompt})) as wire, gateway.scenario() as scenario:

View file

@ -312,7 +312,7 @@ def _chat_caller_stream_usage_event(event: dict[str, JsonValue], model: str) ->
"object": event["object"],
"created": 1,
"model": model,
"choices": [{"index": 0, "delta": {}}],
"choices": [],
"usage": {**_json_object_value(event["usage"]), "cost": 4.05e-6},
}

View file

@ -1,6 +1,7 @@
import json
import re
from collections.abc import Callable, Mapping
from itertools import chain
from typing import Final
from uuid import uuid4
@ -263,13 +264,17 @@ def _assert_chunk_envelope(chunks: tuple[dict[str, JsonValue], ...], model: str,
return next(iter(identities))
def _content_chunks(chunks: tuple[dict[str, JsonValue], ...]) -> tuple[dict[str, JsonValue], ...]:
return tuple(chunk for chunk in chunks if chunk.get("usage") is None)
def _finish_reasons(chunks: tuple[dict[str, JsonValue], ...]) -> list[JsonValue]:
reasons: Final = (_only_choice(chunk).get("finish_reason") for chunk in chunks)
reasons: Final = (_only_choice(chunk).get("finish_reason") for chunk in _content_chunks(chunks))
return [reason for reason in reasons if reason is not None]
def _deltas(chunks: tuple[dict[str, JsonValue], ...]) -> tuple[dict[str, JsonValue], ...]:
return tuple(object_value(_only_choice(chunk)["delta"]) for chunk in chunks)
return tuple(object_value(_only_choice(chunk)["delta"]) for chunk in _content_chunks(chunks))
def _sdk_base(gateway: Gateway) -> str:
@ -301,12 +306,12 @@ def test_chat_completion_on_a_mantle_claude_id_posts_native_messages_and_spends_
@pytest.mark.parametrize(
("stream_options", "usage_chunks"),
[
pytest.param({"stream_options": {"include_usage": True}}, [_USAGE], id="include_usage"),
pytest.param({"stream_options": {"include_usage": True}}, [(True, *_USAGE)], id="include_usage"),
pytest.param({}, [], id="no_usage"),
],
)
def test_chat_stream_on_a_mantle_claude_id_relays_openai_chunks_from_the_native_stream(
gateway: Gateway, stream_options: dict[str, JsonValue], usage_chunks: list[tuple[int, int, int]]
gateway: Gateway, stream_options: dict[str, JsonValue], usage_chunks: list[tuple[JsonValue, ...]]
) -> None:
marker: Final = uuid4().hex
expected: Final = _outbound(_HAIKU, [_user_turn(marker)], stream=True)
@ -322,9 +327,9 @@ def test_chat_stream_on_a_mantle_claude_id_relays_openai_chunks_from_the_native_
identity: Final = _assert_chunk_envelope(chunks, model, response.text)
assert "".join(str(delta.get("content") or "") for delta in _deltas(chunks)) == _answer(marker), response.text
assert _finish_reasons(chunks) == ["stop"], response.text
assert [_token_usage(chunk) for chunk in chunks if chunk.get("usage") is not None] == usage_chunks, (
response.text
)
assert [
(chunk["choices"] == [], *_token_usage(chunk)) for chunk in chunks if chunk.get("usage") is not None
] == usage_chunks, response.text
_assert_success_row(
_spend_row(_BY_REQUEST_ID, identity), model, _HAIKU, "acompletion", _mantle_cost(gateway, _HAIKU)
)
@ -620,15 +625,14 @@ def test_openai_sdk_chat_stream_on_a_mantle_claude_id(gateway: Gateway) -> None:
)
identities: Final = {chunk.id for chunk in chunks}
assert len(identities) == 1 and {chunk.model for chunk in chunks} == {model}, chunks
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _answer(marker), chunks
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"], (
chunks
)
choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks))
assert "".join(choice.delta.content or "" for choice in choices) == _answer(marker), chunks
assert [choice.finish_reason for choice in choices if choice.finish_reason] == ["stop"], chunks
assert [
(chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens)
(len(chunk.choices), chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens)
for chunk in chunks
if chunk.usage is not None
] == [_USAGE], chunks
] == [(0, *_USAGE)], chunks
_assert_success_row(
_spend_row(_BY_REQUEST_ID, next(iter(identities))),
model,
@ -688,15 +692,14 @@ async def test_async_openai_sdk_chat_stream_on_a_mantle_claude_id(gateway: Gatew
chunks: Final = tuple([chunk async for chunk in stream])
identities: Final = {chunk.id for chunk in chunks}
assert len(identities) == 1 and {chunk.model for chunk in chunks} == {model}, chunks
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _answer(marker), chunks
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"], (
chunks
)
choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks))
assert "".join(choice.delta.content or "" for choice in choices) == _answer(marker), chunks
assert [choice.finish_reason for choice in choices if choice.finish_reason] == ["stop"], chunks
assert [
(chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens)
(len(chunk.choices), chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens)
for chunk in chunks
if chunk.usage is not None
] == [_USAGE], chunks
] == [(0, *_USAGE)], chunks
_assert_success_row(
_spend_row(_BY_REQUEST_ID, next(iter(identities))),
model,

View file

@ -15,7 +15,7 @@ import openai
import pytest
from integration._support.client import Gateway, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
@ -723,6 +723,33 @@ def test_chat_sdk_stream_include_usage_masks_request_fields(gateway: Gateway, tm
assert len(post_requests) == 1
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
def test_chat_stream_usage_the_proxy_adds_keeps_its_marker_out_of_the_stored_request(
gateway: Gateway, tmp_path: Path
) -> None:
response_id: Final = f"chatcmpl-proxy-usage-{uuid4()}"
with (
wire_server(
_answering_model_listing(lambda request: _chat_stream(response_id, "sort_key", include_usage=True))
) as wire,
owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
isolated.scenario() as scenario,
):
model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key")
response: Final = isolated.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": "stream marker"}], "stream": True},
)
assert response.status_code == 200, response.text
assert {string_value(event["id"]) for event in _sse_events(response.text)} == {response_id}, response.text
stored_request: Final = object_value(_stored_row(response_id)["proxy_server_request"])
assert stored_request["messages"] == [{"role": "user", "content": "stream marker"}], stored_request
assert "_litellm_stream_usage_injected" not in stored_request, stored_request
observed: Final = _provider_calls(wire.drain())
assert [_json_object(request.body)["stream_options"] for request in observed] == [{"include_usage": True}]
def test_chat_history_keeps_string_tool_arguments_and_tool_content(gateway: Gateway, tmp_path: Path) -> None:
arguments: Final = '{"sort_key":"created_at"}'
tool_content: Final = "tool-result-created_at"

View file

@ -3,6 +3,7 @@ import json
import threading
import uuid
from collections.abc import Callable
from itertools import chain
from pathlib import Path
from queue import SimpleQueue
from typing import Final
@ -13,9 +14,10 @@ from hypothesis import Phase, example, given, settings
from hypothesis import strategies as st
from integration._support.client import Gateway, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.wire import Reply, Request, wire_server
from openai import OpenAI
from openai import OpenAI, omit
from openai.types.chat import ChatCompletionChunk
from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.shares_worker_state
@ -197,6 +199,7 @@ def test_proxy_stream_usage_visibility_keeps_exact_persisted_charge(gateway: Gat
assert {chunk.id for chunk in chunks} == {identity}
usages: Final = tuple(chunk.usage for chunk in chunks if chunk.usage is not None)
assert len(usages) == (1 if include else 0)
assert [len(chunk.choices) for chunk in chunks if chunk.usage is not None] == ([0] if include else [])
if include:
assert usages[0].prompt_tokens == 11 and usages[0].completion_tokens == 4
requests: Final = wire.drain()
@ -214,6 +217,119 @@ def test_proxy_stream_usage_visibility_keeps_exact_persisted_charge(gateway: Gat
assert float(rows[0]["spend"]) == pytest.approx(0.019)
_STREAM_USAGE_INJECTED_FLAG: Final = "_litellm_stream_usage_injected"
def _registered_model_names(gateway: Gateway) -> tuple[JsonValue, ...]:
entries: Final = gateway.get("/model/info")["data"]
assert isinstance(entries, list), entries
return tuple(object_value(entry).get("model_name") for entry in entries)
def _registered(gateway: Gateway, model: str) -> None:
eventually(lambda: _registered_model_names(gateway), lambda model_names: model in model_names, seconds=60)
def _sdk_stream(
gateway: Gateway,
route_prefix: str,
model: str,
identity: str,
*,
include_usage: bool | None,
extra_body: dict[str, JsonValue],
) -> tuple[ChatCompletionChunk, ...]:
with OpenAI(
api_key=gateway.key,
base_url=str(gateway.client.base_url).rstrip("/") + route_prefix,
timeout=30,
max_retries=0,
) as client:
stream: Final = client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": identity}],
stream=True,
stream_options=omit if include_usage is None else {"include_usage": include_usage},
extra_body=extra_body,
)
with stream:
return tuple(stream)
def _streamed_text(chunks: tuple[ChatCompletionChunk, ...]) -> str:
choices: Final = chain.from_iterable(chunk.choices for chunk in chunks)
return "".join(choice.delta.content or "" for choice in choices)
def _usage_chunk_shapes(chunks: tuple[ChatCompletionChunk, ...]) -> list[tuple[int, int]]:
return [(len(chunk.choices), chunk.usage.total_tokens) for chunk in chunks if chunk.usage is not None]
@pytest.mark.parametrize(
("route_prefix", "sends", "extra_body"),
[pytest.param("/v1", 2, {}, id="chat_miss_then_cache_hit"), pytest.param("/queue", 1, {"priority": 0}, id="queue")],
)
def test_caller_requested_stream_usage_has_no_choices_even_with_a_client_sent_injection_flag(
gateway: Gateway, route_prefix: str, sends: int, extra_body: dict[str, JsonValue]
) -> None:
identity: Final = "stream-usage-flag-" + uuid.uuid4().hex
with (
wire_server(lambda request: Reply(content_type="text/event-stream", chunks=text_stream(identity))) as wire,
gateway.scenario() as scenario,
):
model: Final = scenario.model(api_base=wire.url + "/v1")
_registered(gateway, model)
streams: Final = tuple(
_sdk_stream(
gateway,
route_prefix,
model,
identity,
include_usage=True,
extra_body={**extra_body, _STREAM_USAGE_INJECTED_FLAG: True},
)
for _ in range(sends)
)
assert [_streamed_text(chunks) for chunks in streams] == ["Hello 雪 café"] * sends
assert [_usage_chunk_shapes(chunks) for chunks in streams] == [[(0, 15)]] * sends
requests: Final = wire.drain()
assert len(requests) == 1
assert _STREAM_USAGE_INJECTED_FLAG not in json.loads(requests[0].body)
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
def test_stream_usage_the_deployment_adds_keeps_one_empty_choice(gateway: Gateway, tmp_path: Path) -> None:
identity: Final = "deployment-stream-usage-" + uuid.uuid4().hex
with wire_server(lambda request: Reply(content_type="text/event-stream", chunks=text_stream(identity))) as wire:
shared_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config: Final = {
**shared_config,
"model_list": [
{
"model_name": "deployment-usage",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "synthetic-deployment-usage-key",
"api_base": wire.url + "/v1",
"stream_options": {"include_usage": True},
},
}
],
"general_settings": {**shared_config["general_settings"], "always_include_stream_usage": False},
}
path: Final = tmp_path / "deployment-stream-usage.yaml"
path.write_text(yaml.safe_dump(config))
with owned_proxy(gateway, tmp_path, {}, config=path) as candidate:
chunks: Final = _sdk_stream(
candidate, "/v1", "deployment-usage", identity, include_usage=None, extra_body={}
)
assert _streamed_text(chunks) == "Hello 雪 café"
assert _usage_chunk_shapes(chunks) == [(1, 15)]
requests: Final = wire.drain()
assert len(requests) == 1
assert json.loads(requests[0].body)["stream_options"]["include_usage"] is True
@pytest.mark.covers("other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream")
def test_messages_stream_completes_through_trailing_empty_choices_usage_chunk(gateway: Gateway) -> None:
identity: Final = "messages-empty-choices-" + uuid.uuid4().hex

View file

@ -252,6 +252,7 @@ def isolate_host_environment(isolated_aws_config_files: tuple[Path, Path]) -> It
@pytest.fixture(autouse=True)
def isolate_litellm_globals() -> Iterator[None]:
original_callbacks: Final = {name: list(getattr(litellm, name) or []) for name in CALLBACK_LISTS}
original_callback_holders: Final = frozenset(litellm_router_module._routers_holding_global_callbacks)
original_reset: Final = {name: getattr(litellm, name) for name in RESET_TO_NONE_GLOBALS}
original_restored: Final = {name: getattr(litellm, name) for name in RESTORED_GLOBALS if hasattr(litellm, name)}
original_clients: Final = {name: litellm.__dict__[name] for name in MODULE_LEVEL_CLIENTS if name in litellm.__dict__}
@ -263,6 +264,7 @@ def isolate_litellm_globals() -> Iterator[None]:
_flush_client_caches()
for name in CALLBACK_LISTS:
setattr(litellm, name, [])
litellm_router_module._routers_holding_global_callbacks.clear()
for name in RESET_TO_NONE_GLOBALS:
setattr(litellm, name, None)
for name in MODULE_LEVEL_CLIENTS:
@ -277,6 +279,8 @@ def isolate_litellm_globals() -> Iterator[None]:
litellm.__dict__.update(original_clients)
for name, value in (original_callbacks | original_reset | original_restored).items():
setattr(litellm, name, value)
litellm_router_module._routers_holding_global_callbacks.clear()
litellm_router_module._routers_holding_global_callbacks.update(original_callback_holders)
for logger, (level, disabled, propagate, handlers, filters) in original_loggers.items():
logger.setLevel(level)
logger.disabled = disabled

View file

@ -1,11 +1,12 @@
import json
from typing import Final
from typing import Final, Literal
import httpx
import pytest
import respx
import litellm
from litellm.types.utils import LiteLLMFineTuningJob
API_BASE: Final = "https://fine-tuning.test"
BASE_MODEL: Final = "gpt-4.1-nano-2025-04-14"
@ -52,3 +53,42 @@ async def test_create_sends_the_model_without_the_provider_prefix(
)
assert json.loads(sent.calls.last.request.content)["model"] == BASE_MODEL
@pytest.mark.asyncio
@respx.mock
@pytest.mark.parametrize(
("custom_llm_provider", "api_base", "route", "sent_api_version"),
[
("openai", f"{API_BASE}/v1", f"{API_BASE}/v1/fine_tuning/jobs", None),
("azure", API_BASE, f"{API_BASE}/openai/fine_tuning/jobs", "2024-10-21"),
],
)
@pytest.mark.parametrize("is_async", [True, False], ids=["async", "sync"])
async def test_create_with_api_version_reaches_the_provider_and_only_azure_gets_it(
monkeypatch: pytest.MonkeyPatch,
custom_llm_provider: Literal["openai", "azure"],
api_base: str,
route: str,
sent_api_version: str | None,
is_async: bool,
) -> None:
sent: Final = respx.post(url__startswith=route).mock(return_value=httpx.Response(200, json=PROVIDER_JOB))
request: Final = {
"model": BASE_MODEL,
"training_file": "file-abc123",
"custom_llm_provider": custom_llm_provider,
"api_key": "sk-test",
"api_base": api_base,
"api_version": "2024-10-21",
}
async with httpx.AsyncClient() as session:
monkeypatch.setattr(litellm, "aclient_session", session)
job: Final = (
await litellm.acreate_fine_tuning_job(**request) if is_async else litellm.create_fine_tuning_job(**request)
)
assert isinstance(job, LiteLLMFineTuningJob)
assert job.id == PROVIDER_JOB["id"]
assert sent.calls.last.request.url.params.get("api-version") == sent_api_version

View file

@ -272,6 +272,7 @@ async def test_sdk_chat_stream(monkeypatch: pytest.MonkeyPatch) -> None:
assert lines[-1] == "data: [DONE]"
assert json.loads(lines[0][6:])["choices"][0]["delta"]["content"] == "he"
assert calls[0]["stream_options"] == {"include_usage": True}
assert calls[0]["_litellm_stream_usage_injected"] is True
assert (ep.usage.input_tokens, ep.usage.output_tokens) == (6, 2)
assert ep.usage.cost == pytest.approx(6 * 0.001 + 2 * 0.002)

View file

@ -402,6 +402,32 @@ def test_get_combined_callback_list_preserves_insertion_order(logging_obj):
) == ["prometheus", "langfuse", "datadog", "otel", "s3", "gcs_bucket", "arize", "logfire"]
class _InputEventRecorder(CustomLogger):
def __init__(self, label: str, events: list[str]) -> None:
self._label = label
self.events = events
def log_pre_api_call(self, model: object, messages: object, kwargs: object) -> None:
self.events.append(f"{self._label} pre")
def log_post_api_call(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
self.events.append(f"{self._label} post")
def test_input_events_reach_the_replacement_and_not_the_global_callback_it_replaced(
logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch
) -> None:
events: Final[list[str]] = []
monkeypatch.setattr(litellm, "input_callback", [_InputEventRecorder("global", events)])
logging_obj.replace_global_callback(_InputEventRecorder("replacement", events))
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
logging_obj.pre_call(input="", api_key="")
logging_obj.post_call(original_response="ok", input="", api_key="")
assert events == ["replacement pre", "replacement post"]
def test_get_masked_api_base(logging_obj):
api_base = "https://api.openai.com/v1"
masked_api_base = logging_obj._get_masked_api_base(api_base)

View file

@ -145,6 +145,7 @@ def test_remove_callback_from_all_lists():
obj = TestLogger()
manager.add_litellm_callback(obj)
manager.add_litellm_input_callback(obj)
manager.add_litellm_success_callback(obj)
manager.add_litellm_failure_callback(obj)
manager.add_litellm_async_success_callback(obj)
@ -153,6 +154,7 @@ def test_remove_callback_from_all_lists():
manager.remove_callback_from_all_lists(obj)
assert obj not in litellm.callbacks
assert obj not in litellm.input_callback
assert obj not in litellm.success_callback
assert obj not in litellm.failure_callback
assert obj not in litellm._async_success_callback

View file

@ -603,6 +603,91 @@ async def test_streaming_handler_with_usage(
assert chunk_has_usage
def _usage_test_logging(
caller_stream_options: dict[str, bool] | None, stream_usage_injected: bool = False
) -> Logging:
logging_obj: Final = Logging(
model="hosted_vllm/m",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="completion",
start_time=1_790_000_000.0,
litellm_call_id="12345",
function_id="1245",
kwargs={"_litellm_stream_usage_injected": True} if stream_usage_injected else {},
)
logging_obj.update_environment_variables(
litellm_params={}, optional_params={}, model="hosted_vllm/m", stream_options=caller_stream_options
)
return logging_obj
@pytest.mark.parametrize(
("caller_stream_options", "stream_usage_injected", "usage_chunk_choice_count"),
[
({"include_usage": True}, False, 0),
(None, False, 1),
({"include_usage": True}, True, 1),
],
ids=["caller-asked", "provider-handler-injected", "proxy-injected"],
)
@pytest.mark.parametrize("plain_iterator", [False, True])
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_usage_chunk_has_empty_choices_only_for_callers_that_asked(
sync_mode: bool,
plain_iterator: bool,
caller_stream_options: dict[str, bool] | None,
stream_usage_injected: bool,
usage_chunk_choice_count: int,
):
usage: Final = Usage(prompt_tokens=5, completion_tokens=1, total_tokens=6)
provider_chunks: Final = [
ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="ok"))]),
ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]),
ModelResponseStream(id="c1", choices=[], provider_specific_fields={"citations": ["a"]}),
ModelResponseStream(id="c1", choices=[], usage=usage),
]
wrapper: Final = CustomStreamWrapper(
completion_stream=(
iter(provider_chunks) if plain_iterator else ModelResponseListIterator(model_responses=provider_chunks)
),
model="hosted_vllm/m",
custom_llm_provider="hosted_vllm",
logging_obj=_usage_test_logging(caller_stream_options, stream_usage_injected),
stream_options={"include_usage": True},
)
chunks: Final = list(wrapper) if sync_mode else [chunk async for chunk in wrapper]
assert [len(chunk.choices) for chunk in chunks] == [1, 1, 0, usage_chunk_choice_count]
assert chunks[2].provider_specific_fields == {"citations": ["a"]}
assert [getattr(chunk, "usage", None) is not None for chunk in chunks] == [False, False, False, True]
assert chunks[-1].usage.total_tokens == 6
@pytest.mark.asyncio
async def test_plain_iterator_stream_keeps_usage_on_its_chunk_when_usage_was_not_requested():
usage: Final = Usage(prompt_tokens=5, completion_tokens=1, total_tokens=6)
provider_chunks: Final = [
ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="ok"))]),
ModelResponseStream(
id="c1", choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], usage=usage
),
]
wrapper: Final = CustomStreamWrapper(
completion_stream=iter(provider_chunks),
model="hosted_vllm/m",
custom_llm_provider="hosted_vllm",
logging_obj=_usage_test_logging(None),
)
chunks: Final = [chunk async for chunk in wrapper]
assert [chunk.choices[0].finish_reason for chunk in chunks] == [None, "stop"]
assert chunks[-1].usage.total_tokens == 6
@pytest.mark.parametrize("sync_mode", [False])
@pytest.mark.asyncio
@pytest.mark.flaky(reruns=3)

View file

@ -364,7 +364,8 @@ def test_text_completion_stream_forwards_stream_options_and_usage(monkeypatch):
)
)
assert tuple(chunk.choices[0].text for chunk in chunks) == ("hello", None, None)
assert tuple(chunk.choices[0].text for chunk in chunks[:-1]) == ("hello", None)
assert chunks[-1].choices == []
assert tuple(chunk.usage for chunk in chunks[:-1]) == (None, None)
assert (chunks[-1].usage.prompt_tokens, chunks[-1].usage.completion_tokens, chunks[-1].usage.total_tokens) == (
2,

View file

@ -543,8 +543,9 @@ def test_nova_streaming_sends_stream_flag_and_parses_chunks_and_usage(respx_mock
"stream": True,
"stream_options": {"include_usage": True},
}
assert tuple(chunk.choices[0].delta.content for chunk in chunks) == ("Nova ", "stream", None, None)
assert tuple(chunk.choices[0].finish_reason for chunk in chunks) == (None, None, "stop", None)
assert tuple(chunk.choices[0].delta.content for chunk in chunks[:-1]) == ("Nova ", "stream", None)
assert chunks[-1].choices == []
assert tuple(chunk.choices[0].finish_reason for chunk in chunks[:-1]) == (None, None, "stop")
assert (chunks[-1].usage.prompt_tokens, chunks[-1].usage.completion_tokens, chunks[-1].usage.total_tokens) == (
2,
1,

View file

@ -516,9 +516,16 @@ async def test_unary_answer_cut_at_the_callers_cap_is_returned_with_its_token_in
assert response.choices[0].message.provider_specific_fields["continuation_token"] == "dG9rLTE="
@pytest.mark.parametrize(
"choice_count_request",
[{"max_tokens": 100_000, "n": 2}, {"extra_body": {"generationConfig": {"candidateCount": 2}}}],
)
@pytest.mark.parametrize("is_async", [True, False])
@pytest.mark.asyncio
async def test_unary_request_for_more_than_one_choice_is_never_continued(is_async: bool):
async def test_unary_request_for_more_than_one_choice_is_never_continued(
monkeypatch: pytest.MonkeyPatch, is_async: bool, choice_count_request: Mapping[str, object]
):
monkeypatch.setenv(BUDGET_ENV, "100000")
sent: Final[list[dict[str, object]]] = []
# Vertex AI answered 400 "Candidate count greater than 1 is not supported when continuation_token is set"
# on 2026-10-13
@ -526,9 +533,8 @@ async def test_unary_request_for_more_than_one_choice_is_never_continued(is_asyn
kwargs: Final = {
"model": MODEL,
"messages": list(MESSAGES),
"max_tokens": 100_000,
"n": 2,
"api_key": "fake-gemini-key",
**choice_count_request,
}
response: Final = (
@ -548,6 +554,42 @@ async def test_unary_request_for_more_than_one_choice_is_never_continued(is_asyn
]
@pytest.mark.parametrize(
"one_choice_request",
[
{"max_tokens": 100_000, "n": 0},
{"extra_body": {"generationConfig": {"candidateCount": None}}},
{"extra_body": {"generationConfig": {"candidateCount": 0}}},
],
)
@pytest.mark.parametrize("is_async", [True, False])
@pytest.mark.asyncio
async def test_unary_request_whose_candidate_count_google_reads_as_one_is_continued(
monkeypatch: pytest.MonkeyPatch, is_async: bool, one_choice_request: Mapping[str, object]
):
monkeypatch.setenv(BUDGET_ENV, "100000")
sent: Final[list[dict[str, object]]] = []
hops: Final = (httpx.Response(200, json=HOP_1), httpx.Response(200, json=HOP_2))
kwargs: Final = {
"model": MODEL,
"messages": list(MESSAGES),
"api_key": "fake-gemini-key",
**one_choice_request,
}
response: Final = (
await litellm.acompletion(**kwargs, client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay(hops, sent))))
if is_async
else litellm.completion(
**kwargs, client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_replay(hops, sent))))
)
)
assert sent[1] == {**sent[0], "continuationToken": "dG9rLTE="}
assert response.choices[0].message.content == "hello world"
assert response.choices[0].finish_reason == "stop"
@pytest.mark.asyncio
async def test_unary_failed_continuation_hop_returns_the_answer_so_far_with_its_token():
sent: Final[list[dict[str, object]]] = []
@ -1295,6 +1337,468 @@ async def test_vertex_long_decode_keeps_the_callers_own_authorization_header_on_
assert seen == ["Bearer mine", "Bearer mine"]
BUDGET_ENV: Final = "LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS"
ARGON: Final = "gemini/gemini-4-argon"
UNMAPPED_GEMINI: Final = "gemini/gemini-99-unreleased"
OUTPUT_LIMIT_KEYS: Final = ("max_output_tokens", "maxOutputTokens")
def _output_limits(request: Mapping[str, object]) -> dict[str, object]:
generation_config: Final = request.get("generationConfig", {})
assert isinstance(generation_config, dict)
return {key: value for key, value in generation_config.items() if key in OUTPUT_LIMIT_KEYS}
def _cost_map_max_output_tokens(model: str) -> int:
provider, name = model.split("/", 1)
max_output_tokens: Final = litellm.get_model_info(model=name, custom_llm_provider=provider)["max_output_tokens"]
assert isinstance(max_output_tokens, int)
return max_output_tokens
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("is_async", [True, False])
@pytest.mark.asyncio
async def test_request_with_no_output_limit_keeps_continuing_until_the_env_budget_is_spent(
monkeypatch: pytest.MonkeyPatch, is_async: bool, stream: bool
):
monkeypatch.setenv(BUDGET_ENV, "50")
sent: Final[list[dict[str, object]]] = []
hops: Final = (
(_sse(STREAM_FIRST_CHUNK, STREAM_HOP_1_END), _sse(HOP_2_TRUNCATED), _sse(HOP_2))
if stream
else (
httpx.Response(200, json=HOP_1),
httpx.Response(200, json=HOP_2_TRUNCATED),
httpx.Response(200, json=HOP_2),
)
)
kwargs: Final = {"model": ARGON, "messages": list(MESSAGES), "stream": stream, "api_key": "fake-gemini-key"}
transport: Final = httpx.MockTransport(_replay(hops, sent))
response: Final = (
await litellm.acompletion(**kwargs, client=AsyncHTTPHandler(transport=transport))
if is_async
else litellm.completion(**kwargs, client=HTTPHandler(client=httpx.Client(transport=transport)))
)
chunks: Final = ([chunk async for chunk in response] if is_async else list(response)) if stream else []
assert len(sent) == 2
assert _output_limits(sent[0]) == {"max_output_tokens": 50}
if stream:
assert _stream_text(chunks) == "hello world"
assert _stream_finish_reasons(chunks) == ["length"]
assert {"continuation_token": "dG9rLTI="} in _delta_fields(chunks)
else:
assert response.choices[0].message.content == "hello world"
assert response.choices[0].finish_reason == "length"
assert response.choices[0].message.provider_specific_fields["continuation_token"] == "dG9rLTI="
@pytest.mark.parametrize(
"budget, model, request_params, expected_limits",
[
("50", ARGON, {"max_tokens": 30}, {"max_output_tokens": 30}),
("50", ARGON, {"max_completion_tokens": 30}, {"max_output_tokens": 30}),
(
"50",
ARGON,
{"extra_body": {"generationConfig": {"maxOutputTokens": 30}}},
{"maxOutputTokens": 30},
),
(None, ARGON, {}, {}),
("", ARGON, {}, {}),
("a lot", ARGON, {}, {}),
("0", ARGON, {}, {}),
("-5", ARGON, {}, {}),
("50", UNMAPPED_GEMINI, {}, {}),
],
)
@pytest.mark.asyncio
async def test_env_budget_applies_only_to_a_request_that_sets_no_output_limit_of_its_own(
monkeypatch: pytest.MonkeyPatch,
budget: str | None,
model: str,
request_params: Mapping[str, object],
expected_limits: Mapping[str, object],
):
if budget is None:
monkeypatch.delenv(BUDGET_ENV, raising=False)
else:
monkeypatch.setenv(BUDGET_ENV, budget)
sent: Final[list[dict[str, object]]] = []
await litellm.acompletion(
model=model,
messages=list(MESSAGES),
api_key="fake-gemini-key",
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
**request_params,
)
assert _output_limits(sent[0]) == expected_limits
@pytest.mark.parametrize(
"modalities, expected_limits",
[
(["audio"], {}),
(["text", "audio"], {}),
(["text", "image"], {}),
(["text"], {"max_output_tokens": 50}),
],
)
@pytest.mark.asyncio
async def test_env_budget_applies_only_to_a_request_asking_for_text_output(
monkeypatch: pytest.MonkeyPatch, modalities: list[str], expected_limits: Mapping[str, object]
):
monkeypatch.setenv(BUDGET_ENV, "50")
sent: Final[list[dict[str, object]]] = []
await litellm.acompletion(
model=ARGON,
messages=list(MESSAGES),
modalities=modalities,
api_key="fake-gemini-key",
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
)
assert _output_limits(sent[0]) == expected_limits
def _ai_studio_model_in_mode(mode: str) -> str:
return next(
name
for name, row in litellm.model_cost.items()
if name.startswith("gemini/") and row.get("mode") == mode and row.get("max_output_tokens")
)
@pytest.mark.parametrize("mode", ["audio_speech", "image_generation"])
@pytest.mark.asyncio
async def test_env_budget_skips_a_model_the_cost_map_does_not_list_as_a_chat_model(
monkeypatch: pytest.MonkeyPatch, mode: str
):
monkeypatch.setenv(BUDGET_ENV, "50")
sent: Final[list[dict[str, object]]] = []
await litellm.acompletion(
model=_ai_studio_model_in_mode(mode),
messages=list(MESSAGES),
api_key="fake-gemini-key",
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
)
assert _output_limits(sent[0]) == {}
@pytest.mark.parametrize(
"output_modalities, expected_limits",
[
({}, {"max_output_tokens": 50}),
({"supported_output_modalities": ["text"]}, {"max_output_tokens": 50}),
({"supported_output_modalities": ["text", "audio"]}, {"max_output_tokens": 50}),
({"supported_output_modalities": ["audio"]}, {}),
({"supported_output_modalities": "audio"}, {}),
],
)
@pytest.mark.asyncio
async def test_env_budget_applies_only_to_a_chat_model_the_cost_map_lists_as_answering_text(
monkeypatch: pytest.MonkeyPatch, output_modalities: Mapping[str, object], expected_limits: Mapping[str, object]
):
monkeypatch.setenv(BUDGET_ENV, "50")
litellm.register_model(
{
UNMAPPED_GEMINI: {
"litellm_provider": "gemini",
"mode": "chat",
"max_output_tokens": 1000,
**output_modalities,
}
}
)
sent: Final[list[dict[str, object]]] = []
await litellm.acompletion(
model=UNMAPPED_GEMINI,
messages=list(MESSAGES),
api_key="fake-gemini-key",
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
)
assert _output_limits(sent[0]) == expected_limits
@pytest.mark.parametrize(
"extra_body",
[
{"generation_config": {"temperature": 0.1}},
{"generation_config": {"max_output_tokens": 30}},
{"generationConfig": {"response_modalities": ["AUDIO"]}},
],
)
@pytest.mark.asyncio
async def test_env_budget_leaves_a_snake_case_generation_config_from_extra_body_alone(
monkeypatch: pytest.MonkeyPatch, extra_body: Mapping[str, object]
):
monkeypatch.setenv(BUDGET_ENV, "50")
sent: Final[list[dict[str, object]]] = []
await litellm.acompletion(
model=ARGON,
messages=list(MESSAGES),
extra_body=dict(extra_body),
api_key="fake-gemini-key",
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
)
assert _output_limits(sent[0]) == {}
def _gemini_chat_models_answering_no_text() -> tuple[str, ...]:
rows: Final = (
(name, row)
for name, row in litellm.model_cost.items()
if row.get("mode") == "chat" and "text" not in row.get("supported_output_modalities", ("text",))
)
return tuple(
name if name.startswith("gemini/") else f"vertex_ai/{name}"
for name, row in rows
if row.get("litellm_provider") in ("gemini", "vertex_ai-language-models")
)
@pytest.mark.asyncio
async def test_env_budget_skips_every_chat_model_the_cost_map_lists_as_answering_no_text(
monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint
):
monkeypatch.setenv(BUDGET_ENV, "50")
models: Final = _gemini_chat_models_answering_no_text()
sent: Final[list[dict[str, object]]] = []
replies: Final = tuple(httpx.Response(200, json=HOP_2) for _ in models)
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent)))
for model in models:
await litellm.acompletion(
model=model,
messages=list(MESSAGES),
api_key="fake-gemini-key",
vertex_project="p",
vertex_location="us-central1",
vertex_credentials=token_endpoint.credentials,
client=client,
)
assert models
assert {model: _output_limits(request) for model, request in zip(models, sent, strict=True)} == dict.fromkeys(
models, {}
)
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.asyncio
async def test_env_budget_leaves_a_continuation_token_resume_with_no_limit_alone(
monkeypatch: pytest.MonkeyPatch, stream: bool
):
monkeypatch.setenv(BUDGET_ENV, "50")
sent: Final[list[dict[str, object]]] = []
reply: Final = _sse(HOP_2) if stream else httpx.Response(200, json=HOP_2)
response: Final = await litellm.acompletion(
model=ARGON,
messages=list(MESSAGES),
continuation_token="dG9rLTE=",
stream=stream,
api_key="fake-gemini-key",
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((reply,), sent))),
)
if stream:
_ = [chunk async for chunk in response]
assert sent[0]["continuationToken"] == "dG9rLTE="
assert _output_limits(sent[0]) == {}
@pytest.mark.parametrize("model", ["gemini/gemini-2.5-flash", "vertex_ai/gemini-2.5-flash", ARGON])
@pytest.mark.asyncio
async def test_env_budget_above_a_models_output_limit_sends_that_limit_instead(
monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint, model: str
):
budget: Final = 2 * _cost_map_max_output_tokens(model)
monkeypatch.setenv(BUDGET_ENV, str(budget))
sent: Final[list[dict[str, object]]] = []
await litellm.acompletion(
model=model,
messages=list(MESSAGES),
api_key="fake-gemini-key",
vertex_project="p",
vertex_location="us-central1",
vertex_credentials=token_endpoint.credentials,
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
)
assert _output_limits(sent[0]) == {"max_output_tokens": _cost_map_max_output_tokens(model)}
@pytest.mark.asyncio
async def test_env_budget_never_goes_above_the_cost_map_limit_when_a_deployment_overrides_it(
monkeypatch: pytest.MonkeyPatch,
):
model: Final = "gemini/gemini-2.5-flash"
model_limit: Final = _cost_map_max_output_tokens(model)
monkeypatch.setenv(BUDGET_ENV, str(2 * model_limit))
router: Final = litellm.Router(
model_list=[
{"model_name": "plain", "litellm_params": {"model": model, "api_key": "fake-gemini-key"}},
{
"model_name": "raised",
"litellm_params": {"model": model, "api_key": "fake-gemini-key"},
"model_info": {"max_output_tokens": 4 * model_limit},
},
]
)
sent: Final[list[dict[str, object]]] = []
replies: Final = (httpx.Response(200, json=HOP_2), httpx.Response(200, json=HOP_2))
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent)))
for model_name in ("plain", "raised"):
await router.acompletion(model=model_name, messages=list(MESSAGES), client=client)
sent_limits: Final = [_output_limits(request).get("max_output_tokens", 0) for request in sent]
assert len(sent_limits) == 2
assert all(isinstance(limit, int) and limit <= model_limit for limit in sent_limits)
@pytest.mark.asyncio
async def test_env_budget_caps_each_deployment_at_its_own_model_info_limit_and_never_at_a_siblings(
monkeypatch: pytest.MonkeyPatch,
):
model: Final = "gemini/gemini-2.5-flash"
budget: Final = _cost_map_max_output_tokens(model) // 2
low_limit: Final = budget // 4
monkeypatch.setenv(BUDGET_ENV, str(budget))
router: Final = litellm.Router(
model_list=[
{
"model_name": "low",
"litellm_params": {"model": model, "api_key": "fake-gemini-key"},
"model_info": {"max_output_tokens": low_limit},
},
{"model_name": "plain", "litellm_params": {"model": model, "api_key": "fake-gemini-key"}},
]
)
sent: Final[list[dict[str, object]]] = []
replies: Final = (httpx.Response(200, json=HOP_2), httpx.Response(200, json=HOP_2))
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent)))
for model_name in ("plain", "low"):
await router.acompletion(model=model_name, messages=list(MESSAGES), client=client)
assert [_output_limits(request) for request in sent] == [
{"max_output_tokens": budget},
{"max_output_tokens": low_limit},
]
@pytest.mark.parametrize("deployment_limit", [0, -1])
@pytest.mark.asyncio
async def test_env_budget_ignores_a_deployment_model_info_limit_that_is_not_positive(
monkeypatch: pytest.MonkeyPatch, deployment_limit: int
):
model: Final = "gemini/gemini-2.5-flash"
budget: Final = _cost_map_max_output_tokens(model) // 2
monkeypatch.setenv(BUDGET_ENV, str(budget))
router: Final = litellm.Router(
model_list=[
{
"model_name": "odd",
"litellm_params": {"model": model, "api_key": "fake-gemini-key"},
"model_info": {"max_output_tokens": deployment_limit},
}
]
)
sent: Final[list[dict[str, object]]] = []
await router.acompletion(
model="odd",
messages=list(MESSAGES),
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
)
assert _output_limits(sent[0]) == {"max_output_tokens": budget}
@pytest.mark.asyncio
async def test_env_budget_on_a_direct_call_never_goes_above_the_cost_map_limit_after_a_router_raised_it(
monkeypatch: pytest.MonkeyPatch,
):
model: Final = "gemini/gemini-2.5-flash"
model_limit: Final = _cost_map_max_output_tokens(model)
monkeypatch.setenv(BUDGET_ENV, str(2 * model_limit))
litellm.Router(
model_list=[
{
"model_name": "raised",
"litellm_params": {"model": model, "api_key": "fake-gemini-key"},
"model_info": {"max_output_tokens": 4 * model_limit},
}
]
)
sent: Final[list[dict[str, object]]] = []
await litellm.acompletion(
model=model,
messages=list(MESSAGES),
api_key="fake-gemini-key",
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
)
assert _output_limits(sent[0]) == {"max_output_tokens": model_limit}
@pytest.mark.asyncio
async def test_env_budget_on_a_direct_call_follows_a_lower_limit_set_with_register_model(
monkeypatch: pytest.MonkeyPatch,
):
model: Final = "gemini/gemini-2.5-flash"
lowered_limit: Final = _cost_map_max_output_tokens(model) // 4
monkeypatch.setenv(BUDGET_ENV, str(2 * lowered_limit))
litellm.register_model({model: {"litellm_provider": "gemini", "max_output_tokens": lowered_limit}})
sent: Final[list[dict[str, object]]] = []
await litellm.acompletion(
model=model,
messages=list(MESSAGES),
api_key="fake-gemini-key",
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
)
assert _output_limits(sent[0]) == {"max_output_tokens": lowered_limit}
@pytest.mark.parametrize(
"generation_config",
[None, {"responseModalities": None}, {"responseModalities": {"TEXT": 1}}],
)
@pytest.mark.asyncio
async def test_env_budget_sends_a_generation_config_of_an_unexpected_shape_as_it_came(
monkeypatch: pytest.MonkeyPatch, generation_config: object
):
monkeypatch.setenv(BUDGET_ENV, "50")
sent: Final[list[dict[str, object]]] = []
await litellm.acompletion(
model=ARGON,
messages=list(MESSAGES),
extra_body={"generationConfig": generation_config},
api_key="fake-gemini-key",
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
)
assert sent[0]["generationConfig"] == generation_config
def test_initial_prompt_tokens_is_absent_on_a_single_hop_and_the_first_hops_count_otherwise():
assert initial_prompt_tokens_of((HOP_1,)) is None
assert initial_prompt_tokens_of((HOP_1, HOP_2)) == 10

View file

@ -1894,7 +1894,9 @@ def test_gemini_tool_use(respx_mock: respx.MockRouter) -> None:
assert [(call.function.name, json.loads(call.function.arguments)) for call in tool_calls] == [
("get_weather", {"location": "Lima, Peru"})
]
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["tool_calls"]
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices and chunk.choices[0].finish_reason] == [
"tool_calls"
]
@pytest.mark.respx(assert_all_called=True)

View file

@ -632,7 +632,7 @@ class TestVertexGemmaCompletion:
assert chunks[0].choices[0].delta.content == "READY"
assert chunks[1].choices[0].finish_reason == "stop"
if stream_options and stream_options["include_usage"]:
assert chunks[-1].choices[0].delta.content is None
assert chunks[-1].choices == []
assert chunks[-1].usage.total_tokens == 114
assert span.get_metric("_dd.llmobs.total_tokens") == 114
else:

View file

@ -1,6 +1,7 @@
import asyncio
import json
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock
import pytest
@ -1975,6 +1976,20 @@ class TestProxyWiring:
class TestStreamUsage:
@pytest.mark.asyncio
async def test_trailing_flush_survives_a_usage_chunk_with_no_choices(self):
guardrail, _ = _shielded({"[EMAIL_1]": "a@example.com"})
held: Final = ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="Mail [EMAI"))])
usage_chunk: Final = ModelResponseStream(
choices=[], usage=Usage(prompt_tokens=5, completion_tokens=7, total_tokens=12)
)
out: Final = await _restore_stream(guardrail, [held, usage_chunk])
assert out[1] is not None and out[1].choices == []
assert [chunk.choices[0].delta.content for chunk in out[2:]] == ["[EMAI"]
assert all(getattr(chunk, "usage", None) is None for chunk in out[2:])
@pytest.mark.asyncio
async def test_trailing_flush_does_not_repeat_usage(self):
"""With n>=2 and include_usage, the last chunk carries usage; a flush copied from it must not.

View file

@ -8536,6 +8536,7 @@ PROXY_STAMPED_NAMES: Final = frozenset(
"litellm_trusted_callback_vars",
"_litellm_addressed_response_id",
"_litellm_strip_stream_usage",
"_litellm_stream_usage_injected",
"client_side_timeout",
"model_file_id_mapping",
)

View file

@ -6,10 +6,13 @@ Pins (PR2):
from __future__ import annotations
from collections.abc import Callable
from contextlib import AbstractContextManager
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from litellm import Router
from litellm.proxy import proxy_server
@ -85,6 +88,25 @@ def test_queue_chat_completions_happy(client, auth_as, patched_queue):
}
def test_queue_chat_completions_drops_client_sent_stream_usage_flags(
client: TestClient, auth_as: Callable[..., AbstractContextManager[None]], patched_queue: MagicMock
) -> None:
payload: Final = {
"model": "gpt-4",
"messages": [{"role": "user", "content": "hi"}],
"priority": 0,
"_litellm_strip_stream_usage": True,
"_litellm_stream_usage_injected": True,
}
with auth_as():
response: Final = client.post("/queue/chat/completions", json=payload)
assert response.status_code == 200
forwarded: Final = patched_queue.schedule_acompletion.call_args.kwargs
assert "_litellm_strip_stream_usage" not in forwarded
assert "_litellm_stream_usage_injected" not in forwarded
assert forwarded["messages"] == [{"role": "user", "content": "hi"}]
def test_queue_chat_completions_no_router_error(client, auth_as, queue_no_router):
"""Pins ``POST /queue/chat/completions`` (error: no llm_router)."""
payload = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}

View file

@ -9183,6 +9183,7 @@ class TestApplyStreamUsageTracking:
assert data["stream_options"] == {"include_usage": True}
assert data["_litellm_strip_stream_usage"] is True
assert data["_litellm_stream_usage_injected"] is True
def test_default_preserves_other_client_stream_options_keys(self):
data = {"stream": True, "stream_options": {"include_obfuscation": True}}
@ -9191,6 +9192,7 @@ class TestApplyStreamUsageTracking:
assert data["stream_options"] == {"include_obfuscation": True, "include_usage": True}
assert data["_litellm_strip_stream_usage"] is True
assert data["_litellm_stream_usage_injected"] is True
def test_client_requested_usage_is_left_untouched_and_not_stripped(self):
data = {"stream": True, "stream_options": {"include_usage": True}}
@ -9207,6 +9209,7 @@ class TestApplyStreamUsageTracking:
assert data["stream_options"]["include_usage"] is True
assert data["_litellm_strip_stream_usage"] is True
assert data["_litellm_stream_usage_injected"] is True
def test_explicit_false_flag_disables_injection_entirely(self):
data = {"stream": True}
@ -9232,6 +9235,36 @@ class TestApplyStreamUsageTracking:
assert data["stream_options"] == {"include_usage": True}
assert "_litellm_strip_stream_usage" not in data
@pytest.mark.parametrize("client_stream_options", [None, {"include_obfuscation": True}])
def test_flag_true_marks_usage_injected_for_clients_that_did_not_ask(
self, client_stream_options: dict[str, bool] | None
):
data: Final = {
"stream": True,
**({} if client_stream_options is None else {"stream_options": client_stream_options}),
}
_apply_stream_usage_tracking(
data=data,
general_settings={"always_include_stream_usage": True},
route_type="acompletion",
)
assert data["stream_options"]["include_usage"] is True
assert data["_litellm_stream_usage_injected"] is True
def test_flag_true_leaves_usage_unmarked_when_client_asked_and_neutralizes_client_marker(self):
data: Final = {"stream": True, "stream_options": {"include_usage": True}, "_litellm_stream_usage_injected": True}
_apply_stream_usage_tracking(
data=data,
general_settings={"always_include_stream_usage": True},
route_type="acompletion",
)
assert data["stream_options"] == {"include_usage": True}
assert data["_litellm_stream_usage_injected"] is False
def test_flag_true_respects_client_explicit_include_usage_false(self):
data = {"stream": True, "stream_options": {"include_usage": False}}

View file

@ -936,6 +936,26 @@ def test_body_snapshot_excludes_team_callback_credentials() -> None:
}, proxy_request
def test_body_snapshot_keeps_the_stream_usage_injection_marker_out_of_the_stored_request() -> None:
from litellm.proxy.common_request_processing import stream_usage_tracking_updates
from litellm.proxy.litellm_pre_call_utils import refresh_proxy_server_request_body_snapshot
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
request: Final = {"model": "gpt-5-mini", "messages": [{"role": "user", "content": "hi"}], "stream": True}
proxy_request: Final = {"body": dict(request)}
data: Final = {
**request,
**stream_usage_tracking_updates(request, {}, "acompletion", lambda: True),
"proxy_server_request": proxy_request,
}
refresh_proxy_server_request_body_snapshot(data)
assert data[STREAM_USAGE_INJECTED_FIELD] is True
assert proxy_request["body"]["stream_options"] == {"include_usage": True}
assert STREAM_USAGE_INJECTED_FIELD not in proxy_request["body"]
@pytest.mark.asyncio
@pytest.mark.parametrize("pre_call_ran", [False, True])
async def test_post_guardrail_snapshot_preserves_logging_only_masking_in_spend_logs(

View file

@ -1,25 +1,44 @@
import asyncio
import contextlib
import gc
import threading
import weakref
import httpx
import json
import litellm
import pytest
import redis
import respx
from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping
from datetime import datetime
from types import SimpleNamespace
from uuid import uuid4
from typing import Final
from unittest.mock import MagicMock
from fakeredis import TcpFakeServer
from fastapi import HTTPException
from litellm.caching.caching import DualCache
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.route_llm_request import (
JSON_OBJECT_ADAPTER,
MockTestingParamsDisabledError,
ProxyMissingRequiredParamError,
ProxyModelNotFoundError,
UserConfigRouters,
route_request,
user_config_routers,
)
from litellm.types.router import RouterGeneralSettings
from litellm.proxy.utils import ProxyLogging
from litellm.router_strategy.adaptive_router.config import (
ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY,
ADAPTIVE_ROUTER_RESPONSE_HEADER,
)
from litellm.types.router import RouterErrors, RouterGeneralSettings, RouterRateLimitError
@pytest.mark.parametrize(
@ -2684,3 +2703,571 @@ async def test_team_key_search_of_registered_bedrock_store_reaches_bedrock_with_
posted_url
)
assert "Credential=AKIAREGISTRYKEY00000/" in signed_auth, signed_auth
@pytest.fixture(autouse=True)
def discard_cached_user_config_routers() -> Iterator[None]:
yield
user_config_routers.clear()
@pytest.fixture
def fake_redis_port() -> Iterator[int]:
server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis")
thread: Final = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
yield server.server_address[1]
server.shutdown()
server.server_close()
async def _wait_out_a_rate_limit_minute_rollover() -> None:
seconds_left: Final = 60 - datetime.now().second
if seconds_left < 10:
await asyncio.sleep(seconds_left)
def _proxy_logging_obj(model: str = "rate-limited") -> LiteLLMLoggingObj:
logging_obj, _ = litellm.utils.function_setup(
original_function="acompletion",
rules_obj=litellm.utils.Rules(),
start_time=datetime.now(),
model=model,
messages=[{"role": "user", "content": "hi"}],
litellm_call_id=str(uuid4()),
)
assert isinstance(logging_obj, LiteLLMLoggingObj)
return logging_obj
def _mock_deployment(model_name: str, **deployment_limits: int) -> dict[str, object]:
return {
"model_name": model_name,
**deployment_limits,
"litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test", "mock_response": "ok"},
}
async def _user_config_request_outcome(
redis_port: int | None,
routing_settings: Mapping[str, object],
deployment_limits: Mapping[str, int],
logging_obj: LiteLLMLoggingObj | None,
) -> str:
data: Final = {
"model": "rate-limited",
"messages": [{"role": "user", "content": "hi"}],
"num_retries": 0,
**({"litellm_logging_obj": logging_obj} if logging_obj is not None else {}),
"user_config": {
**routing_settings,
**({"redis_host": "127.0.0.1", "redis_port": redis_port} if redis_port is not None else {}),
"model_list": [_mock_deployment("rate-limited", **deployment_limits)],
},
}
try:
await (await route_request(data, None, None, "acompletion"))
except (litellm.RateLimitError, RouterRateLimitError):
return "429"
finally:
await asyncio.sleep(0.3)
return "200"
async def _tpm_for_three_requests() -> int:
messages: Final = [{"role": "user", "content": "hi"}]
probe: Final = await litellm.acompletion(model="openai/gpt-5.5", messages=messages, mock_response="ok")
assert isinstance(probe, litellm.ModelResponse)
probe_usage: Final = getattr(probe, "usage", None)
assert isinstance(probe_usage, litellm.Usage)
return 2 * probe_usage.total_tokens + litellm.token_counter(messages=messages)
async def _request_cost() -> float:
probe: Final = await litellm.acompletion(
model="openai/gpt-5.5", messages=[{"role": "user", "content": "hi"}], mock_response="ok"
)
return litellm.completion_cost(completion_response=probe)
async def _proxy_router_outcomes(
redis_port: int | None,
routing_settings: Mapping[str, object],
deployment_limits: Mapping[str, int],
requests: int,
) -> list[str]:
router: Final = litellm.Router(
**routing_settings,
num_retries=0,
model_list=[_mock_deployment("proxy-rate-limited", **deployment_limits)],
**({"redis_host": "127.0.0.1", "redis_port": redis_port} if redis_port is not None else {}),
)
async def outcome() -> str:
try:
await router.acompletion(model="proxy-rate-limited", messages=[{"role": "user", "content": "hi"}])
except (litellm.RateLimitError, RouterRateLimitError):
return "429"
finally:
await asyncio.sleep(0.3)
return "200"
try:
return [await outcome() for _ in range(requests)]
finally:
router.discard()
@pytest.mark.asyncio
async def test_a_user_config_adaptive_request_answers_with_the_model_it_picked() -> None:
data: Final = {
"model": "smart-cheap-router",
"messages": [{"role": "user", "content": "Write a one line python function that adds two numbers"}],
"metadata": {},
"user_config": {
"model_list": [
{
"model_name": "smart-cheap-router",
"litellm_params": {
"model": "auto_router/adaptive_router",
"adaptive_router_config": {"available_models": ["fast", "smart"]},
},
},
{
**_mock_deployment("fast"),
"model_info": {"adaptive_router_preferences": {"quality_tier": 2, "strengths": []}},
},
{
**_mock_deployment("smart"),
"model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": ["code_generation"]}},
},
]
},
}
await (await route_request(data, None, None, "acompletion"))
headers: Final = await ProxyLogging(user_api_key_cache=DualCache()).post_call_response_headers_hook(
data=data, user_api_key_dict=UserAPIKeyAuth(), response=None
)
assert headers == {ADAPTIVE_ROUTER_RESPONSE_HEADER: data["metadata"][ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY]}
def _global_router_callbacks() -> tuple[list[object], list[object]]:
return list(litellm.callbacks), list(litellm.input_callback)
@pytest.mark.parametrize(
"routing_settings, deployment_limits",
[
({"routing_strategy": "usage-based-routing-v2"}, {"rpm": 2}),
({"routing_strategy": "usage-based-routing-v2"}, {"tpm": 30}),
({"routing_strategy": "usage-based-routing"}, {"rpm": 2}),
({"optional_pre_call_checks": ["enforce_model_rate_limits"]}, {"rpm": 2}),
],
)
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
@pytest.mark.parametrize("with_redis", [True, False])
@pytest.mark.asyncio
async def test_user_config_router_enforces_its_limit_across_proxy_requests(
fake_redis_port: int,
routing_settings: Mapping[str, object],
deployment_limits: Mapping[str, int],
arrives_with_logging_obj: bool,
with_redis: bool,
) -> None:
await _wait_out_a_rate_limit_minute_rollover()
callbacks_before: Final = _global_router_callbacks()
outcomes: Final = [
await _user_config_request_outcome(
fake_redis_port if with_redis else None,
routing_settings,
deployment_limits,
_proxy_logging_obj() if arrives_with_logging_obj else None,
)
for _ in range(6)
]
assert "429" in outcomes, outcomes
assert _global_router_callbacks() == callbacks_before
assert outcomes == await _proxy_router_outcomes(
fake_redis_port if with_redis else None, routing_settings, deployment_limits, len(outcomes)
)
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
@pytest.mark.asyncio
async def test_concurrent_user_config_requests_each_count_once_against_the_tpm_limit(
fake_redis_port: int, arrives_with_logging_obj: bool
) -> None:
tpm_for_three_requests: Final = await _tpm_for_three_requests()
def request() -> Coroutine[None, None, str]:
return _user_config_request_outcome(
fake_redis_port,
{"routing_strategy": "usage-based-routing-v2"},
{"tpm": tpm_for_three_requests},
_proxy_logging_obj() if arrives_with_logging_obj else None,
)
await _wait_out_a_rate_limit_minute_rollover()
concurrent_outcomes: Final = await asyncio.gather(request(), request())
later_outcomes: Final = [await request() for _ in range(2)]
assert [*concurrent_outcomes, *later_outcomes] == ["200", "200", "200", "429"]
@pytest.mark.asyncio
async def test_overlapping_user_config_requests_share_one_rpm_limit(fake_redis_port: int) -> None:
def request() -> Coroutine[None, None, str]:
return _user_config_request_outcome(
fake_redis_port, {"routing_strategy": "usage-based-routing-v2"}, {"rpm": 2}, _proxy_logging_obj()
)
await _wait_out_a_rate_limit_minute_rollover()
outcomes: Final = await asyncio.gather(*(request() for _ in range(4)))
assert outcomes.count("200") <= 2, outcomes
assert "429" in outcomes, outcomes
@pytest.mark.parametrize(
"routing_strategy, limited_by",
[("usage-based-routing-v2", "rpm"), ("usage-based-routing-v2", "tpm"), ("usage-based-routing", "rpm")],
)
@pytest.mark.parametrize("proxy_router_shares_redis", [True, False])
@pytest.mark.asyncio
async def test_a_proxy_router_on_the_same_strategy_leaves_the_user_config_limit_unchanged(
fake_redis_port: int, routing_strategy: str, limited_by: str, proxy_router_shares_redis: bool
) -> None:
deployment_limits: Final = {"rpm": 3} if limited_by == "rpm" else {"tpm": await _tpm_for_three_requests()}
def request() -> Coroutine[None, None, str]:
return _user_config_request_outcome(
fake_redis_port, {"routing_strategy": routing_strategy}, deployment_limits, _proxy_logging_obj()
)
async def outcomes() -> list[str]:
first_instance: Final = [await request() for _ in range(2)]
user_config_routers.clear()
second_instance: Final = [await request() for _ in range(3)]
return [*first_instance, *second_instance]
await _wait_out_a_rate_limit_minute_rollover()
alone: Final = await outcomes()
user_config_routers.clear()
await asyncio.sleep(0.3)
redis.Redis(host="127.0.0.1", port=fake_redis_port).flushall()
proxy_router: Final = litellm.Router(
routing_strategy=routing_strategy,
model_list=[_mock_deployment("proxy-model")],
redis_host="127.0.0.1" if proxy_router_shares_redis else None,
redis_port=fake_redis_port if proxy_router_shares_redis else None,
)
try:
beside_proxy_router: Final = await outcomes()
finally:
proxy_router.discard()
assert "429" in alone, alone
assert beside_proxy_router == alone
@pytest.mark.parametrize("user_config_settings", [{}, {"optional_pre_call_checks": ["router_budget_limiting"]}])
@pytest.mark.asyncio
async def test_user_config_requests_stay_under_the_proxy_provider_budget(
user_config_settings: Mapping[str, object],
) -> None:
proxy_router: Final = litellm.Router(
model_list=[_mock_deployment("proxy-model")],
provider_budget_config={"openai": {"budget_limit": await _request_cost() / 2, "time_period": "1d"}},
)
try:
await proxy_router.acompletion(model="proxy-model", messages=[{"role": "user", "content": "hi"}])
await asyncio.sleep(0.3)
with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_provider_budget_routing.value):
await _user_config_request_outcome(None, user_config_settings, {}, _proxy_logging_obj())
finally:
proxy_router.discard()
@pytest.mark.asyncio
async def test_a_user_config_provider_budget_limits_its_own_requests() -> None:
budget_limit: Final = await _request_cost() / 2
def request() -> Coroutine[None, None, str]:
budget: Final = {"openai": {"budget_limit": budget_limit, "time_period": "1d"}}
return _user_config_request_outcome(None, {"provider_budget_config": budget}, {}, _proxy_logging_obj())
first: Final = await request()
with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_provider_budget_routing.value):
await request()
assert first == "200"
@pytest.mark.asyncio
async def test_a_request_in_flight_when_its_user_config_router_is_dropped_still_counts_against_its_budget(
fake_redis_port: int,
) -> None:
budget_limit: Final = 1.5 * await _request_cost()
async def request() -> None:
await (
await route_request(
{
"model": "budgeted",
"messages": [{"role": "user", "content": "hi"}],
"litellm_logging_obj": _proxy_logging_obj("budgeted"),
"user_config": {
"redis_host": "127.0.0.1",
"redis_port": fake_redis_port,
"provider_budget_config": {"openai": {"budget_limit": budget_limit, "time_period": "1d"}},
"model_list": [
{
"model_name": "budgeted",
"litellm_params": {
"model": "openai/gpt-5.5",
"api_key": "sk-test",
"mock_response": "ok",
"mock_delay": 0.5,
},
}
],
},
},
None,
None,
"acompletion",
)
)
await request()
in_flight: Final = asyncio.create_task(request())
await asyncio.sleep(0.2)
user_config_routers.clear()
await in_flight
await asyncio.sleep(0.3)
with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_provider_budget_routing.value):
await request()
async def _user_config_request(model: str, user_config: Mapping[str, object], arrives_with_logging_obj: bool) -> object:
return await (
await route_request(
{
"model": model,
"messages": list(_HELLO_MESSAGES),
"num_retries": 0,
**({"litellm_logging_obj": _proxy_logging_obj(model)} if arrives_with_logging_obj else {}),
"user_config": user_config,
},
None,
None,
"acompletion",
)
)
def _ordered_pair(model_name: str, first_base: str, second_base: str) -> list[dict[str, object]]:
return [
{
"model_name": model_name,
"litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test", "api_base": api_base, "order": order},
}
for order, api_base in ((1, first_base), (2, second_base))
]
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
@pytest.mark.asyncio
async def test_a_rate_limited_user_config_deployment_cools_down_for_the_next_requests(
monkeypatch: pytest.MonkeyPatch, arrives_with_logging_obj: bool
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
rate_limited_base: Final = "https://rate-limited.example/v1"
healthy_base: Final = "https://healthy.example/v1"
user_config: Final = {"model_list": _ordered_pair("flaky", rate_limited_base, healthy_base)}
with respx.mock(assert_all_called=False) as respx_mock:
rate_limited: Final = respx_mock.post(f"{rate_limited_base}/chat/completions").mock(
return_value=httpx.Response(429, json={"error": {"message": "slow down"}})
)
respx_mock.post(f"{healthy_base}/chat/completions").mock(return_value=_chat_completion_response("gpt-5.5"))
for _ in range(3):
await _user_config_request("flaky", user_config, arrives_with_logging_obj)
await asyncio.sleep(0.3)
assert rate_limited.call_count == 1
async def _remaining_usage_headers_after_429_then_200s(
send: Callable[[], Awaitable[object]], api_base: str
) -> list[tuple[object, object] | str]:
async def remaining_usage() -> tuple[object, object] | str:
try:
response: Final = await send()
except litellm.RateLimitError:
return "429"
finally:
await asyncio.sleep(0.3)
assert isinstance(response, litellm.ModelResponse)
headers: Final = response._hidden_params["additional_headers"]
return headers.get("x-ratelimit-remaining-requests"), headers.get("x-ratelimit-remaining-tokens")
with respx.mock(assert_all_called=False) as respx_mock:
respx_mock.post(f"{api_base}/chat/completions").mock(
side_effect=[
httpx.Response(429, json={"error": {"message": "slow down"}}),
_chat_completion_response("gpt-5.5"),
_chat_completion_response("gpt-5.5"),
]
)
return [await remaining_usage() for _ in range(3)]
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
@pytest.mark.asyncio
async def test_user_config_remaining_usage_headers_match_a_proxy_router_with_the_same_deployment(
monkeypatch: pytest.MonkeyPatch, arrives_with_logging_obj: bool
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
await _wait_out_a_rate_limit_minute_rollover()
proxy_base: Final = "https://proxy-limited.example/v1"
user_config_base: Final = "https://user-config-limited.example/v1"
def deployment(model_name: str, api_base: str) -> dict[str, object]:
return {
"model_name": model_name,
"litellm_params": {
"model": "openai/gpt-5.5",
"api_key": "sk-test",
"api_base": api_base,
"rpm": 10,
"tpm": 1000,
},
}
proxy_router: Final = litellm.Router(model_list=[deployment("proxy-limited", proxy_base)])
try:
proxy_router_headers: Final = await _remaining_usage_headers_after_429_then_200s(
lambda: proxy_router.acompletion(model="proxy-limited", messages=list(_HELLO_MESSAGES), num_retries=0),
proxy_base,
)
finally:
proxy_router.discard()
user_config: Final = {"model_list": [deployment("user-config-limited", user_config_base)]}
user_config_headers: Final = await _remaining_usage_headers_after_429_then_200s(
lambda: _user_config_request("user-config-limited", user_config, arrives_with_logging_obj), user_config_base
)
assert user_config_headers == proxy_router_headers
async def _first_deployment_calls_over_successes_then_failures(
send: Callable[[], Awaitable[object]], first_base: str, second_base: str
) -> int:
successes: Final = [_chat_completion_response("gpt-5.5") for _ in range(6)]
failures: Final = [httpx.Response(500, json={"error": {"message": "boom"}}) for _ in range(6)]
with respx.mock(assert_all_called=False) as respx_mock:
first: Final = respx_mock.post(f"{first_base}/chat/completions").mock(side_effect=[*successes, *failures])
respx_mock.post(f"{second_base}/chat/completions").mock(return_value=_chat_completion_response("gpt-5.5"))
for _ in range(len(successes) + len(failures)):
await send()
await asyncio.sleep(0.3)
return first.call_count
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
@pytest.mark.asyncio
async def test_user_config_successes_keep_a_deployment_out_of_cooldown_like_a_proxy_router(
monkeypatch: pytest.MonkeyPatch, arrives_with_logging_obj: bool
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
await _wait_out_a_rate_limit_minute_rollover()
proxy_router: Final = litellm.Router(
model_list=_ordered_pair("proxy-pair", "https://proxy-first.example/v1", "https://proxy-second.example/v1")
)
try:
proxy_router_calls: Final = await _first_deployment_calls_over_successes_then_failures(
lambda: proxy_router.acompletion(model="proxy-pair", messages=list(_HELLO_MESSAGES), num_retries=0),
"https://proxy-first.example/v1",
"https://proxy-second.example/v1",
)
finally:
proxy_router.discard()
user_config: Final = {
"model_list": _ordered_pair("user-pair", "https://user-first.example/v1", "https://user-second.example/v1")
}
user_config_calls: Final = await _first_deployment_calls_over_successes_then_failures(
lambda: _user_config_request("user-pair", user_config, arrives_with_logging_obj),
"https://user-first.example/v1",
"https://user-second.example/v1",
)
assert user_config_calls == proxy_router_calls
@pytest.mark.asyncio
async def test_user_config_adaptive_router_learns_from_proxy_requests() -> None:
user_config: Final = {
"model_list": [
_mock_deployment("smart"),
_mock_deployment("fast"),
{
"model_name": "adaptive",
"litellm_params": {
"model": "auto_router/adaptive_router",
"adaptive_router_config": {"available_models": ["smart", "fast"]},
},
},
]
}
await (
await route_request(
{
"model": "adaptive",
"messages": [{"role": "user", "content": "write a sorting function"}],
"metadata": {"session_id": "session-1"},
"litellm_logging_obj": _proxy_logging_obj("adaptive"),
"user_config": user_config,
},
None,
None,
"acompletion",
)
)
await asyncio.sleep(0.3)
router: Final = user_config_routers.get(user_config, lambda: pytest.fail("the user_config router was not reused"))
snapshot: Final = await router.adaptive_routers["adaptive"][0].strategy.get_state_snapshot()
assert snapshot["feedback_contexts_live"] == 1, snapshot
@pytest.mark.asyncio
async def test_user_config_routers_discard_the_least_recently_used_router_past_capacity() -> None:
routers: Final = UserConfigRouters(max_size=2)
def config(model_name: str) -> dict[str, object]:
return {"routing_strategy": "usage-based-routing-v2", "model_list": [_mock_deployment(model_name)]}
def build(model_name: str) -> litellm.Router:
return litellm.Router(**config(model_name))
def must_be_cached() -> litellm.Router:
pytest.fail("a cached router was rebuilt")
first_handler: Final = weakref.ref(routers.get(config("first"), lambda: build("first")).lowesttpm_logger_v2)
second_handler: Final = weakref.ref(routers.get(config("second"), lambda: build("second")).lowesttpm_logger_v2)
routers.get(config("first"), must_be_cached)
routers.get(config("third"), lambda: build("third"))
await asyncio.sleep(0.2)
gc.collect()
assert routers.get(config("first"), must_be_cached) is not None
assert first_handler() is not None
assert second_handler() is None
routers.clear()

View file

@ -10,6 +10,7 @@ from litellm.router_strategy.adaptive_router.config import (
)
from litellm.router_strategy.adaptive_router.hooks import (
AdaptiveRouterPostCallHook,
AdaptiveRouterResponseHeaderHook,
_recent_tool_results,
_resolve_session_key,
)
@ -337,8 +338,7 @@ async def test_post_call_response_headers_hook_returns_chosen_model_header():
"""The header hook returns the `x-litellm-adaptive-router-model` header
so proxy header construction picks it up (works for both streaming and
non-streaming; `async_post_call_success_hook` is too late for streaming)."""
hook = _make_hook()
headers = await hook.async_post_call_response_headers_hook(
headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook(
data={"metadata": {"adaptive_router_chosen_model": "smart"}},
user_api_key_dict=MagicMock(),
response=MagicMock(),
@ -348,8 +348,7 @@ async def test_post_call_response_headers_hook_returns_chosen_model_header():
@pytest.mark.asyncio
async def test_post_call_response_headers_hook_noop_when_metadata_missing_key():
hook = _make_hook()
headers = await hook.async_post_call_response_headers_hook(
headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook(
data={"metadata": {"litellm_session_id": "sess-A"}},
user_api_key_dict=MagicMock(),
response=MagicMock(),
@ -359,8 +358,7 @@ async def test_post_call_response_headers_hook_noop_when_metadata_missing_key():
@pytest.mark.asyncio
async def test_post_call_response_headers_hook_noop_when_no_metadata():
hook = _make_hook()
headers = await hook.async_post_call_response_headers_hook(
headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook(
data={},
user_api_key_dict=MagicMock(),
response=MagicMock(),
@ -370,8 +368,7 @@ async def test_post_call_response_headers_hook_noop_when_no_metadata():
@pytest.mark.asyncio
async def test_post_call_response_headers_hook_noop_when_metadata_not_dict():
hook = _make_hook()
headers = await hook.async_post_call_response_headers_hook(
headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook(
data={"metadata": "not-a-dict"},
user_api_key_dict=MagicMock(),
response=MagicMock(),

View file

@ -1,15 +1,20 @@
import json
import logging
from typing import Any, Dict, List, Optional, Set, Union
from typing import Any, Dict, Final, List, Optional, Set, Union
import pytest
import asyncio
import threading
from unittest.mock import MagicMock, patch
from fakeredis import TcpFakeServer
from litellm.caching.caching import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import RedisCache
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError, RedisPipelineIncrementOperation
from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
@ -162,3 +167,24 @@ async def test_push_refused_by_the_open_circuit_breaker_is_not_logged_as_an_erro
assert caplog.records == []
assert base_strategy.redis_increment_operation_queue == []
@pytest.mark.asyncio
async def test_an_increment_counted_after_retire_still_reaches_redis():
server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis")
threading.Thread(target=server.serve_forever, daemon=True).start()
redis_cache: Final = RedisCache(host="127.0.0.1", port=server.server_address[1])
strategy: Final = BaseRoutingStrategy(
dual_cache=DualCache(redis_cache=redis_cache, in_memory_cache=InMemoryCache()),
should_batch_redis_writes=True,
default_sync_interval=60,
)
try:
strategy.retire()
await strategy._increment_value_in_current_window("counted-after-retire", 3, 60)
await asyncio.sleep(0.2)
assert float(await redis_cache.async_get_cache("counted-after-retire")) == 3
finally:
server.shutdown()
server.server_close()

View file

@ -1,4 +1,5 @@
import asyncio
import contextlib
import gc
import logging
from types import SimpleNamespace
@ -492,6 +493,8 @@ def _new_router_budget_limiter(
budget_limiter._redis_increment_queue_lock = queue_lock if queue_lock is not None else asyncio.Lock()
budget_limiter._redis_increment_flush_lock = asyncio.Lock()
budget_limiter._detached_increment_operations = None
budget_limiter._retired = False
budget_limiter._flush_after_retire = None
return budget_limiter
@ -784,3 +787,57 @@ async def test_cancelled_flush_does_not_requeue_an_applied_batch(cancellations:
await limiter._push_in_memory_increments_to_redis()
assert redis_cache.values[_SPEND_KEY] == 10.0
assert limiter.redis_increment_operation_queue == []
@pytest.mark.asyncio
async def test_spend_queued_when_a_limiter_is_retired_reaches_redis_after_a_failed_write() -> None:
pipeline_started: Final = asyncio.Event()
pipeline_completed: Final = asyncio.Event()
redis_cache: Final = _MockRedisCache(
initial_values={},
pipeline_started=pipeline_started,
pipeline_completed=pipeline_completed,
should_fail_pipeline=True,
)
budget_limiter: Final = _new_router_budget_limiter(
redis_cache=redis_cache, redis_increment_operation_queue=[_increment(10.0)]
)
budget_limiter._sync_task = asyncio.create_task(asyncio.sleep(3600))
budget_limiter.retire()
await asyncio.wait_for(pipeline_started.wait(), timeout=1)
assert redis_cache.values == {}
redis_cache.should_fail_pipeline = False
await asyncio.wait_for(pipeline_completed.wait(), timeout=5)
assert redis_cache.values == {_SPEND_KEY: 10.0}
assert budget_limiter.redis_increment_operation_queue == []
@pytest.mark.asyncio
async def test_spend_in_flight_when_a_limiter_is_retired_reaches_redis_after_that_write_fails() -> None:
pipeline_started: Final = asyncio.Event()
allow_pipeline_to_complete: Final = asyncio.Event()
pipeline_completed: Final = asyncio.Event()
redis_cache: Final = _MockRedisCache(
initial_values={},
pipeline_started=pipeline_started,
allow_pipeline_to_complete=allow_pipeline_to_complete,
should_fail_pipeline=True,
pipeline_completed=pipeline_completed,
)
budget_limiter: Final = _new_router_budget_limiter(
redis_cache=redis_cache, redis_increment_operation_queue=[_increment(10.0)]
)
budget_limiter._sync_task = asyncio.create_task(budget_limiter.periodic_sync_in_memory_spend_with_redis())
await asyncio.wait_for(pipeline_started.wait(), timeout=1)
budget_limiter.retire()
allow_pipeline_to_complete.set()
with contextlib.suppress(asyncio.CancelledError):
await budget_limiter._sync_task
redis_cache.should_fail_pipeline = False
await asyncio.wait_for(pipeline_completed.wait(), timeout=5)
assert redis_cache.values == {_SPEND_KEY: 10.0}
assert budget_limiter.redis_increment_operation_queue == []

View file

@ -96,7 +96,6 @@ CI = [".github/workflows/test-unit.yml"]
("ui", BACKEND + CLIENT, "run"),
("cost-map-only", ["model_prices_and_context_window.json"], "run"),
("cost-map-only", ["litellm/model_prices_and_context_window_backup.json"], "run"),
("cost-map-only", ["model_prices_and_context_window.schema.json"], "run"),
(
"cost-map-only",
["model_prices_and_context_window.json", "tests/unit/proxy/test_y.py"],

View file

@ -186,13 +186,6 @@ def test_bot_may_only_touch_the_cost_map_files() -> None:
)
def test_bot_may_update_cost_map_and_legacy_schema() -> None:
head: Final = _snapshot({**BASE_MAP, "openrouter/c": _entry()})
changed: Final = (*guard.GUARDED_PATHS, guard.LEGACY_SCHEMA_PATH)
assert _failures(head, changed_files=changed) == ()
def test_bot_may_not_remove_models() -> None:
head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"})
assert _failures(head, bot=False) == ()

View file

@ -3868,6 +3868,9 @@ def test_stream_chunk_builder_text_completion_combines_text_and_usage():
model="gpt-3.5-turbo-instruct",
choices=[{"text": " world", "index": 0, "logprobs": None, "finish_reason": "stop"}],
),
TextCompletionResponse(
id="cmpl-1", object="text_completion", created=1, model="gpt-3.5-turbo-instruct", choices=[]
),
]
response = stream_chunk_builder_text_completion(chunks=chunks, messages=[{"role": "user", "content": "say hello"}])
@ -8853,7 +8856,8 @@ async def test_text_completion_stream_include_usage_returns_usage_on_final_chunk
"stream": True,
"stream_options": {"include_usage": True},
}
assert "".join(chunk.choices[0].text or "" for chunk in chunks) == "reply"
assert "".join(chunk.choices[0].text or "" for chunk in chunks if chunk.choices) == "reply"
assert chunks[-1].choices == []
final_usage: Final = chunks[-1].usage
assert (final_usage.prompt_tokens, final_usage.completion_tokens, final_usage.total_tokens) == (3, 2, 5)

View file

@ -29670,6 +29670,73 @@ def test_pre_call_checks_ignores_an_image_response_format_string(monkeypatch: py
assert [d["model_info"]["id"] for d in filtered] == ["only"]
@pytest.mark.parametrize(
("deployment_stream_options", "default_stream_options", "request_stream_options", "usage_chunk_choice_count"),
[
({"include_usage": True}, None, None, 1),
(None, {"include_usage": True}, None, 1),
({"include_usage": True}, None, {"include_usage": True}, 0),
(None, None, {"include_usage": True}, 0),
],
ids=["deployment-config", "router-default", "request-over-deployment", "request-only"],
)
@pytest.mark.asyncio
async def test_router_configured_stream_usage_keeps_a_choice_on_the_usage_chunk(
deployment_stream_options: dict[str, bool] | None,
default_stream_options: dict[str, bool] | None,
request_stream_options: dict[str, bool] | None,
usage_chunk_choice_count: int,
):
deployment_params: Final = {"model": "openai/gpt-5-mini", "api_key": "fake", "mock_response": "hi there"}
router: Final = Router(
model_list=[
{
"model_name": "m",
"litellm_params": (
deployment_params
if deployment_stream_options is None
else {**deployment_params, "stream_options": deployment_stream_options}
),
}
],
default_litellm_params=None if default_stream_options is None else {"stream_options": default_stream_options},
)
request_kwargs: Final = {} if request_stream_options is None else {"stream_options": request_stream_options}
response: Final = await router.acompletion(
model="m", messages=[{"role": "user", "content": "hi"}], stream=True, **request_kwargs
)
chunks: Final = [chunk async for chunk in response]
assert getattr(chunks[-1], "usage", None) is not None
assert len(chunks[-1].choices) == usage_chunk_choice_count
assert all(len(chunk.choices) == 1 for chunk in chunks[:-1])
@pytest.mark.parametrize(
("stream_options", "marked_injected"),
[({"include_usage": True}, True), (None, False)],
ids=["include-usage", "no-stream-options"],
)
def test_update_kwargs_with_deployment_reads_stream_options_from_pydantic_litellm_params(
stream_options: dict[str, bool] | None, marked_injected: bool
):
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.router import LiteLLM_Params
kwargs: Final[dict[str, object]] = {"stream": True, "metadata": {}}
Router()._update_kwargs_with_deployment(
deployment={
"model_name": "gpt-5-mini",
"litellm_params": LiteLLM_Params(model="openai/gpt-5-mini", api_key="fake-key", stream_options=stream_options),
"model_info": {"id": "pydantic-params"},
},
kwargs=kwargs,
)
assert (kwargs.get(STREAM_USAGE_INJECTED_FIELD) is True) is marked_injected
@pytest.mark.asyncio
async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_streams_tokens():
"""After a mid-stream hop, a disconnect bills the primary's relayed text plus the fallback's, not the fallback alone."""

View file

@ -1,8 +1,12 @@
import asyncio
import functools
import gc
import json
import os
import weakref
from datetime import datetime, timezone
from collections.abc import Mapping
from collections.abc import Iterator, Mapping
from types import MethodType
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -12,8 +16,12 @@ from litellm import Router
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
from litellm.types.litellm_params import RoutingStrategyName
from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
from litellm.types.utils import (
CallTypes,
ModelResponse,
@ -1852,6 +1860,315 @@ def test_discard(model_list):
assert len(litellm.service_callback) == 0
_CALLBACK_LIST_NAMES: Final = (
"callbacks",
"input_callback",
"success_callback",
"failure_callback",
"_async_success_callback",
"_async_failure_callback",
)
def _callback_ids(list_name: str) -> list[int]:
return [id(callback) for callback in getattr(litellm, list_name)]
@pytest.mark.parametrize(
"routing_strategy, selector_attr",
[
("least-busy", "leastbusy_logger"),
("usage-based-routing", "lowesttpm_logger"),
("usage-based-routing-v2", "lowesttpm_logger_v2"),
("latency-based-routing", "lowestlatency_logger"),
("cost-based-routing", "lowestcost_logger"),
],
)
@pytest.mark.asyncio
async def test_discard_releases_strategy_selector_across_router_lifecycles(
routing_strategy: RoutingStrategyName, selector_attr: str
):
def retire() -> weakref.ReferenceType[object]:
router: Final = Router(model_list=[], routing_strategy=routing_strategy)
selector: Final = getattr(router, selector_attr)
assert any(callback is selector for callback in litellm.callbacks)
router.discard()
return weakref.ref(selector)
selector_refs: Final = tuple(retire() for _ in range(3))
await asyncio.sleep(0)
gc.collect()
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
assert all(selector_ref() is None for selector_ref in selector_refs)
@pytest.mark.parametrize(
"routing_strategy, selector_attr",
[
("least-busy", "leastbusy_logger"),
("usage-based-routing", "lowesttpm_logger"),
("usage-based-routing-v2", "lowesttpm_logger_v2"),
("latency-based-routing", "lowestlatency_logger"),
("cost-based-routing", "lowestcost_logger"),
],
)
@pytest.mark.asyncio
async def test_discard_purges_selector_from_event_lists_after_serving_a_request(
routing_strategy: RoutingStrategyName, selector_attr: str
):
router: Final = Router(
model_list=[{"model_name": "gpt-5.5", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}],
routing_strategy=routing_strategy,
)
selector: Final = getattr(router, selector_attr)
await router.acompletion(model="gpt-5.5", messages=[{"role": "user", "content": "hi"}], mock_response="ok")
assert any(callback is selector for callback in litellm.success_callback)
router.discard()
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
def test_discard_keeps_callbacks_owned_by_other_live_routers():
survivor: Final = Router(model_list=[], routing_strategy="latency-based-routing")
unrelated: Final = CustomLogger()
litellm.logging_callback_manager.add_litellm_callback(unrelated)
before_ids: Final = {list_name: _callback_ids(list_name) for list_name in _CALLBACK_LIST_NAMES}
retired: Final = Router(model_list=[], routing_strategy="least-busy")
assert any(callback is retired.leastbusy_logger for callback in litellm.callbacks)
retired.discard()
assert {list_name: _callback_ids(list_name) for list_name in before_ids} == before_ids
assert any(callback is survivor.lowestlatency_logger for callback in litellm.callbacks)
survivor.discard()
@pytest.mark.parametrize(
"routing_strategy, selector_attr",
[
("usage-based-routing", "lowesttpm_logger"),
("usage-based-routing-v2", "lowesttpm_logger_v2"),
("latency-based-routing", "lowestlatency_logger"),
("cost-based-routing", "lowestcost_logger"),
],
)
@pytest.mark.asyncio
async def test_discard_leaves_same_strategy_survivor_receiving_events(
routing_strategy: RoutingStrategyName, selector_attr: str
):
model_list: Final[list[DeploymentTypedDict]] = [
{"model_name": "gpt-5.5", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}
]
retired: Final = Router(model_list=model_list, routing_strategy=routing_strategy)
survivor: Final = Router(model_list=model_list, routing_strategy=routing_strategy)
retired.discard()
await survivor.acompletion(model="gpt-5.5", messages=[{"role": "user", "content": "hi"}], mock_response="ok")
assert any(callback is getattr(survivor, selector_attr) for callback in litellm.success_callback)
survivor.discard()
def test_discard_drops_routing_group_selectors():
router: Final = Router(
model_list=[
{"model_name": "grouped-model", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}
],
routing_groups=[
{"group_name": "fast", "models": ["grouped-model"], "routing_strategy": "latency-based-routing"}
],
)
_, group_selector = router._get_routing_context("grouped-model")
assert any(callback is group_selector for callback in litellm.callbacks)
router.discard()
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
_GROUPED_MODEL_LIST: Final[list[DeploymentTypedDict]] = [
{"model_name": "grouped-model", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}
]
class _LabelledHook(CustomLogger):
def __init__(self, label: str) -> None:
super().__init__()
self.label = label
def _visit_hooks_reloading_routing_groups_mid_loop(
router: Router, hooks: tuple[_LabelledHook, ...]
) -> Iterator[CustomLogger]:
for callback in litellm.callbacks:
if callback is hooks[0]:
router.update_settings(
routing_groups=[{"group_name": "fast", "models": ["grouped-model"], "routing_strategy": "least-busy"}]
)
if callback in hooks:
yield callback
def test_routing_group_reload_keeps_every_hook_of_an_in_flight_callback_loop():
router: Final = Router(
model_list=_GROUPED_MODEL_LIST,
routing_groups=[{"group_name": "fast", "models": ["grouped-model"], "routing_strategy": "least-busy"}],
)
hooks: Final = tuple(_LabelledHook(label) for label in ("guardrail_a", "guardrail_b", "guardrail_c"))
for hook in hooks:
litellm.logging_callback_manager.add_litellm_callback(hook)
visited: Final = tuple(_visit_hooks_reloading_routing_groups_mid_loop(router, hooks))
assert visited == hooks
router.discard()
def _routers_seen_in_success_loop_discarding_first(first: Router, second: Router) -> Iterator[Router]:
for callback in litellm.success_callback:
if not isinstance(callback, MethodType):
continue
if callback.__self__ is first:
first.discard()
yield first
if callback.__self__ is second:
yield second
def test_discard_mid_success_loop_keeps_the_next_router_callback():
first: Final = Router(model_list=_GROUPED_MODEL_LIST)
second: Final = Router(model_list=_GROUPED_MODEL_LIST)
seen: Final = tuple(_routers_seen_in_success_loop_discarding_first(first, second))
assert seen == (first, second)
second.discard()
@pytest.mark.asyncio
async def test_discard_purges_optional_pre_call_checks_from_event_lists_after_serving_a_request():
router: Final = Router(
model_list=_GROUPED_MODEL_LIST,
optional_pre_call_checks=["enforce_model_rate_limits", "prompt_caching", "deployment_affinity"],
)
await router.acompletion(model="grouped-model", messages=[{"role": "user", "content": "hi"}], mock_response="ok")
router.discard()
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
def test_discard_registers_same_config_survivor_optional_pre_call_checks():
retired: Final = Router(model_list=_GROUPED_MODEL_LIST, optional_pre_call_checks=["enforce_model_rate_limits"])
survivor: Final = Router(model_list=_GROUPED_MODEL_LIST, optional_pre_call_checks=["enforce_model_rate_limits"])
retired.discard()
assert survivor.optional_callbacks
assert all(_is_registered(check) for check in survivor.optional_callbacks)
survivor.discard()
def _is_registered(callback: object) -> bool:
return any(registered is callback for registered in litellm.callbacks)
@pytest.mark.asyncio
async def test_discard_releases_per_request_override_selector():
def build_override_then_discard() -> weakref.ReferenceType[object]:
router: Final = Router(model_list=_GROUPED_MODEL_LIST)
selector: Final = router._get_override_strategy_selector("usage-based-routing-v2")
assert selector is not None
router.discard()
return weakref.ref(selector)
selector_ref: Final = build_override_then_discard()
await asyncio.sleep(0)
gc.collect()
assert selector_ref() is None
def test_discard_registers_the_selector_of_a_same_strategy_survivor_built_without_a_model_list():
retired: Final = Router(model_list=[], routing_strategy="latency-based-routing")
survivor: Final = Router(routing_strategy="latency-based-routing")
assert not _is_registered(survivor.lowestlatency_logger)
retired.discard()
assert _is_registered(survivor.lowestlatency_logger)
survivor.discard()
def test_discard_drops_its_slack_alerting_callbacks():
router: Final = Router(model_list=_GROUPED_MODEL_LIST, alerting_config=AlertingConfig(webhook_url="test"))
assert any(isinstance(callback, SlackAlerting) for callback in litellm.callbacks)
router.discard()
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
def _adaptive_router_model_list(name: str) -> list[DeploymentTypedDict]:
return [
{"model_name": "gpt-5.5", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}},
{
"model_name": name,
"litellm_params": {
"model": "auto_router/adaptive_router",
"adaptive_router_config": {"available_models": ["gpt-5.5"]},
},
},
]
def _registered_adaptive_router_strategies() -> tuple[object, ...]:
return tuple(
callback.adaptive_router for callback in litellm.callbacks if isinstance(callback, AdaptiveRouterPostCallHook)
)
def test_discard_drops_its_adaptive_router_hook():
router: Final = Router(model_list=_adaptive_router_model_list("adaptive"), ignore_invalid_deployments=True)
assert _registered_adaptive_router_strategies()
router.discard()
assert _registered_adaptive_router_strategies() == ()
def test_discard_registers_the_adaptive_router_hook_of_a_survivor():
survivor: Final = Router(model_list=_adaptive_router_model_list("adaptive"), ignore_invalid_deployments=True)
retired: Final = Router(model_list=_adaptive_router_model_list("adaptive"), ignore_invalid_deployments=True)
retired.discard()
survivor_strategies: Final = tuple(tagged.strategy for tagged in survivor.adaptive_routers["adaptive"])
assert _registered_adaptive_router_strategies() == survivor_strategies
survivor.discard()
@pytest.mark.asyncio
async def test_discard_releases_its_router_budget_limiter():
def build_budget_limiter_then_discard() -> weakref.ReferenceType[object]:
router: Final = Router(
model_list=_GROUPED_MODEL_LIST,
provider_budget_config={"openai": {"budget_limit": 10.0, "time_period": "1d"}},
)
budget_limiter: Final = router.get_router_deployment_budget_limiter()
assert budget_limiter is not None
router.discard()
return weakref.ref(budget_limiter)
budget_limiter_ref: Final = build_budget_limiter_then_discard()
await asyncio.sleep(0)
gc.collect()
assert budget_limiter_ref() is None
def test_initialize_assistants_endpoint(model_list):
"""Test if the 'initialize_assistants_endpoint' function is working correctly"""
router = Router(model_list=model_list)

View file

@ -3891,6 +3891,114 @@ class TestCallbackAsyncSyncSeparation:
assert my_sync_cb not in litellm._async_input_callback
class _DropsACallbackFromAnotherThreadWhenComparedToDynamodb:
def __init__(self, dropped: Callable[..., object]) -> None:
self.dropped = dropped
self.drop_thread: threading.Thread | None = None
def __eq__(self, other: object) -> bool:
if isinstance(other, str) and other == "dynamodb" and self.drop_thread is None:
self.drop_thread = threading.Thread(
target=litellm.logging_callback_manager.drop_callbacks, args=((self.dropped,),)
)
self.drop_thread.start()
self.drop_thread.join(timeout=0.2)
return self is other
__hash__ = object.__hash__
def test_function_setup_moves_async_callbacks_while_another_thread_drops_a_callback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def dropped_callback(*args: object, **kwargs: object) -> None:
return None
async def async_callback(*args: object, **kwargs: object) -> None:
return None
drops_on_compare: Final = _DropsACallbackFromAnotherThreadWhenComparedToDynamodb(dropped_callback)
monkeypatch.setattr(litellm.utils, "callback_list", ["initialized"])
for list_name in (
"callbacks",
"input_callback",
"failure_callback",
"service_callback",
"_async_input_callback",
"_async_success_callback",
"_async_failure_callback",
):
monkeypatch.setattr(litellm, list_name, [])
monkeypatch.setattr(litellm, "success_callback", [dropped_callback, drops_on_compare, async_callback])
litellm.utils.function_setup(
"completion",
litellm.utils.Rules(),
datetime.now(),
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
litellm_call_id="callback-drop-race",
)
assert drops_on_compare.drop_thread is not None
drops_on_compare.drop_thread.join()
assert litellm.success_callback == [drops_on_compare]
assert litellm._async_success_callback == [async_callback]
class _DropsACallbackFromAnotherThreadWhenComparedToSentry:
def __init__(self, dropped: Callable[..., object]) -> None:
self.dropped = dropped
self.drop_thread: threading.Thread | None = None
self.drop_finished_during_logger_setup = False
def __eq__(self, other: object) -> bool:
if isinstance(other, str) and other == "sentry" and self.drop_thread is None:
self.drop_thread = threading.Thread(
target=litellm.logging_callback_manager.drop_callbacks, args=((self.dropped,),)
)
self.drop_thread.start()
self.drop_thread.join(timeout=5)
self.drop_finished_during_logger_setup = not self.drop_thread.is_alive()
return self is other
__hash__ = object.__hash__
def test_function_setup_lets_other_threads_drop_callbacks_while_it_sets_up_loggers(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def dropped_callback(*args: object, **kwargs: object) -> None:
return None
drops_on_compare: Final = _DropsACallbackFromAnotherThreadWhenComparedToSentry(dropped_callback)
monkeypatch.setattr(litellm.utils, "callback_list", [])
for list_name in (
"callbacks",
"input_callback",
"failure_callback",
"service_callback",
"_async_input_callback",
"_async_success_callback",
"_async_failure_callback",
):
monkeypatch.setattr(litellm, list_name, [])
monkeypatch.setattr(litellm, "success_callback", [drops_on_compare])
litellm.utils.function_setup(
"completion",
litellm.utils.Rules(),
datetime.now(),
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
litellm_call_id="logger-setup-outside-lock",
)
assert drops_on_compare.drop_thread is not None
drops_on_compare.drop_thread.join()
assert drops_on_compare.drop_finished_during_logger_setup
class TestMetadataNoneHandling:
"""
Test that metadata=None in kwargs doesn't cause TypeError.

View file

@ -263,6 +263,7 @@ INTERNAL_STATE_NAMES: Final = (
"litellm_trusted_callback_vars",
"_litellm_addressed_response_id",
"_litellm_strip_stream_usage",
"_litellm_stream_usage_injected",
"client_side_timeout",
"model_file_id_mapping",
"acompletion",