mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge remote-tracking branch 'origin/main' into lit8011-pr41622
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
This commit is contained in:
commit
7ee528e294
64 changed files with 2990 additions and 1635 deletions
|
|
@ -33,7 +33,7 @@ while IFS= read -r file || [ -n "$file" ]; do
|
|||
*) has_backend=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json)
|
||||
has_cost_map=true ;;
|
||||
tests/unit/proxy/*) : ;;
|
||||
*) outside_cost_map_set=true ;;
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@ Every pull request whose diff against its merge base touches one of the two cost
|
|||
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
|
||||
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
|
||||
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
|
||||
get the file checks and additionally may only touch those two files or the legacy schema
|
||||
path and may only add or update models.
|
||||
get the file checks and additionally may only touch those two files and may only add or update models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -21,7 +20,6 @@ from typing import Final
|
|||
|
||||
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
|
||||
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
|
||||
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
|
||||
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
|
||||
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
|
||||
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
|
|
@ -110,7 +108,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
*(
|
||||
f"bot PRs may only change the cost map files, not {path}"
|
||||
for path in changed_files
|
||||
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
|
||||
if path not in GUARDED_PATHS
|
||||
),
|
||||
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
|
||||
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.harness.context import GatewayTarget
|
|||
from litellm.harness.errors import HarnessError, HarnessInstallFailed
|
||||
from litellm.harness.types import Harness, Usage
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -690,18 +691,21 @@ class ModelEndpoint:
|
|||
async def _invoke_sdk(
|
||||
self,
|
||||
route: str,
|
||||
kwargs: dict[str, Any], # mutable-ok: injects stream_options into the SDK kwargs
|
||||
kwargs: dict[str, Any], # mutable-ok: SDK call kwargs built by _sdk_kwargs
|
||||
) -> object:
|
||||
if route == ROUTE_MESSAGES:
|
||||
return await litellm.anthropic.messages.acreate(**kwargs)
|
||||
if route == ROUTE_CHAT:
|
||||
if kwargs.get("stream"):
|
||||
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
|
||||
kwargs["stream_options"] = { # mutable-ok: JSON field sent to litellm.acompletion
|
||||
"include_usage": True,
|
||||
**stream_options,
|
||||
if not kwargs.get("stream"):
|
||||
return await litellm.acompletion(**kwargs)
|
||||
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
|
||||
return await litellm.acompletion(
|
||||
**{
|
||||
**kwargs,
|
||||
"stream_options": {"include_usage": True, **stream_options},
|
||||
STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options,
|
||||
}
|
||||
return await litellm.acompletion(**kwargs)
|
||||
)
|
||||
return await litellm.aresponses(**kwargs)
|
||||
|
||||
async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response:
|
||||
|
|
|
|||
|
|
@ -138,6 +138,7 @@ from litellm.types.interactions import (
|
|||
InteractionsAPIResponse,
|
||||
InteractionsAPIStreamingResponse,
|
||||
)
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
Batch,
|
||||
|
|
@ -595,6 +596,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
prometheusLogger, \
|
||||
slack_app
|
||||
custom_pricing: bool = False
|
||||
replaced_global_callback_keys: frozenset[str] = frozenset()
|
||||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
streamed_anthropic_message_id: str | None = None
|
||||
|
|
@ -649,11 +651,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
litellm_call_id: str,
|
||||
function_id: str,
|
||||
litellm_trace_id: str | None = None,
|
||||
dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None,
|
||||
dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
|
||||
dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
|
||||
dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
|
||||
applied_guardrails: list[str] | None = None,
|
||||
kwargs: dict | None = None,
|
||||
log_raw_request_response: bool = False,
|
||||
|
|
@ -698,6 +700,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self._pre_call_session_id: str = session_id_var.get()
|
||||
_sid: Final = kwargs.get("litellm_session_id") if kwargs else None
|
||||
self.litellm_session_id: str = str(_sid) if _sid else ""
|
||||
self.stream_usage_injected: Final = kwargs is not None and kwargs.get(STREAM_USAGE_INJECTED_FIELD) is True
|
||||
# supports_correlation_logging is False for calls originating from the
|
||||
# sync client entry point (wrapper() in utils.py): a plain OS thread
|
||||
# has no per-call context isolation the way an asyncio Task does, and
|
||||
|
|
@ -726,15 +729,17 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.raw_request_only = raw_request_only
|
||||
|
||||
# Initialize dynamic callbacks
|
||||
self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks
|
||||
self.dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = dynamic_input_callbacks
|
||||
self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
|
||||
dynamic_success_callbacks
|
||||
)
|
||||
self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
|
||||
dynamic_async_success_callbacks
|
||||
)
|
||||
self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks
|
||||
self.dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = (
|
||||
self.dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
|
||||
dynamic_failure_callbacks
|
||||
)
|
||||
self.dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
|
||||
dynamic_async_failure_callbacks
|
||||
)
|
||||
|
||||
|
|
@ -851,10 +856,32 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.dynamic_async_failure_callbacks, callback
|
||||
)
|
||||
|
||||
def replace_global_callback(self, callback: CustomLogger) -> None:
|
||||
self.add_dynamic_callback(callback)
|
||||
self.replaced_global_callback_keys = self.replaced_global_callback_keys | {
|
||||
litellm.logging_callback_manager.get_custom_logger_key(callback)
|
||||
}
|
||||
|
||||
def add_dynamic_function_callbacks(
|
||||
self,
|
||||
*,
|
||||
async_success: Callable[..., object],
|
||||
failure: Callable[..., object],
|
||||
async_failure: Callable[..., object],
|
||||
) -> None:
|
||||
self.dynamic_async_success_callbacks = self._with_dynamic_callback(
|
||||
self.dynamic_async_success_callbacks, async_success
|
||||
)
|
||||
self.dynamic_failure_callbacks = self._with_dynamic_callback(self.dynamic_failure_callbacks, failure)
|
||||
self.dynamic_async_failure_callbacks = self._with_dynamic_callback(
|
||||
self.dynamic_async_failure_callbacks, async_failure
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _with_dynamic_callback(
|
||||
callbacks: Sequence[str | Callable | CustomLogger] | None, callback: CustomLogger
|
||||
) -> list[str | Callable | CustomLogger]:
|
||||
callbacks: Sequence[str | Callable[..., object] | CustomLogger] | None,
|
||||
callback: Callable[..., object] | CustomLogger,
|
||||
) -> list[str | Callable[..., object] | CustomLogger]:
|
||||
existing: Final = tuple(callbacks or ())
|
||||
return [*existing, *(() if callback in existing else (callback,))]
|
||||
|
||||
|
|
@ -891,9 +918,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
def _process_dynamic_callback_list(
|
||||
self,
|
||||
callback_list: list[str | Callable | CustomLogger] | None,
|
||||
callback_list: list[str | Callable[..., object] | CustomLogger] | None,
|
||||
dynamic_callbacks_type: Literal["input", "success", "failure", "async_success", "async_failure"],
|
||||
) -> list[str | Callable | CustomLogger] | None:
|
||||
) -> list[str | Callable[..., object] | CustomLogger] | None:
|
||||
"""
|
||||
Helper function to initialize CustomLogger compatible callbacks in self.dynamic_* callbacks
|
||||
|
||||
|
|
@ -905,7 +932,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if callback_list is None:
|
||||
return None
|
||||
|
||||
processed_list: Final[list[str | Callable | CustomLogger]] = []
|
||||
processed_list: Final[list[str | Callable[..., object] | CustomLogger]] = []
|
||||
for callback in callback_list:
|
||||
if isinstance(callback, str) and callback in litellm._known_custom_logger_compatible_callbacks:
|
||||
for callback_instance in self._resolve_dynamic_callback_string(callback):
|
||||
|
|
@ -1566,7 +1593,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"api_call_start_time"
|
||||
]
|
||||
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
|
||||
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
|
||||
callbacks: Final = (
|
||||
*self._without_replaced_global_callbacks(litellm.input_callback),
|
||||
*(self.dynamic_input_callbacks or ()),
|
||||
)
|
||||
for callback in callbacks:
|
||||
try:
|
||||
if callback == "supabase" and supabaseClient is not None:
|
||||
|
|
@ -1739,7 +1769,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
|
||||
|
||||
callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or [])
|
||||
callbacks: Final = (
|
||||
*self._without_replaced_global_callbacks(litellm.input_callback),
|
||||
*(self.dynamic_input_callbacks or ()),
|
||||
)
|
||||
for callback in callbacks:
|
||||
try:
|
||||
if callback == "sentry" and add_breadcrumb:
|
||||
|
|
@ -4516,14 +4549,26 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
_filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks)
|
||||
return len(_filtered_failure_callbacks) > 0
|
||||
|
||||
def _without_replaced_global_callbacks(
|
||||
self, global_callbacks: Iterable[str | Callable[..., object] | CustomLogger]
|
||||
) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
|
||||
if not self.replaced_global_callback_keys:
|
||||
return tuple(global_callbacks)
|
||||
return tuple(
|
||||
litellm.logging_callback_manager.without_custom_logger_keys(
|
||||
global_callbacks, self.replaced_global_callback_keys
|
||||
)
|
||||
)
|
||||
|
||||
def get_combined_callback_list(
|
||||
self,
|
||||
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None,
|
||||
global_callbacks: list[str | Callable[..., object] | CustomLogger],
|
||||
) -> list[str | Callable[..., object] | CustomLogger]:
|
||||
kept_global_callbacks: Final = self._without_replaced_global_callbacks(global_callbacks)
|
||||
if dynamic_success_callbacks is None:
|
||||
return list(global_callbacks)
|
||||
return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
|
||||
return list(kept_global_callbacks)
|
||||
return list(dict.fromkeys((*dynamic_success_callbacks, *kept_global_callbacks)))
|
||||
|
||||
def _remove_internal_litellm_callbacks(self, callbacks: list) -> list:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
from collections.abc import Callable
|
||||
import threading
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from types import MethodType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -15,6 +17,7 @@ else:
|
|||
_custom_logger_compatible_callbacks_literal = str
|
||||
|
||||
_generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
|
||||
_callback_lists_lock: Final = threading.RLock()
|
||||
|
||||
|
||||
class LoggingCallbackManager:
|
||||
|
|
@ -43,16 +46,18 @@ class LoggingCallbackManager:
|
|||
Add a input callback to litellm.input_callback.
|
||||
Auto-routes async callbacks to litellm._async_input_callback.
|
||||
"""
|
||||
if not isinstance(callback, str) and self._is_async_callable(callback):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback)
|
||||
with _callback_lists_lock:
|
||||
if not isinstance(callback, str) and self._is_async_callable(callback):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback)
|
||||
|
||||
def add_litellm_service_callback(self, callback: CustomLogger | str | Callable[..., object]):
|
||||
"""
|
||||
Add a service callback to litellm.service_callback
|
||||
"""
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback)
|
||||
with _callback_lists_lock:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback)
|
||||
|
||||
def add_litellm_callback(self, callback: CustomLogger | str | Callable[..., object]):
|
||||
"""
|
||||
|
|
@ -60,10 +65,11 @@ class LoggingCallbackManager:
|
|||
|
||||
Ensures no duplicates are added.
|
||||
"""
|
||||
self._safe_add_callback_to_list(
|
||||
callback=callback,
|
||||
parent_list=litellm.callbacks,
|
||||
)
|
||||
with _callback_lists_lock:
|
||||
self._safe_add_callback_to_list(
|
||||
callback=callback,
|
||||
parent_list=litellm.callbacks,
|
||||
)
|
||||
|
||||
def add_litellm_success_callback(self, callback: CustomLogger | str | Callable[..., object]):
|
||||
"""
|
||||
|
|
@ -71,37 +77,41 @@ class LoggingCallbackManager:
|
|||
Auto-routes async callbacks to litellm._async_success_callback.
|
||||
Special-cases 'dynamodb' and 'openmeter' as async callbacks.
|
||||
"""
|
||||
if (
|
||||
isinstance(callback, str)
|
||||
and callback in ("dynamodb", "openmeter")
|
||||
or not isinstance(callback, str)
|
||||
and self._is_async_callable(callback)
|
||||
):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback)
|
||||
with _callback_lists_lock:
|
||||
if (
|
||||
isinstance(callback, str)
|
||||
and callback in ("dynamodb", "openmeter")
|
||||
or not isinstance(callback, str)
|
||||
and self._is_async_callable(callback)
|
||||
):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback)
|
||||
|
||||
def add_litellm_failure_callback(self, callback: CustomLogger | str | Callable[..., object]):
|
||||
"""
|
||||
Add a failure callback to `litellm.failure_callback`.
|
||||
Auto-routes async callbacks to litellm._async_failure_callback.
|
||||
"""
|
||||
if not isinstance(callback, str) and self._is_async_callable(callback):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback)
|
||||
with _callback_lists_lock:
|
||||
if not isinstance(callback, str) and self._is_async_callable(callback):
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
|
||||
else:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback)
|
||||
|
||||
def add_litellm_async_success_callback(self, callback: CustomLogger | Callable[..., object] | str):
|
||||
"""
|
||||
Add a success callback to litellm._async_success_callback
|
||||
"""
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
|
||||
with _callback_lists_lock:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback)
|
||||
|
||||
def add_litellm_async_failure_callback(self, callback: CustomLogger | Callable[..., object] | str):
|
||||
"""
|
||||
Add a failure callback to litellm._async_failure_callback
|
||||
"""
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
|
||||
with _callback_lists_lock:
|
||||
self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback)
|
||||
|
||||
def remove_callback_from_list_by_object(self, callback_list, obj, require_self=True):
|
||||
"""
|
||||
|
|
@ -110,13 +120,14 @@ class LoggingCallbackManager:
|
|||
if not isinstance(callback_list, list): # Not list -> do nothing
|
||||
return
|
||||
|
||||
if require_self:
|
||||
remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj]
|
||||
else:
|
||||
remove_list = [c for c in callback_list if c == obj]
|
||||
with _callback_lists_lock:
|
||||
if require_self:
|
||||
remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj]
|
||||
else:
|
||||
remove_list = [c for c in callback_list if c == obj]
|
||||
|
||||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
|
||||
def remove_callbacks_by_type(self, callback_list, callback_type):
|
||||
"""
|
||||
|
|
@ -134,10 +145,11 @@ class LoggingCallbackManager:
|
|||
if not isinstance(callback_list, list):
|
||||
return
|
||||
|
||||
remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)]
|
||||
with _callback_lists_lock:
|
||||
remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)]
|
||||
|
||||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
|
||||
def _add_string_callback_to_list(
|
||||
self, callback: str, parent_list: list[CustomLogger | Callable[..., object] | str]
|
||||
|
|
@ -336,6 +348,55 @@ class LoggingCallbackManager:
|
|||
|
||||
return "-".join(key_parts)
|
||||
|
||||
get_custom_logger_key = _get_custom_logger_key
|
||||
callback_lists_lock: Final = _callback_lists_lock
|
||||
|
||||
def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]:
|
||||
dropped_ids: Final = frozenset(map(id, callbacks))
|
||||
owner_ids: Final = frozenset(map(id, method_owners))
|
||||
|
||||
def keep(callback: object) -> bool:
|
||||
if id(callback) in dropped_ids:
|
||||
return False
|
||||
return not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids)
|
||||
|
||||
with _callback_lists_lock:
|
||||
vacated_keys: Final = frozenset(
|
||||
self._get_custom_logger_key(callback)
|
||||
for callback in litellm.callbacks
|
||||
if isinstance(callback, CustomLogger) and id(callback) in dropped_ids
|
||||
)
|
||||
litellm.callbacks = [callback for callback in litellm.callbacks if keep(callback)]
|
||||
litellm.input_callback = [callback for callback in litellm.input_callback if keep(callback)]
|
||||
litellm.success_callback = [callback for callback in litellm.success_callback if keep(callback)]
|
||||
litellm.failure_callback = [callback for callback in litellm.failure_callback if keep(callback)]
|
||||
litellm.service_callback = [callback for callback in litellm.service_callback if keep(callback)]
|
||||
litellm._async_success_callback = [
|
||||
callback for callback in litellm._async_success_callback if keep(callback)
|
||||
]
|
||||
litellm._async_failure_callback = [
|
||||
callback for callback in litellm._async_failure_callback if keep(callback)
|
||||
]
|
||||
return vacated_keys
|
||||
|
||||
def with_custom_logger_keys(
|
||||
self, candidates: Iterable[CustomLogger], keys: frozenset[str]
|
||||
) -> tuple[CustomLogger, ...]:
|
||||
if not keys:
|
||||
return ()
|
||||
class_names: Final = frozenset(key.split("-", 1)[0] for key in keys)
|
||||
same_class: Final = (candidate for candidate in candidates if type(candidate).__name__ in class_names)
|
||||
return tuple(candidate for candidate in same_class if self._get_custom_logger_key(candidate) in keys)
|
||||
|
||||
def without_custom_logger_keys(
|
||||
self, callbacks: Iterable[str | Callable[..., object] | CustomLogger], keys: frozenset[str]
|
||||
) -> Iterator[str | Callable[..., object] | CustomLogger]:
|
||||
return (
|
||||
callback
|
||||
for callback in callbacks
|
||||
if not (isinstance(callback, CustomLogger) and self._get_custom_logger_key(callback) in keys)
|
||||
)
|
||||
|
||||
def _reset_all_callbacks(self):
|
||||
"""
|
||||
Reset all callbacks to an empty list
|
||||
|
|
@ -363,19 +424,21 @@ class LoggingCallbackManager:
|
|||
|
||||
_get_all_callbacks = get_all_callbacks
|
||||
|
||||
def remove_callback_from_all_lists(self, obj, require_self=False) -> None:
|
||||
def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None:
|
||||
"""
|
||||
Remove a callback object from every callback list it may have been
|
||||
promoted into, so a re-initialized callback leaves no stale instance behind.
|
||||
"""
|
||||
for callback_list in (
|
||||
litellm.callbacks,
|
||||
litellm.success_callback,
|
||||
litellm.failure_callback,
|
||||
litellm._async_success_callback,
|
||||
litellm._async_failure_callback,
|
||||
):
|
||||
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
|
||||
with _callback_lists_lock:
|
||||
for callback_list in (
|
||||
litellm.callbacks,
|
||||
litellm.input_callback,
|
||||
litellm.success_callback,
|
||||
litellm.failure_callback,
|
||||
litellm._async_success_callback,
|
||||
litellm._async_failure_callback,
|
||||
):
|
||||
self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self)
|
||||
|
||||
def get_active_additional_logging_utils_from_custom_logger(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -342,6 +342,10 @@ class CustomStreamWrapper:
|
|||
self.messages = getattr(logging_obj, "messages", None)
|
||||
self.sent_stream_usage = False
|
||||
self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False
|
||||
self.caller_requested_stream_usage: Final = (
|
||||
self.check_send_stream_usage(getattr(logging_obj, "stream_options", None))
|
||||
and getattr(logging_obj, "stream_usage_injected", False) is not True
|
||||
)
|
||||
self.tool_call = False
|
||||
# keep track of the returned chunks - used for calculating the input/output tokens for stream options
|
||||
self.chunks: list[ModelResponseStream] = []
|
||||
|
|
@ -2070,7 +2074,7 @@ class CustomStreamWrapper:
|
|||
except StopIteration:
|
||||
if self.sent_last_chunk is True:
|
||||
complete_streaming_response: Final = self._build_complete_streaming_response()
|
||||
response = self.model_response_creator()
|
||||
response = self._new_usage_chunk()
|
||||
if complete_streaming_response is not None:
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
|
||||
|
||||
|
|
@ -2222,27 +2226,10 @@ class CustomStreamWrapper:
|
|||
)
|
||||
|
||||
if _has_usage:
|
||||
# Store a copy ONLY when usage stripping below will mutate
|
||||
# the chunk. For non-usage chunks (vast majority), store
|
||||
# directly to avoid expensive model_copy() per chunk.
|
||||
self.chunks.append(processed_chunk.model_copy())
|
||||
|
||||
# Strip usage from the outgoing chunk so it's not sent twice
|
||||
# (once in the chunk, once in _hidden_params).
|
||||
obj_dict = processed_chunk.model_dump()
|
||||
if "usage" in obj_dict:
|
||||
del obj_dict["usage"]
|
||||
processed_chunk = self.model_response_creator(
|
||||
chunk=obj_dict,
|
||||
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
|
||||
Mapping[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR)
|
||||
),
|
||||
)
|
||||
is_empty = is_model_response_stream_empty(
|
||||
model_response=cast(ModelResponseStream, processed_chunk)
|
||||
)
|
||||
if is_empty:
|
||||
usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk)
|
||||
if usage_free_chunk is None:
|
||||
continue
|
||||
processed_chunk = usage_free_chunk
|
||||
else:
|
||||
# No usage data — safe to store directly without copying
|
||||
self.chunks.append(processed_chunk)
|
||||
|
|
@ -2273,13 +2260,18 @@ class CustomStreamWrapper:
|
|||
if processed_chunk is None:
|
||||
continue
|
||||
|
||||
choice = processed_chunk.choices[0]
|
||||
if isinstance(choice, StreamingChoices):
|
||||
self._accumulate_streamed_delta(choice.delta)
|
||||
if processed_chunk.choices:
|
||||
choice = processed_chunk.choices[0]
|
||||
if isinstance(choice, StreamingChoices):
|
||||
self._accumulate_streamed_delta(choice.delta)
|
||||
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
|
||||
# RETURN RESULT
|
||||
self.chunks.append(processed_chunk)
|
||||
return processed_chunk
|
||||
if not self.send_stream_usage or getattr(processed_chunk, "usage", None) is None:
|
||||
self.chunks.append(processed_chunk)
|
||||
return processed_chunk
|
||||
usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk)
|
||||
if usage_free_chunk is None:
|
||||
continue
|
||||
return usage_free_chunk
|
||||
except (StopAsyncIteration, StopIteration):
|
||||
return await self._finalize_completed_stream(cache_hit=cache_hit)
|
||||
except httpx.TimeoutException as e: # if httpx read timeout error occues
|
||||
|
|
@ -2306,7 +2298,7 @@ class CustomStreamWrapper:
|
|||
async def _finalize_completed_stream(self, cache_hit: bool) -> "ModelResponseStream":
|
||||
if self.sent_last_chunk is True:
|
||||
complete_streaming_response: Final = await self._async_build_complete_streaming_response()
|
||||
response: Final = self.model_response_creator()
|
||||
response: Final = self._new_usage_chunk()
|
||||
if complete_streaming_response is not None:
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
|
||||
|
||||
|
|
@ -2463,6 +2455,22 @@ class CustomStreamWrapper:
|
|||
return False
|
||||
return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
|
||||
|
||||
def _new_usage_chunk(self) -> ModelResponseStream:
|
||||
usage_chunk: Final = self.model_response_creator()
|
||||
if self.caller_requested_stream_usage:
|
||||
usage_chunk.choices = []
|
||||
return usage_chunk
|
||||
|
||||
def _record_and_strip_usage_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
|
||||
self.chunks.append(chunk.model_copy())
|
||||
usage_free_chunk: Final = self.model_response_creator(
|
||||
chunk={key: value for key, value in chunk.model_dump().items() if key != "usage"},
|
||||
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
|
||||
Mapping[str, object], getattr(chunk, HIDDEN_PARAMS_ATTR)
|
||||
),
|
||||
)
|
||||
return None if is_model_response_stream_empty(model_response=usage_free_chunk) else usage_free_chunk
|
||||
|
||||
def _accumulate_streamed_delta(self, delta: object) -> None:
|
||||
# Shared by the sync, async, and non-aiohttp iteration sites so answer
|
||||
# text and the disqualifying-content latch stay in step across all three.
|
||||
|
|
|
|||
|
|
@ -2,17 +2,20 @@ import json
|
|||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, chain
|
||||
from typing import Final, TypeAlias, TypeVar, cast # noqa: TID251 # sum_usage_metadata casts computed keys
|
||||
from typing import Final, Literal, TypeAlias, TypeVar, cast # noqa: TID251 # sum_usage_metadata casts computed keys
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import Timeout
|
||||
from litellm.litellm_core_utils.env_utils import get_env_int_or_none
|
||||
from litellm.types.llms.vertex_ai import (
|
||||
Candidates,
|
||||
Citation,
|
||||
CitationMetadata,
|
||||
GenerateContentResponseBody,
|
||||
GenerationConfig,
|
||||
GroundingMetadata,
|
||||
GroundingSegment,
|
||||
GroundingSupport,
|
||||
|
|
@ -48,8 +51,14 @@ HOP_FAILURES: Final = (VertexAIError, httpx.HTTPError, Timeout)
|
|||
HOP_END_FINISH_REASON: Final = "CONTINUATION"
|
||||
CUT_OFF_FINISH_REASONS: Final = frozenset({"MAX_TOKENS", HOP_END_FINISH_REASON})
|
||||
|
||||
DEFAULT_OUTPUT_BUDGET_ENV: Final = "LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS"
|
||||
_OUTPUT_LIMIT_KEYS: Final = frozenset({"max_output_tokens", "maxOutputTokens"})
|
||||
_REQUEST_KEYS_THAT_SKIP_THE_DEFAULT_BUDGET: Final = frozenset({"generation_config", "continuationToken"})
|
||||
_TEXT_ONLY_MODALITIES: Final = TypeAdapter(list[Literal["TEXT"]])
|
||||
|
||||
_VERTEX_V1_PATH: Final = "/v1/projects/"
|
||||
_VERTEX_PATH_ACCEPTING_CONTINUATION_TOKEN: Final = "/v1beta1/projects/"
|
||||
_CANDIDATE_COUNT_KEYS: Final = ("candidate_count", "candidateCount")
|
||||
|
||||
LongDecodeHops: TypeAlias = tuple[GenerateContentResponseBody, ...]
|
||||
AsyncHopPost: TypeAlias = Callable[[RequestBody], Awaitable[httpx.Response]]
|
||||
|
|
@ -76,15 +85,53 @@ def _int_or_none(value: object) -> int | None:
|
|||
return value if isinstance(value, int) else None
|
||||
|
||||
|
||||
def _asks_for_one_candidate(generation_config: Mapping[str, object]) -> bool:
|
||||
return all(
|
||||
_int_or_none(generation_config[key] or 1) == 1 for key in _CANDIDATE_COUNT_KEYS if key in generation_config
|
||||
)
|
||||
|
||||
|
||||
def continuation_cap_of(request_body: RequestBody) -> int | None:
|
||||
generation_config: Final = request_body.get("generationConfig")
|
||||
if generation_config is None or "max_output_tokens" not in generation_config:
|
||||
return None
|
||||
if "candidate_count" in generation_config and _int_or_none(generation_config["candidate_count"]) != 1:
|
||||
if not _asks_for_one_candidate(generation_config):
|
||||
return None
|
||||
return _int_or_none(generation_config["max_output_tokens"])
|
||||
|
||||
|
||||
def _asks_for_text_only(generation_config: Mapping[str, object]) -> bool:
|
||||
if "response_modalities" in generation_config:
|
||||
return False
|
||||
try:
|
||||
_ = _TEXT_ONLY_MODALITIES.validate_python(generation_config.get("responseModalities", []))
|
||||
except ValidationError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def with_default_output_budget(
|
||||
request_body: RequestBody, model_max_output_tokens: Callable[[], int | None]
|
||||
) -> RequestBody:
|
||||
generation_config: Final = request_body.get("generationConfig", GenerationConfig())
|
||||
request_json: Final[Mapping[str, object]] = request_body
|
||||
budget: Final = get_env_int_or_none(DEFAULT_OUTPUT_BUDGET_ENV)
|
||||
if budget is None or budget <= 0:
|
||||
return request_body
|
||||
if not _REQUEST_KEYS_THAT_SKIP_THE_DEFAULT_BUDGET.isdisjoint(request_body):
|
||||
return request_body
|
||||
if not isinstance(request_json.get("generationConfig", {}), dict):
|
||||
return request_body
|
||||
if not _OUTPUT_LIMIT_KEYS.isdisjoint(generation_config):
|
||||
return request_body
|
||||
if not _asks_for_text_only(generation_config):
|
||||
return request_body
|
||||
model_max: Final = model_max_output_tokens()
|
||||
if model_max is None:
|
||||
return request_body
|
||||
return {**request_body, "generationConfig": {**generation_config, "max_output_tokens": min(budget, model_max)}}
|
||||
|
||||
|
||||
def client_continuation_token(candidate: Candidates) -> str | None:
|
||||
if candidate.get("finishReason") not in CUT_OFF_FINISH_REASONS:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -7,16 +7,19 @@ Why separate file? Make it easy to see how transformation works
|
|||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Mapping
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel, PositiveInt, TypeAdapter, ValidationError
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_image_mime_type_from_url,
|
||||
)
|
||||
|
|
@ -65,6 +68,7 @@ from ..common_utils import (
|
|||
get_supports_response_schema,
|
||||
get_supports_system_message,
|
||||
)
|
||||
from .long_decode import with_default_output_budget
|
||||
|
||||
# VertexBase is imported only for type checking to avoid introducing a
|
||||
# module-load-time cyclic import to vertex_llm_base. The instance is lazily
|
||||
|
|
@ -1304,13 +1308,75 @@ def sync_transform_request_body(
|
|||
vertex_auth_header=vertex_auth_header,
|
||||
)
|
||||
|
||||
return transform_request_body(
|
||||
messages=messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
cached_content=cached_content,
|
||||
optional_params=optional_params,
|
||||
return _with_default_output_budget(
|
||||
transform_request_body(
|
||||
messages=messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
cached_content=cached_content,
|
||||
optional_params=optional_params,
|
||||
),
|
||||
model,
|
||||
custom_llm_provider,
|
||||
litellm_params.get("model_info"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is an untyped dict
|
||||
)
|
||||
|
||||
|
||||
class _CostMapOutputModalities(TypedDict):
|
||||
supported_output_modalities: NotRequired[ReadOnly[tuple[str, ...]]]
|
||||
|
||||
|
||||
_COST_MAP_OUTPUT_MODALITIES: Final = TypeAdapter(_CostMapOutputModalities)
|
||||
|
||||
|
||||
def _answers_text(cost_map_key: str) -> bool:
|
||||
try:
|
||||
row: Final = _COST_MAP_OUTPUT_MODALITIES.validate_python(litellm.model_cost.get(cost_map_key)) # pyright: ignore[reportUnknownMemberType] # cost map rows are untyped JSON
|
||||
except ValidationError:
|
||||
return False
|
||||
return "text" in row.get("supported_output_modalities", ("text",))
|
||||
|
||||
|
||||
class _OutputLimit(TypedDict):
|
||||
max_output_tokens: NotRequired[ReadOnly[PositiveInt | None]]
|
||||
|
||||
|
||||
_OUTPUT_LIMIT: Final = TypeAdapter(_OutputLimit)
|
||||
|
||||
|
||||
def _output_limit(fields: object) -> int | None:
|
||||
try:
|
||||
return _OUTPUT_LIMIT.validate_python(fields).get("max_output_tokens")
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _catalog_capped_output_limit(cost_map_key: str, override: object) -> int | None:
|
||||
limits: Final = (
|
||||
_output_limit(GetModelCostMap.loaded_model_cost_map().get(cost_map_key)),
|
||||
_output_limit(override),
|
||||
)
|
||||
return min((limit for limit in limits if limit is not None), default=None)
|
||||
|
||||
|
||||
def _text_chat_model_max_output_tokens(
|
||||
model: str, custom_llm_provider: str, deployment_model_info: object
|
||||
) -> int | None:
|
||||
info: Final = litellm.utils.lookup_model_info_for_registration(model, custom_llm_provider)
|
||||
if isinstance(info, litellm.utils.ModelInfoLookupMiss) or info.get("mode") != "chat":
|
||||
return None
|
||||
if not _answers_text(info["key"]):
|
||||
return None
|
||||
return _catalog_capped_output_limit(info["key"], deployment_model_info or info)
|
||||
|
||||
|
||||
def _with_default_output_budget(
|
||||
request_body: RequestBody, model: str, custom_llm_provider: str, deployment_model_info: object
|
||||
) -> RequestBody:
|
||||
return with_default_output_budget(
|
||||
request_body,
|
||||
partial(_text_chat_model_max_output_tokens, model, custom_llm_provider, deployment_model_info),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1380,21 +1446,8 @@ async def async_transform_request_body(
|
|||
messages, should_inline=should_inline_remote_media(custom_llm_provider)
|
||||
)
|
||||
|
||||
if _openai_messages_may_need_sync_gcs_metadata_fetch(inlined_messages):
|
||||
# _transform_request_body may issue a sync httpx.get (up to 5s timeout)
|
||||
# via _get_gcs_object_content_type to fetch GCS object metadata. Run the
|
||||
# whole sync transformation on a worker thread so it does not block the
|
||||
# async event loop.
|
||||
return await asyncify(transform_request_body)(
|
||||
messages=inlined_messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
cached_content=cached_content,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
|
||||
return transform_request_body(
|
||||
transform: Final = partial(
|
||||
transform_request_body,
|
||||
messages=inlined_messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -1402,6 +1455,21 @@ async def async_transform_request_body(
|
|||
cached_content=cached_content,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
# _transform_request_body may issue a sync httpx.get (up to 5s timeout)
|
||||
# via _get_gcs_object_content_type to fetch GCS object metadata. Run the
|
||||
# whole sync transformation on a worker thread so it does not block the
|
||||
# async event loop.
|
||||
request_body: Final[RequestBody] = (
|
||||
await asyncify(transform)()
|
||||
if _openai_messages_may_need_sync_gcs_metadata_fetch(inlined_messages)
|
||||
else transform()
|
||||
)
|
||||
return _with_default_output_budget(
|
||||
request_body,
|
||||
model,
|
||||
custom_llm_provider,
|
||||
litellm_params.get("model_info"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is an untyped dict
|
||||
)
|
||||
|
||||
|
||||
def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage:
|
||||
|
|
|
|||
|
|
@ -268,6 +268,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
sanitize_for_log,
|
||||
)
|
||||
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.utils import (
|
||||
CallTypesLiteral,
|
||||
ModelResponse,
|
||||
|
|
@ -555,22 +556,29 @@ def model_deployments_support_stream_options(
|
|||
return all(_litellm_model_supports_stream_options(m) for m in candidate_models)
|
||||
|
||||
|
||||
_STREAM_USAGE_FLAGS: Final = ("_litellm_strip_stream_usage", STREAM_USAGE_INJECTED_FIELD)
|
||||
|
||||
|
||||
def stream_usage_tracking_updates(
|
||||
data: Mapping[str, object],
|
||||
general_settings: Mapping[str, object],
|
||||
route_type: str,
|
||||
supports_stream_options: Callable[[], bool],
|
||||
) -> Mapping[str, object]:
|
||||
scrub: Final = {"_litellm_strip_stream_usage": False} if "_litellm_strip_stream_usage" in data else {}
|
||||
scrub: Final = {flag: False for flag in _STREAM_USAGE_FLAGS if flag in data}
|
||||
if data.get("stream", False) is not True:
|
||||
return scrub
|
||||
always_include: Final = general_settings.get("always_include_stream_usage")
|
||||
stream_options: Final = data.get("stream_options")
|
||||
if always_include is True:
|
||||
if "stream_options" not in data:
|
||||
return {**scrub, "stream_options": {"include_usage": True}}
|
||||
return {**scrub, "stream_options": {"include_usage": True}, STREAM_USAGE_INJECTED_FIELD: True}
|
||||
if isinstance(stream_options, dict) and "include_usage" not in stream_options:
|
||||
return {**scrub, "stream_options": {**stream_options, "include_usage": True}}
|
||||
return {
|
||||
**scrub,
|
||||
"stream_options": {**stream_options, "include_usage": True},
|
||||
STREAM_USAGE_INJECTED_FIELD: True,
|
||||
}
|
||||
return scrub
|
||||
if always_include is False and route_type == "agenerate_content_stream" and "stream_options" not in data:
|
||||
return {**scrub, "stream_options": {"include_usage": False}}
|
||||
|
|
@ -584,6 +592,7 @@ def stream_usage_tracking_updates(
|
|||
return {
|
||||
"stream_options": {**merged_stream_options, "include_usage": True},
|
||||
"_litellm_strip_stream_usage": True,
|
||||
STREAM_USAGE_INJECTED_FIELD: True,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -494,7 +494,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail):
|
|||
sse: Final = AnthropicSSERestorer(step)
|
||||
events: Final = ResponsesStreamRestorer(step, rehydrate)
|
||||
carries: Final[CarryWindows] = {}
|
||||
last_chunk = None # rebind-ok: tracks the most recent chunk for the final flush.
|
||||
last_chunk = None # rebind-ok: tracks the most recent chunk with choices for the final flush.
|
||||
|
||||
async for chunk in response:
|
||||
if isinstance(chunk, (bytes, str)):
|
||||
|
|
@ -506,7 +506,8 @@ class LLMShieldProxyGuardrail(CustomGuardrail):
|
|||
yield event
|
||||
continue
|
||||
restored_chunk = detached(chunk)
|
||||
last_chunk = restored_chunk
|
||||
if getattr(restored_chunk, "choices", None):
|
||||
last_chunk = restored_chunk
|
||||
for choice in attribute_items(restored_chunk, "choices"):
|
||||
await self._restore_choice(choice, carries, session_id)
|
||||
yield restored_chunk
|
||||
|
|
|
|||
|
|
@ -374,6 +374,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
self._redis_increment_queue_lock = asyncio.Lock()
|
||||
self._redis_increment_flush_lock = asyncio.Lock()
|
||||
self._detached_increment_operations = None
|
||||
self._retired = False
|
||||
self._flush_after_retire = None
|
||||
self.deployment_budget_config = None
|
||||
|
||||
@with_service_target("model_budgets")
|
||||
|
|
|
|||
|
|
@ -268,7 +268,7 @@ def sanitize_for_log(value: object) -> str:
|
|||
|
||||
from litellm.router import Router
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES
|
||||
from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES, STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS
|
||||
from litellm.types.router import BaggageMappedTagsStamp, holds_api_key_secret_reference
|
||||
from litellm.types.services import ServiceTypes
|
||||
|
|
@ -2370,7 +2370,15 @@ def refresh_proxy_server_request_body_snapshot(
|
|||
if not isinstance(proxy_server_request, dict):
|
||||
return
|
||||
_body_snapshot_exclude: Final = (
|
||||
frozenset({"secret_fields", "proxy_server_request", "litellm_logging_obj", VECTOR_STORE_CALLER_METADATA_KWARG})
|
||||
frozenset(
|
||||
{
|
||||
"secret_fields",
|
||||
"proxy_server_request",
|
||||
"litellm_logging_obj",
|
||||
VECTOR_STORE_CALLER_METADATA_KWARG,
|
||||
STREAM_USAGE_INJECTED_FIELD,
|
||||
}
|
||||
)
|
||||
| _TRANSPORT_ONLY_CREDENTIAL_KEYS
|
||||
| _CALLBACK_CREDENTIAL_KEYS
|
||||
)
|
||||
|
|
|
|||
|
|
@ -880,8 +880,8 @@ def _build_passthrough_failure_request_payload(
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _TeamCallbackWiring:
|
||||
success_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg
|
||||
failure_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg
|
||||
success_callbacks: "list[str | Callable[..., object] | CustomLogger] | None" = None # mutable-ok: Logging init arg
|
||||
failure_callbacks: "list[str | Callable[..., object] | CustomLogger] | None" = None # mutable-ok: Logging init arg
|
||||
logging_kwargs: dict[str, str | dict[str, str]] | None = None # mutable-ok: Logging.__init__ arg
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -976,6 +976,7 @@ from litellm.secret_managers.main import (
|
|||
)
|
||||
from litellm.tracing.config import is_lens_tracing_enabled
|
||||
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicResponse,
|
||||
|
|
@ -18219,6 +18220,7 @@ async def async_queue_request(
|
|||
try:
|
||||
data = await request.json()
|
||||
data.pop("_litellm_strip_stream_usage", None)
|
||||
data.pop(STREAM_USAGE_INJECTED_FIELD, None)
|
||||
|
||||
# Include original request and headers in the data
|
||||
data["proxy_server_request"] = {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,14 @@
|
|||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from enum import Enum, auto
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, TypeVar
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException, status
|
||||
|
|
@ -12,6 +17,7 @@ from typing_extensions import Never
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.router_utils.common_utils import is_proxy_admin_request
|
||||
|
||||
|
|
@ -58,7 +64,34 @@ _ROUTER_ONLY_SETTINGS: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
def _route_user_config_request(data: dict, route_type: str):
|
||||
class UserConfigRouters:
|
||||
def __init__(self, max_size: int) -> None:
|
||||
self._max_size: Final = max_size
|
||||
self._routers: Final[OrderedDict[str, LitellmRouter]] = OrderedDict() # mutable-ok: LRU evicts in place
|
||||
|
||||
def get(self, config: Mapping[str, object], build: Callable[[], LitellmRouter]) -> LitellmRouter:
|
||||
key: Final = hashlib.sha256(json.dumps(config, sort_keys=True, default=str).encode()).hexdigest()
|
||||
cached: Final = self._routers.get(key)
|
||||
if cached is not None:
|
||||
self._routers.move_to_end(key)
|
||||
return cached
|
||||
router: Final = build()
|
||||
router.serve_through_request_logging()
|
||||
self._routers[key] = router
|
||||
if len(self._routers) > self._max_size:
|
||||
self._routers.popitem(last=False)[1].discard()
|
||||
return router
|
||||
|
||||
def clear(self) -> None:
|
||||
for router in self._routers.values():
|
||||
router.discard()
|
||||
self._routers.clear()
|
||||
|
||||
|
||||
user_config_routers: Final = UserConfigRouters(max_size=32)
|
||||
|
||||
|
||||
async def _route_user_config_request(data: dict, route_type: str):
|
||||
"""Route a request using the user-provided router config."""
|
||||
router_config: Final = data.pop("user_config")
|
||||
|
||||
|
|
@ -67,10 +100,24 @@ def _route_user_config_request(data: dict, route_type: str):
|
|||
valid_args: Final = litellm.Router.get_valid_args()
|
||||
filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
|
||||
|
||||
user_router: Final = litellm.Router(**filtered_config)
|
||||
ret_val: Final = getattr(user_router, f"{route_type}")(**data)
|
||||
user_router.discard()
|
||||
return ret_val
|
||||
logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)
|
||||
cached_router: Final = user_config_routers.get(filtered_config, lambda: litellm.Router(**filtered_config))
|
||||
cached_router.bind_to_request_logging(logging_obj)
|
||||
return await getattr(cached_router, f"{route_type}")(**{**data, "litellm_logging_obj": logging_obj})
|
||||
|
||||
|
||||
def _request_logging_obj(data: Mapping[str, object], route_type: str) -> LiteLLMLoggingObj:
|
||||
arrived_with: Final = data.get("litellm_logging_obj")
|
||||
if isinstance(arrived_with, LiteLLMLoggingObj):
|
||||
return arrived_with
|
||||
logging_obj, _ = litellm.utils.function_setup(
|
||||
original_function=route_type,
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=datetime.now(), # noqa: DTZ005 # Logging subtracts it from naive end times
|
||||
is_async_call=True,
|
||||
**{"litellm_call_id": str(uuid4()), **data},
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _is_a2a_agent_model(model_name: object) -> bool:
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from contextvars import ContextVar
|
|||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from functools import lru_cache, partial
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Generic, Literal, NoReturn, Optional, TypeAlias, TypeVar, Union, cast
|
||||
|
||||
|
|
@ -361,10 +362,11 @@ from litellm.router_utils.routing_groups import (
|
|||
validate_routing_strategy,
|
||||
)
|
||||
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch
|
||||
from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage
|
||||
from litellm.scheduler import FlowItem, Scheduler
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.litellm_params import RoutingStrategyName
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD, RoutingStrategyName
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionToolParam,
|
||||
|
|
@ -552,6 +554,7 @@ _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
|
|||
_CallbackT = TypeVar("_CallbackT")
|
||||
_DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset(
|
||||
{"model", "api_base", "api_key", "api_version", "responses_api_version"}
|
||||
)
|
||||
|
|
@ -1046,6 +1049,8 @@ class RoutingArgs(enum.Enum):
|
|||
# as the per-request one built from a caller-supplied user_config, drops out on its
|
||||
# own rather than leaving entries behind that nothing can withdraw.
|
||||
_live_routers: Final["weakref.WeakSet[Router]"] = weakref.WeakSet()
|
||||
_routers_holding_global_callbacks: Final["weakref.WeakSet[Router]"] = weakref.WeakSet()
|
||||
_routers_holding_global_callbacks_lock: Final = threading.Lock()
|
||||
|
||||
|
||||
def _replay_live_router_model_cost() -> None:
|
||||
|
|
@ -1578,22 +1583,18 @@ class Router:
|
|||
self._init_routing_groups(self._routing_groups_input)
|
||||
self._override_selectors: dict[str, RouterStrategySelector | None] = {}
|
||||
self._override_selectors_lock = threading.Lock()
|
||||
self._request_scoped_selectors: tuple[CustomLogger, ...] = ()
|
||||
self._request_scoped_policies: tuple[CustomLogger, ...] = ()
|
||||
self.access_groups = None
|
||||
with _routers_holding_global_callbacks_lock:
|
||||
_routers_holding_global_callbacks.add(self)
|
||||
## USAGE TRACKING ##
|
||||
if isinstance(litellm._async_success_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success)
|
||||
else:
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success)
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success)
|
||||
if isinstance(litellm.success_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(self.sync_deployment_callback_on_success)
|
||||
else:
|
||||
litellm.success_callback = [self.sync_deployment_callback_on_success]
|
||||
if isinstance(litellm._async_failure_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(
|
||||
self.async_deployment_callback_on_failure
|
||||
)
|
||||
else:
|
||||
litellm._async_failure_callback = [self.async_deployment_callback_on_failure]
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(self.async_deployment_callback_on_failure)
|
||||
## COOLDOWNS ##
|
||||
if isinstance(litellm.failure_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_failure_callback(self.deployment_callback_on_failure)
|
||||
|
|
@ -1689,20 +1690,116 @@ class Router:
|
|||
# Stop contributing to cost-map rebuilds straight away rather than waiting
|
||||
# for this router to be collected.
|
||||
_live_routers.discard(self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm._async_success_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.success_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm._async_failure_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.failure_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.input_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.service_callback, self)
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, self)
|
||||
self.retire_routing_state()
|
||||
self._release_global_callbacks()
|
||||
|
||||
# Remove ForwardClientSideHeadersByModelGroup if it exists
|
||||
if self.optional_callbacks is not None:
|
||||
for callback in self.optional_callbacks:
|
||||
litellm.logging_callback_manager.remove_callback_from_list_by_object(
|
||||
litellm.callbacks, callback, require_self=False
|
||||
)
|
||||
def _release_global_callbacks(self) -> None:
|
||||
with _routers_holding_global_callbacks_lock:
|
||||
_routers_holding_global_callbacks.discard(self)
|
||||
slack_alerting_loggers: Final = self._slack_alerting_loggers()
|
||||
adaptive_router_hooks: Final = self._adaptive_router_hooks()
|
||||
vacated_keys: Final = litellm.logging_callback_manager.drop_callbacks(
|
||||
(
|
||||
*self._callback_selectors(),
|
||||
*self._override_selectors.values(),
|
||||
*(self.optional_callbacks or ()),
|
||||
*slack_alerting_loggers,
|
||||
*adaptive_router_hooks,
|
||||
),
|
||||
method_owners=(self, *slack_alerting_loggers),
|
||||
)
|
||||
with _routers_holding_global_callbacks_lock:
|
||||
survivors: Final = tuple(_routers_holding_global_callbacks)
|
||||
candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors)
|
||||
for candidate in litellm.logging_callback_manager.with_custom_logger_keys(candidates, vacated_keys):
|
||||
self._register_router_selector(candidate)
|
||||
adaptive_keys: Final = frozenset(
|
||||
map(litellm.logging_callback_manager.get_custom_logger_key, adaptive_router_hooks)
|
||||
)
|
||||
adaptive_survivor: Final = next((router for router in survivors if router.adaptive_routers), None)
|
||||
if adaptive_survivor is not None and vacated_keys & adaptive_keys:
|
||||
adaptive_survivor._sync_adaptive_router_hooks()
|
||||
|
||||
def serve_through_request_logging(self) -> None:
|
||||
self._request_scoped_selectors = (*self._callback_selectors(), *self._request_adaptive_router_hooks())
|
||||
self._request_scoped_policies = (*self._optional_loggers(), *self._slack_alerting_loggers())
|
||||
self._release_global_callbacks()
|
||||
|
||||
def bind_to_request_logging(self, logging_obj: LiteLLMLogging) -> None:
|
||||
logging_obj.add_dynamic_function_callbacks(
|
||||
async_success=self.deployment_callback_on_success,
|
||||
failure=self.deployment_callback_on_failure,
|
||||
async_failure=self.async_deployment_callback_on_failure,
|
||||
)
|
||||
for selector in self._request_scoped_selectors:
|
||||
logging_obj.replace_global_callback(selector)
|
||||
for policy in self._request_policies_not_set_globally():
|
||||
logging_obj.add_dynamic_callback(policy)
|
||||
|
||||
def _request_policies_not_set_globally(self) -> tuple[CustomLogger, ...]:
|
||||
global_keys: Final = frozenset(
|
||||
map(
|
||||
litellm.logging_callback_manager.get_custom_logger_key,
|
||||
(callback for callback in litellm.callbacks if isinstance(callback, CustomLogger)),
|
||||
)
|
||||
)
|
||||
return tuple(
|
||||
policy
|
||||
for policy in self._request_scoped_policies
|
||||
if litellm.logging_callback_manager.get_custom_logger_key(policy) not in global_keys
|
||||
)
|
||||
|
||||
def _request_adaptive_router_hooks(self) -> tuple[CustomLogger, ...]:
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
)
|
||||
|
||||
return tuple(
|
||||
AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)
|
||||
for tagged in chain.from_iterable(self.adaptive_routers.values())
|
||||
)
|
||||
|
||||
def _routing_callbacks(self) -> Iterable[object]:
|
||||
if not (self._request_scoped_selectors or self._request_scoped_policies):
|
||||
return litellm.callbacks
|
||||
replaced_keys: Final = frozenset(
|
||||
map(litellm.logging_callback_manager.get_custom_logger_key, self._request_scoped_selectors)
|
||||
)
|
||||
return chain(
|
||||
litellm.logging_callback_manager.without_custom_logger_keys(litellm.callbacks, replaced_keys),
|
||||
self._request_scoped_selectors,
|
||||
self._request_policies_not_set_globally(),
|
||||
)
|
||||
|
||||
def retire_routing_state(self) -> None:
|
||||
for strategy in (*self._global_callback_candidates(), *self._override_selectors.values()):
|
||||
if isinstance(strategy, BaseRoutingStrategy | RouterBudgetLimiting):
|
||||
strategy.retire()
|
||||
|
||||
def _slack_alerting_loggers(self) -> tuple[CustomLogger, ...]:
|
||||
slack_alerting_logger: Final = getattr(self, "slack_alerting_logger", None)
|
||||
return (slack_alerting_logger,) if isinstance(slack_alerting_logger, CustomLogger) else ()
|
||||
|
||||
def _adaptive_router_hooks(self) -> tuple[CustomLogger, ...]:
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
)
|
||||
|
||||
own_strategy_ids: Final = frozenset(
|
||||
id(tagged.strategy) for tagged in chain.from_iterable(self.adaptive_routers.values())
|
||||
)
|
||||
hooks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook)
|
||||
return tuple(
|
||||
hook
|
||||
for hook in hooks
|
||||
if isinstance(hook, AdaptiveRouterPostCallHook) and id(hook.adaptive_router) in own_strategy_ids
|
||||
)
|
||||
|
||||
def _optional_loggers(self) -> tuple[CustomLogger, ...]:
|
||||
return tuple(callback for callback in self.optional_callbacks or () if isinstance(callback, CustomLogger))
|
||||
|
||||
def _global_callback_candidates(self) -> tuple[CustomLogger, ...]:
|
||||
return (*self._callback_selectors(), *self._optional_loggers(), *self._slack_alerting_loggers())
|
||||
|
||||
@staticmethod
|
||||
def _create_redis_cache(
|
||||
|
|
@ -1808,7 +1905,7 @@ class Router:
|
|||
return selector
|
||||
|
||||
@staticmethod
|
||||
def _register_router_selector(selector: RouterStrategySelector) -> None:
|
||||
def _register_router_selector(selector: CustomLogger) -> None:
|
||||
if isinstance(selector, LeastBusyLoggingHandler):
|
||||
if isinstance(litellm.input_callback, list):
|
||||
litellm.logging_callback_manager.add_litellm_input_callback(selector)
|
||||
|
|
@ -1817,23 +1914,28 @@ class Router:
|
|||
if isinstance(litellm.callbacks, list):
|
||||
litellm.logging_callback_manager.add_litellm_callback(selector)
|
||||
|
||||
def _callback_selectors(self) -> tuple[RouterStrategySelector, ...]:
|
||||
return tuple(
|
||||
selector
|
||||
for selector in (
|
||||
*(getattr(self, attr, None) for attr in self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.values()),
|
||||
*chain.from_iterable(selectors.values() for selectors in self._group_selectors.values()),
|
||||
)
|
||||
if selector is not None
|
||||
)
|
||||
|
||||
def _unregister_router_selectors(self, selectors: Sequence[object]) -> None:
|
||||
"""
|
||||
Drop router-owned strategy selectors from litellm's global callback
|
||||
lists by identity. Used before re-init (`routing_strategy_init` /
|
||||
`_init_routing_groups`) so repeated `update_settings` calls don't
|
||||
accumulate dead selectors that keep receiving callback events.
|
||||
lists. Used before re-init (`routing_strategy_init` /
|
||||
`_init_routing_groups`) and by `discard` so dead selectors stop
|
||||
receiving callback events. `function_setup` copies `litellm.callbacks`
|
||||
into the per-event lists on every request, so those are purged too.
|
||||
"""
|
||||
for selector in selectors:
|
||||
if isinstance(selector, BaseRoutingStrategy):
|
||||
selector.retire()
|
||||
selector_ids: Final = {id(s) for s in selectors if s is not None}
|
||||
if not selector_ids:
|
||||
return
|
||||
if isinstance(litellm.callbacks, list):
|
||||
litellm.callbacks = [c for c in litellm.callbacks if id(c) not in selector_ids]
|
||||
if isinstance(litellm.input_callback, list):
|
||||
litellm.input_callback = [c for c in litellm.input_callback if id(c) not in selector_ids]
|
||||
litellm.logging_callback_manager.drop_callbacks(selectors)
|
||||
|
||||
def _apply_updated_routing_strategy_args(self) -> None:
|
||||
"""
|
||||
|
|
@ -4673,7 +4775,8 @@ class Router:
|
|||
deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
|
||||
):
|
||||
kwargs.pop(key, None)
|
||||
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=self.default_litellm_params)
|
||||
defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params)
|
||||
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults)
|
||||
|
||||
model_info = deployment.get("model_info", {}).copy()
|
||||
deployment_litellm_model_name = deployment["litellm_params"]["model"]
|
||||
|
|
@ -4817,6 +4920,19 @@ class Router:
|
|||
data=_DEPLOYMENT_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]),
|
||||
)
|
||||
|
||||
deployment_litellm_params: Final = _DEPLOYMENT_PARAMS_ADAPTER.validate_python(deployment["litellm_params"])
|
||||
if router_config_requests_stream_usage(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
|
||||
(
|
||||
_LITELLM_PARAMS_ADAPTER.validate_python(
|
||||
deployment_litellm_params
|
||||
if isinstance(deployment_litellm_params, Mapping)
|
||||
else deployment_litellm_params.model_dump(exclude_none=True)
|
||||
),
|
||||
defaults,
|
||||
),
|
||||
):
|
||||
kwargs[STREAM_USAGE_INJECTED_FIELD] = True
|
||||
self._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name)
|
||||
hop_kwargs: Final = _FALLBACK_HOP_ADAPTER.validate_python(kwargs)
|
||||
if _is_fallback_hop(hop_kwargs):
|
||||
|
|
@ -9789,7 +9905,7 @@ class Router:
|
|||
Raises:
|
||||
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
|
||||
"""
|
||||
for _callback in litellm.callbacks:
|
||||
for _callback in self._routing_callbacks():
|
||||
if isinstance(_callback, CustomLogger):
|
||||
_callback.pre_call_check(deployment)
|
||||
|
||||
|
|
@ -9810,7 +9926,7 @@ class Router:
|
|||
Raises:
|
||||
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
|
||||
"""
|
||||
for _callback in litellm.callbacks:
|
||||
for _callback in self._routing_callbacks():
|
||||
if isinstance(_callback, CustomLogger):
|
||||
try:
|
||||
await _callback.async_pre_call_check(deployment, parent_otel_span)
|
||||
|
|
@ -9947,7 +10063,7 @@ class Router:
|
|||
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
|
||||
"""
|
||||
returned_healthy_deployments = healthy_deployments
|
||||
for _callback in litellm.callbacks:
|
||||
for _callback in self._routing_callbacks():
|
||||
if isinstance(_callback, CustomLogger):
|
||||
try:
|
||||
returned_healthy_deployments = await _callback.async_filter_deployments(
|
||||
|
|
@ -10602,6 +10718,7 @@ class Router:
|
|||
changes, otherwise a released router keeps recording turns through its hook."""
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
AdaptiveRouterResponseHeaderHook,
|
||||
)
|
||||
|
||||
for callback in litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook):
|
||||
|
|
@ -10611,6 +10728,8 @@ class Router:
|
|||
litellm.logging_callback_manager.add_litellm_callback(
|
||||
AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)
|
||||
)
|
||||
if self.adaptive_routers:
|
||||
litellm.logging_callback_manager.add_litellm_callback(AdaptiveRouterResponseHeaderHook())
|
||||
|
||||
def init_adaptive_router_deployment(self, deployment: Deployment) -> None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
"""Adaptive router strategy. See README.md for design overview."""
|
||||
|
||||
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
|
||||
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
AdaptiveRouterResponseHeaderHook,
|
||||
)
|
||||
|
||||
__all__ = ["AdaptiveRouter", "AdaptiveRouterPostCallHook"]
|
||||
__all__ = ["AdaptiveRouter", "AdaptiveRouterPostCallHook", "AdaptiveRouterResponseHeaderHook"]
|
||||
|
|
|
|||
|
|
@ -170,12 +170,7 @@ def _assistant_content_and_tool_calls(response_obj: Any) -> tuple[object, Sequen
|
|||
return content, tool_calls
|
||||
|
||||
|
||||
class AdaptiveRouterPostCallHook(CustomLogger):
|
||||
"""One hook instance per AdaptiveRouter. Registered into litellm.callbacks."""
|
||||
|
||||
def __init__(self, adaptive_router: AdaptiveRouter) -> None:
|
||||
self.adaptive_router = adaptive_router
|
||||
|
||||
class AdaptiveRouterResponseHeaderHook(CustomLogger):
|
||||
async def async_post_call_response_headers_hook(
|
||||
self,
|
||||
data: Mapping[str, object],
|
||||
|
|
@ -200,6 +195,13 @@ class AdaptiveRouterPostCallHook(CustomLogger):
|
|||
return None
|
||||
return {ADAPTIVE_ROUTER_RESPONSE_HEADER: chosen}
|
||||
|
||||
|
||||
class AdaptiveRouterPostCallHook(CustomLogger):
|
||||
"""One hook instance per AdaptiveRouter. Registered into litellm.callbacks."""
|
||||
|
||||
def __init__(self, adaptive_router: AdaptiveRouter) -> None:
|
||||
self.adaptive_router = adaptive_router
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
await self._record(kwargs, response_obj, response_status=200)
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ class BaseRoutingStrategy(ABC):
|
|||
self.dual_cache = dual_cache
|
||||
self.redis_increment_operation_queue: list[RedisPipelineIncrementOperation] = []
|
||||
self._sync_task: asyncio.Task[None] | None = None
|
||||
self._retired = False
|
||||
self._flush_after_retire: asyncio.Task[None] | None = None
|
||||
if should_batch_redis_writes:
|
||||
self.setup_sync_task(default_sync_interval)
|
||||
|
||||
|
|
@ -47,13 +49,25 @@ class BaseRoutingStrategy(ABC):
|
|||
|
||||
def retire(self) -> None:
|
||||
self.cancel_sync_task()
|
||||
if not self.redis_increment_operation_queue:
|
||||
self._retired = True
|
||||
self._schedule_flush_after_retire()
|
||||
|
||||
def _schedule_flush_after_retire(self) -> None:
|
||||
if self._flush_after_retire is not None and not self._flush_after_retire.done():
|
||||
return
|
||||
if not self.redis_increment_operation_queue or self.dual_cache.redis_cache is None:
|
||||
return
|
||||
try:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
loop.create_task(self._push_in_memory_increments_to_redis())
|
||||
self._flush_after_retire = loop.create_task(self._drain_increments_after_retire())
|
||||
|
||||
async def _drain_increments_after_retire(self) -> None:
|
||||
if not self.redis_increment_operation_queue:
|
||||
return
|
||||
await self._push_in_memory_increments_to_redis()
|
||||
await self._drain_increments_after_retire()
|
||||
|
||||
async def cleanup(self):
|
||||
"""Cleanup method to be called when shutting down"""
|
||||
|
|
@ -98,6 +112,8 @@ class BaseRoutingStrategy(ABC):
|
|||
|
||||
self.redis_increment_operation_queue.append(increment_op)
|
||||
self.add_to_in_memory_keys_to_update(key=key)
|
||||
if self._retired:
|
||||
self._schedule_flush_after_retire()
|
||||
return result
|
||||
|
||||
@with_service_target("router_usage")
|
||||
|
|
|
|||
|
|
@ -128,7 +128,9 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
self._redis_increment_queue_lock = asyncio.Lock()
|
||||
self._redis_increment_flush_lock = asyncio.Lock()
|
||||
self._detached_increment_operations: tuple[RedisPipelineIncrementOperation, ...] | None = None
|
||||
asyncio.create_task(self.periodic_sync_in_memory_spend_with_redis())
|
||||
self._sync_task: asyncio.Task[None] = asyncio.create_task(self.periodic_sync_in_memory_spend_with_redis())
|
||||
self._retired = False
|
||||
self._flush_after_retire: asyncio.Task[None] | None = None
|
||||
self.provider_budget_config: GenericBudgetConfigType | None = provider_budget_config
|
||||
self.deployment_budget_config: GenericBudgetConfigType | None = None
|
||||
self.tag_budget_config: GenericBudgetConfigType | None = None
|
||||
|
|
@ -140,6 +142,30 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
if isinstance(litellm.callbacks, list):
|
||||
litellm.logging_callback_manager.add_litellm_callback(self)
|
||||
|
||||
def retire(self) -> None:
|
||||
self._sync_task.cancel()
|
||||
self._retired = True
|
||||
self._schedule_flush_after_retire()
|
||||
|
||||
def _schedule_flush_after_retire(self) -> None:
|
||||
if self._flush_after_retire is not None and not self._flush_after_retire.done():
|
||||
return
|
||||
if not self._has_unwritten_increments():
|
||||
return
|
||||
try:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
self._flush_after_retire = loop.create_task(self._drain_increments_after_retire())
|
||||
|
||||
def _has_unwritten_increments(self) -> bool:
|
||||
return bool(self.redis_increment_operation_queue) or self._detached_increment_operations is not None
|
||||
|
||||
async def _drain_increments_after_retire(self) -> None:
|
||||
while self._has_unwritten_increments():
|
||||
if not await self._push_in_memory_increments_to_redis():
|
||||
await asyncio.sleep(DEFAULT_REDIS_SYNC_INTERVAL)
|
||||
|
||||
@with_service_target("router_budgets")
|
||||
async def async_filter_deployments(
|
||||
self,
|
||||
|
|
@ -427,6 +453,8 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
if self.dual_cache.redis_cache is None:
|
||||
return
|
||||
self.redis_increment_operation_queue.append(increment_op)
|
||||
if self._retired:
|
||||
self._schedule_flush_after_retire()
|
||||
|
||||
def _get_redis_increment_queue_lock(self) -> asyncio.Lock:
|
||||
return self._redis_increment_queue_lock
|
||||
|
|
|
|||
17
litellm/router_utils/stream_usage_injection.py
Normal file
17
litellm/router_utils/stream_usage_injection.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
|
||||
|
||||
def router_config_requests_stream_usage(
|
||||
request_kwargs: Mapping[str, object], configured_params: Sequence[Mapping[str, object]]
|
||||
) -> bool:
|
||||
if request_kwargs.get("stream") is not True or "stream_options" in request_kwargs:
|
||||
return False
|
||||
return any(_includes_usage(params.get("stream_options")) for params in configured_params)
|
||||
|
||||
|
||||
def _includes_usage(stream_options: object) -> bool:
|
||||
match stream_options:
|
||||
case {"include_usage": True}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
|
@ -54,6 +54,7 @@ RoutingStrategyName: TypeAlias = Literal[
|
|||
|
||||
TRUSTED_CALLBACK_VARS_FIELD: Final = "litellm_trusted_callback_vars"
|
||||
ADDRESSED_RESPONSE_ID_FIELD: Final = "_litellm_addressed_response_id"
|
||||
STREAM_USAGE_INJECTED_FIELD: Final = "_litellm_stream_usage_injected"
|
||||
|
||||
WIRE_NAME: Final = "wire_name"
|
||||
|
||||
|
|
@ -397,6 +398,7 @@ class ProxyRequestState:
|
|||
trusted_callback_vars: Mapping[str, str] | None = field(default=None, metadata=wire(TRUSTED_CALLBACK_VARS_FIELD))
|
||||
addressed_response_id: str | None = field(default=None, metadata=wire(ADDRESSED_RESPONSE_ID_FIELD))
|
||||
strip_stream_usage: bool | None = field(default=None, metadata=wire("_litellm_strip_stream_usage"))
|
||||
stream_usage_injected: bool | None = field(default=None, metadata=wire(STREAM_USAGE_INJECTED_FIELD))
|
||||
client_side_timeout: bool | None = None
|
||||
model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None
|
||||
|
||||
|
|
|
|||
171
litellm/utils.py
171
litellm/utils.py
|
|
@ -1126,83 +1126,90 @@ def function_setup(
|
|||
dynamic_callbacks: Final[list[str | Callable | CustomLogger] | None] = kwargs.pop("callbacks", None)
|
||||
all_callbacks: Final = get_dynamic_callbacks(dynamic_callbacks=dynamic_callbacks)
|
||||
|
||||
if len(all_callbacks) > 0:
|
||||
for callback in all_callbacks:
|
||||
# check if callback is a string - e.g. "lago", "openmeter"
|
||||
if isinstance(callback, str):
|
||||
callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(
|
||||
callback,
|
||||
internal_usage_cache=None,
|
||||
llm_router=None,
|
||||
)
|
||||
if callback is None or any(
|
||||
type(cb) is type(callback) for cb in litellm._async_success_callback
|
||||
): # don't double add a callback
|
||||
continue
|
||||
if callback not in litellm.input_callback:
|
||||
litellm.input_callback.append(callback)
|
||||
if callback not in litellm.success_callback:
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(callback)
|
||||
if callback not in litellm.failure_callback:
|
||||
litellm.logging_callback_manager.add_litellm_failure_callback(callback)
|
||||
if callback not in litellm._async_success_callback:
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
if callback not in litellm._async_failure_callback:
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
|
||||
print_verbose(f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}")
|
||||
with litellm.logging_callback_manager.callback_lists_lock:
|
||||
if len(all_callbacks) > 0:
|
||||
for callback in all_callbacks:
|
||||
# check if callback is a string - e.g. "lago", "openmeter"
|
||||
if isinstance(callback, str):
|
||||
callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class(
|
||||
callback,
|
||||
internal_usage_cache=None,
|
||||
llm_router=None,
|
||||
)
|
||||
if callback is None or any(
|
||||
type(cb) is type(callback) for cb in litellm._async_success_callback
|
||||
): # don't double add a callback
|
||||
continue
|
||||
if callback not in litellm.input_callback:
|
||||
litellm.input_callback.append(callback)
|
||||
if callback not in litellm.success_callback:
|
||||
litellm.logging_callback_manager.add_litellm_success_callback(callback)
|
||||
if callback not in litellm.failure_callback:
|
||||
litellm.logging_callback_manager.add_litellm_failure_callback(callback)
|
||||
if callback not in litellm._async_success_callback:
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
if callback not in litellm._async_failure_callback:
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
|
||||
print_verbose(
|
||||
f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}"
|
||||
)
|
||||
|
||||
if (
|
||||
len(litellm.input_callback) > 0 or len(litellm.success_callback) > 0 or len(litellm.failure_callback) > 0
|
||||
) and len(callback_list) == 0:
|
||||
callback_list = list(set(litellm.input_callback + litellm.success_callback + litellm.failure_callback))
|
||||
callbacks_to_set_up: Final = (
|
||||
list(set(litellm.input_callback + litellm.success_callback + litellm.failure_callback))
|
||||
if len(callback_list) == 0
|
||||
else []
|
||||
)
|
||||
if callbacks_to_set_up:
|
||||
callback_list = callbacks_to_set_up
|
||||
## ASYNC CALLBACKS - safety net for callbacks added via direct append
|
||||
if len(litellm.input_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.input_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm._async_input_callback.append(callback)
|
||||
removed_async_items.append(index)
|
||||
|
||||
# Pop the async items from input_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.input_callback.pop(index)
|
||||
if len(litellm.success_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.success_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback == "dynamodb" or callback == "openmeter":
|
||||
# dynamo is an async callback, it's used for the proxy and needs to be async
|
||||
# we only support async dynamo db logging for acompletion/aembedding since that's used on proxy
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
|
||||
add_custom_logger_callback_to_specific_event(callback, "success")
|
||||
|
||||
# Pop the async items from success_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.success_callback.pop(index)
|
||||
|
||||
if len(litellm.failure_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.failure_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
|
||||
add_custom_logger_callback_to_specific_event(callback, "failure")
|
||||
|
||||
# Pop the async items from failure_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.failure_callback.pop(index)
|
||||
if callbacks_to_set_up:
|
||||
get_set_callbacks: Final[Callable[[], Callable[..., None]]] = litellm_utils.get_set_callbacks
|
||||
get_set_callbacks()(callback_list=callback_list, function_id=function_id)
|
||||
## ASYNC CALLBACKS - safety net for callbacks added via direct append
|
||||
if len(litellm.input_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.input_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm._async_input_callback.append(callback)
|
||||
removed_async_items.append(index)
|
||||
|
||||
# Pop the async items from input_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.input_callback.pop(index)
|
||||
if len(litellm.success_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.success_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback == "dynamodb" or callback == "openmeter":
|
||||
# dynamo is an async callback, it's used for the proxy and needs to be async
|
||||
# we only support async dynamo db logging for acompletion/aembedding since that's used on proxy
|
||||
litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
|
||||
add_custom_logger_callback_to_specific_event(callback, "success")
|
||||
|
||||
# Pop the async items from success_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.success_callback.pop(index)
|
||||
|
||||
if len(litellm.failure_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.failure_callback):
|
||||
if coroutine_checker.is_async_callable(callback):
|
||||
litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
|
||||
removed_async_items.append(index)
|
||||
elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str):
|
||||
add_custom_logger_callback_to_specific_event(callback, "failure")
|
||||
|
||||
# Pop the async items from failure_callback in reverse order to avoid index issues
|
||||
for index in reversed(removed_async_items):
|
||||
litellm.failure_callback.pop(index)
|
||||
get_set_callbacks()(callback_list=callbacks_to_set_up, function_id=function_id)
|
||||
### DYNAMIC CALLBACKS ###
|
||||
dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = None
|
||||
dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = None
|
||||
dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None
|
||||
dynamic_async_failure_callbacks: Final[list[str | Callable | CustomLogger] | None] = None
|
||||
dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None
|
||||
dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None
|
||||
dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None
|
||||
dynamic_async_failure_callbacks: Final[list[str | Callable[..., object] | CustomLogger] | None] = None
|
||||
if kwargs.get("success_callback", None) is not None and isinstance(kwargs["success_callback"], list):
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(kwargs["success_callback"]):
|
||||
|
|
@ -7962,12 +7969,16 @@ class TextCompletionStreamWrapper:
|
|||
text_choices: Final = TextChoices()
|
||||
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
|
||||
raise Exception
|
||||
delta: Final = chunk["choices"][0]["delta"]
|
||||
text_choices["text"] = delta["content"]
|
||||
text_choices["reasoning_content"] = delta.get("reasoning_content")
|
||||
text_choices["index"] = chunk["choices"][0]["index"]
|
||||
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
|
||||
response["choices"] = [text_choices]
|
||||
|
||||
if chunk["choices"]:
|
||||
delta: Final = chunk["choices"][0]["delta"]
|
||||
text_choices["text"] = delta["content"]
|
||||
text_choices["reasoning_content"] = delta.get("reasoning_content")
|
||||
text_choices["index"] = chunk["choices"][0]["index"]
|
||||
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
|
||||
response["choices"] = [text_choices]
|
||||
else:
|
||||
response["choices"] = []
|
||||
|
||||
# only pass usage when stream_options["include_usage"] is True
|
||||
if self.stream_options and self.stream_options.get("include_usage", False) is True:
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -146,6 +146,18 @@ ignored_function_names = [
|
|||
"async_admit_deployment_turn", # Bound as the WebSocket turn_admission partial, tested through it in test_router_helper_utils.py
|
||||
"_deployment_as_configured_now", # Read by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py
|
||||
"_admit_turn_through_routing_strategy", # Run by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py
|
||||
"_callback_selectors", # Run by discard, tested through the discard tests in test_router_helper_utils.py
|
||||
"bind_to_request_logging", # Run by the proxy user_config route, tested through the user_config tests in test_route_llm_request.py
|
||||
"serve_through_request_logging", # Run by the proxy user_config route, tested through the user_config tests in test_route_llm_request.py
|
||||
"_request_adaptive_router_hooks", # Run by serve_through_request_logging, tested through the user_config adaptive test
|
||||
"_release_global_callbacks", # Run by discard, tested through the discard tests in test_router_helper_utils.py
|
||||
"_routing_callbacks", # Read by the pre-call checks, tested through the user_config limit tests in test_route_llm_request.py
|
||||
"_request_policies_not_set_globally", # Read by bind_to_request_logging, tested through the user_config budget test in test_route_llm_request.py
|
||||
"_optional_loggers", # Read by discard and serve_through_request_logging, tested through the budget limiter discard test
|
||||
"retire_routing_state", # Run by discard, tested through the budget limiter discard test
|
||||
"_slack_alerting_loggers", # Run by discard, tested through the Slack alerting discard test in test_router_helper_utils.py
|
||||
"_adaptive_router_hooks", # Run by discard, tested through the adaptive router discard tests in test_router_helper_utils.py
|
||||
"_global_callback_candidates", # Run by discard, tested through the survivor discard tests in test_router_helper_utils.py
|
||||
"without_deployments_that_cannot_batch", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
|
||||
"batch_create_routing", # Tested through the mixed-group acreate_batch tests in test_router.py
|
||||
"is_batch_create_routing", # Tested through the chat-after-batch-create test in test_router.py
|
||||
|
|
|
|||
|
|
@ -46,6 +46,11 @@ model_list:
|
|||
model: openai/gpt-6-luna
|
||||
api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
|
||||
api_key: perf-mock-not-checked
|
||||
- model_name: gpt-6-luna-responses
|
||||
litellm_params:
|
||||
model: openai/responses/gpt-6-luna
|
||||
api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
|
||||
api_key: perf-mock-not-checked
|
||||
- model_name: claude-sonnet-5
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-5
|
||||
|
|
@ -58,6 +63,13 @@ model_list:
|
|||
aws_region_name: us-east-1
|
||||
aws_access_key_id: perf-mock-not-checked
|
||||
aws_secret_access_key: perf-mock-not-checked
|
||||
- model_name: claude-sonnet-5-bedrock-converse
|
||||
litellm_params:
|
||||
model: bedrock/converse/anthropic.claude-sonnet-5
|
||||
aws_bedrock_runtime_endpoint: os.environ/E2E_PERF_MOCK_BEDROCK_URL
|
||||
aws_region_name: us-east-1
|
||||
aws_access_key_id: perf-mock-not-checked
|
||||
aws_secret_access_key: perf-mock-not-checked
|
||||
guardrails:
|
||||
- guardrail_name: perf-pre
|
||||
litellm_params:
|
||||
|
|
|
|||
15
tests/e2e/perf/cases/chat_completions_2rps.py
Normal file
15
tests/e2e/perf/cases/chat_completions_2rps.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
"""`/v1/chat/completions` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's chat completions
|
||||
route, `gpt-6-luna-responses` (`openai/responses/gpt-6-luna`) is bridged to its Responses route."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
from .direct_vs_bridge import direct_vs_bridge
|
||||
|
||||
CASE: Final = direct_vs_bridge(
|
||||
"chat_completions_2rps",
|
||||
"chat_completions",
|
||||
direct="gpt-6-luna",
|
||||
bridge="gpt-6-luna-responses",
|
||||
)
|
||||
31
tests/e2e/perf/cases/direct_vs_bridge.py
Normal file
31
tests/e2e/perf/cases/direct_vs_bridge.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""The mixed case's traffic on one public API, half of it on a deployment the gateway serves on
|
||||
that provider's own API (direct) and half on a deployment it has to translate to another provider
|
||||
API (bridge). Same 2 rps, request count, shapes, stream share, callers, guardrails and limits as
|
||||
`mixed_size_requests_2rps`, so a gap between the two halves and the mixed baseline is the
|
||||
translation's cost."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
from setup.performance_test_case import Endpoint, PerformanceTestCase, VirtualKey
|
||||
|
||||
from .mixed_size_requests_2rps import CASE as MIXED
|
||||
|
||||
|
||||
def _scoped(key: VirtualKey, direct: str) -> VirtualKey:
|
||||
return replace(key, models=(direct,)) if key.models else key
|
||||
|
||||
|
||||
def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) -> PerformanceTestCase:
|
||||
return replace(
|
||||
MIXED,
|
||||
name=name,
|
||||
teams=(
|
||||
replace(MIXED.team("team-a"), models=(direct, bridge)),
|
||||
replace(MIXED.team("team-b"), models=(bridge,)),
|
||||
),
|
||||
virtual_keys=tuple(_scoped(key, direct) for key in MIXED.virtual_keys),
|
||||
model_groups=(direct, bridge),
|
||||
traffic=replace(MIXED.traffic, endpoints={endpoint: 1.0}),
|
||||
)
|
||||
|
|
@ -8,8 +8,8 @@ new hook that deep-copies the request inline all land here as a new row. Runs on
|
|||
`PYTHONASYNCIODEBUG=1` and `LITELLM_EVENT_LOOP=asyncio` (uvloop reports nothing and crashed under debug locally),
|
||||
and the runner `E2E_PERF_EVENT_LOOP=1`, so the nightly build, which sets neither, skips it. The allowlist counts
|
||||
are twice the highest count any of ten back-to-back local runs showed on that row (rows that showed nothing in
|
||||
those ten but did in the earlier before/after legs for 46925 and 46967 get 2), so each run sits at or under half
|
||||
of every cap and a listed location doubling in frequency is a failure, not a flake. The
|
||||
those ten but did in the earlier before/after legs for 46925 and 46967, or in the first scheduled-lane run, get 2),
|
||||
so each run sits at or under half of every cap and a listed location doubling in frequency is a failure, not a flake. The
|
||||
`garbage collection` bucket is the gen-2 passes the collector logs in debug mode (0.6 to 0.7 s each, one every
|
||||
minute or two of this traffic, plus the ramp while the heap fills on a local whole-file run), which asyncio would
|
||||
otherwise charge to a random task; its count is the signal for a change that makes much more garbage."""
|
||||
|
|
@ -44,6 +44,7 @@ CASE: Final = replace(
|
|||
KnownStall("run_coroutine_job", max_count=2),
|
||||
KnownStall("user_api_key_auth.py", max_count=2),
|
||||
KnownStall("route_streaming_logging_to_handler", max_count=2),
|
||||
KnownStall("TaskHandle._run_coro", max_count=2),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
16
tests/e2e/perf/cases/messages_2rps.py
Normal file
16
tests/e2e/perf/cases/messages_2rps.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
"""`/v1/messages` at 2 rps: `claude-sonnet-5` is served by the Anthropic mock's Messages route,
|
||||
`claude-sonnet-5-bedrock-converse` (`bedrock/converse/anthropic.claude-sonnet-5`) is bridged to the
|
||||
Bedrock mock's Converse route."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
from .direct_vs_bridge import direct_vs_bridge
|
||||
|
||||
CASE: Final = direct_vs_bridge(
|
||||
"messages_2rps",
|
||||
"messages",
|
||||
direct="claude-sonnet-5",
|
||||
bridge="claude-sonnet-5-bedrock-converse",
|
||||
)
|
||||
15
tests/e2e/perf/cases/responses_2rps.py
Normal file
15
tests/e2e/perf/cases/responses_2rps.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
"""`/v1/responses` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's Responses route,
|
||||
`claude-sonnet-5` is bridged to the Anthropic mock's Messages route."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
from .direct_vs_bridge import direct_vs_bridge
|
||||
|
||||
CASE: Final = direct_vs_bridge(
|
||||
"responses_2rps",
|
||||
"responses",
|
||||
direct="gpt-6-luna",
|
||||
bridge="claude-sonnet-5",
|
||||
)
|
||||
|
|
@ -565,6 +565,7 @@ def _diagnostics(observation: Observation) -> list[str]:
|
|||
return [
|
||||
" diagnostics (not limits):",
|
||||
f" bodies off reply {observation.bodies_off_reply()}",
|
||||
f" mock routes {_mock_routes(observation)}",
|
||||
f" guardrail calls per request {len(observation.guardrail_calls) / count:.3f}",
|
||||
f" guardrail missed {observation.guardrail_missed() if observation.guardrail_calls else 'n/a'}",
|
||||
f" retried share {len(observation.retried_markers) / count:.4f}",
|
||||
|
|
@ -575,6 +576,12 @@ def _diagnostics(observation: Observation) -> list[str]:
|
|||
]
|
||||
|
||||
|
||||
def _mock_routes(observation: Observation) -> str:
|
||||
"""Which provider route each request landed on, so a direct-vs-bridge case shows its split."""
|
||||
counts: Final = Counter(f"{seen.provider}/{seen.route}" for seen in observation.mock_requests)
|
||||
return ", ".join(f"{route} {count}" for route, count in sorted(counts.items())) or "n/a"
|
||||
|
||||
|
||||
def _datastore_lines(observation: Observation) -> list[str]:
|
||||
"""What the per-request Postgres and Redis counts are made of, so a red run names the statement or
|
||||
command that grew. `plumbing` (BEGIN, COMMIT, session settings) and `unclassified` count in neither limit."""
|
||||
|
|
|
|||
|
|
@ -459,7 +459,6 @@ async def test_openai_clients_parse_text_completion_streams_and_choices(gateway:
|
|||
|
||||
|
||||
def test_text_completion_usage_chunk_has_empty_choices(gateway: Gateway) -> None:
|
||||
pytest.skip("BUG: the /v1/completions include_usage chunk carries choices [{index: 0}] instead of []")
|
||||
identity: Final = f"cmpl-usage-{uuid.uuid4().hex}"
|
||||
prompt: Final = f"raw-stream {identity}"
|
||||
with wire_server(_text_provider(identity, {prompt})) as wire, gateway.scenario() as scenario:
|
||||
|
|
|
|||
|
|
@ -312,7 +312,7 @@ def _chat_caller_stream_usage_event(event: dict[str, JsonValue], model: str) ->
|
|||
"object": event["object"],
|
||||
"created": 1,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "delta": {}}],
|
||||
"choices": [],
|
||||
"usage": {**_json_object_value(event["usage"]), "cost": 4.05e-6},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import json
|
||||
import re
|
||||
from collections.abc import Callable, Mapping
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
|
|
@ -263,13 +264,17 @@ def _assert_chunk_envelope(chunks: tuple[dict[str, JsonValue], ...], model: str,
|
|||
return next(iter(identities))
|
||||
|
||||
|
||||
def _content_chunks(chunks: tuple[dict[str, JsonValue], ...]) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(chunk for chunk in chunks if chunk.get("usage") is None)
|
||||
|
||||
|
||||
def _finish_reasons(chunks: tuple[dict[str, JsonValue], ...]) -> list[JsonValue]:
|
||||
reasons: Final = (_only_choice(chunk).get("finish_reason") for chunk in chunks)
|
||||
reasons: Final = (_only_choice(chunk).get("finish_reason") for chunk in _content_chunks(chunks))
|
||||
return [reason for reason in reasons if reason is not None]
|
||||
|
||||
|
||||
def _deltas(chunks: tuple[dict[str, JsonValue], ...]) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(object_value(_only_choice(chunk)["delta"]) for chunk in chunks)
|
||||
return tuple(object_value(_only_choice(chunk)["delta"]) for chunk in _content_chunks(chunks))
|
||||
|
||||
|
||||
def _sdk_base(gateway: Gateway) -> str:
|
||||
|
|
@ -301,12 +306,12 @@ def test_chat_completion_on_a_mantle_claude_id_posts_native_messages_and_spends_
|
|||
@pytest.mark.parametrize(
|
||||
("stream_options", "usage_chunks"),
|
||||
[
|
||||
pytest.param({"stream_options": {"include_usage": True}}, [_USAGE], id="include_usage"),
|
||||
pytest.param({"stream_options": {"include_usage": True}}, [(True, *_USAGE)], id="include_usage"),
|
||||
pytest.param({}, [], id="no_usage"),
|
||||
],
|
||||
)
|
||||
def test_chat_stream_on_a_mantle_claude_id_relays_openai_chunks_from_the_native_stream(
|
||||
gateway: Gateway, stream_options: dict[str, JsonValue], usage_chunks: list[tuple[int, int, int]]
|
||||
gateway: Gateway, stream_options: dict[str, JsonValue], usage_chunks: list[tuple[JsonValue, ...]]
|
||||
) -> None:
|
||||
marker: Final = uuid4().hex
|
||||
expected: Final = _outbound(_HAIKU, [_user_turn(marker)], stream=True)
|
||||
|
|
@ -322,9 +327,9 @@ def test_chat_stream_on_a_mantle_claude_id_relays_openai_chunks_from_the_native_
|
|||
identity: Final = _assert_chunk_envelope(chunks, model, response.text)
|
||||
assert "".join(str(delta.get("content") or "") for delta in _deltas(chunks)) == _answer(marker), response.text
|
||||
assert _finish_reasons(chunks) == ["stop"], response.text
|
||||
assert [_token_usage(chunk) for chunk in chunks if chunk.get("usage") is not None] == usage_chunks, (
|
||||
response.text
|
||||
)
|
||||
assert [
|
||||
(chunk["choices"] == [], *_token_usage(chunk)) for chunk in chunks if chunk.get("usage") is not None
|
||||
] == usage_chunks, response.text
|
||||
_assert_success_row(
|
||||
_spend_row(_BY_REQUEST_ID, identity), model, _HAIKU, "acompletion", _mantle_cost(gateway, _HAIKU)
|
||||
)
|
||||
|
|
@ -620,15 +625,14 @@ def test_openai_sdk_chat_stream_on_a_mantle_claude_id(gateway: Gateway) -> None:
|
|||
)
|
||||
identities: Final = {chunk.id for chunk in chunks}
|
||||
assert len(identities) == 1 and {chunk.model for chunk in chunks} == {model}, chunks
|
||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _answer(marker), chunks
|
||||
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"], (
|
||||
chunks
|
||||
)
|
||||
choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks))
|
||||
assert "".join(choice.delta.content or "" for choice in choices) == _answer(marker), chunks
|
||||
assert [choice.finish_reason for choice in choices if choice.finish_reason] == ["stop"], chunks
|
||||
assert [
|
||||
(chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens)
|
||||
(len(chunk.choices), chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens)
|
||||
for chunk in chunks
|
||||
if chunk.usage is not None
|
||||
] == [_USAGE], chunks
|
||||
] == [(0, *_USAGE)], chunks
|
||||
_assert_success_row(
|
||||
_spend_row(_BY_REQUEST_ID, next(iter(identities))),
|
||||
model,
|
||||
|
|
@ -688,15 +692,14 @@ async def test_async_openai_sdk_chat_stream_on_a_mantle_claude_id(gateway: Gatew
|
|||
chunks: Final = tuple([chunk async for chunk in stream])
|
||||
identities: Final = {chunk.id for chunk in chunks}
|
||||
assert len(identities) == 1 and {chunk.model for chunk in chunks} == {model}, chunks
|
||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _answer(marker), chunks
|
||||
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"], (
|
||||
chunks
|
||||
)
|
||||
choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks))
|
||||
assert "".join(choice.delta.content or "" for choice in choices) == _answer(marker), chunks
|
||||
assert [choice.finish_reason for choice in choices if choice.finish_reason] == ["stop"], chunks
|
||||
assert [
|
||||
(chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens)
|
||||
(len(chunk.choices), chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens)
|
||||
for chunk in chunks
|
||||
if chunk.usage is not None
|
||||
] == [_USAGE], chunks
|
||||
] == [(0, *_USAGE)], chunks
|
||||
_assert_success_row(
|
||||
_spend_row(_BY_REQUEST_ID, next(iter(identities))),
|
||||
model,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import openai
|
|||
import pytest
|
||||
from integration._support.client import Gateway, eventually, object_value, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
|
|
@ -723,6 +723,33 @@ def test_chat_sdk_stream_include_usage_masks_request_fields(gateway: Gateway, tm
|
|||
assert len(post_requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
def test_chat_stream_usage_the_proxy_adds_keeps_its_marker_out_of_the_stored_request(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
response_id: Final = f"chatcmpl-proxy-usage-{uuid4()}"
|
||||
with (
|
||||
wire_server(
|
||||
_answering_model_listing(lambda request: _chat_stream(response_id, "sort_key", include_usage=True))
|
||||
) as wire,
|
||||
owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
|
||||
isolated.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key")
|
||||
response: Final = isolated.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": "stream marker"}], "stream": True},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert {string_value(event["id"]) for event in _sse_events(response.text)} == {response_id}, response.text
|
||||
stored_request: Final = object_value(_stored_row(response_id)["proxy_server_request"])
|
||||
assert stored_request["messages"] == [{"role": "user", "content": "stream marker"}], stored_request
|
||||
assert "_litellm_stream_usage_injected" not in stored_request, stored_request
|
||||
observed: Final = _provider_calls(wire.drain())
|
||||
assert [_json_object(request.body)["stream_options"] for request in observed] == [{"include_usage": True}]
|
||||
|
||||
|
||||
def test_chat_history_keeps_string_tool_arguments_and_tool_content(gateway: Gateway, tmp_path: Path) -> None:
|
||||
arguments: Final = '{"sort_key":"created_at"}'
|
||||
tool_content: Final = "tool-result-created_at"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import json
|
|||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from queue import SimpleQueue
|
||||
from typing import Final
|
||||
|
|
@ -13,9 +14,10 @@ from hypothesis import Phase, example, given, settings
|
|||
from hypothesis import strategies as st
|
||||
from integration._support.client import Gateway, eventually, object_value, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from openai import OpenAI
|
||||
from openai import OpenAI, omit
|
||||
from openai.types.chat import ChatCompletionChunk
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.shares_worker_state
|
||||
|
|
@ -197,6 +199,7 @@ def test_proxy_stream_usage_visibility_keeps_exact_persisted_charge(gateway: Gat
|
|||
assert {chunk.id for chunk in chunks} == {identity}
|
||||
usages: Final = tuple(chunk.usage for chunk in chunks if chunk.usage is not None)
|
||||
assert len(usages) == (1 if include else 0)
|
||||
assert [len(chunk.choices) for chunk in chunks if chunk.usage is not None] == ([0] if include else [])
|
||||
if include:
|
||||
assert usages[0].prompt_tokens == 11 and usages[0].completion_tokens == 4
|
||||
requests: Final = wire.drain()
|
||||
|
|
@ -214,6 +217,119 @@ def test_proxy_stream_usage_visibility_keeps_exact_persisted_charge(gateway: Gat
|
|||
assert float(rows[0]["spend"]) == pytest.approx(0.019)
|
||||
|
||||
|
||||
_STREAM_USAGE_INJECTED_FLAG: Final = "_litellm_stream_usage_injected"
|
||||
|
||||
|
||||
def _registered_model_names(gateway: Gateway) -> tuple[JsonValue, ...]:
|
||||
entries: Final = gateway.get("/model/info")["data"]
|
||||
assert isinstance(entries, list), entries
|
||||
return tuple(object_value(entry).get("model_name") for entry in entries)
|
||||
|
||||
|
||||
def _registered(gateway: Gateway, model: str) -> None:
|
||||
eventually(lambda: _registered_model_names(gateway), lambda model_names: model in model_names, seconds=60)
|
||||
|
||||
|
||||
def _sdk_stream(
|
||||
gateway: Gateway,
|
||||
route_prefix: str,
|
||||
model: str,
|
||||
identity: str,
|
||||
*,
|
||||
include_usage: bool | None,
|
||||
extra_body: dict[str, JsonValue],
|
||||
) -> tuple[ChatCompletionChunk, ...]:
|
||||
with OpenAI(
|
||||
api_key=gateway.key,
|
||||
base_url=str(gateway.client.base_url).rstrip("/") + route_prefix,
|
||||
timeout=30,
|
||||
max_retries=0,
|
||||
) as client:
|
||||
stream: Final = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": identity}],
|
||||
stream=True,
|
||||
stream_options=omit if include_usage is None else {"include_usage": include_usage},
|
||||
extra_body=extra_body,
|
||||
)
|
||||
with stream:
|
||||
return tuple(stream)
|
||||
|
||||
|
||||
def _streamed_text(chunks: tuple[ChatCompletionChunk, ...]) -> str:
|
||||
choices: Final = chain.from_iterable(chunk.choices for chunk in chunks)
|
||||
return "".join(choice.delta.content or "" for choice in choices)
|
||||
|
||||
|
||||
def _usage_chunk_shapes(chunks: tuple[ChatCompletionChunk, ...]) -> list[tuple[int, int]]:
|
||||
return [(len(chunk.choices), chunk.usage.total_tokens) for chunk in chunks if chunk.usage is not None]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route_prefix", "sends", "extra_body"),
|
||||
[pytest.param("/v1", 2, {}, id="chat_miss_then_cache_hit"), pytest.param("/queue", 1, {"priority": 0}, id="queue")],
|
||||
)
|
||||
def test_caller_requested_stream_usage_has_no_choices_even_with_a_client_sent_injection_flag(
|
||||
gateway: Gateway, route_prefix: str, sends: int, extra_body: dict[str, JsonValue]
|
||||
) -> None:
|
||||
identity: Final = "stream-usage-flag-" + uuid.uuid4().hex
|
||||
with (
|
||||
wire_server(lambda request: Reply(content_type="text/event-stream", chunks=text_stream(identity))) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(api_base=wire.url + "/v1")
|
||||
_registered(gateway, model)
|
||||
streams: Final = tuple(
|
||||
_sdk_stream(
|
||||
gateway,
|
||||
route_prefix,
|
||||
model,
|
||||
identity,
|
||||
include_usage=True,
|
||||
extra_body={**extra_body, _STREAM_USAGE_INJECTED_FLAG: True},
|
||||
)
|
||||
for _ in range(sends)
|
||||
)
|
||||
assert [_streamed_text(chunks) for chunks in streams] == ["Hello 雪 café"] * sends
|
||||
assert [_usage_chunk_shapes(chunks) for chunks in streams] == [[(0, 15)]] * sends
|
||||
requests: Final = wire.drain()
|
||||
assert len(requests) == 1
|
||||
assert _STREAM_USAGE_INJECTED_FLAG not in json.loads(requests[0].body)
|
||||
|
||||
|
||||
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
def test_stream_usage_the_deployment_adds_keeps_one_empty_choice(gateway: Gateway, tmp_path: Path) -> None:
|
||||
identity: Final = "deployment-stream-usage-" + uuid.uuid4().hex
|
||||
with wire_server(lambda request: Reply(content_type="text/event-stream", chunks=text_stream(identity))) as wire:
|
||||
shared_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config: Final = {
|
||||
**shared_config,
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": "deployment-usage",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "synthetic-deployment-usage-key",
|
||||
"api_base": wire.url + "/v1",
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
}
|
||||
],
|
||||
"general_settings": {**shared_config["general_settings"], "always_include_stream_usage": False},
|
||||
}
|
||||
path: Final = tmp_path / "deployment-stream-usage.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with owned_proxy(gateway, tmp_path, {}, config=path) as candidate:
|
||||
chunks: Final = _sdk_stream(
|
||||
candidate, "/v1", "deployment-usage", identity, include_usage=None, extra_body={}
|
||||
)
|
||||
assert _streamed_text(chunks) == "Hello 雪 café"
|
||||
assert _usage_chunk_shapes(chunks) == [(1, 15)]
|
||||
requests: Final = wire.drain()
|
||||
assert len(requests) == 1
|
||||
assert json.loads(requests[0].body)["stream_options"]["include_usage"] is True
|
||||
|
||||
|
||||
@pytest.mark.covers("other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream")
|
||||
def test_messages_stream_completes_through_trailing_empty_choices_usage_chunk(gateway: Gateway) -> None:
|
||||
identity: Final = "messages-empty-choices-" + uuid.uuid4().hex
|
||||
|
|
|
|||
|
|
@ -252,6 +252,7 @@ def isolate_host_environment(isolated_aws_config_files: tuple[Path, Path]) -> It
|
|||
@pytest.fixture(autouse=True)
|
||||
def isolate_litellm_globals() -> Iterator[None]:
|
||||
original_callbacks: Final = {name: list(getattr(litellm, name) or []) for name in CALLBACK_LISTS}
|
||||
original_callback_holders: Final = frozenset(litellm_router_module._routers_holding_global_callbacks)
|
||||
original_reset: Final = {name: getattr(litellm, name) for name in RESET_TO_NONE_GLOBALS}
|
||||
original_restored: Final = {name: getattr(litellm, name) for name in RESTORED_GLOBALS if hasattr(litellm, name)}
|
||||
original_clients: Final = {name: litellm.__dict__[name] for name in MODULE_LEVEL_CLIENTS if name in litellm.__dict__}
|
||||
|
|
@ -263,6 +264,7 @@ def isolate_litellm_globals() -> Iterator[None]:
|
|||
_flush_client_caches()
|
||||
for name in CALLBACK_LISTS:
|
||||
setattr(litellm, name, [])
|
||||
litellm_router_module._routers_holding_global_callbacks.clear()
|
||||
for name in RESET_TO_NONE_GLOBALS:
|
||||
setattr(litellm, name, None)
|
||||
for name in MODULE_LEVEL_CLIENTS:
|
||||
|
|
@ -277,6 +279,8 @@ def isolate_litellm_globals() -> Iterator[None]:
|
|||
litellm.__dict__.update(original_clients)
|
||||
for name, value in (original_callbacks | original_reset | original_restored).items():
|
||||
setattr(litellm, name, value)
|
||||
litellm_router_module._routers_holding_global_callbacks.clear()
|
||||
litellm_router_module._routers_holding_global_callbacks.update(original_callback_holders)
|
||||
for logger, (level, disabled, propagate, handlers, filters) in original_loggers.items():
|
||||
logger.setLevel(level)
|
||||
logger.disabled = disabled
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import json
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.types.utils import LiteLLMFineTuningJob
|
||||
|
||||
API_BASE: Final = "https://fine-tuning.test"
|
||||
BASE_MODEL: Final = "gpt-4.1-nano-2025-04-14"
|
||||
|
|
@ -52,3 +53,42 @@ async def test_create_sends_the_model_without_the_provider_prefix(
|
|||
)
|
||||
|
||||
assert json.loads(sent.calls.last.request.content)["model"] == BASE_MODEL
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
@pytest.mark.parametrize(
|
||||
("custom_llm_provider", "api_base", "route", "sent_api_version"),
|
||||
[
|
||||
("openai", f"{API_BASE}/v1", f"{API_BASE}/v1/fine_tuning/jobs", None),
|
||||
("azure", API_BASE, f"{API_BASE}/openai/fine_tuning/jobs", "2024-10-21"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("is_async", [True, False], ids=["async", "sync"])
|
||||
async def test_create_with_api_version_reaches_the_provider_and_only_azure_gets_it(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
custom_llm_provider: Literal["openai", "azure"],
|
||||
api_base: str,
|
||||
route: str,
|
||||
sent_api_version: str | None,
|
||||
is_async: bool,
|
||||
) -> None:
|
||||
sent: Final = respx.post(url__startswith=route).mock(return_value=httpx.Response(200, json=PROVIDER_JOB))
|
||||
request: Final = {
|
||||
"model": BASE_MODEL,
|
||||
"training_file": "file-abc123",
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"api_key": "sk-test",
|
||||
"api_base": api_base,
|
||||
"api_version": "2024-10-21",
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient() as session:
|
||||
monkeypatch.setattr(litellm, "aclient_session", session)
|
||||
job: Final = (
|
||||
await litellm.acreate_fine_tuning_job(**request) if is_async else litellm.create_fine_tuning_job(**request)
|
||||
)
|
||||
|
||||
assert isinstance(job, LiteLLMFineTuningJob)
|
||||
assert job.id == PROVIDER_JOB["id"]
|
||||
assert sent.calls.last.request.url.params.get("api-version") == sent_api_version
|
||||
|
|
|
|||
|
|
@ -272,6 +272,7 @@ async def test_sdk_chat_stream(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
assert lines[-1] == "data: [DONE]"
|
||||
assert json.loads(lines[0][6:])["choices"][0]["delta"]["content"] == "he"
|
||||
assert calls[0]["stream_options"] == {"include_usage": True}
|
||||
assert calls[0]["_litellm_stream_usage_injected"] is True
|
||||
assert (ep.usage.input_tokens, ep.usage.output_tokens) == (6, 2)
|
||||
assert ep.usage.cost == pytest.approx(6 * 0.001 + 2 * 0.002)
|
||||
|
||||
|
|
|
|||
|
|
@ -402,6 +402,32 @@ def test_get_combined_callback_list_preserves_insertion_order(logging_obj):
|
|||
) == ["prometheus", "langfuse", "datadog", "otel", "s3", "gcs_bucket", "arize", "logfire"]
|
||||
|
||||
|
||||
class _InputEventRecorder(CustomLogger):
|
||||
def __init__(self, label: str, events: list[str]) -> None:
|
||||
self._label = label
|
||||
self.events = events
|
||||
|
||||
def log_pre_api_call(self, model: object, messages: object, kwargs: object) -> None:
|
||||
self.events.append(f"{self._label} pre")
|
||||
|
||||
def log_post_api_call(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
|
||||
self.events.append(f"{self._label} post")
|
||||
|
||||
|
||||
def test_input_events_reach_the_replacement_and_not_the_global_callback_it_replaced(
|
||||
logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
events: Final[list[str]] = []
|
||||
monkeypatch.setattr(litellm, "input_callback", [_InputEventRecorder("global", events)])
|
||||
logging_obj.replace_global_callback(_InputEventRecorder("replacement", events))
|
||||
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
|
||||
|
||||
logging_obj.pre_call(input="", api_key="")
|
||||
logging_obj.post_call(original_response="ok", input="", api_key="")
|
||||
|
||||
assert events == ["replacement pre", "replacement post"]
|
||||
|
||||
|
||||
def test_get_masked_api_base(logging_obj):
|
||||
api_base = "https://api.openai.com/v1"
|
||||
masked_api_base = logging_obj._get_masked_api_base(api_base)
|
||||
|
|
|
|||
|
|
@ -145,6 +145,7 @@ def test_remove_callback_from_all_lists():
|
|||
|
||||
obj = TestLogger()
|
||||
manager.add_litellm_callback(obj)
|
||||
manager.add_litellm_input_callback(obj)
|
||||
manager.add_litellm_success_callback(obj)
|
||||
manager.add_litellm_failure_callback(obj)
|
||||
manager.add_litellm_async_success_callback(obj)
|
||||
|
|
@ -153,6 +154,7 @@ def test_remove_callback_from_all_lists():
|
|||
manager.remove_callback_from_all_lists(obj)
|
||||
|
||||
assert obj not in litellm.callbacks
|
||||
assert obj not in litellm.input_callback
|
||||
assert obj not in litellm.success_callback
|
||||
assert obj not in litellm.failure_callback
|
||||
assert obj not in litellm._async_success_callback
|
||||
|
|
|
|||
|
|
@ -603,6 +603,91 @@ async def test_streaming_handler_with_usage(
|
|||
assert chunk_has_usage
|
||||
|
||||
|
||||
def _usage_test_logging(
|
||||
caller_stream_options: dict[str, bool] | None, stream_usage_injected: bool = False
|
||||
) -> Logging:
|
||||
logging_obj: Final = Logging(
|
||||
model="hosted_vllm/m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=1_790_000_000.0,
|
||||
litellm_call_id="12345",
|
||||
function_id="1245",
|
||||
kwargs={"_litellm_stream_usage_injected": True} if stream_usage_injected else {},
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
litellm_params={}, optional_params={}, model="hosted_vllm/m", stream_options=caller_stream_options
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("caller_stream_options", "stream_usage_injected", "usage_chunk_choice_count"),
|
||||
[
|
||||
({"include_usage": True}, False, 0),
|
||||
(None, False, 1),
|
||||
({"include_usage": True}, True, 1),
|
||||
],
|
||||
ids=["caller-asked", "provider-handler-injected", "proxy-injected"],
|
||||
)
|
||||
@pytest.mark.parametrize("plain_iterator", [False, True])
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_usage_chunk_has_empty_choices_only_for_callers_that_asked(
|
||||
sync_mode: bool,
|
||||
plain_iterator: bool,
|
||||
caller_stream_options: dict[str, bool] | None,
|
||||
stream_usage_injected: bool,
|
||||
usage_chunk_choice_count: int,
|
||||
):
|
||||
usage: Final = Usage(prompt_tokens=5, completion_tokens=1, total_tokens=6)
|
||||
provider_chunks: Final = [
|
||||
ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="ok"))]),
|
||||
ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]),
|
||||
ModelResponseStream(id="c1", choices=[], provider_specific_fields={"citations": ["a"]}),
|
||||
ModelResponseStream(id="c1", choices=[], usage=usage),
|
||||
]
|
||||
wrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=(
|
||||
iter(provider_chunks) if plain_iterator else ModelResponseListIterator(model_responses=provider_chunks)
|
||||
),
|
||||
model="hosted_vllm/m",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
logging_obj=_usage_test_logging(caller_stream_options, stream_usage_injected),
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
|
||||
chunks: Final = list(wrapper) if sync_mode else [chunk async for chunk in wrapper]
|
||||
|
||||
assert [len(chunk.choices) for chunk in chunks] == [1, 1, 0, usage_chunk_choice_count]
|
||||
assert chunks[2].provider_specific_fields == {"citations": ["a"]}
|
||||
assert [getattr(chunk, "usage", None) is not None for chunk in chunks] == [False, False, False, True]
|
||||
assert chunks[-1].usage.total_tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_iterator_stream_keeps_usage_on_its_chunk_when_usage_was_not_requested():
|
||||
usage: Final = Usage(prompt_tokens=5, completion_tokens=1, total_tokens=6)
|
||||
provider_chunks: Final = [
|
||||
ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="ok"))]),
|
||||
ModelResponseStream(
|
||||
id="c1", choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], usage=usage
|
||||
),
|
||||
]
|
||||
wrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=iter(provider_chunks),
|
||||
model="hosted_vllm/m",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
logging_obj=_usage_test_logging(None),
|
||||
)
|
||||
|
||||
chunks: Final = [chunk async for chunk in wrapper]
|
||||
|
||||
assert [chunk.choices[0].finish_reason for chunk in chunks] == [None, "stop"]
|
||||
assert chunks[-1].usage.total_tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [False])
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
|
|
|
|||
|
|
@ -364,7 +364,8 @@ def test_text_completion_stream_forwards_stream_options_and_usage(monkeypatch):
|
|||
)
|
||||
)
|
||||
|
||||
assert tuple(chunk.choices[0].text for chunk in chunks) == ("hello", None, None)
|
||||
assert tuple(chunk.choices[0].text for chunk in chunks[:-1]) == ("hello", None)
|
||||
assert chunks[-1].choices == []
|
||||
assert tuple(chunk.usage for chunk in chunks[:-1]) == (None, None)
|
||||
assert (chunks[-1].usage.prompt_tokens, chunks[-1].usage.completion_tokens, chunks[-1].usage.total_tokens) == (
|
||||
2,
|
||||
|
|
|
|||
|
|
@ -543,8 +543,9 @@ def test_nova_streaming_sends_stream_flag_and_parses_chunks_and_usage(respx_mock
|
|||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
assert tuple(chunk.choices[0].delta.content for chunk in chunks) == ("Nova ", "stream", None, None)
|
||||
assert tuple(chunk.choices[0].finish_reason for chunk in chunks) == (None, None, "stop", None)
|
||||
assert tuple(chunk.choices[0].delta.content for chunk in chunks[:-1]) == ("Nova ", "stream", None)
|
||||
assert chunks[-1].choices == []
|
||||
assert tuple(chunk.choices[0].finish_reason for chunk in chunks[:-1]) == (None, None, "stop")
|
||||
assert (chunks[-1].usage.prompt_tokens, chunks[-1].usage.completion_tokens, chunks[-1].usage.total_tokens) == (
|
||||
2,
|
||||
1,
|
||||
|
|
|
|||
|
|
@ -516,9 +516,16 @@ async def test_unary_answer_cut_at_the_callers_cap_is_returned_with_its_token_in
|
|||
assert response.choices[0].message.provider_specific_fields["continuation_token"] == "dG9rLTE="
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"choice_count_request",
|
||||
[{"max_tokens": 100_000, "n": 2}, {"extra_body": {"generationConfig": {"candidateCount": 2}}}],
|
||||
)
|
||||
@pytest.mark.parametrize("is_async", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_unary_request_for_more_than_one_choice_is_never_continued(is_async: bool):
|
||||
async def test_unary_request_for_more_than_one_choice_is_never_continued(
|
||||
monkeypatch: pytest.MonkeyPatch, is_async: bool, choice_count_request: Mapping[str, object]
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "100000")
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
# Vertex AI answered 400 "Candidate count greater than 1 is not supported when continuation_token is set"
|
||||
# on 2026-10-13
|
||||
|
|
@ -526,9 +533,8 @@ async def test_unary_request_for_more_than_one_choice_is_never_continued(is_asyn
|
|||
kwargs: Final = {
|
||||
"model": MODEL,
|
||||
"messages": list(MESSAGES),
|
||||
"max_tokens": 100_000,
|
||||
"n": 2,
|
||||
"api_key": "fake-gemini-key",
|
||||
**choice_count_request,
|
||||
}
|
||||
|
||||
response: Final = (
|
||||
|
|
@ -548,6 +554,42 @@ async def test_unary_request_for_more_than_one_choice_is_never_continued(is_asyn
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"one_choice_request",
|
||||
[
|
||||
{"max_tokens": 100_000, "n": 0},
|
||||
{"extra_body": {"generationConfig": {"candidateCount": None}}},
|
||||
{"extra_body": {"generationConfig": {"candidateCount": 0}}},
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("is_async", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_unary_request_whose_candidate_count_google_reads_as_one_is_continued(
|
||||
monkeypatch: pytest.MonkeyPatch, is_async: bool, one_choice_request: Mapping[str, object]
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "100000")
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
hops: Final = (httpx.Response(200, json=HOP_1), httpx.Response(200, json=HOP_2))
|
||||
kwargs: Final = {
|
||||
"model": MODEL,
|
||||
"messages": list(MESSAGES),
|
||||
"api_key": "fake-gemini-key",
|
||||
**one_choice_request,
|
||||
}
|
||||
|
||||
response: Final = (
|
||||
await litellm.acompletion(**kwargs, client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay(hops, sent))))
|
||||
if is_async
|
||||
else litellm.completion(
|
||||
**kwargs, client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_replay(hops, sent))))
|
||||
)
|
||||
)
|
||||
|
||||
assert sent[1] == {**sent[0], "continuationToken": "dG9rLTE="}
|
||||
assert response.choices[0].message.content == "hello world"
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unary_failed_continuation_hop_returns_the_answer_so_far_with_its_token():
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
|
@ -1295,6 +1337,468 @@ async def test_vertex_long_decode_keeps_the_callers_own_authorization_header_on_
|
|||
assert seen == ["Bearer mine", "Bearer mine"]
|
||||
|
||||
|
||||
BUDGET_ENV: Final = "LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS"
|
||||
ARGON: Final = "gemini/gemini-4-argon"
|
||||
UNMAPPED_GEMINI: Final = "gemini/gemini-99-unreleased"
|
||||
OUTPUT_LIMIT_KEYS: Final = ("max_output_tokens", "maxOutputTokens")
|
||||
|
||||
|
||||
def _output_limits(request: Mapping[str, object]) -> dict[str, object]:
|
||||
generation_config: Final = request.get("generationConfig", {})
|
||||
assert isinstance(generation_config, dict)
|
||||
return {key: value for key, value in generation_config.items() if key in OUTPUT_LIMIT_KEYS}
|
||||
|
||||
|
||||
def _cost_map_max_output_tokens(model: str) -> int:
|
||||
provider, name = model.split("/", 1)
|
||||
max_output_tokens: Final = litellm.get_model_info(model=name, custom_llm_provider=provider)["max_output_tokens"]
|
||||
assert isinstance(max_output_tokens, int)
|
||||
return max_output_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("is_async", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_with_no_output_limit_keeps_continuing_until_the_env_budget_is_spent(
|
||||
monkeypatch: pytest.MonkeyPatch, is_async: bool, stream: bool
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "50")
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
hops: Final = (
|
||||
(_sse(STREAM_FIRST_CHUNK, STREAM_HOP_1_END), _sse(HOP_2_TRUNCATED), _sse(HOP_2))
|
||||
if stream
|
||||
else (
|
||||
httpx.Response(200, json=HOP_1),
|
||||
httpx.Response(200, json=HOP_2_TRUNCATED),
|
||||
httpx.Response(200, json=HOP_2),
|
||||
)
|
||||
)
|
||||
kwargs: Final = {"model": ARGON, "messages": list(MESSAGES), "stream": stream, "api_key": "fake-gemini-key"}
|
||||
transport: Final = httpx.MockTransport(_replay(hops, sent))
|
||||
|
||||
response: Final = (
|
||||
await litellm.acompletion(**kwargs, client=AsyncHTTPHandler(transport=transport))
|
||||
if is_async
|
||||
else litellm.completion(**kwargs, client=HTTPHandler(client=httpx.Client(transport=transport)))
|
||||
)
|
||||
chunks: Final = ([chunk async for chunk in response] if is_async else list(response)) if stream else []
|
||||
|
||||
assert len(sent) == 2
|
||||
assert _output_limits(sent[0]) == {"max_output_tokens": 50}
|
||||
if stream:
|
||||
assert _stream_text(chunks) == "hello world"
|
||||
assert _stream_finish_reasons(chunks) == ["length"]
|
||||
assert {"continuation_token": "dG9rLTI="} in _delta_fields(chunks)
|
||||
else:
|
||||
assert response.choices[0].message.content == "hello world"
|
||||
assert response.choices[0].finish_reason == "length"
|
||||
assert response.choices[0].message.provider_specific_fields["continuation_token"] == "dG9rLTI="
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"budget, model, request_params, expected_limits",
|
||||
[
|
||||
("50", ARGON, {"max_tokens": 30}, {"max_output_tokens": 30}),
|
||||
("50", ARGON, {"max_completion_tokens": 30}, {"max_output_tokens": 30}),
|
||||
(
|
||||
"50",
|
||||
ARGON,
|
||||
{"extra_body": {"generationConfig": {"maxOutputTokens": 30}}},
|
||||
{"maxOutputTokens": 30},
|
||||
),
|
||||
(None, ARGON, {}, {}),
|
||||
("", ARGON, {}, {}),
|
||||
("a lot", ARGON, {}, {}),
|
||||
("0", ARGON, {}, {}),
|
||||
("-5", ARGON, {}, {}),
|
||||
("50", UNMAPPED_GEMINI, {}, {}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_applies_only_to_a_request_that_sets_no_output_limit_of_its_own(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
budget: str | None,
|
||||
model: str,
|
||||
request_params: Mapping[str, object],
|
||||
expected_limits: Mapping[str, object],
|
||||
):
|
||||
if budget is None:
|
||||
monkeypatch.delenv(BUDGET_ENV, raising=False)
|
||||
else:
|
||||
monkeypatch.setenv(BUDGET_ENV, budget)
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await litellm.acompletion(
|
||||
model=model,
|
||||
messages=list(MESSAGES),
|
||||
api_key="fake-gemini-key",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
**request_params,
|
||||
)
|
||||
|
||||
assert _output_limits(sent[0]) == expected_limits
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"modalities, expected_limits",
|
||||
[
|
||||
(["audio"], {}),
|
||||
(["text", "audio"], {}),
|
||||
(["text", "image"], {}),
|
||||
(["text"], {"max_output_tokens": 50}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_applies_only_to_a_request_asking_for_text_output(
|
||||
monkeypatch: pytest.MonkeyPatch, modalities: list[str], expected_limits: Mapping[str, object]
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "50")
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await litellm.acompletion(
|
||||
model=ARGON,
|
||||
messages=list(MESSAGES),
|
||||
modalities=modalities,
|
||||
api_key="fake-gemini-key",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
)
|
||||
|
||||
assert _output_limits(sent[0]) == expected_limits
|
||||
|
||||
|
||||
def _ai_studio_model_in_mode(mode: str) -> str:
|
||||
return next(
|
||||
name
|
||||
for name, row in litellm.model_cost.items()
|
||||
if name.startswith("gemini/") and row.get("mode") == mode and row.get("max_output_tokens")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["audio_speech", "image_generation"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_skips_a_model_the_cost_map_does_not_list_as_a_chat_model(
|
||||
monkeypatch: pytest.MonkeyPatch, mode: str
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "50")
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await litellm.acompletion(
|
||||
model=_ai_studio_model_in_mode(mode),
|
||||
messages=list(MESSAGES),
|
||||
api_key="fake-gemini-key",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
)
|
||||
|
||||
assert _output_limits(sent[0]) == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output_modalities, expected_limits",
|
||||
[
|
||||
({}, {"max_output_tokens": 50}),
|
||||
({"supported_output_modalities": ["text"]}, {"max_output_tokens": 50}),
|
||||
({"supported_output_modalities": ["text", "audio"]}, {"max_output_tokens": 50}),
|
||||
({"supported_output_modalities": ["audio"]}, {}),
|
||||
({"supported_output_modalities": "audio"}, {}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_applies_only_to_a_chat_model_the_cost_map_lists_as_answering_text(
|
||||
monkeypatch: pytest.MonkeyPatch, output_modalities: Mapping[str, object], expected_limits: Mapping[str, object]
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "50")
|
||||
litellm.register_model(
|
||||
{
|
||||
UNMAPPED_GEMINI: {
|
||||
"litellm_provider": "gemini",
|
||||
"mode": "chat",
|
||||
"max_output_tokens": 1000,
|
||||
**output_modalities,
|
||||
}
|
||||
}
|
||||
)
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await litellm.acompletion(
|
||||
model=UNMAPPED_GEMINI,
|
||||
messages=list(MESSAGES),
|
||||
api_key="fake-gemini-key",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
)
|
||||
|
||||
assert _output_limits(sent[0]) == expected_limits
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra_body",
|
||||
[
|
||||
{"generation_config": {"temperature": 0.1}},
|
||||
{"generation_config": {"max_output_tokens": 30}},
|
||||
{"generationConfig": {"response_modalities": ["AUDIO"]}},
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_leaves_a_snake_case_generation_config_from_extra_body_alone(
|
||||
monkeypatch: pytest.MonkeyPatch, extra_body: Mapping[str, object]
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "50")
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await litellm.acompletion(
|
||||
model=ARGON,
|
||||
messages=list(MESSAGES),
|
||||
extra_body=dict(extra_body),
|
||||
api_key="fake-gemini-key",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
)
|
||||
|
||||
assert _output_limits(sent[0]) == {}
|
||||
|
||||
|
||||
def _gemini_chat_models_answering_no_text() -> tuple[str, ...]:
|
||||
rows: Final = (
|
||||
(name, row)
|
||||
for name, row in litellm.model_cost.items()
|
||||
if row.get("mode") == "chat" and "text" not in row.get("supported_output_modalities", ("text",))
|
||||
)
|
||||
return tuple(
|
||||
name if name.startswith("gemini/") else f"vertex_ai/{name}"
|
||||
for name, row in rows
|
||||
if row.get("litellm_provider") in ("gemini", "vertex_ai-language-models")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_skips_every_chat_model_the_cost_map_lists_as_answering_no_text(
|
||||
monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "50")
|
||||
models: Final = _gemini_chat_models_answering_no_text()
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
replies: Final = tuple(httpx.Response(200, json=HOP_2) for _ in models)
|
||||
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent)))
|
||||
|
||||
for model in models:
|
||||
await litellm.acompletion(
|
||||
model=model,
|
||||
messages=list(MESSAGES),
|
||||
api_key="fake-gemini-key",
|
||||
vertex_project="p",
|
||||
vertex_location="us-central1",
|
||||
vertex_credentials=token_endpoint.credentials,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert models
|
||||
assert {model: _output_limits(request) for model, request in zip(models, sent, strict=True)} == dict.fromkeys(
|
||||
models, {}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_leaves_a_continuation_token_resume_with_no_limit_alone(
|
||||
monkeypatch: pytest.MonkeyPatch, stream: bool
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "50")
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
reply: Final = _sse(HOP_2) if stream else httpx.Response(200, json=HOP_2)
|
||||
|
||||
response: Final = await litellm.acompletion(
|
||||
model=ARGON,
|
||||
messages=list(MESSAGES),
|
||||
continuation_token="dG9rLTE=",
|
||||
stream=stream,
|
||||
api_key="fake-gemini-key",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((reply,), sent))),
|
||||
)
|
||||
if stream:
|
||||
_ = [chunk async for chunk in response]
|
||||
|
||||
assert sent[0]["continuationToken"] == "dG9rLTE="
|
||||
assert _output_limits(sent[0]) == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gemini/gemini-2.5-flash", "vertex_ai/gemini-2.5-flash", ARGON])
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_above_a_models_output_limit_sends_that_limit_instead(
|
||||
monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint, model: str
|
||||
):
|
||||
budget: Final = 2 * _cost_map_max_output_tokens(model)
|
||||
monkeypatch.setenv(BUDGET_ENV, str(budget))
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await litellm.acompletion(
|
||||
model=model,
|
||||
messages=list(MESSAGES),
|
||||
api_key="fake-gemini-key",
|
||||
vertex_project="p",
|
||||
vertex_location="us-central1",
|
||||
vertex_credentials=token_endpoint.credentials,
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
)
|
||||
|
||||
assert _output_limits(sent[0]) == {"max_output_tokens": _cost_map_max_output_tokens(model)}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_never_goes_above_the_cost_map_limit_when_a_deployment_overrides_it(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
model: Final = "gemini/gemini-2.5-flash"
|
||||
model_limit: Final = _cost_map_max_output_tokens(model)
|
||||
monkeypatch.setenv(BUDGET_ENV, str(2 * model_limit))
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "plain", "litellm_params": {"model": model, "api_key": "fake-gemini-key"}},
|
||||
{
|
||||
"model_name": "raised",
|
||||
"litellm_params": {"model": model, "api_key": "fake-gemini-key"},
|
||||
"model_info": {"max_output_tokens": 4 * model_limit},
|
||||
},
|
||||
]
|
||||
)
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
replies: Final = (httpx.Response(200, json=HOP_2), httpx.Response(200, json=HOP_2))
|
||||
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent)))
|
||||
|
||||
for model_name in ("plain", "raised"):
|
||||
await router.acompletion(model=model_name, messages=list(MESSAGES), client=client)
|
||||
|
||||
sent_limits: Final = [_output_limits(request).get("max_output_tokens", 0) for request in sent]
|
||||
assert len(sent_limits) == 2
|
||||
assert all(isinstance(limit, int) and limit <= model_limit for limit in sent_limits)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_caps_each_deployment_at_its_own_model_info_limit_and_never_at_a_siblings(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
model: Final = "gemini/gemini-2.5-flash"
|
||||
budget: Final = _cost_map_max_output_tokens(model) // 2
|
||||
low_limit: Final = budget // 4
|
||||
monkeypatch.setenv(BUDGET_ENV, str(budget))
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "low",
|
||||
"litellm_params": {"model": model, "api_key": "fake-gemini-key"},
|
||||
"model_info": {"max_output_tokens": low_limit},
|
||||
},
|
||||
{"model_name": "plain", "litellm_params": {"model": model, "api_key": "fake-gemini-key"}},
|
||||
]
|
||||
)
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
replies: Final = (httpx.Response(200, json=HOP_2), httpx.Response(200, json=HOP_2))
|
||||
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent)))
|
||||
|
||||
for model_name in ("plain", "low"):
|
||||
await router.acompletion(model=model_name, messages=list(MESSAGES), client=client)
|
||||
|
||||
assert [_output_limits(request) for request in sent] == [
|
||||
{"max_output_tokens": budget},
|
||||
{"max_output_tokens": low_limit},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deployment_limit", [0, -1])
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_ignores_a_deployment_model_info_limit_that_is_not_positive(
|
||||
monkeypatch: pytest.MonkeyPatch, deployment_limit: int
|
||||
):
|
||||
model: Final = "gemini/gemini-2.5-flash"
|
||||
budget: Final = _cost_map_max_output_tokens(model) // 2
|
||||
monkeypatch.setenv(BUDGET_ENV, str(budget))
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "odd",
|
||||
"litellm_params": {"model": model, "api_key": "fake-gemini-key"},
|
||||
"model_info": {"max_output_tokens": deployment_limit},
|
||||
}
|
||||
]
|
||||
)
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await router.acompletion(
|
||||
model="odd",
|
||||
messages=list(MESSAGES),
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
)
|
||||
|
||||
assert _output_limits(sent[0]) == {"max_output_tokens": budget}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_on_a_direct_call_never_goes_above_the_cost_map_limit_after_a_router_raised_it(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
model: Final = "gemini/gemini-2.5-flash"
|
||||
model_limit: Final = _cost_map_max_output_tokens(model)
|
||||
monkeypatch.setenv(BUDGET_ENV, str(2 * model_limit))
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "raised",
|
||||
"litellm_params": {"model": model, "api_key": "fake-gemini-key"},
|
||||
"model_info": {"max_output_tokens": 4 * model_limit},
|
||||
}
|
||||
]
|
||||
)
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await litellm.acompletion(
|
||||
model=model,
|
||||
messages=list(MESSAGES),
|
||||
api_key="fake-gemini-key",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
)
|
||||
|
||||
assert _output_limits(sent[0]) == {"max_output_tokens": model_limit}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_on_a_direct_call_follows_a_lower_limit_set_with_register_model(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
model: Final = "gemini/gemini-2.5-flash"
|
||||
lowered_limit: Final = _cost_map_max_output_tokens(model) // 4
|
||||
monkeypatch.setenv(BUDGET_ENV, str(2 * lowered_limit))
|
||||
litellm.register_model({model: {"litellm_provider": "gemini", "max_output_tokens": lowered_limit}})
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await litellm.acompletion(
|
||||
model=model,
|
||||
messages=list(MESSAGES),
|
||||
api_key="fake-gemini-key",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
)
|
||||
|
||||
assert _output_limits(sent[0]) == {"max_output_tokens": lowered_limit}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"generation_config",
|
||||
[None, {"responseModalities": None}, {"responseModalities": {"TEXT": 1}}],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_budget_sends_a_generation_config_of_an_unexpected_shape_as_it_came(
|
||||
monkeypatch: pytest.MonkeyPatch, generation_config: object
|
||||
):
|
||||
monkeypatch.setenv(BUDGET_ENV, "50")
|
||||
sent: Final[list[dict[str, object]]] = []
|
||||
|
||||
await litellm.acompletion(
|
||||
model=ARGON,
|
||||
messages=list(MESSAGES),
|
||||
extra_body={"generationConfig": generation_config},
|
||||
api_key="fake-gemini-key",
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))),
|
||||
)
|
||||
|
||||
assert sent[0]["generationConfig"] == generation_config
|
||||
|
||||
|
||||
def test_initial_prompt_tokens_is_absent_on_a_single_hop_and_the_first_hops_count_otherwise():
|
||||
assert initial_prompt_tokens_of((HOP_1,)) is None
|
||||
assert initial_prompt_tokens_of((HOP_1, HOP_2)) == 10
|
||||
|
|
|
|||
|
|
@ -1894,7 +1894,9 @@ def test_gemini_tool_use(respx_mock: respx.MockRouter) -> None:
|
|||
assert [(call.function.name, json.loads(call.function.arguments)) for call in tool_calls] == [
|
||||
("get_weather", {"location": "Lima, Peru"})
|
||||
]
|
||||
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["tool_calls"]
|
||||
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices and chunk.choices[0].finish_reason] == [
|
||||
"tool_calls"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
|
|
|
|||
|
|
@ -632,7 +632,7 @@ class TestVertexGemmaCompletion:
|
|||
assert chunks[0].choices[0].delta.content == "READY"
|
||||
assert chunks[1].choices[0].finish_reason == "stop"
|
||||
if stream_options and stream_options["include_usage"]:
|
||||
assert chunks[-1].choices[0].delta.content is None
|
||||
assert chunks[-1].choices == []
|
||||
assert chunks[-1].usage.total_tokens == 114
|
||||
assert span.get_metric("_dd.llmobs.total_tokens") == 114
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -1975,6 +1976,20 @@ class TestProxyWiring:
|
|||
|
||||
|
||||
class TestStreamUsage:
|
||||
@pytest.mark.asyncio
|
||||
async def test_trailing_flush_survives_a_usage_chunk_with_no_choices(self):
|
||||
guardrail, _ = _shielded({"[EMAIL_1]": "a@example.com"})
|
||||
held: Final = ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="Mail [EMAI"))])
|
||||
usage_chunk: Final = ModelResponseStream(
|
||||
choices=[], usage=Usage(prompt_tokens=5, completion_tokens=7, total_tokens=12)
|
||||
)
|
||||
|
||||
out: Final = await _restore_stream(guardrail, [held, usage_chunk])
|
||||
|
||||
assert out[1] is not None and out[1].choices == []
|
||||
assert [chunk.choices[0].delta.content for chunk in out[2:]] == ["[EMAI"]
|
||||
assert all(getattr(chunk, "usage", None) is None for chunk in out[2:])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trailing_flush_does_not_repeat_usage(self):
|
||||
"""With n>=2 and include_usage, the last chunk carries usage; a flush copied from it must not.
|
||||
|
|
|
|||
|
|
@ -8536,6 +8536,7 @@ PROXY_STAMPED_NAMES: Final = frozenset(
|
|||
"litellm_trusted_callback_vars",
|
||||
"_litellm_addressed_response_id",
|
||||
"_litellm_strip_stream_usage",
|
||||
"_litellm_stream_usage_injected",
|
||||
"client_side_timeout",
|
||||
"model_file_id_mapping",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,10 +6,13 @@ Pins (PR2):
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm import Router
|
||||
from litellm.proxy import proxy_server
|
||||
|
|
@ -85,6 +88,25 @@ def test_queue_chat_completions_happy(client, auth_as, patched_queue):
|
|||
}
|
||||
|
||||
|
||||
def test_queue_chat_completions_drops_client_sent_stream_usage_flags(
|
||||
client: TestClient, auth_as: Callable[..., AbstractContextManager[None]], patched_queue: MagicMock
|
||||
) -> None:
|
||||
payload: Final = {
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"priority": 0,
|
||||
"_litellm_strip_stream_usage": True,
|
||||
"_litellm_stream_usage_injected": True,
|
||||
}
|
||||
with auth_as():
|
||||
response: Final = client.post("/queue/chat/completions", json=payload)
|
||||
assert response.status_code == 200
|
||||
forwarded: Final = patched_queue.schedule_acompletion.call_args.kwargs
|
||||
assert "_litellm_strip_stream_usage" not in forwarded
|
||||
assert "_litellm_stream_usage_injected" not in forwarded
|
||||
assert forwarded["messages"] == [{"role": "user", "content": "hi"}]
|
||||
|
||||
|
||||
def test_queue_chat_completions_no_router_error(client, auth_as, queue_no_router):
|
||||
"""Pins ``POST /queue/chat/completions`` (error: no llm_router)."""
|
||||
payload = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}
|
||||
|
|
|
|||
|
|
@ -9183,6 +9183,7 @@ class TestApplyStreamUsageTracking:
|
|||
|
||||
assert data["stream_options"] == {"include_usage": True}
|
||||
assert data["_litellm_strip_stream_usage"] is True
|
||||
assert data["_litellm_stream_usage_injected"] is True
|
||||
|
||||
def test_default_preserves_other_client_stream_options_keys(self):
|
||||
data = {"stream": True, "stream_options": {"include_obfuscation": True}}
|
||||
|
|
@ -9191,6 +9192,7 @@ class TestApplyStreamUsageTracking:
|
|||
|
||||
assert data["stream_options"] == {"include_obfuscation": True, "include_usage": True}
|
||||
assert data["_litellm_strip_stream_usage"] is True
|
||||
assert data["_litellm_stream_usage_injected"] is True
|
||||
|
||||
def test_client_requested_usage_is_left_untouched_and_not_stripped(self):
|
||||
data = {"stream": True, "stream_options": {"include_usage": True}}
|
||||
|
|
@ -9207,6 +9209,7 @@ class TestApplyStreamUsageTracking:
|
|||
|
||||
assert data["stream_options"]["include_usage"] is True
|
||||
assert data["_litellm_strip_stream_usage"] is True
|
||||
assert data["_litellm_stream_usage_injected"] is True
|
||||
|
||||
def test_explicit_false_flag_disables_injection_entirely(self):
|
||||
data = {"stream": True}
|
||||
|
|
@ -9232,6 +9235,36 @@ class TestApplyStreamUsageTracking:
|
|||
assert data["stream_options"] == {"include_usage": True}
|
||||
assert "_litellm_strip_stream_usage" not in data
|
||||
|
||||
@pytest.mark.parametrize("client_stream_options", [None, {"include_obfuscation": True}])
|
||||
def test_flag_true_marks_usage_injected_for_clients_that_did_not_ask(
|
||||
self, client_stream_options: dict[str, bool] | None
|
||||
):
|
||||
data: Final = {
|
||||
"stream": True,
|
||||
**({} if client_stream_options is None else {"stream_options": client_stream_options}),
|
||||
}
|
||||
|
||||
_apply_stream_usage_tracking(
|
||||
data=data,
|
||||
general_settings={"always_include_stream_usage": True},
|
||||
route_type="acompletion",
|
||||
)
|
||||
|
||||
assert data["stream_options"]["include_usage"] is True
|
||||
assert data["_litellm_stream_usage_injected"] is True
|
||||
|
||||
def test_flag_true_leaves_usage_unmarked_when_client_asked_and_neutralizes_client_marker(self):
|
||||
data: Final = {"stream": True, "stream_options": {"include_usage": True}, "_litellm_stream_usage_injected": True}
|
||||
|
||||
_apply_stream_usage_tracking(
|
||||
data=data,
|
||||
general_settings={"always_include_stream_usage": True},
|
||||
route_type="acompletion",
|
||||
)
|
||||
|
||||
assert data["stream_options"] == {"include_usage": True}
|
||||
assert data["_litellm_stream_usage_injected"] is False
|
||||
|
||||
def test_flag_true_respects_client_explicit_include_usage_false(self):
|
||||
data = {"stream": True, "stream_options": {"include_usage": False}}
|
||||
|
||||
|
|
|
|||
|
|
@ -936,6 +936,26 @@ def test_body_snapshot_excludes_team_callback_credentials() -> None:
|
|||
}, proxy_request
|
||||
|
||||
|
||||
def test_body_snapshot_keeps_the_stream_usage_injection_marker_out_of_the_stored_request() -> None:
|
||||
from litellm.proxy.common_request_processing import stream_usage_tracking_updates
|
||||
from litellm.proxy.litellm_pre_call_utils import refresh_proxy_server_request_body_snapshot
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
|
||||
request: Final = {"model": "gpt-5-mini", "messages": [{"role": "user", "content": "hi"}], "stream": True}
|
||||
proxy_request: Final = {"body": dict(request)}
|
||||
data: Final = {
|
||||
**request,
|
||||
**stream_usage_tracking_updates(request, {}, "acompletion", lambda: True),
|
||||
"proxy_server_request": proxy_request,
|
||||
}
|
||||
|
||||
refresh_proxy_server_request_body_snapshot(data)
|
||||
|
||||
assert data[STREAM_USAGE_INJECTED_FIELD] is True
|
||||
assert proxy_request["body"]["stream_options"] == {"include_usage": True}
|
||||
assert STREAM_USAGE_INJECTED_FIELD not in proxy_request["body"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("pre_call_ran", [False, True])
|
||||
async def test_post_guardrail_snapshot_preserves_logging_only_masking_in_spend_logs(
|
||||
|
|
|
|||
|
|
@ -1,25 +1,44 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import gc
|
||||
import threading
|
||||
import weakref
|
||||
import httpx
|
||||
import json
|
||||
import litellm
|
||||
import pytest
|
||||
import redis
|
||||
import respx
|
||||
|
||||
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from fakeredis import TcpFakeServer
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.route_llm_request import (
|
||||
JSON_OBJECT_ADAPTER,
|
||||
MockTestingParamsDisabledError,
|
||||
ProxyMissingRequiredParamError,
|
||||
ProxyModelNotFoundError,
|
||||
UserConfigRouters,
|
||||
route_request,
|
||||
user_config_routers,
|
||||
)
|
||||
from litellm.types.router import RouterGeneralSettings
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.router_strategy.adaptive_router.config import (
|
||||
ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY,
|
||||
ADAPTIVE_ROUTER_RESPONSE_HEADER,
|
||||
)
|
||||
from litellm.types.router import RouterErrors, RouterGeneralSettings, RouterRateLimitError
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -2684,3 +2703,571 @@ async def test_team_key_search_of_registered_bedrock_store_reaches_bedrock_with_
|
|||
posted_url
|
||||
)
|
||||
assert "Credential=AKIAREGISTRYKEY00000/" in signed_auth, signed_auth
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def discard_cached_user_config_routers() -> Iterator[None]:
|
||||
yield
|
||||
user_config_routers.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_redis_port() -> Iterator[int]:
|
||||
server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis")
|
||||
thread: Final = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield server.server_address[1]
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
async def _wait_out_a_rate_limit_minute_rollover() -> None:
|
||||
seconds_left: Final = 60 - datetime.now().second
|
||||
if seconds_left < 10:
|
||||
await asyncio.sleep(seconds_left)
|
||||
|
||||
|
||||
def _proxy_logging_obj(model: str = "rate-limited") -> LiteLLMLoggingObj:
|
||||
logging_obj, _ = litellm.utils.function_setup(
|
||||
original_function="acompletion",
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=datetime.now(),
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
litellm_call_id=str(uuid4()),
|
||||
)
|
||||
assert isinstance(logging_obj, LiteLLMLoggingObj)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _mock_deployment(model_name: str, **deployment_limits: int) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": model_name,
|
||||
**deployment_limits,
|
||||
"litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test", "mock_response": "ok"},
|
||||
}
|
||||
|
||||
|
||||
async def _user_config_request_outcome(
|
||||
redis_port: int | None,
|
||||
routing_settings: Mapping[str, object],
|
||||
deployment_limits: Mapping[str, int],
|
||||
logging_obj: LiteLLMLoggingObj | None,
|
||||
) -> str:
|
||||
data: Final = {
|
||||
"model": "rate-limited",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"num_retries": 0,
|
||||
**({"litellm_logging_obj": logging_obj} if logging_obj is not None else {}),
|
||||
"user_config": {
|
||||
**routing_settings,
|
||||
**({"redis_host": "127.0.0.1", "redis_port": redis_port} if redis_port is not None else {}),
|
||||
"model_list": [_mock_deployment("rate-limited", **deployment_limits)],
|
||||
},
|
||||
}
|
||||
try:
|
||||
await (await route_request(data, None, None, "acompletion"))
|
||||
except (litellm.RateLimitError, RouterRateLimitError):
|
||||
return "429"
|
||||
finally:
|
||||
await asyncio.sleep(0.3)
|
||||
return "200"
|
||||
|
||||
|
||||
async def _tpm_for_three_requests() -> int:
|
||||
messages: Final = [{"role": "user", "content": "hi"}]
|
||||
probe: Final = await litellm.acompletion(model="openai/gpt-5.5", messages=messages, mock_response="ok")
|
||||
assert isinstance(probe, litellm.ModelResponse)
|
||||
probe_usage: Final = getattr(probe, "usage", None)
|
||||
assert isinstance(probe_usage, litellm.Usage)
|
||||
return 2 * probe_usage.total_tokens + litellm.token_counter(messages=messages)
|
||||
|
||||
|
||||
async def _request_cost() -> float:
|
||||
probe: Final = await litellm.acompletion(
|
||||
model="openai/gpt-5.5", messages=[{"role": "user", "content": "hi"}], mock_response="ok"
|
||||
)
|
||||
return litellm.completion_cost(completion_response=probe)
|
||||
|
||||
|
||||
async def _proxy_router_outcomes(
|
||||
redis_port: int | None,
|
||||
routing_settings: Mapping[str, object],
|
||||
deployment_limits: Mapping[str, int],
|
||||
requests: int,
|
||||
) -> list[str]:
|
||||
router: Final = litellm.Router(
|
||||
**routing_settings,
|
||||
num_retries=0,
|
||||
model_list=[_mock_deployment("proxy-rate-limited", **deployment_limits)],
|
||||
**({"redis_host": "127.0.0.1", "redis_port": redis_port} if redis_port is not None else {}),
|
||||
)
|
||||
|
||||
async def outcome() -> str:
|
||||
try:
|
||||
await router.acompletion(model="proxy-rate-limited", messages=[{"role": "user", "content": "hi"}])
|
||||
except (litellm.RateLimitError, RouterRateLimitError):
|
||||
return "429"
|
||||
finally:
|
||||
await asyncio.sleep(0.3)
|
||||
return "200"
|
||||
|
||||
try:
|
||||
return [await outcome() for _ in range(requests)]
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_user_config_adaptive_request_answers_with_the_model_it_picked() -> None:
|
||||
data: Final = {
|
||||
"model": "smart-cheap-router",
|
||||
"messages": [{"role": "user", "content": "Write a one line python function that adds two numbers"}],
|
||||
"metadata": {},
|
||||
"user_config": {
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": "smart-cheap-router",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/adaptive_router",
|
||||
"adaptive_router_config": {"available_models": ["fast", "smart"]},
|
||||
},
|
||||
},
|
||||
{
|
||||
**_mock_deployment("fast"),
|
||||
"model_info": {"adaptive_router_preferences": {"quality_tier": 2, "strengths": []}},
|
||||
},
|
||||
{
|
||||
**_mock_deployment("smart"),
|
||||
"model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": ["code_generation"]}},
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
await (await route_request(data, None, None, "acompletion"))
|
||||
headers: Final = await ProxyLogging(user_api_key_cache=DualCache()).post_call_response_headers_hook(
|
||||
data=data, user_api_key_dict=UserAPIKeyAuth(), response=None
|
||||
)
|
||||
|
||||
assert headers == {ADAPTIVE_ROUTER_RESPONSE_HEADER: data["metadata"][ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY]}
|
||||
|
||||
|
||||
def _global_router_callbacks() -> tuple[list[object], list[object]]:
|
||||
return list(litellm.callbacks), list(litellm.input_callback)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"routing_settings, deployment_limits",
|
||||
[
|
||||
({"routing_strategy": "usage-based-routing-v2"}, {"rpm": 2}),
|
||||
({"routing_strategy": "usage-based-routing-v2"}, {"tpm": 30}),
|
||||
({"routing_strategy": "usage-based-routing"}, {"rpm": 2}),
|
||||
({"optional_pre_call_checks": ["enforce_model_rate_limits"]}, {"rpm": 2}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
|
||||
@pytest.mark.parametrize("with_redis", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_config_router_enforces_its_limit_across_proxy_requests(
|
||||
fake_redis_port: int,
|
||||
routing_settings: Mapping[str, object],
|
||||
deployment_limits: Mapping[str, int],
|
||||
arrives_with_logging_obj: bool,
|
||||
with_redis: bool,
|
||||
) -> None:
|
||||
await _wait_out_a_rate_limit_minute_rollover()
|
||||
callbacks_before: Final = _global_router_callbacks()
|
||||
outcomes: Final = [
|
||||
await _user_config_request_outcome(
|
||||
fake_redis_port if with_redis else None,
|
||||
routing_settings,
|
||||
deployment_limits,
|
||||
_proxy_logging_obj() if arrives_with_logging_obj else None,
|
||||
)
|
||||
for _ in range(6)
|
||||
]
|
||||
|
||||
assert "429" in outcomes, outcomes
|
||||
assert _global_router_callbacks() == callbacks_before
|
||||
assert outcomes == await _proxy_router_outcomes(
|
||||
fake_redis_port if with_redis else None, routing_settings, deployment_limits, len(outcomes)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_user_config_requests_each_count_once_against_the_tpm_limit(
|
||||
fake_redis_port: int, arrives_with_logging_obj: bool
|
||||
) -> None:
|
||||
tpm_for_three_requests: Final = await _tpm_for_three_requests()
|
||||
|
||||
def request() -> Coroutine[None, None, str]:
|
||||
return _user_config_request_outcome(
|
||||
fake_redis_port,
|
||||
{"routing_strategy": "usage-based-routing-v2"},
|
||||
{"tpm": tpm_for_three_requests},
|
||||
_proxy_logging_obj() if arrives_with_logging_obj else None,
|
||||
)
|
||||
|
||||
await _wait_out_a_rate_limit_minute_rollover()
|
||||
concurrent_outcomes: Final = await asyncio.gather(request(), request())
|
||||
later_outcomes: Final = [await request() for _ in range(2)]
|
||||
|
||||
assert [*concurrent_outcomes, *later_outcomes] == ["200", "200", "200", "429"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overlapping_user_config_requests_share_one_rpm_limit(fake_redis_port: int) -> None:
|
||||
def request() -> Coroutine[None, None, str]:
|
||||
return _user_config_request_outcome(
|
||||
fake_redis_port, {"routing_strategy": "usage-based-routing-v2"}, {"rpm": 2}, _proxy_logging_obj()
|
||||
)
|
||||
|
||||
await _wait_out_a_rate_limit_minute_rollover()
|
||||
outcomes: Final = await asyncio.gather(*(request() for _ in range(4)))
|
||||
|
||||
assert outcomes.count("200") <= 2, outcomes
|
||||
assert "429" in outcomes, outcomes
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"routing_strategy, limited_by",
|
||||
[("usage-based-routing-v2", "rpm"), ("usage-based-routing-v2", "tpm"), ("usage-based-routing", "rpm")],
|
||||
)
|
||||
@pytest.mark.parametrize("proxy_router_shares_redis", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_proxy_router_on_the_same_strategy_leaves_the_user_config_limit_unchanged(
|
||||
fake_redis_port: int, routing_strategy: str, limited_by: str, proxy_router_shares_redis: bool
|
||||
) -> None:
|
||||
deployment_limits: Final = {"rpm": 3} if limited_by == "rpm" else {"tpm": await _tpm_for_three_requests()}
|
||||
|
||||
def request() -> Coroutine[None, None, str]:
|
||||
return _user_config_request_outcome(
|
||||
fake_redis_port, {"routing_strategy": routing_strategy}, deployment_limits, _proxy_logging_obj()
|
||||
)
|
||||
|
||||
async def outcomes() -> list[str]:
|
||||
first_instance: Final = [await request() for _ in range(2)]
|
||||
user_config_routers.clear()
|
||||
second_instance: Final = [await request() for _ in range(3)]
|
||||
return [*first_instance, *second_instance]
|
||||
|
||||
await _wait_out_a_rate_limit_minute_rollover()
|
||||
alone: Final = await outcomes()
|
||||
user_config_routers.clear()
|
||||
await asyncio.sleep(0.3)
|
||||
redis.Redis(host="127.0.0.1", port=fake_redis_port).flushall()
|
||||
proxy_router: Final = litellm.Router(
|
||||
routing_strategy=routing_strategy,
|
||||
model_list=[_mock_deployment("proxy-model")],
|
||||
redis_host="127.0.0.1" if proxy_router_shares_redis else None,
|
||||
redis_port=fake_redis_port if proxy_router_shares_redis else None,
|
||||
)
|
||||
try:
|
||||
beside_proxy_router: Final = await outcomes()
|
||||
finally:
|
||||
proxy_router.discard()
|
||||
|
||||
assert "429" in alone, alone
|
||||
assert beside_proxy_router == alone
|
||||
|
||||
|
||||
@pytest.mark.parametrize("user_config_settings", [{}, {"optional_pre_call_checks": ["router_budget_limiting"]}])
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_config_requests_stay_under_the_proxy_provider_budget(
|
||||
user_config_settings: Mapping[str, object],
|
||||
) -> None:
|
||||
proxy_router: Final = litellm.Router(
|
||||
model_list=[_mock_deployment("proxy-model")],
|
||||
provider_budget_config={"openai": {"budget_limit": await _request_cost() / 2, "time_period": "1d"}},
|
||||
)
|
||||
try:
|
||||
await proxy_router.acompletion(model="proxy-model", messages=[{"role": "user", "content": "hi"}])
|
||||
await asyncio.sleep(0.3)
|
||||
with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_provider_budget_routing.value):
|
||||
await _user_config_request_outcome(None, user_config_settings, {}, _proxy_logging_obj())
|
||||
finally:
|
||||
proxy_router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_user_config_provider_budget_limits_its_own_requests() -> None:
|
||||
budget_limit: Final = await _request_cost() / 2
|
||||
|
||||
def request() -> Coroutine[None, None, str]:
|
||||
budget: Final = {"openai": {"budget_limit": budget_limit, "time_period": "1d"}}
|
||||
return _user_config_request_outcome(None, {"provider_budget_config": budget}, {}, _proxy_logging_obj())
|
||||
|
||||
first: Final = await request()
|
||||
with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_provider_budget_routing.value):
|
||||
await request()
|
||||
assert first == "200"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_request_in_flight_when_its_user_config_router_is_dropped_still_counts_against_its_budget(
|
||||
fake_redis_port: int,
|
||||
) -> None:
|
||||
budget_limit: Final = 1.5 * await _request_cost()
|
||||
|
||||
async def request() -> None:
|
||||
await (
|
||||
await route_request(
|
||||
{
|
||||
"model": "budgeted",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"litellm_logging_obj": _proxy_logging_obj("budgeted"),
|
||||
"user_config": {
|
||||
"redis_host": "127.0.0.1",
|
||||
"redis_port": fake_redis_port,
|
||||
"provider_budget_config": {"openai": {"budget_limit": budget_limit, "time_period": "1d"}},
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": "budgeted",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.5",
|
||||
"api_key": "sk-test",
|
||||
"mock_response": "ok",
|
||||
"mock_delay": 0.5,
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
None,
|
||||
None,
|
||||
"acompletion",
|
||||
)
|
||||
)
|
||||
|
||||
await request()
|
||||
in_flight: Final = asyncio.create_task(request())
|
||||
await asyncio.sleep(0.2)
|
||||
user_config_routers.clear()
|
||||
await in_flight
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_provider_budget_routing.value):
|
||||
await request()
|
||||
|
||||
|
||||
async def _user_config_request(model: str, user_config: Mapping[str, object], arrives_with_logging_obj: bool) -> object:
|
||||
return await (
|
||||
await route_request(
|
||||
{
|
||||
"model": model,
|
||||
"messages": list(_HELLO_MESSAGES),
|
||||
"num_retries": 0,
|
||||
**({"litellm_logging_obj": _proxy_logging_obj(model)} if arrives_with_logging_obj else {}),
|
||||
"user_config": user_config,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
"acompletion",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _ordered_pair(model_name: str, first_base: str, second_base: str) -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"model_name": model_name,
|
||||
"litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test", "api_base": api_base, "order": order},
|
||||
}
|
||||
for order, api_base in ((1, first_base), (2, second_base))
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_rate_limited_user_config_deployment_cools_down_for_the_next_requests(
|
||||
monkeypatch: pytest.MonkeyPatch, arrives_with_logging_obj: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
rate_limited_base: Final = "https://rate-limited.example/v1"
|
||||
healthy_base: Final = "https://healthy.example/v1"
|
||||
user_config: Final = {"model_list": _ordered_pair("flaky", rate_limited_base, healthy_base)}
|
||||
|
||||
with respx.mock(assert_all_called=False) as respx_mock:
|
||||
rate_limited: Final = respx_mock.post(f"{rate_limited_base}/chat/completions").mock(
|
||||
return_value=httpx.Response(429, json={"error": {"message": "slow down"}})
|
||||
)
|
||||
respx_mock.post(f"{healthy_base}/chat/completions").mock(return_value=_chat_completion_response("gpt-5.5"))
|
||||
for _ in range(3):
|
||||
await _user_config_request("flaky", user_config, arrives_with_logging_obj)
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
assert rate_limited.call_count == 1
|
||||
|
||||
|
||||
async def _remaining_usage_headers_after_429_then_200s(
|
||||
send: Callable[[], Awaitable[object]], api_base: str
|
||||
) -> list[tuple[object, object] | str]:
|
||||
async def remaining_usage() -> tuple[object, object] | str:
|
||||
try:
|
||||
response: Final = await send()
|
||||
except litellm.RateLimitError:
|
||||
return "429"
|
||||
finally:
|
||||
await asyncio.sleep(0.3)
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
headers: Final = response._hidden_params["additional_headers"]
|
||||
return headers.get("x-ratelimit-remaining-requests"), headers.get("x-ratelimit-remaining-tokens")
|
||||
|
||||
with respx.mock(assert_all_called=False) as respx_mock:
|
||||
respx_mock.post(f"{api_base}/chat/completions").mock(
|
||||
side_effect=[
|
||||
httpx.Response(429, json={"error": {"message": "slow down"}}),
|
||||
_chat_completion_response("gpt-5.5"),
|
||||
_chat_completion_response("gpt-5.5"),
|
||||
]
|
||||
)
|
||||
return [await remaining_usage() for _ in range(3)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_config_remaining_usage_headers_match_a_proxy_router_with_the_same_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch, arrives_with_logging_obj: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
await _wait_out_a_rate_limit_minute_rollover()
|
||||
proxy_base: Final = "https://proxy-limited.example/v1"
|
||||
user_config_base: Final = "https://user-config-limited.example/v1"
|
||||
|
||||
def deployment(model_name: str, api_base: str) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.5",
|
||||
"api_key": "sk-test",
|
||||
"api_base": api_base,
|
||||
"rpm": 10,
|
||||
"tpm": 1000,
|
||||
},
|
||||
}
|
||||
|
||||
proxy_router: Final = litellm.Router(model_list=[deployment("proxy-limited", proxy_base)])
|
||||
try:
|
||||
proxy_router_headers: Final = await _remaining_usage_headers_after_429_then_200s(
|
||||
lambda: proxy_router.acompletion(model="proxy-limited", messages=list(_HELLO_MESSAGES), num_retries=0),
|
||||
proxy_base,
|
||||
)
|
||||
finally:
|
||||
proxy_router.discard()
|
||||
|
||||
user_config: Final = {"model_list": [deployment("user-config-limited", user_config_base)]}
|
||||
user_config_headers: Final = await _remaining_usage_headers_after_429_then_200s(
|
||||
lambda: _user_config_request("user-config-limited", user_config, arrives_with_logging_obj), user_config_base
|
||||
)
|
||||
|
||||
assert user_config_headers == proxy_router_headers
|
||||
|
||||
|
||||
async def _first_deployment_calls_over_successes_then_failures(
|
||||
send: Callable[[], Awaitable[object]], first_base: str, second_base: str
|
||||
) -> int:
|
||||
successes: Final = [_chat_completion_response("gpt-5.5") for _ in range(6)]
|
||||
failures: Final = [httpx.Response(500, json={"error": {"message": "boom"}}) for _ in range(6)]
|
||||
with respx.mock(assert_all_called=False) as respx_mock:
|
||||
first: Final = respx_mock.post(f"{first_base}/chat/completions").mock(side_effect=[*successes, *failures])
|
||||
respx_mock.post(f"{second_base}/chat/completions").mock(return_value=_chat_completion_response("gpt-5.5"))
|
||||
for _ in range(len(successes) + len(failures)):
|
||||
await send()
|
||||
await asyncio.sleep(0.3)
|
||||
return first.call_count
|
||||
|
||||
|
||||
@pytest.mark.parametrize("arrives_with_logging_obj", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_config_successes_keep_a_deployment_out_of_cooldown_like_a_proxy_router(
|
||||
monkeypatch: pytest.MonkeyPatch, arrives_with_logging_obj: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
await _wait_out_a_rate_limit_minute_rollover()
|
||||
proxy_router: Final = litellm.Router(
|
||||
model_list=_ordered_pair("proxy-pair", "https://proxy-first.example/v1", "https://proxy-second.example/v1")
|
||||
)
|
||||
try:
|
||||
proxy_router_calls: Final = await _first_deployment_calls_over_successes_then_failures(
|
||||
lambda: proxy_router.acompletion(model="proxy-pair", messages=list(_HELLO_MESSAGES), num_retries=0),
|
||||
"https://proxy-first.example/v1",
|
||||
"https://proxy-second.example/v1",
|
||||
)
|
||||
finally:
|
||||
proxy_router.discard()
|
||||
|
||||
user_config: Final = {
|
||||
"model_list": _ordered_pair("user-pair", "https://user-first.example/v1", "https://user-second.example/v1")
|
||||
}
|
||||
|
||||
user_config_calls: Final = await _first_deployment_calls_over_successes_then_failures(
|
||||
lambda: _user_config_request("user-pair", user_config, arrives_with_logging_obj),
|
||||
"https://user-first.example/v1",
|
||||
"https://user-second.example/v1",
|
||||
)
|
||||
|
||||
assert user_config_calls == proxy_router_calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_config_adaptive_router_learns_from_proxy_requests() -> None:
|
||||
user_config: Final = {
|
||||
"model_list": [
|
||||
_mock_deployment("smart"),
|
||||
_mock_deployment("fast"),
|
||||
{
|
||||
"model_name": "adaptive",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/adaptive_router",
|
||||
"adaptive_router_config": {"available_models": ["smart", "fast"]},
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
await (
|
||||
await route_request(
|
||||
{
|
||||
"model": "adaptive",
|
||||
"messages": [{"role": "user", "content": "write a sorting function"}],
|
||||
"metadata": {"session_id": "session-1"},
|
||||
"litellm_logging_obj": _proxy_logging_obj("adaptive"),
|
||||
"user_config": user_config,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
"acompletion",
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
router: Final = user_config_routers.get(user_config, lambda: pytest.fail("the user_config router was not reused"))
|
||||
snapshot: Final = await router.adaptive_routers["adaptive"][0].strategy.get_state_snapshot()
|
||||
assert snapshot["feedback_contexts_live"] == 1, snapshot
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_config_routers_discard_the_least_recently_used_router_past_capacity() -> None:
|
||||
routers: Final = UserConfigRouters(max_size=2)
|
||||
|
||||
def config(model_name: str) -> dict[str, object]:
|
||||
return {"routing_strategy": "usage-based-routing-v2", "model_list": [_mock_deployment(model_name)]}
|
||||
|
||||
def build(model_name: str) -> litellm.Router:
|
||||
return litellm.Router(**config(model_name))
|
||||
|
||||
def must_be_cached() -> litellm.Router:
|
||||
pytest.fail("a cached router was rebuilt")
|
||||
|
||||
first_handler: Final = weakref.ref(routers.get(config("first"), lambda: build("first")).lowesttpm_logger_v2)
|
||||
second_handler: Final = weakref.ref(routers.get(config("second"), lambda: build("second")).lowesttpm_logger_v2)
|
||||
routers.get(config("first"), must_be_cached)
|
||||
routers.get(config("third"), lambda: build("third"))
|
||||
await asyncio.sleep(0.2)
|
||||
gc.collect()
|
||||
|
||||
assert routers.get(config("first"), must_be_cached) is not None
|
||||
assert first_handler() is not None
|
||||
assert second_handler() is None
|
||||
routers.clear()
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from litellm.router_strategy.adaptive_router.config import (
|
|||
)
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
AdaptiveRouterResponseHeaderHook,
|
||||
_recent_tool_results,
|
||||
_resolve_session_key,
|
||||
)
|
||||
|
|
@ -337,8 +338,7 @@ async def test_post_call_response_headers_hook_returns_chosen_model_header():
|
|||
"""The header hook returns the `x-litellm-adaptive-router-model` header
|
||||
so proxy header construction picks it up (works for both streaming and
|
||||
non-streaming; `async_post_call_success_hook` is too late for streaming)."""
|
||||
hook = _make_hook()
|
||||
headers = await hook.async_post_call_response_headers_hook(
|
||||
headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook(
|
||||
data={"metadata": {"adaptive_router_chosen_model": "smart"}},
|
||||
user_api_key_dict=MagicMock(),
|
||||
response=MagicMock(),
|
||||
|
|
@ -348,8 +348,7 @@ async def test_post_call_response_headers_hook_returns_chosen_model_header():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_response_headers_hook_noop_when_metadata_missing_key():
|
||||
hook = _make_hook()
|
||||
headers = await hook.async_post_call_response_headers_hook(
|
||||
headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook(
|
||||
data={"metadata": {"litellm_session_id": "sess-A"}},
|
||||
user_api_key_dict=MagicMock(),
|
||||
response=MagicMock(),
|
||||
|
|
@ -359,8 +358,7 @@ async def test_post_call_response_headers_hook_noop_when_metadata_missing_key():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_response_headers_hook_noop_when_no_metadata():
|
||||
hook = _make_hook()
|
||||
headers = await hook.async_post_call_response_headers_hook(
|
||||
headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook(
|
||||
data={},
|
||||
user_api_key_dict=MagicMock(),
|
||||
response=MagicMock(),
|
||||
|
|
@ -370,8 +368,7 @@ async def test_post_call_response_headers_hook_noop_when_no_metadata():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_response_headers_hook_noop_when_metadata_not_dict():
|
||||
hook = _make_hook()
|
||||
headers = await hook.async_post_call_response_headers_hook(
|
||||
headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook(
|
||||
data={"metadata": "not-a-dict"},
|
||||
user_api_key_dict=MagicMock(),
|
||||
response=MagicMock(),
|
||||
|
|
|
|||
|
|
@ -1,15 +1,20 @@
|
|||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional, Set, Union
|
||||
from typing import Any, Dict, Final, List, Optional, Set, Union
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from fakeredis import TcpFakeServer
|
||||
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.caching.redis_cache import RedisCache
|
||||
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError, RedisPipelineIncrementOperation
|
||||
from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
|
||||
|
||||
|
|
@ -162,3 +167,24 @@ async def test_push_refused_by_the_open_circuit_breaker_is_not_logged_as_an_erro
|
|||
|
||||
assert caplog.records == []
|
||||
assert base_strategy.redis_increment_operation_queue == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_increment_counted_after_retire_still_reaches_redis():
|
||||
server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis")
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
redis_cache: Final = RedisCache(host="127.0.0.1", port=server.server_address[1])
|
||||
strategy: Final = BaseRoutingStrategy(
|
||||
dual_cache=DualCache(redis_cache=redis_cache, in_memory_cache=InMemoryCache()),
|
||||
should_batch_redis_writes=True,
|
||||
default_sync_interval=60,
|
||||
)
|
||||
try:
|
||||
strategy.retire()
|
||||
await strategy._increment_value_in_current_window("counted-after-retire", 3, 60)
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
assert float(await redis_cache.async_get_cache("counted-after-retire")) == 3
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import gc
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
|
@ -492,6 +493,8 @@ def _new_router_budget_limiter(
|
|||
budget_limiter._redis_increment_queue_lock = queue_lock if queue_lock is not None else asyncio.Lock()
|
||||
budget_limiter._redis_increment_flush_lock = asyncio.Lock()
|
||||
budget_limiter._detached_increment_operations = None
|
||||
budget_limiter._retired = False
|
||||
budget_limiter._flush_after_retire = None
|
||||
return budget_limiter
|
||||
|
||||
|
||||
|
|
@ -784,3 +787,57 @@ async def test_cancelled_flush_does_not_requeue_an_applied_batch(cancellations:
|
|||
await limiter._push_in_memory_increments_to_redis()
|
||||
assert redis_cache.values[_SPEND_KEY] == 10.0
|
||||
assert limiter.redis_increment_operation_queue == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_queued_when_a_limiter_is_retired_reaches_redis_after_a_failed_write() -> None:
|
||||
pipeline_started: Final = asyncio.Event()
|
||||
pipeline_completed: Final = asyncio.Event()
|
||||
redis_cache: Final = _MockRedisCache(
|
||||
initial_values={},
|
||||
pipeline_started=pipeline_started,
|
||||
pipeline_completed=pipeline_completed,
|
||||
should_fail_pipeline=True,
|
||||
)
|
||||
budget_limiter: Final = _new_router_budget_limiter(
|
||||
redis_cache=redis_cache, redis_increment_operation_queue=[_increment(10.0)]
|
||||
)
|
||||
budget_limiter._sync_task = asyncio.create_task(asyncio.sleep(3600))
|
||||
|
||||
budget_limiter.retire()
|
||||
await asyncio.wait_for(pipeline_started.wait(), timeout=1)
|
||||
assert redis_cache.values == {}
|
||||
redis_cache.should_fail_pipeline = False
|
||||
await asyncio.wait_for(pipeline_completed.wait(), timeout=5)
|
||||
|
||||
assert redis_cache.values == {_SPEND_KEY: 10.0}
|
||||
assert budget_limiter.redis_increment_operation_queue == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_in_flight_when_a_limiter_is_retired_reaches_redis_after_that_write_fails() -> None:
|
||||
pipeline_started: Final = asyncio.Event()
|
||||
allow_pipeline_to_complete: Final = asyncio.Event()
|
||||
pipeline_completed: Final = asyncio.Event()
|
||||
redis_cache: Final = _MockRedisCache(
|
||||
initial_values={},
|
||||
pipeline_started=pipeline_started,
|
||||
allow_pipeline_to_complete=allow_pipeline_to_complete,
|
||||
should_fail_pipeline=True,
|
||||
pipeline_completed=pipeline_completed,
|
||||
)
|
||||
budget_limiter: Final = _new_router_budget_limiter(
|
||||
redis_cache=redis_cache, redis_increment_operation_queue=[_increment(10.0)]
|
||||
)
|
||||
budget_limiter._sync_task = asyncio.create_task(budget_limiter.periodic_sync_in_memory_spend_with_redis())
|
||||
await asyncio.wait_for(pipeline_started.wait(), timeout=1)
|
||||
|
||||
budget_limiter.retire()
|
||||
allow_pipeline_to_complete.set()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await budget_limiter._sync_task
|
||||
redis_cache.should_fail_pipeline = False
|
||||
await asyncio.wait_for(pipeline_completed.wait(), timeout=5)
|
||||
|
||||
assert redis_cache.values == {_SPEND_KEY: 10.0}
|
||||
assert budget_limiter.redis_increment_operation_queue == []
|
||||
|
|
|
|||
|
|
@ -96,7 +96,6 @@ CI = [".github/workflows/test-unit.yml"]
|
|||
("ui", BACKEND + CLIENT, "run"),
|
||||
("cost-map-only", ["model_prices_and_context_window.json"], "run"),
|
||||
("cost-map-only", ["litellm/model_prices_and_context_window_backup.json"], "run"),
|
||||
("cost-map-only", ["model_prices_and_context_window.schema.json"], "run"),
|
||||
(
|
||||
"cost-map-only",
|
||||
["model_prices_and_context_window.json", "tests/unit/proxy/test_y.py"],
|
||||
|
|
|
|||
|
|
@ -186,13 +186,6 @@ def test_bot_may_only_touch_the_cost_map_files() -> None:
|
|||
)
|
||||
|
||||
|
||||
def test_bot_may_update_cost_map_and_legacy_schema() -> None:
|
||||
head: Final = _snapshot({**BASE_MAP, "openrouter/c": _entry()})
|
||||
changed: Final = (*guard.GUARDED_PATHS, guard.LEGACY_SCHEMA_PATH)
|
||||
|
||||
assert _failures(head, changed_files=changed) == ()
|
||||
|
||||
|
||||
def test_bot_may_not_remove_models() -> None:
|
||||
head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"})
|
||||
assert _failures(head, bot=False) == ()
|
||||
|
|
|
|||
|
|
@ -3868,6 +3868,9 @@ def test_stream_chunk_builder_text_completion_combines_text_and_usage():
|
|||
model="gpt-3.5-turbo-instruct",
|
||||
choices=[{"text": " world", "index": 0, "logprobs": None, "finish_reason": "stop"}],
|
||||
),
|
||||
TextCompletionResponse(
|
||||
id="cmpl-1", object="text_completion", created=1, model="gpt-3.5-turbo-instruct", choices=[]
|
||||
),
|
||||
]
|
||||
|
||||
response = stream_chunk_builder_text_completion(chunks=chunks, messages=[{"role": "user", "content": "say hello"}])
|
||||
|
|
@ -8853,7 +8856,8 @@ async def test_text_completion_stream_include_usage_returns_usage_on_final_chunk
|
|||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
assert "".join(chunk.choices[0].text or "" for chunk in chunks) == "reply"
|
||||
assert "".join(chunk.choices[0].text or "" for chunk in chunks if chunk.choices) == "reply"
|
||||
assert chunks[-1].choices == []
|
||||
final_usage: Final = chunks[-1].usage
|
||||
assert (final_usage.prompt_tokens, final_usage.completion_tokens, final_usage.total_tokens) == (3, 2, 5)
|
||||
|
||||
|
|
|
|||
|
|
@ -29670,6 +29670,73 @@ def test_pre_call_checks_ignores_an_image_response_format_string(monkeypatch: py
|
|||
assert [d["model_info"]["id"] for d in filtered] == ["only"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_stream_options", "default_stream_options", "request_stream_options", "usage_chunk_choice_count"),
|
||||
[
|
||||
({"include_usage": True}, None, None, 1),
|
||||
(None, {"include_usage": True}, None, 1),
|
||||
({"include_usage": True}, None, {"include_usage": True}, 0),
|
||||
(None, None, {"include_usage": True}, 0),
|
||||
],
|
||||
ids=["deployment-config", "router-default", "request-over-deployment", "request-only"],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_configured_stream_usage_keeps_a_choice_on_the_usage_chunk(
|
||||
deployment_stream_options: dict[str, bool] | None,
|
||||
default_stream_options: dict[str, bool] | None,
|
||||
request_stream_options: dict[str, bool] | None,
|
||||
usage_chunk_choice_count: int,
|
||||
):
|
||||
deployment_params: Final = {"model": "openai/gpt-5-mini", "api_key": "fake", "mock_response": "hi there"}
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "m",
|
||||
"litellm_params": (
|
||||
deployment_params
|
||||
if deployment_stream_options is None
|
||||
else {**deployment_params, "stream_options": deployment_stream_options}
|
||||
),
|
||||
}
|
||||
],
|
||||
default_litellm_params=None if default_stream_options is None else {"stream_options": default_stream_options},
|
||||
)
|
||||
request_kwargs: Final = {} if request_stream_options is None else {"stream_options": request_stream_options}
|
||||
|
||||
response: Final = await router.acompletion(
|
||||
model="m", messages=[{"role": "user", "content": "hi"}], stream=True, **request_kwargs
|
||||
)
|
||||
chunks: Final = [chunk async for chunk in response]
|
||||
|
||||
assert getattr(chunks[-1], "usage", None) is not None
|
||||
assert len(chunks[-1].choices) == usage_chunk_choice_count
|
||||
assert all(len(chunk.choices) == 1 for chunk in chunks[:-1])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stream_options", "marked_injected"),
|
||||
[({"include_usage": True}, True), (None, False)],
|
||||
ids=["include-usage", "no-stream-options"],
|
||||
)
|
||||
def test_update_kwargs_with_deployment_reads_stream_options_from_pydantic_litellm_params(
|
||||
stream_options: dict[str, bool] | None, marked_injected: bool
|
||||
):
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
kwargs: Final[dict[str, object]] = {"stream": True, "metadata": {}}
|
||||
Router()._update_kwargs_with_deployment(
|
||||
deployment={
|
||||
"model_name": "gpt-5-mini",
|
||||
"litellm_params": LiteLLM_Params(model="openai/gpt-5-mini", api_key="fake-key", stream_options=stream_options),
|
||||
"model_info": {"id": "pydantic-params"},
|
||||
},
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert (kwargs.get(STREAM_USAGE_INJECTED_FIELD) is True) is marked_injected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_streams_tokens():
|
||||
"""After a mid-stream hop, a disconnect bills the primary's relayed text plus the fallback's, not the fallback alone."""
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
import asyncio
|
||||
import functools
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import weakref
|
||||
from datetime import datetime, timezone
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Iterator, Mapping
|
||||
from types import MethodType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -12,8 +16,12 @@ from litellm import Router
|
|||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
|
||||
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
|
||||
from litellm.types.litellm_params import RoutingStrategyName
|
||||
from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
ModelResponse,
|
||||
|
|
@ -1852,6 +1860,315 @@ def test_discard(model_list):
|
|||
assert len(litellm.service_callback) == 0
|
||||
|
||||
|
||||
_CALLBACK_LIST_NAMES: Final = (
|
||||
"callbacks",
|
||||
"input_callback",
|
||||
"success_callback",
|
||||
"failure_callback",
|
||||
"_async_success_callback",
|
||||
"_async_failure_callback",
|
||||
)
|
||||
|
||||
|
||||
def _callback_ids(list_name: str) -> list[int]:
|
||||
return [id(callback) for callback in getattr(litellm, list_name)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"routing_strategy, selector_attr",
|
||||
[
|
||||
("least-busy", "leastbusy_logger"),
|
||||
("usage-based-routing", "lowesttpm_logger"),
|
||||
("usage-based-routing-v2", "lowesttpm_logger_v2"),
|
||||
("latency-based-routing", "lowestlatency_logger"),
|
||||
("cost-based-routing", "lowestcost_logger"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_discard_releases_strategy_selector_across_router_lifecycles(
|
||||
routing_strategy: RoutingStrategyName, selector_attr: str
|
||||
):
|
||||
def retire() -> weakref.ReferenceType[object]:
|
||||
router: Final = Router(model_list=[], routing_strategy=routing_strategy)
|
||||
selector: Final = getattr(router, selector_attr)
|
||||
assert any(callback is selector for callback in litellm.callbacks)
|
||||
router.discard()
|
||||
return weakref.ref(selector)
|
||||
|
||||
selector_refs: Final = tuple(retire() for _ in range(3))
|
||||
await asyncio.sleep(0)
|
||||
gc.collect()
|
||||
|
||||
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
|
||||
assert all(selector_ref() is None for selector_ref in selector_refs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"routing_strategy, selector_attr",
|
||||
[
|
||||
("least-busy", "leastbusy_logger"),
|
||||
("usage-based-routing", "lowesttpm_logger"),
|
||||
("usage-based-routing-v2", "lowesttpm_logger_v2"),
|
||||
("latency-based-routing", "lowestlatency_logger"),
|
||||
("cost-based-routing", "lowestcost_logger"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_discard_purges_selector_from_event_lists_after_serving_a_request(
|
||||
routing_strategy: RoutingStrategyName, selector_attr: str
|
||||
):
|
||||
router: Final = Router(
|
||||
model_list=[{"model_name": "gpt-5.5", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}],
|
||||
routing_strategy=routing_strategy,
|
||||
)
|
||||
selector: Final = getattr(router, selector_attr)
|
||||
await router.acompletion(model="gpt-5.5", messages=[{"role": "user", "content": "hi"}], mock_response="ok")
|
||||
assert any(callback is selector for callback in litellm.success_callback)
|
||||
|
||||
router.discard()
|
||||
|
||||
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
|
||||
|
||||
|
||||
def test_discard_keeps_callbacks_owned_by_other_live_routers():
|
||||
survivor: Final = Router(model_list=[], routing_strategy="latency-based-routing")
|
||||
unrelated: Final = CustomLogger()
|
||||
litellm.logging_callback_manager.add_litellm_callback(unrelated)
|
||||
before_ids: Final = {list_name: _callback_ids(list_name) for list_name in _CALLBACK_LIST_NAMES}
|
||||
retired: Final = Router(model_list=[], routing_strategy="least-busy")
|
||||
assert any(callback is retired.leastbusy_logger for callback in litellm.callbacks)
|
||||
|
||||
retired.discard()
|
||||
|
||||
assert {list_name: _callback_ids(list_name) for list_name in before_ids} == before_ids
|
||||
assert any(callback is survivor.lowestlatency_logger for callback in litellm.callbacks)
|
||||
survivor.discard()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"routing_strategy, selector_attr",
|
||||
[
|
||||
("usage-based-routing", "lowesttpm_logger"),
|
||||
("usage-based-routing-v2", "lowesttpm_logger_v2"),
|
||||
("latency-based-routing", "lowestlatency_logger"),
|
||||
("cost-based-routing", "lowestcost_logger"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_discard_leaves_same_strategy_survivor_receiving_events(
|
||||
routing_strategy: RoutingStrategyName, selector_attr: str
|
||||
):
|
||||
model_list: Final[list[DeploymentTypedDict]] = [
|
||||
{"model_name": "gpt-5.5", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}
|
||||
]
|
||||
retired: Final = Router(model_list=model_list, routing_strategy=routing_strategy)
|
||||
survivor: Final = Router(model_list=model_list, routing_strategy=routing_strategy)
|
||||
|
||||
retired.discard()
|
||||
await survivor.acompletion(model="gpt-5.5", messages=[{"role": "user", "content": "hi"}], mock_response="ok")
|
||||
|
||||
assert any(callback is getattr(survivor, selector_attr) for callback in litellm.success_callback)
|
||||
survivor.discard()
|
||||
|
||||
|
||||
def test_discard_drops_routing_group_selectors():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "grouped-model", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}
|
||||
],
|
||||
routing_groups=[
|
||||
{"group_name": "fast", "models": ["grouped-model"], "routing_strategy": "latency-based-routing"}
|
||||
],
|
||||
)
|
||||
_, group_selector = router._get_routing_context("grouped-model")
|
||||
assert any(callback is group_selector for callback in litellm.callbacks)
|
||||
|
||||
router.discard()
|
||||
|
||||
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
|
||||
|
||||
|
||||
_GROUPED_MODEL_LIST: Final[list[DeploymentTypedDict]] = [
|
||||
{"model_name": "grouped-model", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}
|
||||
]
|
||||
|
||||
|
||||
class _LabelledHook(CustomLogger):
|
||||
def __init__(self, label: str) -> None:
|
||||
super().__init__()
|
||||
self.label = label
|
||||
|
||||
|
||||
def _visit_hooks_reloading_routing_groups_mid_loop(
|
||||
router: Router, hooks: tuple[_LabelledHook, ...]
|
||||
) -> Iterator[CustomLogger]:
|
||||
for callback in litellm.callbacks:
|
||||
if callback is hooks[0]:
|
||||
router.update_settings(
|
||||
routing_groups=[{"group_name": "fast", "models": ["grouped-model"], "routing_strategy": "least-busy"}]
|
||||
)
|
||||
if callback in hooks:
|
||||
yield callback
|
||||
|
||||
|
||||
def test_routing_group_reload_keeps_every_hook_of_an_in_flight_callback_loop():
|
||||
router: Final = Router(
|
||||
model_list=_GROUPED_MODEL_LIST,
|
||||
routing_groups=[{"group_name": "fast", "models": ["grouped-model"], "routing_strategy": "least-busy"}],
|
||||
)
|
||||
hooks: Final = tuple(_LabelledHook(label) for label in ("guardrail_a", "guardrail_b", "guardrail_c"))
|
||||
for hook in hooks:
|
||||
litellm.logging_callback_manager.add_litellm_callback(hook)
|
||||
|
||||
visited: Final = tuple(_visit_hooks_reloading_routing_groups_mid_loop(router, hooks))
|
||||
|
||||
assert visited == hooks
|
||||
router.discard()
|
||||
|
||||
|
||||
def _routers_seen_in_success_loop_discarding_first(first: Router, second: Router) -> Iterator[Router]:
|
||||
for callback in litellm.success_callback:
|
||||
if not isinstance(callback, MethodType):
|
||||
continue
|
||||
if callback.__self__ is first:
|
||||
first.discard()
|
||||
yield first
|
||||
if callback.__self__ is second:
|
||||
yield second
|
||||
|
||||
|
||||
def test_discard_mid_success_loop_keeps_the_next_router_callback():
|
||||
first: Final = Router(model_list=_GROUPED_MODEL_LIST)
|
||||
second: Final = Router(model_list=_GROUPED_MODEL_LIST)
|
||||
|
||||
seen: Final = tuple(_routers_seen_in_success_loop_discarding_first(first, second))
|
||||
|
||||
assert seen == (first, second)
|
||||
second.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discard_purges_optional_pre_call_checks_from_event_lists_after_serving_a_request():
|
||||
router: Final = Router(
|
||||
model_list=_GROUPED_MODEL_LIST,
|
||||
optional_pre_call_checks=["enforce_model_rate_limits", "prompt_caching", "deployment_affinity"],
|
||||
)
|
||||
await router.acompletion(model="grouped-model", messages=[{"role": "user", "content": "hi"}], mock_response="ok")
|
||||
|
||||
router.discard()
|
||||
|
||||
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
|
||||
|
||||
|
||||
def test_discard_registers_same_config_survivor_optional_pre_call_checks():
|
||||
retired: Final = Router(model_list=_GROUPED_MODEL_LIST, optional_pre_call_checks=["enforce_model_rate_limits"])
|
||||
survivor: Final = Router(model_list=_GROUPED_MODEL_LIST, optional_pre_call_checks=["enforce_model_rate_limits"])
|
||||
|
||||
retired.discard()
|
||||
|
||||
assert survivor.optional_callbacks
|
||||
assert all(_is_registered(check) for check in survivor.optional_callbacks)
|
||||
survivor.discard()
|
||||
|
||||
|
||||
def _is_registered(callback: object) -> bool:
|
||||
return any(registered is callback for registered in litellm.callbacks)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discard_releases_per_request_override_selector():
|
||||
def build_override_then_discard() -> weakref.ReferenceType[object]:
|
||||
router: Final = Router(model_list=_GROUPED_MODEL_LIST)
|
||||
selector: Final = router._get_override_strategy_selector("usage-based-routing-v2")
|
||||
assert selector is not None
|
||||
router.discard()
|
||||
return weakref.ref(selector)
|
||||
|
||||
selector_ref: Final = build_override_then_discard()
|
||||
await asyncio.sleep(0)
|
||||
gc.collect()
|
||||
|
||||
assert selector_ref() is None
|
||||
|
||||
|
||||
|
||||
def test_discard_registers_the_selector_of_a_same_strategy_survivor_built_without_a_model_list():
|
||||
retired: Final = Router(model_list=[], routing_strategy="latency-based-routing")
|
||||
survivor: Final = Router(routing_strategy="latency-based-routing")
|
||||
assert not _is_registered(survivor.lowestlatency_logger)
|
||||
|
||||
retired.discard()
|
||||
|
||||
assert _is_registered(survivor.lowestlatency_logger)
|
||||
survivor.discard()
|
||||
|
||||
|
||||
def test_discard_drops_its_slack_alerting_callbacks():
|
||||
router: Final = Router(model_list=_GROUPED_MODEL_LIST, alerting_config=AlertingConfig(webhook_url="test"))
|
||||
assert any(isinstance(callback, SlackAlerting) for callback in litellm.callbacks)
|
||||
|
||||
router.discard()
|
||||
|
||||
assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES)
|
||||
|
||||
|
||||
def _adaptive_router_model_list(name: str) -> list[DeploymentTypedDict]:
|
||||
return [
|
||||
{"model_name": "gpt-5.5", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}},
|
||||
{
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": "auto_router/adaptive_router",
|
||||
"adaptive_router_config": {"available_models": ["gpt-5.5"]},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _registered_adaptive_router_strategies() -> tuple[object, ...]:
|
||||
return tuple(
|
||||
callback.adaptive_router for callback in litellm.callbacks if isinstance(callback, AdaptiveRouterPostCallHook)
|
||||
)
|
||||
|
||||
|
||||
def test_discard_drops_its_adaptive_router_hook():
|
||||
router: Final = Router(model_list=_adaptive_router_model_list("adaptive"), ignore_invalid_deployments=True)
|
||||
assert _registered_adaptive_router_strategies()
|
||||
|
||||
router.discard()
|
||||
|
||||
assert _registered_adaptive_router_strategies() == ()
|
||||
|
||||
|
||||
def test_discard_registers_the_adaptive_router_hook_of_a_survivor():
|
||||
survivor: Final = Router(model_list=_adaptive_router_model_list("adaptive"), ignore_invalid_deployments=True)
|
||||
retired: Final = Router(model_list=_adaptive_router_model_list("adaptive"), ignore_invalid_deployments=True)
|
||||
|
||||
retired.discard()
|
||||
|
||||
survivor_strategies: Final = tuple(tagged.strategy for tagged in survivor.adaptive_routers["adaptive"])
|
||||
assert _registered_adaptive_router_strategies() == survivor_strategies
|
||||
survivor.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discard_releases_its_router_budget_limiter():
|
||||
def build_budget_limiter_then_discard() -> weakref.ReferenceType[object]:
|
||||
router: Final = Router(
|
||||
model_list=_GROUPED_MODEL_LIST,
|
||||
provider_budget_config={"openai": {"budget_limit": 10.0, "time_period": "1d"}},
|
||||
)
|
||||
budget_limiter: Final = router.get_router_deployment_budget_limiter()
|
||||
assert budget_limiter is not None
|
||||
router.discard()
|
||||
return weakref.ref(budget_limiter)
|
||||
|
||||
budget_limiter_ref: Final = build_budget_limiter_then_discard()
|
||||
await asyncio.sleep(0)
|
||||
gc.collect()
|
||||
|
||||
assert budget_limiter_ref() is None
|
||||
|
||||
def test_initialize_assistants_endpoint(model_list):
|
||||
"""Test if the 'initialize_assistants_endpoint' function is working correctly"""
|
||||
router = Router(model_list=model_list)
|
||||
|
|
|
|||
|
|
@ -3891,6 +3891,114 @@ class TestCallbackAsyncSyncSeparation:
|
|||
assert my_sync_cb not in litellm._async_input_callback
|
||||
|
||||
|
||||
class _DropsACallbackFromAnotherThreadWhenComparedToDynamodb:
|
||||
def __init__(self, dropped: Callable[..., object]) -> None:
|
||||
self.dropped = dropped
|
||||
self.drop_thread: threading.Thread | None = None
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if isinstance(other, str) and other == "dynamodb" and self.drop_thread is None:
|
||||
self.drop_thread = threading.Thread(
|
||||
target=litellm.logging_callback_manager.drop_callbacks, args=((self.dropped,),)
|
||||
)
|
||||
self.drop_thread.start()
|
||||
self.drop_thread.join(timeout=0.2)
|
||||
return self is other
|
||||
|
||||
__hash__ = object.__hash__
|
||||
|
||||
|
||||
def test_function_setup_moves_async_callbacks_while_another_thread_drops_a_callback(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
def dropped_callback(*args: object, **kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
async def async_callback(*args: object, **kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
drops_on_compare: Final = _DropsACallbackFromAnotherThreadWhenComparedToDynamodb(dropped_callback)
|
||||
monkeypatch.setattr(litellm.utils, "callback_list", ["initialized"])
|
||||
for list_name in (
|
||||
"callbacks",
|
||||
"input_callback",
|
||||
"failure_callback",
|
||||
"service_callback",
|
||||
"_async_input_callback",
|
||||
"_async_success_callback",
|
||||
"_async_failure_callback",
|
||||
):
|
||||
monkeypatch.setattr(litellm, list_name, [])
|
||||
monkeypatch.setattr(litellm, "success_callback", [dropped_callback, drops_on_compare, async_callback])
|
||||
|
||||
litellm.utils.function_setup(
|
||||
"completion",
|
||||
litellm.utils.Rules(),
|
||||
datetime.now(),
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
litellm_call_id="callback-drop-race",
|
||||
)
|
||||
assert drops_on_compare.drop_thread is not None
|
||||
drops_on_compare.drop_thread.join()
|
||||
|
||||
assert litellm.success_callback == [drops_on_compare]
|
||||
assert litellm._async_success_callback == [async_callback]
|
||||
|
||||
|
||||
class _DropsACallbackFromAnotherThreadWhenComparedToSentry:
|
||||
def __init__(self, dropped: Callable[..., object]) -> None:
|
||||
self.dropped = dropped
|
||||
self.drop_thread: threading.Thread | None = None
|
||||
self.drop_finished_during_logger_setup = False
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if isinstance(other, str) and other == "sentry" and self.drop_thread is None:
|
||||
self.drop_thread = threading.Thread(
|
||||
target=litellm.logging_callback_manager.drop_callbacks, args=((self.dropped,),)
|
||||
)
|
||||
self.drop_thread.start()
|
||||
self.drop_thread.join(timeout=5)
|
||||
self.drop_finished_during_logger_setup = not self.drop_thread.is_alive()
|
||||
return self is other
|
||||
|
||||
__hash__ = object.__hash__
|
||||
|
||||
|
||||
def test_function_setup_lets_other_threads_drop_callbacks_while_it_sets_up_loggers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
def dropped_callback(*args: object, **kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
drops_on_compare: Final = _DropsACallbackFromAnotherThreadWhenComparedToSentry(dropped_callback)
|
||||
monkeypatch.setattr(litellm.utils, "callback_list", [])
|
||||
for list_name in (
|
||||
"callbacks",
|
||||
"input_callback",
|
||||
"failure_callback",
|
||||
"service_callback",
|
||||
"_async_input_callback",
|
||||
"_async_success_callback",
|
||||
"_async_failure_callback",
|
||||
):
|
||||
monkeypatch.setattr(litellm, list_name, [])
|
||||
monkeypatch.setattr(litellm, "success_callback", [drops_on_compare])
|
||||
|
||||
litellm.utils.function_setup(
|
||||
"completion",
|
||||
litellm.utils.Rules(),
|
||||
datetime.now(),
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
litellm_call_id="logger-setup-outside-lock",
|
||||
)
|
||||
assert drops_on_compare.drop_thread is not None
|
||||
drops_on_compare.drop_thread.join()
|
||||
|
||||
assert drops_on_compare.drop_finished_during_logger_setup
|
||||
|
||||
|
||||
class TestMetadataNoneHandling:
|
||||
"""
|
||||
Test that metadata=None in kwargs doesn't cause TypeError.
|
||||
|
|
|
|||
|
|
@ -263,6 +263,7 @@ INTERNAL_STATE_NAMES: Final = (
|
|||
"litellm_trusted_callback_vars",
|
||||
"_litellm_addressed_response_id",
|
||||
"_litellm_strip_stream_usage",
|
||||
"_litellm_stream_usage_injected",
|
||||
"client_side_timeout",
|
||||
"model_file_id_mapping",
|
||||
"acompletion",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue