scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

add async_pre_routing_hook

Browse source
This commit is contained in:
Ishaan Jaff 2025-07-24 13:40:48 -07:00
parent e3e38bdcc8
commit 33e1b3b651
2 changed files with 56 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
litellm/integrations/custom_logger.py
View file

@ -125,6 +125,21 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
Allows usage-based-routing-v2 to run pre-call rpm checks within the picked deployment's semaphore (concurrency-safe tpm/rpm checks).
"""
async def async_pre_routing_hook(
self,
model: str,
request_kwargs: Dict,
messages: Optional[List[Dict[str, str]]] = None,
input: Optional[Union[str, List]] = None,
specific_deployment: Optional[bool] = False,
):
"""
This hook is called before the routing decision is made.
Used for the litellm auto-router to modify the request before the routing decision is made.
"""
pass
async def async_filter_deployments(
self,
model: str,

41
litellm/router.py
View file

@ -6462,6 +6462,19 @@ class Router:
)
try:
parent_otel_span = _get_parent_otel_span_from_kwargs(request_kwargs)
#########################################################
# Execute Pre-Routing Hooks
#########################################################
await self.async_pre_routing_hook(
model=model,
request_kwargs=request_kwargs,
messages=messages,
input=input,
specific_deployment=specific_deployment,
)
healthy_deployments = await self.async_get_healthy_deployments(
model=model,
request_kwargs=request_kwargs,
@ -6572,6 +6585,34 @@ class Router:
)
raise e
async def async_pre_routing_hook(
self,
model: str,
request_kwargs: Dict,
messages: Optional[List[Dict[str, str]]] = None,
input: Optional[Union[str, List]] = None,
specific_deployment: Optional[bool] = False,
):
"""
This hook is called before the routing decision is made.
Used for the litellm auto-router to modify the request before the routing decision is made.
"""
custom_logger_callbacks: List[CustomLogger] = litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=CustomLogger,
)
for callback in custom_logger_callbacks:
await callback.async_pre_routing_hook(
model=model,
request_kwargs=request_kwargs,
messages=messages,
input=input,
specific_deployment=specific_deployment,
)
def get_available_deployment(
self,
model: str,