From 33e1b3b65196bcf61e9de22bb5051d6326b12dae Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 24 Jul 2025 13:40:48 -0700 Subject: [PATCH] add async_pre_routing_hook --- litellm/integrations/custom_logger.py | 15 ++++++++++ litellm/router.py | 41 +++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index a73dc6791e2..35e1193275b 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -125,6 +125,21 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac Allows usage-based-routing-v2 to run pre-call rpm checks within the picked deployment's semaphore (concurrency-safe tpm/rpm checks). """ + async def async_pre_routing_hook( + self, + model: str, + request_kwargs: Dict, + messages: Optional[List[Dict[str, str]]] = None, + input: Optional[Union[str, List]] = None, + specific_deployment: Optional[bool] = False, + ): + """ + This hook is called before the routing decision is made. + + Used for the litellm auto-router to modify the request before the routing decision is made. + """ + pass + async def async_filter_deployments( self, model: str, diff --git a/litellm/router.py b/litellm/router.py index 8eeb6e24170..89c2b166abd 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6462,6 +6462,19 @@ class Router: ) try: parent_otel_span = _get_parent_otel_span_from_kwargs(request_kwargs) + + ######################################################### + # Execute Pre-Routing Hooks + ######################################################### + await self.async_pre_routing_hook( + model=model, + request_kwargs=request_kwargs, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) + + healthy_deployments = await self.async_get_healthy_deployments( model=model, request_kwargs=request_kwargs, @@ -6572,6 +6585,34 @@ class Router: ) raise e + async def async_pre_routing_hook( + self, + model: str, + request_kwargs: Dict, + messages: Optional[List[Dict[str, str]]] = None, + input: Optional[Union[str, List]] = None, + specific_deployment: Optional[bool] = False, + ): + """ + This hook is called before the routing decision is made. + + Used for the litellm auto-router to modify the request before the routing decision is made. + """ + custom_logger_callbacks: List[CustomLogger] = litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=CustomLogger, + ) + + for callback in custom_logger_callbacks: + await callback.async_pre_routing_hook( + model=model, + request_kwargs=request_kwargs, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) + + + def get_available_deployment( self, model: str,