diff --git a/litellm/router.py b/litellm/router.py index 47f76d00f9d..ba63813b747 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4067,28 +4067,23 @@ class Router: verbose_router_logger.debug( f"Inside _aspeech()- model: {model}; kwargs: {kwargs}" ) + parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) deployment = await self.async_get_available_deployment( model=model, messages=[{"role": "user", "content": "prompt"}], specific_deployment=kwargs.pop("specific_deployment", None), request_kwargs=kwargs, ) - data = deployment["litellm_params"].copy() - for k, v in self.default_litellm_params.items(): - if ( - k not in kwargs - ): # prioritize model-specific params > default router params - kwargs[k] = v - elif k == "metadata": - kwargs[k].update(v) + self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) + data = deployment["litellm_params"].copy() model_client = self._get_async_openai_model_client( deployment=deployment, kwargs=kwargs, ) self.total_calls[model_name] += 1 - response = await litellm.aspeech( + response = litellm.aspeech( **{ **data, "input": input, @@ -4098,6 +4093,31 @@ class Router: } ) + ### CONCURRENCY-SAFE RPM CHECKS ### + rpm_semaphore = self._get_client( + deployment=deployment, + kwargs=kwargs, + client_type="max_parallel_requests", + ) + + if rpm_semaphore is not None and isinstance( + rpm_semaphore, asyncio.Semaphore + ): + async with rpm_semaphore: + """ + - Check rpm limits before making the call + - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) + """ + await self.async_routing_strategy_pre_call_checks( + deployment=deployment, parent_otel_span=parent_otel_span + ) + response = await response + else: + await self.async_routing_strategy_pre_call_checks( + deployment=deployment, parent_otel_span=parent_otel_span + ) + response = await response + self.success_calls[model_name] += 1 verbose_router_logger.info( f"litellm.aspeech(model={model_name})\033[32m 200 OK\033[0m" diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py index 03a9baa9864..f4c6fbefe05 100644 --- a/tests/router_unit_tests/test_router_endpoints.py +++ b/tests/router_unit_tests/test_router_endpoints.py @@ -262,6 +262,32 @@ async def test_aspeech_success_returns_response(): assert mock_aspeech.call_args.kwargs["model"] == "openai/tts-1" +@pytest.mark.asyncio +async def test_aspeech_sets_deployment_metadata(): + router = Router( + model_list=[ + { + "model_name": "tts", + "litellm_params": {"model": "openai/tts-1", "api_key": "fake-key"}, + }, + ] + ) + + mock_response = MagicMock() + with patch("litellm.aspeech", return_value=mock_response) as mock_aspeech: + response = await router._aspeech( + model="tts", + input="the quick brown fox jumped over the lazy dogs", + voice="alloy", + ) + + assert response is mock_response + metadata = mock_aspeech.call_args.kwargs["metadata"] + assert metadata["deployment"] == "openai/tts-1" + assert metadata["deployment_model_name"] == "tts" + assert metadata["model_info"]["id"] is not None + + @pytest.mark.asyncio() async def test_rerank_endpoint(model_list): from litellm.types.utils import RerankResponse