From 1f9a7e0a3fc627fb2bfeb2440a9fca071ab02d19 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 2 Jan 2025 17:12:07 -0800 Subject: [PATCH] use global client --- litellm/llms/openai_like/chat/handler.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/litellm/llms/openai_like/chat/handler.py b/litellm/llms/openai_like/chat/handler.py index 11bf632b76b..db3941985aa 100644 --- a/litellm/llms/openai_like/chat/handler.py +++ b/litellm/llms/openai_like/chat/handler.py @@ -23,6 +23,8 @@ from litellm.utils import CustomStreamWrapper, ProviderConfigManager from ..common_utils import OpenAILikeBase, OpenAILikeError from .transformation import OpenAILikeChatConfig +global_aiohttp_client = None + async def make_call( client: Optional[AsyncHTTPHandler], @@ -175,16 +177,12 @@ class OpenAILikeChatHandler(OpenAILikeBase): timeout: Optional[Union[float, aiohttp.ClientTimeout]] = None, json_mode: bool = False, ) -> ModelResponse: - if timeout is None: - timeout = aiohttp.ClientTimeout(total=600.0, connect=5.0) - - should_close_client = False - if client is None: - client = aiohttp.ClientSession(timeout=timeout) - should_close_client = True - + global global_aiohttp_client + if global_aiohttp_client is None: + global_aiohttp_client = aiohttp.ClientSession() try: - async with client.post( + + async with global_aiohttp_client.post( api_base, headers=headers, data=json.dumps(data) ) as response: response.raise_for_status() @@ -216,9 +214,6 @@ class OpenAILikeChatHandler(OpenAILikeBase): ) except Exception as e: raise OpenAILikeError(status_code=500, message=str(e)) - finally: - if should_close_client: - await client.close() def completion( self,