From b4def8899f2f41ffb6cf272f3030c31dd93975d6 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Tue, 4 Nov 2025 18:53:54 -0800 Subject: [PATCH] add: shared_session support to responses API (#16260) This change enables HTTP client session reuse by: - Adding shared_session parameter to all responses API methods (responses, delete_responses, get_responses, list_input_items, cancel_responses) - Passing shared_session to get_async_httpx_client() for connection pooling - Adding debug logging to track shared session usage This helps reduce memory overhead by reusing HTTP connections instead of creating new clients for each request, which is particularly important for high-throughput proxy scenarios. --- litellm/llms/custom_httpx/llm_http_handler.py | 35 +++++++++++++++++++ litellm/responses/main.py | 5 +++ 2 files changed, 40 insertions(+) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index d2aab6adf17..dd39c29203a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1945,6 +1945,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, fake_stream: bool = False, litellm_metadata: Optional[Dict[str, Any]] = None, + shared_session: Optional["ClientSession"] = None, ) -> Union[ ResponsesAPIResponse, BaseResponsesAPIStreamingIterator, @@ -1973,6 +1974,7 @@ class BaseLLMHTTPHandler: client=client if isinstance(client, AsyncHTTPHandler) else None, fake_stream=fake_stream, litellm_metadata=litellm_metadata, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): @@ -2090,15 +2092,20 @@ class BaseLLMHTTPHandler: client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, litellm_metadata: Optional[Dict[str, Any]] = None, + shared_session: Optional["ClientSession"] = None, ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: """ Async version of the responses API handler. Uses async HTTP client to make requests. """ if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for responses API with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -2211,15 +2218,20 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> DeleteResponseResult: """ Async version of the delete response API handler. Uses async HTTP client to make requests. """ if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for delete_response with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -2282,6 +2294,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> Union[DeleteResponseResult, Coroutine[Any, Any, DeleteResponseResult]]: """ Async version of the responses API handler. @@ -2298,6 +2311,7 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): sync_httpx_client = _get_httpx_client( @@ -2364,6 +2378,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: """ Get a response by ID @@ -2380,6 +2395,7 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): @@ -2443,14 +2459,19 @@ class BaseLLMHTTPHandler: extra_body: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + shared_session: Optional["ClientSession"] = None, ) -> ResponsesAPIResponse: """ Async version of get_responses """ if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for get_responses with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -2521,6 +2542,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> Union[Dict, Coroutine[Any, Any, Dict]]: if _is_async: return self.async_list_responses_input_items( @@ -2537,6 +2559,7 @@ class BaseLLMHTTPHandler: extra_headers=extra_headers, timeout=timeout, client=client, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): @@ -2605,11 +2628,16 @@ class BaseLLMHTTPHandler: extra_headers: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + shared_session: Optional["ClientSession"] = None, ) -> Dict: if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for list_input_items with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -3296,6 +3324,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: """ Async version of the responses API handler. @@ -3312,6 +3341,7 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): sync_httpx_client = _get_httpx_client( @@ -3378,15 +3408,20 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> ResponsesAPIResponse: """ Async version of the cancel response API handler. Uses async HTTP client to make requests. """ if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for cancel_response with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client diff --git a/litellm/responses/main.py b/litellm/responses/main.py index c297a87077b..6450f550402 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -648,6 +648,7 @@ def responses( model=model, stream=stream, custom_llm_provider=custom_llm_provider ), litellm_metadata=kwargs.get("litellm_metadata", {}), + shared_session=kwargs.get("shared_session"), ) # Update the responses_api_response_id with the model_id @@ -815,6 +816,7 @@ def delete_responses( timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), + shared_session=kwargs.get("shared_session"), ) return response @@ -994,6 +996,7 @@ def get_responses( timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), + shared_session=kwargs.get("shared_session"), ) # Update the responses_api_response_id with the model_id @@ -1148,6 +1151,7 @@ def list_input_items( timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), + shared_session=kwargs.get("shared_session"), ) return response @@ -1307,6 +1311,7 @@ def cancel_responses( timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), + shared_session=kwargs.get("shared_session"), ) return response