scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-04 02:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 122

add: shared_session support to responses API (#16260)

Browse source 
This change enables HTTP client session reuse by:
- Adding shared_session parameter to all responses API methods (responses, delete_responses, get_responses, list_input_items, cancel_responses)
- Passing shared_session to get_async_httpx_client() for connection pooling
- Adding debug logging to track shared session usage

This helps reduce memory overhead by reusing HTTP connections instead of creating new clients for each request, which is particularly important for high-throughput proxy scenarios.
This commit is contained in:
Alexsander Hamir 2025-11-04 18:53:54 -08:00 • committed by GitHub
parent b564ed81f5
commit b4def8899f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 40 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -1945,6 +1945,7 @@ class BaseLLMHTTPHandler:
_is_async: bool = False,
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
shared_session: Optional["ClientSession"] = None,
) -> Union[
ResponsesAPIResponse,
BaseResponsesAPIStreamingIterator,
@ -1973,6 +1974,7 @@ class BaseLLMHTTPHandler:
client=client if isinstance(client, AsyncHTTPHandler) else None,
fake_stream=fake_stream,
litellm_metadata=litellm_metadata,
shared_session=shared_session,
)
if client is None or not isinstance(client, HTTPHandler):
@ -2090,15 +2092,20 @@ class BaseLLMHTTPHandler:
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
shared_session: Optional["ClientSession"] = None,
) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]:
"""
Async version of the responses API handler.
Uses async HTTP client to make requests.
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
verbose_logger.debug(
f"Creating HTTP client for responses API with shared_session: {id(shared_session) if shared_session else None}"
)
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
shared_session=shared_session,
)
else:
async_httpx_client = client
@ -2211,15 +2218,20 @@ class BaseLLMHTTPHandler:
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
_is_async: bool = False,
shared_session: Optional["ClientSession"] = None,
) -> DeleteResponseResult:
"""
Async version of the delete response API handler.
Uses async HTTP client to make requests.
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
verbose_logger.debug(
f"Creating HTTP client for delete_response with shared_session: {id(shared_session) if shared_session else None}"
)
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
shared_session=shared_session,
)
else:
async_httpx_client = client
@ -2282,6 +2294,7 @@ class BaseLLMHTTPHandler:
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
_is_async: bool = False,
shared_session: Optional["ClientSession"] = None,
) -> Union[DeleteResponseResult, Coroutine[Any, Any, DeleteResponseResult]]:
"""
Async version of the responses API handler.
@ -2298,6 +2311,7 @@ class BaseLLMHTTPHandler:
extra_body=extra_body,
timeout=timeout,
client=client,
shared_session=shared_session,
)
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client(
@ -2364,6 +2378,7 @@ class BaseLLMHTTPHandler:
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
_is_async: bool = False,
shared_session: Optional["ClientSession"] = None,
) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]:
"""
Get a response by ID
@ -2380,6 +2395,7 @@ class BaseLLMHTTPHandler:
extra_body=extra_body,
timeout=timeout,
client=client,
shared_session=shared_session,
)
if client is None or not isinstance(client, HTTPHandler):
@ -2443,14 +2459,19 @@ class BaseLLMHTTPHandler:
extra_body: Optional[Dict[str, Any]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
shared_session: Optional["ClientSession"] = None,
) -> ResponsesAPIResponse:
"""
Async version of get_responses
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
verbose_logger.debug(
f"Creating HTTP client for get_responses with shared_session: {id(shared_session) if shared_session else None}"
)
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
shared_session=shared_session,
)
else:
async_httpx_client = client
@ -2521,6 +2542,7 @@ class BaseLLMHTTPHandler:
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
_is_async: bool = False,
shared_session: Optional["ClientSession"] = None,
) -> Union[Dict, Coroutine[Any, Any, Dict]]:
if _is_async:
return self.async_list_responses_input_items(
@ -2537,6 +2559,7 @@ class BaseLLMHTTPHandler:
extra_headers=extra_headers,
timeout=timeout,
client=client,
shared_session=shared_session,
)
if client is None or not isinstance(client, HTTPHandler):
@ -2605,11 +2628,16 @@ class BaseLLMHTTPHandler:
extra_headers: Optional[Dict[str, Any]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
shared_session: Optional["ClientSession"] = None,
) -> Dict:
if client is None or not isinstance(client, AsyncHTTPHandler):
verbose_logger.debug(
f"Creating HTTP client for list_input_items with shared_session: {id(shared_session) if shared_session else None}"
)
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
shared_session=shared_session,
)
else:
async_httpx_client = client
@ -3296,6 +3324,7 @@ class BaseLLMHTTPHandler:
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
_is_async: bool = False,
shared_session: Optional["ClientSession"] = None,
) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]:
"""
Async version of the responses API handler.
@ -3312,6 +3341,7 @@ class BaseLLMHTTPHandler:
extra_body=extra_body,
timeout=timeout,
client=client,
shared_session=shared_session,
)
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client(
@ -3378,15 +3408,20 @@ class BaseLLMHTTPHandler:
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
_is_async: bool = False,
shared_session: Optional["ClientSession"] = None,
) -> ResponsesAPIResponse:
"""
Async version of the cancel response API handler.
Uses async HTTP client to make requests.
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
verbose_logger.debug(
f"Creating HTTP client for cancel_response with shared_session: {id(shared_session) if shared_session else None}"
)
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
shared_session=shared_session,
)
else:
async_httpx_client = client

5
litellm/responses/main.py
View file

@ -648,6 +648,7 @@ def responses(
model=model, stream=stream, custom_llm_provider=custom_llm_provider
),
litellm_metadata=kwargs.get("litellm_metadata", {}),
shared_session=kwargs.get("shared_session"),
)
# Update the responses_api_response_id with the model_id
@ -815,6 +816,7 @@ def delete_responses(
timeout=timeout or request_timeout,
_is_async=_is_async,
client=kwargs.get("client"),
shared_session=kwargs.get("shared_session"),
)
return response
@ -994,6 +996,7 @@ def get_responses(
timeout=timeout or request_timeout,
_is_async=_is_async,
client=kwargs.get("client"),
shared_session=kwargs.get("shared_session"),
)
# Update the responses_api_response_id with the model_id
@ -1148,6 +1151,7 @@ def list_input_items(
timeout=timeout or request_timeout,
_is_async=_is_async,
client=kwargs.get("client"),
shared_session=kwargs.get("shared_session"),
)
return response
@ -1307,6 +1311,7 @@ def cancel_responses(
timeout=timeout or request_timeout,
_is_async=_is_async,
client=kwargs.get("client"),
shared_session=kwargs.get("shared_session"),
)
return response