From f411443e585245118a5411e6efcc7b18e51bce76 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Fri, 19 Apr 2024 16:09:44 -0500 Subject: [PATCH] fix: Stream completion responses from anthropic. (Fix 3129) --- litellm/llms/anthropic.py | 3 ++- litellm/llms/custom_httpx/http_handler.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index d836ed8db54..24d889b0f46 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -258,8 +258,9 @@ class AnthropicChatCompletion(BaseLLM): self.async_handler = AsyncHTTPHandler( timeout=httpx.Timeout(timeout=600.0, connect=5.0) ) + data["stream"] = True response = await self.async_handler.post( - api_base, headers=headers, data=json.dumps(data) + api_base, headers=headers, data=json.dumps(data), stream=True ) if response.status_code != 200: diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index dd03e7dbecd..3ab85772362 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -41,13 +41,16 @@ class AsyncHTTPHandler: data: Optional[Union[dict, str]] = None, # type: ignore params: Optional[dict] = None, headers: Optional[dict] = None, + stream: Optional[bool] = False ): - response = await self.client.post( + req = self.client.build_request( + "POST", url, data=data, # type: ignore params=params, - headers=headers, + headers=headers ) + response = await self.client.send(req, stream=stream) return response def __del__(self) -> None: