From 57998c28dc9a3af079e6efa1b15f2f6348a7afd9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 14:37:29 -0800 Subject: [PATCH 1/4] fix(proxy_server.py): drop none values in streaming response --- litellm/proxy/proxy_server.py | 5 ++--- litellm/utils.py | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 52f2783a82f..bc6f8ccc5a2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2115,10 +2115,9 @@ async def async_data_generator(response, user_api_key_dict): try: start_time = time.time() async for chunk in response: - verbose_proxy_logger.debug(f"returned chunk: {chunk}") - assert isinstance(chunk, litellm.ModelResponse) + chunk = chunk.model_dump_json(exclude_none=True) try: - yield f"data: {json.dumps(chunk.model_dump(exclude_none=True))}\n\n" + yield f"data: {chunk}\n\n" except Exception as e: yield f"data: {str(e)}\n\n" diff --git a/litellm/utils.py b/litellm/utils.py index 6bc229e96ee..d8986be6bb2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -205,18 +205,18 @@ def map_finish_reason( class FunctionCall(OpenAIObject): arguments: str - name: str + name: Optional[str] = None class Function(OpenAIObject): arguments: str - name: str + name: Optional[str] = None class ChatCompletionDeltaToolCall(OpenAIObject): - id: str + id: Optional[str] = None function: Function - type: str + type: Optional[str] = None index: int @@ -275,8 +275,11 @@ class Delta(OpenAIObject): super(Delta, self).__init__(**params) self.content = content self.role = role - self.function_call = function_call - if tool_calls is not None and isinstance(tool_calls, dict): + if function_call is not None and isinstance(function_call, dict): + self.function_call = FunctionCall(**function_call) + else: + self.function_call = function_call + if tool_calls is not None and isinstance(tool_calls, list): self.tool_calls = [] for tool_call in tool_calls: if tool_call.get("index", None) is None: @@ -8727,7 +8730,7 @@ class CustomStreamWrapper: or original_chunk.choices[0].delta.tool_calls is not None ): try: - delta = dict(original_chunk.choices[0].delta) + delta = original_chunk.choices[0].delta model_response.system_fingerprint = ( original_chunk.system_fingerprint ) @@ -8762,7 +8765,9 @@ class CustomStreamWrapper: is None ): t.function.arguments = "" - model_response.choices[0].delta = Delta(**delta) + _json_delta = delta.model_dump() + print_verbose(f"_json_delta: {_json_delta}") + model_response.choices[0].delta = Delta(**_json_delta) except Exception as e: traceback.print_exc() model_response.choices[0].delta = Delta() From f3a6225f6110321a919bfc22d3f1f33a8a594593 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 14:38:40 -0800 Subject: [PATCH 2/4] =?UTF-8?q?bump:=20version=201.27.11=20=E2=86=92=201.2?= =?UTF-8?q?7.12.dev1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e914e69414a..bfe45dbafcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.27.11" +version = "1.27.12.dev1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -74,7 +74,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.27.11" +version = "1.27.12.dev1" version_files = [ "pyproject.toml:^version" ] From 39d8645596c54e5c1495714f56ae2e7f546e1c4f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 14:53:46 -0800 Subject: [PATCH 3/4] test(test_proxy_custom_logger.py): fix format test --- litellm/tests/test_proxy_custom_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_proxy_custom_logger.py b/litellm/tests/test_proxy_custom_logger.py index 34e427ef4d9..64ed08897f1 100644 --- a/litellm/tests/test_proxy_custom_logger.py +++ b/litellm/tests/test_proxy_custom_logger.py @@ -274,7 +274,7 @@ def test_chat_completion_stream(client): print("\n\n decode_data", data) # Access the content of choices[0]['message']['content'] - content = data["choices"][0]["delta"]["content"] or "" + content = data["choices"][0]["delta"].get("content", None) or "" # Process the content as needed print("Content:", content) From 94f4f969943ca8304b09661c86d7086a681fabb0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 14:57:50 -0800 Subject: [PATCH 4/4] fix(utils.py): fix streaming issue --- litellm/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index d8986be6bb2..658ebba6564 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -282,9 +282,12 @@ class Delta(OpenAIObject): if tool_calls is not None and isinstance(tool_calls, list): self.tool_calls = [] for tool_call in tool_calls: - if tool_call.get("index", None) is None: - tool_call["index"] = 0 - self.tool_calls.append(ChatCompletionDeltaToolCall(**tool_call)) + if isinstance(tool_call, dict): + if tool_call.get("index", None) is None: + tool_call["index"] = 0 + self.tool_calls.append(ChatCompletionDeltaToolCall(**tool_call)) + elif isinstance(tool_call, ChatCompletionDeltaToolCall): + self.tool_calls.append(tool_call) else: self.tool_calls = tool_calls