scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

Merge pull request #2225 from BerriAI/litellm_streaming_format_fix

Browse source 
Drop None values from server streaming response
This commit is contained in:
Krish Dholakia 2024-02-27 20:16:15 -08:00 • committed by GitHub
commit 551669ad61
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 21 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/proxy/proxy_server.py
View file

@ -2115,10 +2115,9 @@ async def async_data_generator(response, user_api_key_dict):
try:
start_time = time.time()
async for chunk in response:
verbose_proxy_logger.debug(f"returned chunk: {chunk}")
assert isinstance(chunk, litellm.ModelResponse)
chunk = chunk.model_dump_json(exclude_none=True)
try:
yield f"data: {json.dumps(chunk.model_dump(exclude_none=True))}\n\n"
yield f"data: {chunk}\n\n"
except Exception as e:
yield f"data: {str(e)}\n\n"

30
litellm/utils.py
View file

@ -205,18 +205,18 @@ def map_finish_reason(
class FunctionCall(OpenAIObject):
arguments: str
name: str
name: Optional[str] = None
class Function(OpenAIObject):
arguments: str
name: str
name: Optional[str] = None
class ChatCompletionDeltaToolCall(OpenAIObject):
id: str
id: Optional[str] = None
function: Function
type: str
type: Optional[str] = None
index: int
@ -275,13 +275,19 @@ class Delta(OpenAIObject):
super(Delta, self).__init__(**params)
self.content = content
self.role = role
self.function_call = function_call
if tool_calls is not None and isinstance(tool_calls, dict):
if function_call is not None and isinstance(function_call, dict):
self.function_call = FunctionCall(**function_call)
else:
self.function_call = function_call
if tool_calls is not None and isinstance(tool_calls, list):
self.tool_calls = []
for tool_call in tool_calls:
if tool_call.get("index", None) is None:
tool_call["index"] = 0
self.tool_calls.append(ChatCompletionDeltaToolCall(**tool_call))
if isinstance(tool_call, dict):
if tool_call.get("index", None) is None:
tool_call["index"] = 0
self.tool_calls.append(ChatCompletionDeltaToolCall(**tool_call))
elif isinstance(tool_call, ChatCompletionDeltaToolCall):
self.tool_calls.append(tool_call)
else:
self.tool_calls = tool_calls
@ -8728,7 +8734,7 @@ class CustomStreamWrapper:
or original_chunk.choices[0].delta.tool_calls is not None
):
try:
delta = dict(original_chunk.choices[0].delta)
delta = original_chunk.choices[0].delta
model_response.system_fingerprint = (
original_chunk.system_fingerprint
)
@ -8763,7 +8769,9 @@ class CustomStreamWrapper:
is None
):
t.function.arguments = ""
model_response.choices[0].delta = Delta(**delta)
_json_delta = delta.model_dump()
print_verbose(f"_json_delta: {_json_delta}")
model_response.choices[0].delta = Delta(**_json_delta)
except Exception as e:
traceback.print_exc()
model_response.choices[0].delta = Delta()