From 1e3370f3cb53da3e2e9901174019784d98dd551c Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Wed, 8 Jan 2025 19:45:19 -0800 Subject: [PATCH] LiteLLM Minor Fixes & Improvements (01/08/2025) - p2 (#7643) * fix(streaming_chunk_builder_utils.py): add test for groq tool calling + streaming + combine chunks Addresses https://github.com/BerriAI/litellm/issues/7621 * fix(streaming_utils.py): fix modelresponseiterator for openai like chunk parser ensures chunk parser uses the correct tool call id when translating the chunk Fixes https://github.com/BerriAI/litellm/issues/7621 * build(model_hub.tsx): display cost pricing on model hub * build(model_hub.tsx): show cost per token pricing + complete model information * fix(types/utils.py): fix usage object handling --- .../streaming_chunk_builder_utils.py | 4 +- litellm/llms/databricks/streaming_utils.py | 4 +- litellm/proxy/_experimental/out/404.html | 1 - .../proxy/_experimental/out/model_hub.html | 1 - .../proxy/_experimental/out/onboarding.html | 1 - litellm/proxy/_new_secret_config.yaml | 4 - litellm/types/utils.py | 7 + .../test_stream_chunk_builder.py | 171 ++++++++++++++++++ .../src/components/model_hub.tsx | 34 ++-- 9 files changed, 206 insertions(+), 21 deletions(-) delete mode 100644 litellm/proxy/_experimental/out/404.html delete mode 100644 litellm/proxy/_experimental/out/model_hub.html delete mode 100644 litellm/proxy/_experimental/out/onboarding.html diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 7d28c156691..e78b10c2892 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -103,7 +103,8 @@ class ChunkProcessor: def get_combined_tool_content( self, tool_call_chunks: List[Dict[str, Any]] ) -> List[ChatCompletionMessageToolCall]: - argument_list: List = [] + + argument_list: List[str] = [] delta = tool_call_chunks[0]["choices"][0]["delta"] id = None name = None @@ -171,6 +172,7 @@ class ChunkProcessor: ), ) ) + return tool_calls_list def get_combined_function_call_content( diff --git a/litellm/llms/databricks/streaming_utils.py b/litellm/llms/databricks/streaming_utils.py index 8c75145d2b9..0deaa06988a 100644 --- a/litellm/llms/databricks/streaming_utils.py +++ b/litellm/llms/databricks/streaming_utils.py @@ -17,7 +17,7 @@ class ModelResponseIterator: def chunk_parser(self, chunk: dict) -> GenericStreamingChunk: try: - processed_chunk = litellm.ModelResponse(**chunk, stream=True) # type: ignore + processed_chunk = litellm.ModelResponseStream(**chunk) text = "" tool_use: Optional[ChatCompletionToolCallChunk] = None @@ -46,7 +46,7 @@ class ModelResponseIterator: .delta.tool_calls[0] # type: ignore .function.arguments, ), - index=processed_chunk.choices[0].index, + index=processed_chunk.choices[0].delta.tool_calls[0].index, ) if processed_chunk.choices[0].finish_reason is not None: diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html deleted file mode 100644 index 968b0afe96a..00000000000 --- a/litellm/proxy/_experimental/out/404.html +++ /dev/null @@ -1 +0,0 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/model_hub.html b/litellm/proxy/_experimental/out/model_hub.html deleted file mode 100644 index 0efb9b40725..00000000000 --- a/litellm/proxy/_experimental/out/model_hub.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index f8951f2e45d..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 99a3b4d3a1d..670f2506775 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -25,7 +25,3 @@ model_list: identifier: deepseek-ai/DeepSeek-V3-Base revision: main auth_token: os.environ/HUGGINGFACE_API_KEY - - service_callback: ["prometheus_system"] - callbacks: ["prometheus"] - cache: true \ No newline at end of file diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d80b75f73a5..58719687cd6 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -847,6 +847,13 @@ class ModelResponseStream(ModelResponseBase): else: created = created + if ( + "usage" in kwargs + and kwargs["usage"] is not None + and isinstance(kwargs["usage"], dict) + ): + kwargs["usage"] = Usage(**kwargs["usage"]) + kwargs["id"] = id kwargs["created"] = created kwargs["object"] = "chat.completion.chunk" diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py index 45407991860..2c3eb204062 100644 --- a/tests/local_testing/test_stream_chunk_builder.py +++ b/tests/local_testing/test_stream_chunk_builder.py @@ -747,6 +747,125 @@ def test_stream_chunk_builder_empty_initial_chunk(): assert id == "1" +def test_stream_chunk_builder_tool_calls_list(): + from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( + ChunkProcessor, + ) + from litellm.types.utils import ( + ChatCompletionMessageToolCall, + Function, + ModelResponseStream, + Delta, + StreamingChoices, + ChatCompletionDeltaToolCall, + ) + + chunks = [ + ModelResponseStream( + id="chatcmpl-f323f7a5-2da0-4f86-8ed7-c653c5a359d9", + created=1736388417, + model="llama-3.3-70b-versatile", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + content="", + role="assistant", + function_call=None, + tool_calls=[ + ChatCompletionDeltaToolCall( + id="call_9y79", + function=Function( + arguments='{"location": "San Francisco", "unit": "celsius"}', + name="get_current_weather", + ), + type="function", + index=0, + ) + ], + audio=None, + ), + logprobs=None, + ) + ], + stream_options=None, + ), + ModelResponseStream( + id="chatcmpl-f323f7a5-2da0-4f86-8ed7-c653c5a359d9", + created=1736388417, + model="llama-3.3-70b-versatile", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + content="", + role=None, + function_call=None, + tool_calls=[ + ChatCompletionDeltaToolCall( + id="call_pfp7", + function=Function( + arguments='{"location": "Tokyo", "unit": "celsius"}', + name="get_current_weather", + ), + type="function", + index=1, + ) + ], + audio=None, + ), + logprobs=None, + ) + ], + stream_options=None, + ), + ModelResponseStream( + id="chatcmpl-f323f7a5-2da0-4f86-8ed7-c653c5a359d9", + created=1736388417, + model="llama-3.3-70b-versatile", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + content="", + role=None, + function_call=None, + tool_calls=[ + ChatCompletionDeltaToolCall( + id="call_hyj5", + function=Function( + arguments='{"location": "Paris", "unit": "celsius"}', + name="get_current_weather", + ), + type="function", + index=2, + ) + ], + audio=None, + ), + logprobs=None, + ) + ], + stream_options=None, + ), + ] + + processor = ChunkProcessor(chunks=chunks) + + tool_calls = processor.get_combined_tool_content(tool_call_chunks=chunks) + print(f"tool_calls: {tool_calls}") + assert len(tool_calls) == 3 + + import json @@ -762,3 +881,55 @@ def get_current_weather(location, unit="fahrenheit"): return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"}) else: return json.dumps({"location": location, "temperature": "unknown"}) + + +@pytest.fixture(scope="module", autouse=True) +def load_env(): + messages = [ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What's the weather like in San Francisco, Tokyo, and Paris?", + }, + ] + tools = [ + { + "type": "function", + "function": litellm.utils.function_to_dict(get_current_weather), + } + ] + OPENAI_GPT4oMINI = { + "messages": messages, + "model": "gpt-4o-mini", + "temperature": 0.0, + "tools": tools, + "stream": True, + } + LLAMA3_3 = { + "messages": messages, + "model": "groq/llama-3.3-70b-versatile", + "api_base": "https://api.groq.com/openai/v1", + "temperature": 0.0, + "tools": tools, + "stream": True, + } + return OPENAI_GPT4oMINI, LLAMA3_3 + + +def execute_completion(opts: dict): + partial_streaming_chunks = [] + response_gen = litellm.completion(**opts) + for i, part in enumerate(response_gen): + partial_streaming_chunks.append(part) + assembly = litellm.stream_chunk_builder(partial_streaming_chunks) + print(assembly.choices[0].message.tool_calls) + assert len(assembly.choices[0].message.tool_calls) == 3, ( + assembly.choices[0].message.tool_calls[0].function.arguments[0] + ) + print(assembly.choices[0].message.tool_calls) + + +def test_grok_bug(load_env): + litellm.set_verbose = True + _, LLAMA3_3 = load_env + execute_completion(LLAMA3_3) diff --git a/ui/litellm-dashboard/src/components/model_hub.tsx b/ui/litellm-dashboard/src/components/model_hub.tsx index 3d5e515b05b..8787d598b44 100644 --- a/ui/litellm-dashboard/src/components/model_hub.tsx +++ b/ui/litellm-dashboard/src/components/model_hub.tsx @@ -34,6 +34,8 @@ interface ModelInfo { supports_vision: boolean; max_input_tokens?: number; max_output_tokens?: number; + input_cost_per_token?: number; + output_cost_per_token?: number; supported_openai_params?: string[]; } @@ -161,26 +163,29 @@ const ModelHub: React.FC = ({
- Mode: {model.mode} - - Supports Function Calling:{" "} - {model?.supports_function_calling == true ? "Yes" : "No"} - - - Supports Vision:{" "} - {model?.supports_vision == true ? "Yes" : "No"} - Max Input Tokens:{" "} {model?.max_input_tokens ? model?.max_input_tokens - : "N/A"} + : "Unknown"} Max Output Tokens:{" "} {model?.max_output_tokens ? model?.max_output_tokens - : "N/A"} + : "Unknown"} + + + Input Cost Per Token:{" "} + {model?.input_cost_per_token + ? `$${(model.input_cost_per_token * 1_000_000).toFixed(2)}` + : "Unknown"} + + + Output Cost Per Token:{" "} + {model?.output_cost_per_token + ? `$${(model.output_cost_per_token * 1_000_000).toFixed(2)}` + : "Unknown"}
@@ -245,12 +250,19 @@ const ModelHub: React.FC = ({ + Model Information OpenAI Python SDK Supported OpenAI Params LlamaIndex Langchain Py + + + Model Group: + 
{JSON.stringify(selectedModel, null, 2)}
+ + {`