From ce210ddaf638d9022d6e8b8f0a6d0781da20b2df Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 25 Jul 2024 22:30:55 -0700 Subject: [PATCH] fix(vertex_ai_llama3.py): Fix llama3 streaming issue Closes https://github.com/BerriAI/litellm/issues/4885 --- litellm/llms/vertex_ai_llama.py | 3 ++- litellm/tests/test_amazing_vertex_completion.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/llms/vertex_ai_llama.py b/litellm/llms/vertex_ai_llama.py index f33c127f742..cc4786c4b48 100644 --- a/litellm/llms/vertex_ai_llama.py +++ b/litellm/llms/vertex_ai_llama.py @@ -103,7 +103,8 @@ class VertexAILlama3Config: for param, value in non_default_params.items(): if param == "max_tokens": optional_params["max_tokens"] = value - + if param == "stream": + optional_params["stream"] = value return optional_params diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index aa0ea471ad9..bebe5d0312d 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -937,8 +937,6 @@ async def test_llama_3_httpx(model, sync_mode): response_format_tests(response=response) print(f"response: {response}") - - assert False except litellm.RateLimitError as e: pass except Exception as e: