From 05a4b6188e5287a63160e5fb73d32e2d86d08181 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 19 Jun 2024 17:47:36 -0700 Subject: [PATCH] fix(main.py): fix text-completion vllm routing --- litellm/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index efc7df0e411..a1b8a42abb9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1080,7 +1080,6 @@ def completion( ) elif ( custom_llm_provider == "text-completion-openai" - or (text_completion is True and custom_llm_provider == "openai") or "ft:babbage-002" in model or "ft:davinci-002" in model # support for finetuned completion models ): @@ -3782,6 +3781,12 @@ def text_completion( ) kwargs.pop("prompt", None) + + if model is not None and model.startswith( + "openai/" + ): # for openai compatible endpoints - e.g. vllm, call the native /v1/completions endpoint for text completion calls + model = model.replace("openai/", "text-completion-openai/") + kwargs["text_completion"] = True response = completion( model=model,