scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

feat(utils.py): accept context window fallback dictionary

Browse source
This commit is contained in:
Krrish Dholakia 2023-10-31 22:32:29 -07:00
parent 2f6fed30ad
commit 7762ae7762
4 changed files with 32 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
litellm/main.py
View file

@ -255,9 +255,10 @@ def completion(
fallbacks = kwargs.get('fallbacks', None)
headers = kwargs.get("headers", None)
num_retries = kwargs.get("num_retries", None)
context_window_fallback_dict = kwargs.get("context_window_fallback_dict", None)
######## end of unpacking kwargs ###########
openai_params = ["functions", "function_call", "temperature", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user", "request_timeout", "api_base", "api_version", "api_key"]
litellm_params = ["metadata", "acompletion", "caching", "return_async", "mock_response", "api_key", "api_version", "api_base", "force_timeout", "logger_fn", "verbose", "custom_llm_provider", "litellm_logging_obj", "litellm_call_id", "use_client", "id", "fallbacks", "azure", "headers", "model_list", "num_retries"]
litellm_params = ["metadata", "acompletion", "caching", "return_async", "mock_response", "api_key", "api_version", "api_base", "force_timeout", "logger_fn", "verbose", "custom_llm_provider", "litellm_logging_obj", "litellm_call_id", "use_client", "id", "fallbacks", "azure", "headers", "model_list", "num_retries", "context_window_fallback_dict"]
default_params = openai_params + litellm_params
non_default_params = {k: v for k,v in kwargs.items() if k not in default_params} # model-specific params - pass them straight to the model/provider
if mock_response:
@ -1326,18 +1327,9 @@ def completion(
return response
except Exception as e:
## Map to OpenAI Exception
try:
raise exception_type(
raise exception_type(
model=model, custom_llm_provider=custom_llm_provider, original_exception=e, completion_kwargs=args,
)
except Exception as e:
if num_retries:
if (isinstance(e, openai.error.APIError)
or isinstance(e, openai.error.Timeout)
or isinstance(e, openai.error.ServiceUnavailableError)):
return completion_with_retries(num_retries=num_retries, **args)
else:
raise e
def completion_with_retries(*args, **kwargs):

2
litellm/tests/test_completion_with_retries.py
View file

@ -49,3 +49,5 @@ def test_completion_with_num_retries():
pass
except Exception as e:
pytest.fail(f"Unmapped exception occurred")
# test_completion_with_num_retries()

11
litellm/tests/test_exceptions.py
View file

@ -44,7 +44,16 @@ def test_context_window(model):
with pytest.raises(ContextWindowExceededError):
completion(model=model, messages=messages)
test_context_window(model="command-nightly")
@pytest.mark.parametrize("model", models)
def test_context_window_with_fallbacks(model):
ctx_window_fallback_dict = {"command-nightly": "claude-2"}
sample_text = "how does a court case get to the Supreme Court?" * 1000
messages = [{"content": sample_text, "role": "user"}]
completion(model=model, messages=messages, context_window_fallback_dict=ctx_window_fallback_dict)
# test_context_window(model="command-nightly")
test_context_window_with_fallbacks(model="command-nightly")
# Test 2: InvalidAuth Errors
@pytest.mark.parametrize("model", models)
def invalid_auth(model): # set the model key to an invalid key, depending on the model

17
litellm/utils.py
View file

@ -824,6 +824,23 @@ def client(original_function):
result._response_ms = (end_time - start_time).total_seconds() * 1000 # return response latency in ms like openai
return result
except Exception as e:
call_type = original_function.__name__
if call_type == CallTypes.completion.value:
num_retries = kwargs.get("num_retries", None)
context_window_fallback_dict = kwargs.get("context_window_fallback_dict", {})
if num_retries:
if (isinstance(e, openai.error.APIError)
or isinstance(e, openai.error.Timeout)
or isinstance(e, openai.error.ServiceUnavailableError)):
kwargs["num_retries"] = num_retries
return litellm.completion_with_retries(*args, **kwargs)
elif isinstance(e, litellm.exceptions.ContextWindowExceededError) and context_window_fallback_dict and model in context_window_fallback_dict:
if len(args) > 0:
args[0] = context_window_fallback_dict[model]
else:
kwargs["model"] = context_window_fallback_dict[model]
return original_function(*args, **kwargs)
traceback_exception = traceback.format_exc()
crash_reporting(*args, **kwargs, exception=traceback_exception)
end_time = datetime.datetime.now()