scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 86

Merge pull request #2241 from BerriAI/litellm_track_models_support_function_calling

Browse source 
[FEAT] Track which models support function calling
This commit is contained in:
Ishaan Jaff 2024-02-28 17:43:12 -08:00 • committed by GitHub
commit da2287fe79
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 264 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
docs/my-website/docs/completion/function_call.md
View file

@ -1,18 +1,25 @@
# Function Calling
Function calling is supported with the following models on OpenAI, Azure OpenAI
- gpt-4
- gpt-4-1106-preview
- gpt-4-0613
- gpt-3.5-turbo
- gpt-3.5-turbo-1106
- gpt-3.5-turbo-0613
- Non OpenAI LLMs (litellm adds the function call to the prompt for these llms)
## Checking if a model supports function calling
In addition, parallel function calls is supported on the following models:
- gpt-4-1106-preview
- gpt-3.5-turbo-1106
Use `litellm.supports_function_calling(model="")` -> returns `True` if model supports Function calling, `False` if not
```python
assert litellm.supports_function_calling(model="gpt-3.5-turbo") == True
assert litellm.supports_function_calling(model="azure/gpt-4-1106-preview") == True
assert litellm.supports_function_calling(model="palm/chat-bison") == False
assert litellm.supports_function_calling(model="ollama/llama2") == False
```
## Checking if a model supports parallel function calling
Use `litellm.supports_parallel_function_calling(model="")` -> returns `True` if model supports parallel function calling, `False` if not
```python
assert litellm.supports_parallel_function_calling(model="gpt-4-turbo-preview") == True
assert litellm.supports_parallel_function_calling(model="gpt-4") == False
```
## Parallel Function calling
Parallel function calling is the model's ability to perform multiple function calls together, allowing the effects and results of these function calls to be resolved in parallel

2
litellm/__init__.py
View file

@ -549,6 +549,8 @@ from .utils import (
token_counter,
cost_per_token,
completion_cost,
supports_function_calling,
supports_parallel_function_calling,
get_litellm_params,
Logging,
acreate,

105
litellm/model_prices_and_context_window_backup.json
View file

@ -6,7 +6,8 @@
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gpt-4-turbo-preview": {
"max_tokens": 8192,
@ -15,7 +16,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-4-0314": {
"max_tokens": 8192,
@ -33,7 +36,8 @@
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gpt-4-32k": {
"max_tokens": 32768,
@ -69,7 +73,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-4-0125-preview": {
"max_tokens": 128000,
@ -78,7 +84,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-4-vision-preview": {
"max_tokens": 128000,
@ -105,7 +113,8 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gpt-3.5-turbo-0301": {
"max_tokens": 4097,
@ -123,7 +132,8 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gpt-3.5-turbo-1106": {
"max_tokens": 16385,
@ -132,7 +142,9 @@
"input_cost_per_token": 0.0000010,
"output_cost_per_token": 0.0000020,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-3.5-turbo-0125": {
"max_tokens": 16385,
@ -141,7 +153,9 @@
"input_cost_per_token": 0.0000005,
"output_cost_per_token": 0.0000015,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-3.5-turbo-16k": {
"max_tokens": 16385,
@ -286,7 +300,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-4-1106-preview": {
"max_tokens": 128000,
@ -295,7 +311,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-4-0613": {
"max_tokens": 8192,
@ -304,7 +322,8 @@
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"azure/gpt-4-32k-0613": {
"max_tokens": 32768,
@ -331,7 +350,8 @@
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"azure/gpt-4-turbo": {
"max_tokens": 128000,
@ -340,7 +360,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-4-turbo-vision-preview": {
"max_tokens": 128000,
@ -358,7 +380,8 @@
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000004,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"azure/gpt-35-turbo-1106": {
"max_tokens": 16384,
@ -367,7 +390,20 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-35-turbo-0125": {
"max_tokens": 16384,
"max_input_tokens": 16384,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000005,
"output_cost_per_token": 0.0000015,
"litellm_provider": "azure",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-35-turbo-16k": {
"max_tokens": 16385,
@ -385,7 +421,8 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"azure/ada": {
"max_tokens": 8191,
@ -518,7 +555,8 @@
"input_cost_per_token": 0.000008,
"output_cost_per_token": 0.000024,
"litellm_provider": "mistral",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"mistral/mistral-embed": {
"max_tokens": 8192,
@ -676,7 +714,8 @@
"input_cost_per_token": 0.00000025,
"output_cost_per_token": 0.0000005,
"litellm_provider": "vertex_ai-language-models",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gemini-1.5-pro": {
"max_tokens": 8192,
@ -1738,6 +1777,23 @@
"output_cost_per_token": 0.0000009,
"litellm_provider": "together_ai"
},
"together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": {
"input_cost_per_token": 0.0000006,
"output_cost_per_token": 0.0000006,
"litellm_provider": "together_ai",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"together_ai/mistralai/Mistral-7B-Instruct-v0.1": {
"litellm_provider": "together_ai",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"together_ai/togethercomputer/CodeLlama-34b-Instruct": {
"litellm_provider": "together_ai",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"ollama/llama2": {
"max_tokens": 4096,
"input_cost_per_token": 0.0,
@ -1990,7 +2046,16 @@
"input_cost_per_token": 0.00000015,
"output_cost_per_token": 0.00000015,
"litellm_provider": "anyscale",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"anyscale/Mixtral-8x7B-Instruct-v0.1": {
"max_tokens": 16384,
"input_cost_per_token": 0.00000015,
"output_cost_per_token": 0.00000015,
"litellm_provider": "anyscale",
"mode": "chat",
"supports_function_calling": true
},
"anyscale/HuggingFaceH4/zephyr-7b-beta": {
"max_tokens": 16384,

2
litellm/tests/test_function_calling.py
View file

@ -71,7 +71,7 @@ def test_parallel_function_call():
}
]
response = litellm.completion(
model="gpt-3.5-turbo-1106",
model="mistral/mistral-medium-latest",
messages=messages,
tools=tools,
tool_choice="auto", # auto is default, but we'll be explicit

24
litellm/tests/test_utils.py
View file

@ -317,3 +317,27 @@ def test_token_counter():
# test_token_counter()
def test_supports_function_calling():
try:
assert litellm.supports_function_calling(model="gpt-3.5-turbo") == True
assert (
litellm.supports_function_calling(model="azure/gpt-4-1106-preview") == True
)
assert (
litellm.supports_function_calling(
model="bedrock/anthropic.claude-instant-v1"
)
== False
)
assert litellm.supports_function_calling(model="palm/chat-bison") == False
assert litellm.supports_function_calling(model="ollama/llama2") == False
assert (
litellm.supports_function_calling(model="anthropic.claude-instant-v1")
== False
)
assert litellm.supports_function_calling(model="claude-2") == False
except Exception as e:
pytest.fail(f"Error occurred: {e}")

49
litellm/utils.py
View file

@ -3720,6 +3720,54 @@ def completion_cost(
raise e
def supports_function_calling(model: str):
"""
Check if the given model supports function calling and return a boolean value.
Parameters:
model (str): The model name to be checked.
Returns:
bool: True if the model supports function calling, False otherwise.
Raises:
Exception: If the given model is not found in model_prices_and_context_window.json.
"""
if model in litellm.model_cost:
model_info = litellm.model_cost[model]
if model_info.get("supports_function_calling", False):
return True
return False
else:
raise Exception(
f"Model not in model_prices_and_context_window.json. You passed model={model}."
)
def supports_parallel_function_calling(model: str):
"""
Check if the given model supports parallel function calling and return True if it does, False otherwise.
Parameters:
model (str): The model to check for support of parallel function calling.
Returns:
bool: True if the model supports parallel function calling, False otherwise.
Raises:
Exception: If the model is not found in the model_cost dictionary.
"""
if model in litellm.model_cost:
model_info = litellm.model_cost[model]
if model_info.get("supports_parallel_function_calling", False):
return True
return False
else:
raise Exception(
f"Model not in model_prices_and_context_window.json. You passed model={model}."
)
####### HELPER FUNCTIONS ################
def register_model(model_cost: Union[str, dict]):
"""
@ -4048,6 +4096,7 @@ def get_optional_params(
and custom_llm_provider != "vertex_ai"
and custom_llm_provider != "anyscale"
and custom_llm_provider != "together_ai"
and custom_llm_provider != "mistral"
):
if custom_llm_provider == "ollama" or custom_llm_provider == "ollama_chat":
# ollama actually supports json output

105
model_prices_and_context_window.json
View file

@ -6,7 +6,8 @@
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gpt-4-turbo-preview": {
"max_tokens": 8192,
@ -15,7 +16,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-4-0314": {
"max_tokens": 8192,
@ -33,7 +36,8 @@
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gpt-4-32k": {
"max_tokens": 32768,
@ -69,7 +73,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-4-0125-preview": {
"max_tokens": 128000,
@ -78,7 +84,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-4-vision-preview": {
"max_tokens": 128000,
@ -105,7 +113,8 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gpt-3.5-turbo-0301": {
"max_tokens": 4097,
@ -123,7 +132,8 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gpt-3.5-turbo-1106": {
"max_tokens": 16385,
@ -132,7 +142,9 @@
"input_cost_per_token": 0.0000010,
"output_cost_per_token": 0.0000020,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-3.5-turbo-0125": {
"max_tokens": 16385,
@ -141,7 +153,9 @@
"input_cost_per_token": 0.0000005,
"output_cost_per_token": 0.0000015,
"litellm_provider": "openai",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"gpt-3.5-turbo-16k": {
"max_tokens": 16385,
@ -286,7 +300,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-4-1106-preview": {
"max_tokens": 128000,
@ -295,7 +311,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-4-0613": {
"max_tokens": 8192,
@ -304,7 +322,8 @@
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"azure/gpt-4-32k-0613": {
"max_tokens": 32768,
@ -331,7 +350,8 @@
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"azure/gpt-4-turbo": {
"max_tokens": 128000,
@ -340,7 +360,9 @@
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-4-turbo-vision-preview": {
"max_tokens": 128000,
@ -358,7 +380,8 @@
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000004,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"azure/gpt-35-turbo-1106": {
"max_tokens": 16384,
@ -367,7 +390,20 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-35-turbo-0125": {
"max_tokens": 16384,
"max_input_tokens": 16384,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000005,
"output_cost_per_token": 0.0000015,
"litellm_provider": "azure",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"azure/gpt-35-turbo-16k": {
"max_tokens": 16385,
@ -385,7 +421,8 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "azure",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"azure/ada": {
"max_tokens": 8191,
@ -518,7 +555,8 @@
"input_cost_per_token": 0.000008,
"output_cost_per_token": 0.000024,
"litellm_provider": "mistral",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"mistral/mistral-embed": {
"max_tokens": 8192,
@ -676,7 +714,8 @@
"input_cost_per_token": 0.00000025,
"output_cost_per_token": 0.0000005,
"litellm_provider": "vertex_ai-language-models",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"gemini-1.5-pro": {
"max_tokens": 8192,
@ -1738,6 +1777,23 @@
"output_cost_per_token": 0.0000009,
"litellm_provider": "together_ai"
},
"together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": {
"input_cost_per_token": 0.0000006,
"output_cost_per_token": 0.0000006,
"litellm_provider": "together_ai",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"together_ai/mistralai/Mistral-7B-Instruct-v0.1": {
"litellm_provider": "together_ai",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"together_ai/togethercomputer/CodeLlama-34b-Instruct": {
"litellm_provider": "together_ai",
"supports_function_calling": true,
"supports_parallel_function_calling": true
},
"ollama/llama2": {
"max_tokens": 4096,
"input_cost_per_token": 0.0,
@ -1990,7 +2046,16 @@
"input_cost_per_token": 0.00000015,
"output_cost_per_token": 0.00000015,
"litellm_provider": "anyscale",
"mode": "chat"
"mode": "chat",
"supports_function_calling": true
},
"anyscale/Mixtral-8x7B-Instruct-v0.1": {
"max_tokens": 16384,
"input_cost_per_token": 0.00000015,
"output_cost_per_token": 0.00000015,
"litellm_provider": "anyscale",
"mode": "chat",
"supports_function_calling": true
},
"anyscale/HuggingFaceH4/zephyr-7b-beta": {
"max_tokens": 16384,