From 493a737787120f4d2e20edfc3201b51e6fbee6a8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 26 Jun 2024 06:29:21 -0700 Subject: [PATCH 1/6] feat - add fireworks ai config for param mapping --- litellm/llms/fireworks_ai.py | 107 ++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 24 ++++ 2 files changed, 131 insertions(+) create mode 100644 litellm/llms/fireworks_ai.py diff --git a/litellm/llms/fireworks_ai.py b/litellm/llms/fireworks_ai.py new file mode 100644 index 00000000000..18309f4c2e0 --- /dev/null +++ b/litellm/llms/fireworks_ai.py @@ -0,0 +1,107 @@ +import types +from typing import Literal, Optional, Union + +import litellm + + +class FireworksAIConfig: + """ + Reference: https://docs.fireworks.ai/api-reference/post-chatcompletions + + The class `FireworksAIConfig` provides configuration for the Fireworks's Chat Completions API interface. Below are the parameters: + """ + + tools: Optional[list] = None + tool_choice: Optional[Union[str, dict]] = None + max_tokens: Optional[int] = None + temperature: Optional[int] = None + top_p: Optional[int] = None + top_k: Optional[int] = None + frequency_penalty: Optional[int] = None + presence_penalty: Optional[int] = None + n: Optional[int] = None + stop: Optional[Union[str, list]] = None + response_format: Optional[dict] = None + user: Optional[str] = None + + # Non OpenAI parameters - Fireworks AI only params + prompt_truncate_length: Optional[int] = None + context_length_exceeded_behavior: Optional[Literal["error", "truncate"]] = None + + def __init__( + self, + tools: Optional[list] = None, + tool_choice: Optional[Union[str, dict]] = None, + max_tokens: Optional[int] = None, + temperature: Optional[int] = None, + top_p: Optional[int] = None, + top_k: Optional[int] = None, + frequency_penalty: Optional[int] = None, + presence_penalty: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, list]] = None, + response_format: Optional[dict] = None, + user: Optional[str] = None, + prompt_truncate_length: Optional[int] = None, + context_length_exceeded_behavior: Optional[Literal["error", "truncate"]] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self): + return [ + "stream", + "tools", + "tool_choice", + "max_tokens", + "temperature", + "top_p", + "top_k", + "frequency_penalty", + "presence_penalty", + "n", + "stop", + "response_format", + "user", + "prompt_truncate_length", + "context_length_exceeded_behavior", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_openai_params = self.get_supported_openai_params() + for param, value in non_default_params.items(): + if param == "tool_choice": + if value == "required": + # relevant issue: https://github.com/BerriAI/litellm/issues/4416 + optional_params["tools"] = "any" + + if param in supported_openai_params: + if value is not None: + optional_params[param] = value + return optional_params diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 415d220f214..d7a7a7dc809 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2073,6 +2073,30 @@ "supports_function_calling": true, "supports_vision": true }, + "openrouter/anthropic/claude-3-haiku-20240307": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000125, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 264 + }, + "openrouter/anthropic/claude-3.5-sonnet": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "openrouter/anthropic/claude-3-sonnet": { "max_tokens": 200000, "input_cost_per_token": 0.000003, From dcdf266f366720551184c76a0f0b5469295b8dc5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 26 Jun 2024 06:40:44 -0700 Subject: [PATCH 2/6] fix fireworks ai config --- litellm/llms/fireworks_ai.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/llms/fireworks_ai.py b/litellm/llms/fireworks_ai.py index 18309f4c2e0..7c2d3b72ad1 100644 --- a/litellm/llms/fireworks_ai.py +++ b/litellm/llms/fireworks_ai.py @@ -92,16 +92,15 @@ class FireworksAIConfig: non_default_params: dict, optional_params: dict, model: str, - drop_params: bool, ) -> dict: supported_openai_params = self.get_supported_openai_params() for param, value in non_default_params.items(): if param == "tool_choice": if value == "required": # relevant issue: https://github.com/BerriAI/litellm/issues/4416 - optional_params["tools"] = "any" + optional_params["tool_choice"] = "any" - if param in supported_openai_params: + elif param in supported_openai_params: if value is not None: optional_params[param] = value return optional_params From 1cfe03c8204158d281060fb38ebae6d61d2bd449 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 26 Jun 2024 06:43:18 -0700 Subject: [PATCH 3/6] add fireworks ai param mapping --- litellm/__init__.py | 1 + litellm/utils.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 08ee84aaad5..cee80a32df4 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -817,6 +817,7 @@ from .llms.openai import ( AzureAIStudioConfig, ) from .llms.nvidia_nim import NvidiaNimConfig +from .llms.fireworks_ai import FireworksAIConfig from .llms.text_completion_codestral import MistralTextCompletionConfig from .llms.azure import ( AzureOpenAIConfig, diff --git a/litellm/utils.py b/litellm/utils.py index beae7ba4ab1..a33a160e4de 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3079,6 +3079,16 @@ def get_optional_params( optional_params = litellm.NvidiaNimConfig().map_openai_params( non_default_params=non_default_params, optional_params=optional_params ) + elif custom_llm_provider == "fireworks_ai": + supported_params = get_supported_openai_params( + model=model, custom_llm_provider=custom_llm_provider + ) + _check_valid_arg(supported_params=supported_params) + optional_params = litellm.FireworksAIConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + ) elif custom_llm_provider == "groq": supported_params = get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider @@ -3645,6 +3655,8 @@ def get_supported_openai_params( return litellm.OllamaChatConfig().get_supported_openai_params() elif custom_llm_provider == "anthropic": return litellm.AnthropicConfig().get_supported_openai_params() + elif custom_llm_provider == "fireworks_ai": + return litellm.FireworksAIConfig().get_supported_openai_params() elif custom_llm_provider == "nvidia_nim": return litellm.NvidiaNimConfig().get_supported_openai_params() elif custom_llm_provider == "groq": From 6c388dc05aa159add44fd33447ba228dfb82a457 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 26 Jun 2024 06:45:29 -0700 Subject: [PATCH 4/6] test fireworks ai tool calling --- litellm/tests/test_completion.py | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 30ae1d0ab13..a3b0e6ea269 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -1222,6 +1222,44 @@ def test_completion_fireworks_ai(): pytest.fail(f"Error occurred: {e}") +def test_fireworks_ai_tool_calling(): + litellm.set_verbose = True + model_name = "fireworks_ai/accounts/fireworks/models/firefunction-v2" + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + messages = [ + { + "role": "user", + "content": "What's the weather like in Boston today in Fahrenheit?", + } + ] + response = completion( + model=model_name, + messages=messages, + tools=tools, + tool_choice="required", + ) + print(response) + + @pytest.mark.skip(reason="this test is flaky") def test_completion_perplexity_api(): try: From 8a7f2921f21dbb6679a20bc603b5372c10e462e7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 26 Jun 2024 06:57:04 -0700 Subject: [PATCH 5/6] fix + test fireworks ai param mapping for tools --- litellm/llms/fireworks_ai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/fireworks_ai.py b/litellm/llms/fireworks_ai.py index 7c2d3b72ad1..e9caf887ad4 100644 --- a/litellm/llms/fireworks_ai.py +++ b/litellm/llms/fireworks_ai.py @@ -99,7 +99,9 @@ class FireworksAIConfig: if value == "required": # relevant issue: https://github.com/BerriAI/litellm/issues/4416 optional_params["tool_choice"] = "any" - + else: + # pass through the value of tool choice + optional_params["tool_choice"] = value elif param in supported_openai_params: if value is not None: optional_params[param] = value From 829dece9aa9d0fd07b4d7619f30631b616759b45 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 26 Jun 2024 06:58:00 -0700 Subject: [PATCH 6/6] test - fireworks ai param mapping --- litellm/tests/test_fireworks_ai.py | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 litellm/tests/test_fireworks_ai.py diff --git a/litellm/tests/test_fireworks_ai.py b/litellm/tests/test_fireworks_ai.py new file mode 100644 index 00000000000..c7c1f54453c --- /dev/null +++ b/litellm/tests/test_fireworks_ai.py @@ -0,0 +1,32 @@ +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +from litellm.llms.fireworks_ai import FireworksAIConfig + +fireworks = FireworksAIConfig() + + +def test_map_openai_params_tool_choice(): + # Test case 1: tool_choice is "required" + result = fireworks.map_openai_params({"tool_choice": "required"}, {}, "some_model") + assert result == {"tool_choice": "any"} + + # Test case 2: tool_choice is "auto" + result = fireworks.map_openai_params({"tool_choice": "auto"}, {}, "some_model") + assert result == {"tool_choice": "auto"} + + # Test case 3: tool_choice is not present + result = fireworks.map_openai_params( + {"some_other_param": "value"}, {}, "some_model" + ) + assert result == {} + + # Test case 4: tool_choice is None + result = fireworks.map_openai_params({"tool_choice": None}, {}, "some_model") + assert result == {"tool_choice": None}