diff --git a/README.md b/README.md index 8f95669aeea..01a60310522 100644 --- a/README.md +++ b/README.md @@ -334,7 +334,7 @@ curl 'http://0.0.0.0:4000/key/generate' \ | [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | ✅ | | | | [Galadriel](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | ✅ | | | | [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | ✅ | | | - +| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | ✅ | | | [**Read the Docs**](https://docs.litellm.ai/docs/) ## Contributing diff --git a/docs/my-website/docs/providers/featherless_ai.md b/docs/my-website/docs/providers/featherless_ai.md new file mode 100644 index 00000000000..5b9312e435d --- /dev/null +++ b/docs/my-website/docs/providers/featherless_ai.md @@ -0,0 +1,56 @@ +# Featherless AI +https://featherless.ai/ + +:::tip + +**We support ALL Featherless AI models, just set `model=featherless_ai/` as a prefix when sending litellm requests. For the complete supported model list, visit https://featherless.ai/models ** + +::: + + +## API Key +```python +# env variable +os.environ['FEATHERLESS_AI_API_KEY'] +``` + +## Sample Usage +```python +from litellm import completion +import os + +os.environ['FEATHERLESS_AI_API_KEY'] = "" +response = completion( + model="featherless_ai/featherless-ai/Qwerky-72B", + messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}] +) +``` + +## Sample Usage - Streaming +```python +from litellm import completion +import os + +os.environ['FEATHERLESS_AI_API_KEY'] = "" +response = completion( + model="featherless_ai/featherless-ai/Qwerky-72B", + messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}], + stream=True +) + +for chunk in response: + print(chunk) +``` + +## Chat Models +| Model Name | Function Call | +|---------------------------------------------|-----------------------------------------------------------------------------------------------| +| featherless-ai/Qwerky-72B | `completion(model="featherless_ai/featherless-ai/Qwerky-72B", messages)` | +| featherless-ai/Qwerky-QwQ-32B | `completion(model="featherless_ai/featherless-ai/Qwerky-QwQ-32B", messages)` | +| Qwen/Qwen2.5-72B-Instruct | `completion(model="featherless_ai/Qwen/Qwen2.5-72B-Instruct", messages)` | +| all-hands/openhands-lm-32b-v0.1 | `completion(model="featherless_ai/all-hands/openhands-lm-32b-v0.1", messages)` | +| Qwen/Qwen2.5-Coder-32B-Instruct | `completion(model="featherless_ai/Qwen/Qwen2.5-Coder-32B-Instruct", messages)` | +| deepseek-ai/DeepSeek-V3-0324 | `completion(model="featherless_ai/deepseek-ai/DeepSeek-V3-0324", messages)` | +| mistralai/Mistral-Small-24B-Instruct-2501 | `completion(model="featherless_ai/mistralai/Mistral-Small-24B-Instruct-2501", messages)` | +| mistralai/Mistral-Nemo-Instruct-2407 | `completion(model="featherless_ai/mistralai/Mistral-Nemo-Instruct-2407", messages)` | +| ProdeusUnity/Stellar-Odyssey-12b-v0.0 | `completion(model="featherless_ai/ProdeusUnity/Stellar-Odyssey-12b-v0.0", messages)` | diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index edda8a5acb4..59bf42c9302 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -370,7 +370,8 @@ const sidebars = { "providers/sambanova", "providers/custom_llm_server", "providers/petals", - "providers/snowflake" + "providers/snowflake", + "providers/featherless_ai" ], }, { diff --git a/litellm/__init__.py b/litellm/__init__.py index 65cb61749c7..2bba6a0233f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -433,6 +433,7 @@ databricks_models: List = [] cloudflare_models: List = [] codestral_models: List = [] friendliai_models: List = [] +featherless_ai_models: List = [] palm_models: List = [] groq_models: List = [] azure_models: List = [] @@ -608,6 +609,8 @@ def add_known_models(): jina_ai_models.append(key) elif value.get("litellm_provider") == "snowflake": snowflake_models.append(key) + elif value.get("litellm_provider") == "featherless_ai": + featherless_ai_models.append(key) add_known_models() @@ -686,6 +689,7 @@ model_list = ( + jina_ai_models + snowflake_models + llama_models + + featherless_ai_models + nscale_models ) @@ -747,6 +751,7 @@ models_by_provider: dict = { "snowflake": snowflake_models, "meta_llama": llama_models, "nscale": nscale_models, + "featherless_ai": featherless_ai_models, } # mapping for those models which have larger equivalents @@ -1019,6 +1024,7 @@ from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig nvidiaNimConfig = NvidiaNimConfig() nvidiaNimEmbeddingConfig = NvidiaNimEmbeddingConfig() +from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig from .llms.cerebras.chat import CerebrasConfig from .llms.sambanova.chat import SambanovaConfig from .llms.ai21.chat.transformation import AI21ChatConfig diff --git a/litellm/constants.py b/litellm/constants.py index 8c72f60bee0..cf12ec60f07 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -223,6 +223,7 @@ LITELLM_CHAT_PROVIDERS = [ "galadriel", "novita", "meta_llama", + "featherless_ai", "nscale", ] @@ -292,6 +293,7 @@ openai_compatible_endpoints: List = [ "api.x.ai/v1", "api.galadriel.ai/v1", "api.llama.com/compat/v1/", + "api.featherless.ai/v1", "inference.api.nscale.com/v1", ] @@ -325,6 +327,7 @@ openai_compatible_providers: List = [ "galadriel", "novita", "meta_llama", + "featherless_ai", "nscale", ] openai_text_completion_compatible_providers: List = ( @@ -334,6 +337,7 @@ openai_text_completion_compatible_providers: List = ( "hosted_vllm", "meta_llama", "llamafile", + "featherless_ai", ] ) _openai_like_providers: List = [ @@ -480,6 +484,18 @@ baseten_models: List = [ "31dxrj3", ] # FALCON 7B # WizardLM # Mosaic ML +featherless_ai_models: List = [ + "featherless-ai/Qwerky-72B", + "featherless-ai/Qwerky-QwQ-32B", + "Qwen/Qwen2.5-72B-Instruct", + "all-hands/openhands-lm-32b-v0.1", + "Qwen/Qwen2.5-Coder-32B-Instruct", + "deepseek-ai/DeepSeek-V3-0324", + "mistralai/Mistral-Small-24B-Instruct-2501", + "mistralai/Mistral-Nemo-Instruct-2407", + "ProdeusUnity/Stellar-Odyssey-12b-v0.0", +] + BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "cohere", "anthropic", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 7543d1722c1..f792d249b3f 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -225,6 +225,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.llama.com/compat/v1": custom_llm_provider = "meta_llama" dynamic_api_key = api_key or get_secret_str("LLAMA_API_KEY") + elif endpoint == "https://api.featherless.ai/v1": + custom_llm_provider = "featherless_ai" + dynamic_api_key = get_secret_str("FEATHERLESS_AI_API_KEY") elif endpoint == litellm.NscaleConfig.API_BASE_URL: custom_llm_provider = "nscale" dynamic_api_key = litellm.NscaleConfig.get_api_key() @@ -618,6 +621,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or f"https://{get_secret('SNOWFLAKE_ACCOUNT_ID')}.snowflakecomputing.com/api/v2/cortex/inference:complete" ) # type: ignore dynamic_api_key = api_key or get_secret_str("SNOWFLAKE_JWT") + elif custom_llm_provider == "featherless_ai": + ( + api_base, + dynamic_api_key, + ) = litellm.FeatherlessAIConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "nscale": ( api_base, diff --git a/litellm/llms/featherless_ai/chat/transformation.py b/litellm/llms/featherless_ai/chat/transformation.py new file mode 100644 index 00000000000..aac887d7357 --- /dev/null +++ b/litellm/llms/featherless_ai/chat/transformation.py @@ -0,0 +1,132 @@ +from typing import Optional, Tuple, Union + +import litellm +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.secret_managers.main import get_secret_str + + +class FeatherlessAIConfig(OpenAIGPTConfig): + """ + Reference: https://featherless.ai/docs/completions + + The class `FeatherlessAI` provides configuration for the FeatherlessAI's Chat Completions API interface. Below are the parameters: + """ + + frequency_penalty: Optional[int] = None + function_call: Optional[Union[str, dict]] = None + functions: Optional[list] = None + logit_bias: Optional[dict] = None + max_tokens: Optional[int] = None + n: Optional[int] = None + presence_penalty: Optional[int] = None + stop: Optional[Union[str, list]] = None + temperature: Optional[int] = None + top_p: Optional[int] = None + response_format: Optional[dict] = None + tool_choice: Optional[str] = None + tools: Optional[list] = None + + + def __init__( + self, + frequency_penalty: Optional[int] = None, + function_call: Optional[Union[str, dict]] = None, + functions: Optional[list] = None, + logit_bias: Optional[dict] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + presence_penalty: Optional[int] = None, + stop: Optional[Union[str, list]] = None, + temperature: Optional[int] = None, + top_p: Optional[int] = None, + response_format: Optional[dict] = None, + tool_choice: Optional[str] = None, + tools: Optional[list] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return super().get_config() + + def get_supported_openai_params(self, model: str): + return [ + "stream", + "frequency_penalty", + "function_call", + "functions", + "logit_bias", + "max_tokens", + "max_completion_tokens", + "n", + "presence_penalty", + "stop", + "temperature", + "top_p", + "response_format", + "tool_choice", + "tools" + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_openai_params = self.get_supported_openai_params(model=model) + for param, value in non_default_params.items(): + if param == "tool_choice" or param == "tools": + if param == "tool_choice" and (value == "auto" or value == "none"): + # These values are supported, so add them to optional_params + optional_params[param] = value + else: # https://featherless.ai/docs/completions + ## UNSUPPORTED TOOL CHOICE VALUE + if litellm.drop_params is True or drop_params is True: + value = None + else: + error_message = f"Featherless AI doesn't support {param}={value}. To drop unsupported openai params from the call, set `litellm.drop_params = True`" + raise litellm.utils.UnsupportedParamsError( + message=error_message, + status_code=400, + ) + elif param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_openai_params: + if value is not None: + optional_params[param] = value + return optional_params + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + # FeatherlessAI is openai compatible, set to custom_openai and use FeatherlessAI's endpoint + api_base = ( + api_base + or get_secret_str("FEATHERLESS_API_BASE") + or "https://api.featherless.ai/v1" + ) + dynamic_api_key = api_key or get_secret_str("FEATHERLESS_API_KEY") + return api_base, dynamic_api_key + + def validate_environment( + self, + headers: dict, + model: str, + messages: list, + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + if not api_key: + raise ValueError("Missing Featherless AI API Key") + + headers["Authorization"] = f"Bearer {api_key}" + headers["Content-Type"] = "application/json" + + return headers diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f96f35d1b29..9c70900d034 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2176,6 +2176,7 @@ class LlmProviders(str, Enum): XINFERENCE = "xinference" FIREWORKS_AI = "fireworks_ai" FRIENDLIAI = "friendliai" + FEATHERLESS_AI = "featherless_ai" WATSONX = "watsonx" WATSONX_TEXT = "watsonx_text" TRITON = "triton" diff --git a/litellm/utils.py b/litellm/utils.py index 16479940470..49e75bcf544 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4885,6 +4885,11 @@ def validate_environment( # noqa: PLR0915 keys_in_environment = True else: missing_keys.append("DEEPINFRA_API_KEY") + elif custom_llm_provider == "featherless_ai": + if "FEATHERLESS_AI_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("FEATHERLESS_AI_API_KEY") elif custom_llm_provider == "gemini": if "GEMINI_API_KEY" in os.environ: keys_in_environment = True @@ -6410,6 +6415,8 @@ class ProviderConfigManager: return litellm.TritonConfig() elif litellm.LlmProviders.PETALS == provider: return litellm.PetalsConfig() + elif litellm.LlmProviders.FEATHERLESS_AI == provider: + return litellm.FeatherlessAIConfig() elif litellm.LlmProviders.NOVITA == provider: return litellm.NovitaConfig() elif litellm.LlmProviders.BEDROCK == provider: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e7f63d18e13..aec5d2e314c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12643,5 +12643,19 @@ "/v1/images/generations" ], "source": "https://docs.nscale.com/docs/inference/serverless-models/current#image-models" + }, + "featherless_ai/featherless-ai/Qwerky-72B": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "litellm_provider": "featherless_ai", + "mode": "chat" + }, + "featherless_ai/featherless-ai/Qwerky-QwQ-32B": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "litellm_provider": "featherless_ai", + "mode": "chat" } } diff --git a/tests/litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py b/tests/litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py new file mode 100644 index 00000000000..171996016f5 --- /dev/null +++ b/tests/litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py @@ -0,0 +1,227 @@ +""" +Unit tests for Featherless AI configuration. + +These tests validate the FeatherlessAIConfig class which extends OpenAIGPTConfig. +Featherless AI is an OpenAI-compatible provider with a few customizations. +""" + +import os +import sys +from typing import Dict, List, Optional +from unittest.mock import patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.featherless_ai.chat.transformation import FeatherlessAIConfig + + +class TestFeatherlessAIConfig: + """Test class for FeatherlessAIConfig functionality""" + + def test_validate_environment(self): + """Test that validate_environment adds correct headers""" + config = FeatherlessAIConfig() + headers = {} + api_key = "fake-featherless-key" + + result = config.validate_environment( + headers=headers, + model="featherless-ai/Qwerky-72B", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=api_key, + api_base="https://api.featherless.ai/v1/", + ) + + # Verify headers + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" + + def test_missing_api_key(self): + """Test error handling when API key is missing""" + config = FeatherlessAIConfig() + + with pytest.raises(ValueError) as excinfo: + config.validate_environment( + headers={}, + model="featherless-ai/Qwerky-72B", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base="https://api.featherless.ai/v1/", + ) + + assert "Missing Featherless AI API Key" in str(excinfo.value) + + def test_inheritance(self): + """Test proper inheritance from OpenAIGPTConfig""" + config = FeatherlessAIConfig() + + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + assert isinstance(config, OpenAIGPTConfig) + assert hasattr(config, "get_supported_openai_params") + + def test_map_openai_params_with_tool_choice(self): + """Test map_openai_params handles tool_choice parameter correctly""" + config = FeatherlessAIConfig() + + # Test with auto value (supported) + non_default_params = {"tool_choice": "auto"} + optional_params = {} + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="featherless-ai/Qwerky-72B", + drop_params=False + ) + assert "tool_choice" in result + assert result["tool_choice"] == "auto" + + # Test with none value (supported) + non_default_params = {"tool_choice": "none"} + optional_params = {} + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="featherless-ai/Qwerky-72B", + drop_params=False + ) + assert "tool_choice" in result + assert result["tool_choice"] == "none" + + # Test with unsupported value and drop_params=True + non_default_params = {"tool_choice": {"type": "function", "function": {"name": "get_weather"}}} + optional_params = {} + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="featherless-ai/Qwerky-72B", + drop_params=True + ) + assert "tool_choice" not in result + + # Test with unsupported value and drop_params=False + non_default_params = {"tool_choice": {"type": "function", "function": {"name": "get_weather"}}} + optional_params = {} + with pytest.raises(Exception) as excinfo: + config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="featherless-ai/Qwerky-72B", + drop_params=False + ) + assert "Featherless AI doesn't support tool_choice=" in str(excinfo.value) + + def test_map_openai_params_with_tools(self): + """Test map_openai_params handles tools parameter correctly""" + config = FeatherlessAIConfig() + + # Test with tools and drop_params=True + tools = [{"type": "function", "function": {"name": "get_weather"}}] + non_default_params = {"tools": tools} + optional_params = {} + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="featherless-ai/Qwerky-72B", + drop_params=True + ) + assert "tools" not in result + + # Test with tools and drop_params=False + with pytest.raises(Exception) as excinfo: + config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="featherless-ai/Qwerky-72B", + drop_params=False + ) + assert "Featherless AI doesn't support tools=" in str(excinfo.value) + + def test_default_api_base(self): + """Test that default API base is used when none is provided""" + config = FeatherlessAIConfig() + headers = {} + api_key = "fake-featherless-key" + + # Call validate_environment without specifying api_base + result = config.validate_environment( + headers=headers, + model="featherless-ai/Qwerky-72B", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=api_key, + api_base=None, # Not providing api_base + ) + + # Verify headers are still set correctly + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" + + # We can't directly test the api_base value here since validate_environment + # only returns the headers, but we can verify it doesn't raise an exception + # which would happen if api_base handling was incorrect + + def test_featherless_ai_completion_mock(self, respx_mock): + """ + Mock test for Featherless AI completion using the model format from docs. + This test mocks the actual HTTP request to test the integration properly. + """ + import respx + from litellm import completion + + # Set up environment variables for the test + api_key = "fake-featherless-key" + api_base = "https://api.featherless.ai/v1" + model = "featherless_ai/featherless-ai/Qwerky-72B" + model_name = "Qwerky-72B" # The actual model name without provider prefix + + # Mock the HTTP request to the Featherless AI API + respx_mock.post(f"{api_base}/chat/completions").respond( + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": model_name, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "```python\nprint(\"Hi from LiteLLM!\")\n```\n\nThis simple Python code prints a greeting message from LiteLLM.", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21}, + }, + status_code=200 + ) + + # Make the actual API call through LiteLLM + response = completion( + model=model, + messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}], + api_key=api_key, + api_base=api_base + ) + + # Verify response structure + assert response is not None + assert hasattr(response, "choices") + assert len(response.choices) > 0 + assert hasattr(response.choices[0], "message") + assert hasattr(response.choices[0].message, "content") + assert response.choices[0].message.content is not None + + # Check for specific content in the response + assert "```python" in response.choices[0].message.content + assert "Hi from LiteLLM" in response.choices[0].message.content