diff --git a/docs/my-website/docs/providers/friendliai.md b/docs/my-website/docs/providers/friendliai.md index 137c3dde380..6d4015f9ab5 100644 --- a/docs/my-website/docs/providers/friendliai.md +++ b/docs/my-website/docs/providers/friendliai.md @@ -1,23 +1,32 @@ # FriendliAI -https://suite.friendli.ai/ +:::info **We support ALL FriendliAI models, just set `friendliai/` as a prefix when sending completion requests** +::: + +| Property | Details | +| -------------------------- | ----------------------------------------------------------------------------------------------- | +| Description | The fastest and most efficient inference engine to build production-ready, compound AI systems. | +| Provider Route on LiteLLM | `friendliai/` | +| Provider Doc | [FriendliAI ↗](https://friendli.ai/docs/sdk/integrations/litellm) | +| Supported OpenAI Endpoints | `/chat/completions`, `/completions` | ## API Key + ```python # env variable os.environ['FRIENDLI_TOKEN'] -os.environ['FRIENDLI_API_BASE'] # Optional. Set this when using dedicated endpoint. ``` ## Sample Usage + ```python from litellm import completion import os os.environ['FRIENDLI_TOKEN'] = "" response = completion( - model="friendliai/mixtral-8x7b-instruct-v0-1", + model="friendliai/meta-llama-3.1-8b-instruct", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -26,13 +35,14 @@ print(response) ``` ## Sample Usage - Streaming + ```python from litellm import completion import os os.environ['FRIENDLI_TOKEN'] = "" response = completion( - model="friendliai/mixtral-8x7b-instruct-v0-1", + model="friendliai/meta-llama-3.1-8b-instruct", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -43,18 +53,11 @@ for chunk in response: print(chunk) ``` - ## Supported Models -### Serverless Endpoints + We support ALL FriendliAI AI models, just set `friendliai/` as a prefix when sending completion requests -| Model Name | Function Call | -|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| mixtral-8x7b-instruct | `completion(model="friendliai/mixtral-8x7b-instruct-v0-1", messages)` | -| meta-llama-3-8b-instruct | `completion(model="friendliai/meta-llama-3-8b-instruct", messages)` | -| meta-llama-3-70b-instruct | `completion(model="friendliai/meta-llama-3-70b-instruct", messages)` | - -### Dedicated Endpoints -``` -model="friendliai/$ENDPOINT_ID:$ADAPTER_ROUTE" -``` +| Model Name | Function Call | +| --------------------------- | ---------------------------------------------------------------------- | +| meta-llama-3.1-8b-instruct | `completion(model="friendliai/meta-llama-3.1-8b-instruct", messages)` | +| meta-llama-3.1-70b-instruct | `completion(model="friendliai/meta-llama-3.1-70b-instruct", messages)` | diff --git a/litellm/__init__.py b/litellm/__init__.py index 83e6dc8c534..539632c4efd 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -609,7 +609,7 @@ openai_compatible_endpoints: List = [ "api.deepseek.com/v1", "api.together.xyz/v1", "app.empower.dev/api/v1", - "inference.friendli.ai/v1", + "https://api.friendli.ai/serverless/v1", "api.sambanova.ai/v1", "api.x.ai/v1", "api.galadriel.ai/v1", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index cb26955fb17..74e0ddc3623 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -208,11 +208,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "api.deepseek.com/v1": custom_llm_provider = "deepseek" dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY") - elif endpoint == "inference.friendli.ai/v1": + elif endpoint == "https://api.friendli.ai/serverless/v1": custom_llm_provider = "friendliai" - dynamic_api_key = get_secret_str( - "FRIENDLIAI_API_KEY" - ) or get_secret("FRIENDLI_TOKEN") + dynamic_api_key = get_secret_str("FRIENDLIAI_API_KEY") or get_secret("FRIENDLI_TOKEN") elif endpoint == "api.galadriel.com/v1": custom_llm_provider = "galadriel" dynamic_api_key = get_secret_str("GALADRIEL_API_KEY") @@ -550,7 +548,7 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 api_base = ( api_base or get_secret("FRIENDLI_API_BASE") - or "https://inference.friendli.ai/v1" + or "https://api.friendli.ai/serverless/v1" ) # type: ignore dynamic_api_key = ( api_key diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bdf761753e1..f8815e4f5c4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2264,17 +2264,7 @@ "mode": "chat", "supports_function_calling": true }, - "friendliai/mixtral-8x7b-instruct-v0-1": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004, - "litellm_provider": "friendliai", - "mode": "chat", - "supports_function_calling": true - }, - "friendliai/meta-llama-3-8b-instruct": { + "friendliai/meta-llama-3.1-8b-instruct": { "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, @@ -2282,17 +2272,23 @@ "output_cost_per_token": 0.0000001, "litellm_provider": "friendliai", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true }, - "friendliai/meta-llama-3-70b-instruct": { + "friendliai/meta-llama-3.1-70b-instruct": { "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000008, + "input_cost_per_token": 0.0000006, + "output_cost_per_token": 0.0000006, "litellm_provider": "friendliai", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true }, "claude-instant-1.2": { "max_tokens": 8191, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bdf761753e1..f8815e4f5c4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2264,17 +2264,7 @@ "mode": "chat", "supports_function_calling": true }, - "friendliai/mixtral-8x7b-instruct-v0-1": { - "max_tokens": 32768, - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004, - "litellm_provider": "friendliai", - "mode": "chat", - "supports_function_calling": true - }, - "friendliai/meta-llama-3-8b-instruct": { + "friendliai/meta-llama-3.1-8b-instruct": { "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, @@ -2282,17 +2272,23 @@ "output_cost_per_token": 0.0000001, "litellm_provider": "friendliai", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true }, - "friendliai/meta-llama-3-70b-instruct": { + "friendliai/meta-llama-3.1-70b-instruct": { "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000008, + "input_cost_per_token": 0.0000006, + "output_cost_per_token": 0.0000006, "litellm_provider": "friendliai", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_response_schema": true }, "claude-instant-1.2": { "max_tokens": 8191,