scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 79

FriendliAI: Documentation Updates (#7517)

Browse source 
* docs(friendliai.md): update FriendliAI documentation and model details

* docs(friendliai.md): remove unused imports for cleaner documentation

* feat: add support for parallel function calling, system messages, and response schema in model configuration
This commit is contained in:
minpeter 2025-01-05 15:44:24 +09:00 • committed by GitHub
parent 2b0738ecc4
commit f7931b659b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 47 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
docs/my-website/docs/providers/friendliai.md
View file

@ -1,23 +1,32 @@
# FriendliAI
https://suite.friendli.ai/
:::info
**We support ALL FriendliAI models, just set `friendliai/` as a prefix when sending completion requests**
:::
| Property | Details |
| -------------------------- | ----------------------------------------------------------------------------------------------- |
| Description | The fastest and most efficient inference engine to build production-ready, compound AI systems. |
| Provider Route on LiteLLM | `friendliai/` |
| Provider Doc | [FriendliAI ↗](https://friendli.ai/docs/sdk/integrations/litellm) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions` |
## API Key
```python
# env variable
os.environ['FRIENDLI_TOKEN']
os.environ['FRIENDLI_API_BASE'] # Optional. Set this when using dedicated endpoint.
```
## Sample Usage
```python
from litellm import completion
import os
os.environ['FRIENDLI_TOKEN'] = ""
response = completion(
model="friendliai/mixtral-8x7b-instruct-v0-1",
model="friendliai/meta-llama-3.1-8b-instruct",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@ -26,13 +35,14 @@ print(response)
```
## Sample Usage - Streaming
```python
from litellm import completion
import os
os.environ['FRIENDLI_TOKEN'] = ""
response = completion(
model="friendliai/mixtral-8x7b-instruct-v0-1",
model="friendliai/meta-llama-3.1-8b-instruct",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@ -43,18 +53,11 @@ for chunk in response:
print(chunk)
```
## Supported Models
### Serverless Endpoints
We support ALL FriendliAI AI models, just set `friendliai/` as a prefix when sending completion requests
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| mixtral-8x7b-instruct | `completion(model="friendliai/mixtral-8x7b-instruct-v0-1", messages)` |
| meta-llama-3-8b-instruct | `completion(model="friendliai/meta-llama-3-8b-instruct", messages)` |
| meta-llama-3-70b-instruct | `completion(model="friendliai/meta-llama-3-70b-instruct", messages)` |
### Dedicated Endpoints
```
model="friendliai/$ENDPOINT_ID:$ADAPTER_ROUTE"
```
| Model Name | Function Call |
| --------------------------- | ---------------------------------------------------------------------- |
| meta-llama-3.1-8b-instruct | `completion(model="friendliai/meta-llama-3.1-8b-instruct", messages)` |
| meta-llama-3.1-70b-instruct | `completion(model="friendliai/meta-llama-3.1-70b-instruct", messages)` |

2
litellm/__init__.py
View file

@ -609,7 +609,7 @@ openai_compatible_endpoints: List = [
"api.deepseek.com/v1",
"api.together.xyz/v1",
"app.empower.dev/api/v1",
"inference.friendli.ai/v1",
"https://api.friendli.ai/serverless/v1",
"api.sambanova.ai/v1",
"api.x.ai/v1",
"api.galadriel.ai/v1",

8
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -208,11 +208,9 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "api.deepseek.com/v1":
custom_llm_provider = "deepseek"
dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
elif endpoint == "inference.friendli.ai/v1":
elif endpoint == "https://api.friendli.ai/serverless/v1":
custom_llm_provider = "friendliai"
dynamic_api_key = get_secret_str(
"FRIENDLIAI_API_KEY"
) or get_secret("FRIENDLI_TOKEN")
dynamic_api_key = get_secret_str("FRIENDLIAI_API_KEY") or get_secret("FRIENDLI_TOKEN")
elif endpoint == "api.galadriel.com/v1":
custom_llm_provider = "galadriel"
dynamic_api_key = get_secret_str("GALADRIEL_API_KEY")
@ -550,7 +548,7 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
api_base = (
api_base
or get_secret("FRIENDLI_API_BASE")
or "https://inference.friendli.ai/v1"
or "https://api.friendli.ai/serverless/v1"
) # type: ignore
dynamic_api_key = (
api_key

28
litellm/model_prices_and_context_window_backup.json
View file

@ -2264,17 +2264,7 @@
"mode": "chat",
"supports_function_calling": true
},
"friendliai/mixtral-8x7b-instruct-v0-1": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
"input_cost_per_token": 0.0000004,
"output_cost_per_token": 0.0000004,
"litellm_provider": "friendliai",
"mode": "chat",
"supports_function_calling": true
},
"friendliai/meta-llama-3-8b-instruct": {
"friendliai/meta-llama-3.1-8b-instruct": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
@ -2282,17 +2272,23 @@
"output_cost_per_token": 0.0000001,
"litellm_provider": "friendliai",
"mode": "chat",
"supports_function_calling": true
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_response_schema": true
},
"friendliai/meta-llama-3-70b-instruct": {
"friendliai/meta-llama-3.1-70b-instruct": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 0.0000008,
"output_cost_per_token": 0.0000008,
"input_cost_per_token": 0.0000006,
"output_cost_per_token": 0.0000006,
"litellm_provider": "friendliai",
"mode": "chat",
"supports_function_calling": true
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_response_schema": true
},
"claude-instant-1.2": {
"max_tokens": 8191,

28
model_prices_and_context_window.json
View file

@ -2264,17 +2264,7 @@
"mode": "chat",
"supports_function_calling": true
},
"friendliai/mixtral-8x7b-instruct-v0-1": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 32768,
"input_cost_per_token": 0.0000004,
"output_cost_per_token": 0.0000004,
"litellm_provider": "friendliai",
"mode": "chat",
"supports_function_calling": true
},
"friendliai/meta-llama-3-8b-instruct": {
"friendliai/meta-llama-3.1-8b-instruct": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
@ -2282,17 +2272,23 @@
"output_cost_per_token": 0.0000001,
"litellm_provider": "friendliai",
"mode": "chat",
"supports_function_calling": true
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_response_schema": true
},
"friendliai/meta-llama-3-70b-instruct": {
"friendliai/meta-llama-3.1-70b-instruct": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"input_cost_per_token": 0.0000008,
"output_cost_per_token": 0.0000008,
"input_cost_per_token": 0.0000006,
"output_cost_per_token": 0.0000006,
"litellm_provider": "friendliai",
"mode": "chat",
"supports_function_calling": true
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_response_schema": true
},
"claude-instant-1.2": {
"max_tokens": 8191,