mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
Add Foundry Local provider and docs
Add Foundry Local as a first-class OpenAI-compatible provider, wire it into the dashboard and proxy metadata, include the official upstream logo, and document the current SDK-managed local endpoint flow for Foundry Local. Files changed: litellm/llms/foundry_local/, litellm/utils.py, litellm/litellm_core_utils/get_llm_provider_logic.py, tests/test_litellm/llms/foundry_local/test_foundry_local_chat_transformation.py, docs/my-website/docs/providers/foundry_local.md, provider_endpoints_support.json, ui/litellm-dashboard/src/components/provider_info_helpers.tsx, litellm/proxy/public_endpoints/provider_create_fields.json Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Bhagirath Mehta
parent
5be0797d24
commit
9ceb46f937
16 changed files with 486 additions and 0 deletions
222
docs/my-website/docs/providers/foundry_local.md
Normal file
222
docs/my-website/docs/providers/foundry_local.md
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Foundry Local
|
||||
|
||||
https://github.com/microsoft/Foundry-Local
|
||||
|
||||
:::tip
|
||||
|
||||
**We support ALL Foundry Local models, just set `model=foundry_local/<any-model-alias>` as a prefix when sending litellm requests**
|
||||
|
||||
:::
|
||||
|
||||
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | Run AI models on-device with Microsoft Foundry Local. |
|
||||
| Provider Route on LiteLLM | `foundry_local/` |
|
||||
| Provider Doc | [Foundry Local ↗](https://learn.microsoft.com/en-us/azure/foundry-local/) |
|
||||
| Supported OpenAI Endpoints | `/chat/completions` |
|
||||
|
||||
## Quick Start
|
||||
|
||||
Foundry Local itself is SDK-first and does **not** need to run as a web server for in-process apps. LiteLLM uses the optional OpenAI-compatible web service, so point `FOUNDRY_LOCAL_API_BASE` at `manager.urls[0]/v1`.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="python" label="Python SDK">
|
||||
|
||||
```bash
|
||||
# Windows (recommended for hardware acceleration)
|
||||
pip install foundry-local-sdk-winml
|
||||
|
||||
# macOS/Linux
|
||||
pip install foundry-local-sdk
|
||||
```
|
||||
|
||||
```python
|
||||
import os
|
||||
from foundry_local_sdk import Configuration, FoundryLocalManager
|
||||
|
||||
FoundryLocalManager.initialize(Configuration(app_name="litellm-foundry-local"))
|
||||
manager = FoundryLocalManager.instance
|
||||
|
||||
model = manager.catalog.get_model("qwen2.5-0.5b")
|
||||
model.download()
|
||||
model.load()
|
||||
manager.start_web_service()
|
||||
|
||||
os.environ["FOUNDRY_LOCAL_API_BASE"] = f"{manager.urls[0].rstrip('/')}/v1"
|
||||
print(os.environ["FOUNDRY_LOCAL_API_BASE"])
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="javascript" label="JavaScript SDK">
|
||||
|
||||
```bash
|
||||
# Windows (recommended for hardware acceleration)
|
||||
npm install foundry-local-sdk-winml
|
||||
|
||||
# macOS/Linux
|
||||
npm install foundry-local-sdk
|
||||
```
|
||||
|
||||
```javascript
|
||||
import { FoundryLocalManager } from "foundry-local-sdk";
|
||||
|
||||
const manager = FoundryLocalManager.create({
|
||||
appName: "litellm-foundry-local",
|
||||
});
|
||||
|
||||
const model = await manager.catalog.getModel("qwen2.5-0.5b");
|
||||
await model.download();
|
||||
await model.load();
|
||||
manager.startWebService();
|
||||
|
||||
const apiBase = `${manager.urls[0].replace(/\/$/, "")}/v1`;
|
||||
console.log(apiBase);
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
:::tip
|
||||
|
||||
Foundry Local 1.2 also adds cancellable model and EP downloads (`threading.Event` in Python, `AbortController` in JavaScript) and serves the OpenAI Responses API from the same `/v1` endpoint. LiteLLM currently uses `/chat/completions`, so the local server is optional in general but required for this HTTP integration path.
|
||||
|
||||
:::
|
||||
|
||||
## API Key
|
||||
```python
|
||||
# env variable
|
||||
os.environ['FOUNDRY_LOCAL_API_BASE'] # e.g. http://127.0.0.1:5272/v1
|
||||
os.environ['FOUNDRY_LOCAL_API_KEY'] # optional, not required for local use
|
||||
```
|
||||
|
||||
## Sample Usage
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1"
|
||||
|
||||
response = completion(
|
||||
model="foundry_local/qwen2.5-0.5b",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in Boston today in Fahrenheit?",
|
||||
}
|
||||
]
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Sample Usage - Streaming
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1"
|
||||
|
||||
response = completion(
|
||||
model="foundry_local/qwen2.5-0.5b",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in Boston today in Fahrenheit?",
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
## Usage with DSPy
|
||||
|
||||
Foundry Local works seamlessly with [DSPy](https://dspy.ai/) via LiteLLM:
|
||||
|
||||
```python
|
||||
import dspy
|
||||
import os
|
||||
|
||||
os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1"
|
||||
|
||||
lm = dspy.LM("foundry_local/qwen2.5-0.5b")
|
||||
dspy.configure(lm=lm)
|
||||
```
|
||||
|
||||
## Usage with LiteLLM Proxy Server
|
||||
|
||||
Here's how to call a Foundry Local model with the LiteLLM Proxy Server
|
||||
|
||||
1. Modify the config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: my-model
|
||||
litellm_params:
|
||||
model: foundry_local/qwen2.5-0.5b
|
||||
api_base: http://127.0.0.1:5272/v1
|
||||
```
|
||||
|
||||
|
||||
2. Start the proxy
|
||||
|
||||
```bash
|
||||
$ litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Send Request to LiteLLM Proxy Server
|
||||
|
||||
<Tabs>
|
||||
|
||||
<TabItem value="openai" label="OpenAI Python v1.0.0+">
|
||||
|
||||
```python
|
||||
import openai
|
||||
client = openai.OpenAI(
|
||||
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
|
||||
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="my-model",
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="curl" label="curl">
|
||||
|
||||
```shell
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Authorization: Bearer sk-1234' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"model": "my-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what llm are you"
|
||||
}
|
||||
],
|
||||
}'
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
|
||||
|
||||
## Supported Parameters
|
||||
|
||||
See [Supported Parameters](../completion/input.md#translated-openai-params) for supported parameters.
|
||||
|
|
@ -1803,6 +1803,9 @@ if TYPE_CHECKING:
|
|||
from .llms.lm_studio.chat.transformation import (
|
||||
LMStudioChatConfig as _LMStudioChatConfig,
|
||||
)
|
||||
from .llms.foundry_local.chat.transformation import (
|
||||
FoundryLocalChatConfig as _FoundryLocalChatConfig,
|
||||
)
|
||||
from .llms.lm_studio.embed.transformation import (
|
||||
LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig,
|
||||
)
|
||||
|
|
@ -1827,6 +1830,7 @@ if TYPE_CHECKING:
|
|||
DeepInfraConfig: Type[_DeepInfraConfig]
|
||||
LlamafileChatConfig: Type[_LlamafileChatConfig]
|
||||
LMStudioChatConfig: Type[_LMStudioChatConfig]
|
||||
FoundryLocalChatConfig: Type[_FoundryLocalChatConfig]
|
||||
LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]
|
||||
IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig]
|
||||
VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig
|
||||
|
|
|
|||
|
|
@ -283,6 +283,7 @@ LLM_CONFIG_NAMES = (
|
|||
"VLLMConfig",
|
||||
"DeepSeekChatConfig",
|
||||
"LMStudioChatConfig",
|
||||
"FoundryLocalChatConfig",
|
||||
"LmStudioEmbeddingConfig",
|
||||
"NscaleConfig",
|
||||
"PerplexityChatConfig",
|
||||
|
|
@ -1083,6 +1084,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
|
|||
"VLLMConfig": (".llms.vllm.completion.transformation", "VLLMConfig"),
|
||||
"DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"),
|
||||
"LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"),
|
||||
"FoundryLocalChatConfig": (
|
||||
".llms.foundry_local.chat.transformation",
|
||||
"FoundryLocalChatConfig",
|
||||
),
|
||||
"LmStudioEmbeddingConfig": (
|
||||
".llms.lm_studio.embed.transformation",
|
||||
"LmStudioEmbeddingConfig",
|
||||
|
|
|
|||
|
|
@ -603,6 +603,7 @@ LITELLM_CHAT_PROVIDERS = [
|
|||
"hosted_vllm",
|
||||
"llamafile",
|
||||
"lm_studio",
|
||||
"foundry_local",
|
||||
"galadriel",
|
||||
"gradient_ai",
|
||||
"github_copilot", # GitHub Copilot Chat API
|
||||
|
|
@ -813,6 +814,7 @@ openai_compatible_providers: List = [
|
|||
"hosted_vllm",
|
||||
"llamafile",
|
||||
"lm_studio",
|
||||
"foundry_local",
|
||||
"galadriel",
|
||||
"github_copilot", # GitHub Copilot Chat API
|
||||
"chatgpt", # ChatGPT subscription API
|
||||
|
|
|
|||
|
|
@ -731,6 +731,14 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
|
|||
) = litellm.LMStudioChatConfig()._get_openai_compatible_provider_info(
|
||||
api_base, api_key
|
||||
)
|
||||
elif custom_llm_provider == "foundry_local":
|
||||
# foundry_local is openai compatible, we just need to set this to custom_openai
|
||||
(
|
||||
api_base,
|
||||
dynamic_api_key,
|
||||
) = litellm.FoundryLocalChatConfig()._get_openai_compatible_provider_info(
|
||||
api_base, api_key
|
||||
)
|
||||
elif custom_llm_provider == "deepseek":
|
||||
# deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1
|
||||
api_base = (
|
||||
|
|
|
|||
0
litellm/llms/foundry_local/__init__.py
Normal file
0
litellm/llms/foundry_local/__init__.py
Normal file
0
litellm/llms/foundry_local/chat/__init__.py
Normal file
0
litellm/llms/foundry_local/chat/__init__.py
Normal file
23
litellm/llms/foundry_local/chat/transformation.py
Normal file
23
litellm/llms/foundry_local/chat/transformation.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
"""
|
||||
Translate from OpenAI's `/v1/chat/completions` to Foundry Local's `/v1/chat/completions`
|
||||
|
||||
Foundry Local (https://github.com/microsoft/Foundry-Local) runs LLMs on-device
|
||||
and exposes an OpenAI-compatible REST API endpoint.
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
|
||||
class FoundryLocalChatConfig(OpenAIGPTConfig):
|
||||
def _get_openai_compatible_provider_info(
|
||||
self, api_base: Optional[str], api_key: Optional[str]
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
api_base = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE") # type: ignore
|
||||
dynamic_api_key = (
|
||||
api_key or get_secret_str("FOUNDRY_LOCAL_API_KEY") or "fake-api-key"
|
||||
) # Foundry Local does not require an api key
|
||||
return api_base, dynamic_api_key
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
<svg width="256" height="256" viewBox="0 0 256 256" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_1197_17983)">
|
||||
<rect width="256" height="256" fill="white"/>
|
||||
<path d="M154.903 33.0322H35.785C34.2618 33.0322 33.0323 34.2618 33.0323 35.7849V63.3118C33.0323 64.835 34.2618 66.0645 35.785 66.0645H181.677C186.238 66.0645 189.936 69.7623 189.936 74.3225V70.9C189.936 51.3101 175.319 33.0322 154.903 33.0322Z" fill="url(#paint0_linear_1197_17983)"/>
|
||||
<path d="M178.255 42.7031C185.733 50.1996 189.936 60.3112 189.936 70.8998V220.215C189.936 221.738 191.165 222.967 192.688 222.967H220.215C221.738 222.967 222.968 221.738 222.968 220.215V101.106C222.968 92.3433 219.49 83.9476 213.297 77.7449L178.255 42.7031Z" fill="url(#paint1_linear_1197_17983)"/>
|
||||
<path d="M113.613 74.3228H35.785C34.2618 74.3228 33.0323 75.5523 33.0323 77.0754V104.602C33.0323 106.125 34.2618 107.355 35.785 107.355H140.387C144.947 107.355 148.645 111.053 148.645 115.613V112.191C148.645 92.6006 134.028 74.3228 113.613 74.3228Z" fill="url(#paint2_linear_1197_17983)"/>
|
||||
<path d="M136.965 83.9937C144.443 91.4901 148.645 101.602 148.645 112.19V220.215C148.645 221.738 149.875 222.968 151.398 222.968H178.925C180.448 222.968 181.677 221.738 181.677 220.215V142.397C181.677 133.634 178.2 125.238 172.006 119.035L136.965 83.9937Z" fill="url(#paint3_linear_1197_17983)"/>
|
||||
<path d="M72.3223 115.613H35.785C34.2618 115.613 33.0323 116.842 33.0323 118.365V145.892C33.0323 147.416 34.2618 148.645 35.785 148.645H99.0968C103.657 148.645 107.355 152.343 107.355 156.903V153.481C107.355 133.891 92.7381 115.613 72.3223 115.613Z" fill="url(#paint4_linear_1197_17983)"/>
|
||||
<path d="M95.6743 125.284C103.152 132.781 107.355 142.892 107.355 153.481V220.215C107.355 221.738 108.584 222.968 110.108 222.968H137.634C139.158 222.968 140.387 221.738 140.387 220.215V183.687C140.387 174.924 136.91 166.529 130.716 160.326L95.6743 125.284Z" fill="url(#paint5_linear_1197_17983)"/>
|
||||
</g>
|
||||
<defs>
|
||||
<linearGradient id="paint0_linear_1197_17983" x1="189.936" y1="70.4672" x2="33.0323" y2="70.4672" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#2C08AC"/>
|
||||
<stop offset="0.8" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint1_linear_1197_17983" x1="201.96" y1="42.7031" x2="311.288" y2="175.174" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.3" stop-color="#7274FF"/>
|
||||
<stop offset="1" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint2_linear_1197_17983" x1="148.645" y1="111.758" x2="33.0323" y2="111.758" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#2C08AC"/>
|
||||
<stop offset="0.8" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint3_linear_1197_17983" x1="160.67" y1="83.9937" x2="238.429" y2="206.207" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.3" stop-color="#7274FF"/>
|
||||
<stop offset="1" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint4_linear_1197_17983" x1="107.355" y1="150.688" x2="33.0323" y2="150.688" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#2C08AC"/>
|
||||
<stop offset="0.8" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint5_linear_1197_17983" x1="119.38" y1="125.284" x2="164.354" y2="225.849" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.3" stop-color="#7274FF"/>
|
||||
<stop offset="1" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<clipPath id="clip0_1197_17983">
|
||||
<rect width="256" height="256" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.3 KiB |
|
|
@ -1618,6 +1618,34 @@
|
|||
],
|
||||
"default_model_placeholder": "gpt-3.5-turbo"
|
||||
},
|
||||
{
|
||||
"provider": "FOUNDRY_LOCAL",
|
||||
"provider_display_name": "Foundry Local",
|
||||
"litellm_provider": "foundry_local",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "http://localhost:5272/v1",
|
||||
"tooltip": "Foundry Local endpoint (from foundry-local-sdk or manually set)",
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": "Optional - Foundry Local does not require authentication",
|
||||
"required": false,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "phi-3.5-mini"
|
||||
},
|
||||
{
|
||||
"provider": "MARITALK",
|
||||
"provider_display_name": "Maritalk",
|
||||
|
|
|
|||
|
|
@ -3317,6 +3317,7 @@ class LlmProviders(str, Enum):
|
|||
HOSTED_VLLM = "hosted_vllm"
|
||||
LLAMAFILE = "llamafile"
|
||||
LM_STUDIO = "lm_studio"
|
||||
FOUNDRY_LOCAL = "foundry_local"
|
||||
GALADRIEL = "galadriel"
|
||||
NEBIUS = "nebius"
|
||||
INFINITY = "infinity"
|
||||
|
|
|
|||
|
|
@ -8272,6 +8272,10 @@ class ProviderConfigManager:
|
|||
LlmProviders.HOSTED_VLLM: (lambda: litellm.HostedVLLMChatConfig(), False),
|
||||
LlmProviders.LLAMAFILE: (lambda: litellm.LlamafileChatConfig(), False),
|
||||
LlmProviders.LM_STUDIO: (lambda: litellm.LMStudioChatConfig(), False),
|
||||
LlmProviders.FOUNDRY_LOCAL: (
|
||||
lambda: litellm.FoundryLocalChatConfig(),
|
||||
False,
|
||||
),
|
||||
LlmProviders.GALADRIEL: (lambda: litellm.GaladrielChatConfig(), False),
|
||||
LlmProviders.REPLICATE: (lambda: litellm.ReplicateConfig(), False),
|
||||
LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False),
|
||||
|
|
|
|||
|
|
@ -1395,6 +1395,24 @@
|
|||
"interactions": true
|
||||
}
|
||||
},
|
||||
"foundry_local": {
|
||||
"display_name": "Foundry Local (`foundry_local`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/foundry_local",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"messages": true,
|
||||
"responses": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"moderations": false,
|
||||
"batches": false,
|
||||
"rerank": false,
|
||||
"a2a": true,
|
||||
"interactions": true
|
||||
}
|
||||
},
|
||||
"maritalk": {
|
||||
"display_name": "Maritalk (`maritalk`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/maritalk",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))
|
||||
)
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.foundry_local.chat.transformation import FoundryLocalChatConfig
|
||||
|
||||
|
||||
class TestFoundryLocalChatConfig:
|
||||
def test_should_resolve_provider_from_model_prefix(self):
|
||||
"""get_llm_provider should return foundry_local for 'foundry_local/model' strings."""
|
||||
_, provider, _, _ = get_llm_provider("foundry_local/phi-3.5-mini")
|
||||
assert provider == "foundry_local"
|
||||
|
||||
def test_should_return_fake_api_key_when_none_provided(self):
|
||||
"""Foundry Local does not require auth; a fake key is returned for the OpenAI client."""
|
||||
config = FoundryLocalChatConfig()
|
||||
_, api_key = config._get_openai_compatible_provider_info(None, None)
|
||||
assert api_key == "fake-api-key"
|
||||
|
||||
def test_should_use_explicit_api_key_when_provided(self):
|
||||
config = FoundryLocalChatConfig()
|
||||
_, api_key = config._get_openai_compatible_provider_info(None, "my-key")
|
||||
assert api_key == "my-key"
|
||||
|
||||
def test_should_use_explicit_api_base_when_provided(self):
|
||||
config = FoundryLocalChatConfig()
|
||||
api_base, _ = config._get_openai_compatible_provider_info(
|
||||
"http://localhost:5272/v1", None
|
||||
)
|
||||
assert api_base == "http://localhost:5272/v1"
|
||||
|
||||
def test_should_read_env_vars_for_api_base_and_key(self):
|
||||
config = FoundryLocalChatConfig()
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{
|
||||
"FOUNDRY_LOCAL_API_BASE": "http://localhost:5272/v1",
|
||||
"FOUNDRY_LOCAL_API_KEY": "env-key",
|
||||
},
|
||||
):
|
||||
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
|
||||
assert api_base == "http://localhost:5272/v1"
|
||||
assert api_key == "env-key"
|
||||
|
||||
def test_should_prefer_explicit_over_env_vars(self):
|
||||
config = FoundryLocalChatConfig()
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{
|
||||
"FOUNDRY_LOCAL_API_BASE": "http://env-base:5272/v1",
|
||||
"FOUNDRY_LOCAL_API_KEY": "env-key",
|
||||
},
|
||||
):
|
||||
api_base, api_key = config._get_openai_compatible_provider_info(
|
||||
"http://explicit:9999/v1", "explicit-key"
|
||||
)
|
||||
assert api_base == "http://explicit:9999/v1"
|
||||
assert api_key == "explicit-key"
|
||||
|
||||
def test_should_be_in_openai_compatible_providers(self):
|
||||
"""foundry_local must be listed as an OpenAI-compatible provider."""
|
||||
assert "foundry_local" in litellm.openai_compatible_providers
|
||||
|
||||
def test_should_be_in_provider_list(self):
|
||||
"""foundry_local must appear in the global provider list."""
|
||||
assert "foundry_local" in litellm.provider_list
|
||||
|
||||
def test_should_resolve_provider_info_in_get_llm_provider(self):
|
||||
"""get_llm_provider should resolve api_base and api_key via env vars."""
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{
|
||||
"FOUNDRY_LOCAL_API_BASE": "http://localhost:5272/v1",
|
||||
},
|
||||
):
|
||||
model, provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
"foundry_local/phi-3.5-mini"
|
||||
)
|
||||
assert model == "phi-3.5-mini"
|
||||
assert provider == "foundry_local"
|
||||
assert api_base == "http://localhost:5272/v1"
|
||||
assert dynamic_api_key == "fake-api-key"
|
||||
40
ui/litellm-dashboard/public/assets/logos/foundry_local.svg
Normal file
40
ui/litellm-dashboard/public/assets/logos/foundry_local.svg
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
<svg width="256" height="256" viewBox="0 0 256 256" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_1197_17983)">
|
||||
<rect width="256" height="256" fill="white"/>
|
||||
<path d="M154.903 33.0322H35.785C34.2618 33.0322 33.0323 34.2618 33.0323 35.7849V63.3118C33.0323 64.835 34.2618 66.0645 35.785 66.0645H181.677C186.238 66.0645 189.936 69.7623 189.936 74.3225V70.9C189.936 51.3101 175.319 33.0322 154.903 33.0322Z" fill="url(#paint0_linear_1197_17983)"/>
|
||||
<path d="M178.255 42.7031C185.733 50.1996 189.936 60.3112 189.936 70.8998V220.215C189.936 221.738 191.165 222.967 192.688 222.967H220.215C221.738 222.967 222.968 221.738 222.968 220.215V101.106C222.968 92.3433 219.49 83.9476 213.297 77.7449L178.255 42.7031Z" fill="url(#paint1_linear_1197_17983)"/>
|
||||
<path d="M113.613 74.3228H35.785C34.2618 74.3228 33.0323 75.5523 33.0323 77.0754V104.602C33.0323 106.125 34.2618 107.355 35.785 107.355H140.387C144.947 107.355 148.645 111.053 148.645 115.613V112.191C148.645 92.6006 134.028 74.3228 113.613 74.3228Z" fill="url(#paint2_linear_1197_17983)"/>
|
||||
<path d="M136.965 83.9937C144.443 91.4901 148.645 101.602 148.645 112.19V220.215C148.645 221.738 149.875 222.968 151.398 222.968H178.925C180.448 222.968 181.677 221.738 181.677 220.215V142.397C181.677 133.634 178.2 125.238 172.006 119.035L136.965 83.9937Z" fill="url(#paint3_linear_1197_17983)"/>
|
||||
<path d="M72.3223 115.613H35.785C34.2618 115.613 33.0323 116.842 33.0323 118.365V145.892C33.0323 147.416 34.2618 148.645 35.785 148.645H99.0968C103.657 148.645 107.355 152.343 107.355 156.903V153.481C107.355 133.891 92.7381 115.613 72.3223 115.613Z" fill="url(#paint4_linear_1197_17983)"/>
|
||||
<path d="M95.6743 125.284C103.152 132.781 107.355 142.892 107.355 153.481V220.215C107.355 221.738 108.584 222.968 110.108 222.968H137.634C139.158 222.968 140.387 221.738 140.387 220.215V183.687C140.387 174.924 136.91 166.529 130.716 160.326L95.6743 125.284Z" fill="url(#paint5_linear_1197_17983)"/>
|
||||
</g>
|
||||
<defs>
|
||||
<linearGradient id="paint0_linear_1197_17983" x1="189.936" y1="70.4672" x2="33.0323" y2="70.4672" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#2C08AC"/>
|
||||
<stop offset="0.8" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint1_linear_1197_17983" x1="201.96" y1="42.7031" x2="311.288" y2="175.174" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.3" stop-color="#7274FF"/>
|
||||
<stop offset="1" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint2_linear_1197_17983" x1="148.645" y1="111.758" x2="33.0323" y2="111.758" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#2C08AC"/>
|
||||
<stop offset="0.8" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint3_linear_1197_17983" x1="160.67" y1="83.9937" x2="238.429" y2="206.207" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.3" stop-color="#7274FF"/>
|
||||
<stop offset="1" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint4_linear_1197_17983" x1="107.355" y1="150.688" x2="33.0323" y2="150.688" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#2C08AC"/>
|
||||
<stop offset="0.8" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint5_linear_1197_17983" x1="119.38" y1="125.284" x2="164.354" y2="225.849" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.3" stop-color="#7274FF"/>
|
||||
<stop offset="1" stop-color="#4F42FD"/>
|
||||
</linearGradient>
|
||||
<clipPath id="clip0_1197_17983">
|
||||
<rect width="256" height="256" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.3 KiB |
|
|
@ -36,6 +36,7 @@ export enum Providers {
|
|||
ElevenLabs = "ElevenLabs",
|
||||
EMPOWER = "Empower",
|
||||
FalAI = "Fal AI",
|
||||
FOUNDRY_LOCAL = "Foundry Local",
|
||||
FEATHERLESS_AI = "Featherless Ai",
|
||||
FireworksAI = "Fireworks AI",
|
||||
FRIENDLIAI = "Friendliai",
|
||||
|
|
@ -143,6 +144,7 @@ export const provider_map: Record<string, string> = {
|
|||
ElevenLabs: "elevenlabs",
|
||||
EMPOWER: "empower",
|
||||
FalAI: "fal_ai",
|
||||
FOUNDRY_LOCAL: "foundry_local",
|
||||
FEATHERLESS_AI: "featherless_ai",
|
||||
FireworksAI: "fireworks_ai",
|
||||
FRIENDLIAI: "friendliai",
|
||||
|
|
@ -246,6 +248,7 @@ export const providerLogoMap: Record<string, string> = {
|
|||
[Providers.DeepInfra]: `${asset_logos_folder}deepinfra.png`,
|
||||
[Providers.ElevenLabs]: `${asset_logos_folder}elevenlabs.png`,
|
||||
[Providers.FalAI]: `${asset_logos_folder}fal_ai.jpg`,
|
||||
[Providers.FOUNDRY_LOCAL]: `${asset_logos_folder}foundry_local.svg`,
|
||||
[Providers.FEATHERLESS_AI]: `${asset_logos_folder}featherless.svg`,
|
||||
[Providers.FireworksAI]: `${asset_logos_folder}fireworks.svg`,
|
||||
[Providers.FRIENDLIAI]: `${asset_logos_folder}friendli.svg`,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue