From b010c3c13d4e290e227461047c2031dc4d0c038c Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Tue, 20 Oct 2026 20:07:21 -0500 Subject: [PATCH 1/5] feat: support Foundry Local with SDK 2.2.0 setup Allow on-device models to use a named OpenAI-compatible provider without adding the native SDK as a LiteLLM dependency. Replace the stale provider proposal with a contribution based on current main, and document the unified 2.2.0 Python wheel and explicit execution-provider registration. Fixes #47710 Replaces #29449 Files changed: - litellm/__init__.py - litellm/_lazy_imports_registry.py - litellm/constants.py - litellm/litellm_core_utils/get_llm_provider_logic.py - litellm/llms/foundry_local/__init__.py - litellm/llms/foundry_local/chat/__init__.py - litellm/llms/foundry_local/chat/transformation.py - litellm/proxy/public_endpoints/provider_create_fields.json - litellm/proxy/_experimental/out/assets/logos/foundry_local.svg - litellm/types/utils.py - litellm/utils.py - provider_endpoints_support.json - tests/unit/llms/foundry_local/chat/test_transformation.py - docs/my-website/docs/providers/foundry_local.md - ui/litellm-dashboard/public/assets/logos/foundry_local.svg - ui/litellm-dashboard/src/components/provider_info_helpers.tsx - ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cf44fcd-0425-4d84-8ee0-ef774b78ec80 --- .../docs/providers/foundry_local.md | 240 ++++++++++++++++++ litellm/__init__.py | 4 + litellm/_lazy_imports_registry.py | 5 + litellm/constants.py | 2 + .../get_llm_provider_logic.py | 5 + litellm/llms/foundry_local/__init__.py | 0 litellm/llms/foundry_local/chat/__init__.py | 0 .../llms/foundry_local/chat/transformation.py | 21 ++ .../out/assets/logos/foundry_local.svg | 40 +++ .../provider_create_fields.json | 28 ++ litellm/types/utils.py | 1 + litellm/utils.py | 1 + provider_endpoints_support.json | 18 ++ .../foundry_local/chat/test_transformation.py | 141 ++++++++++ .../public/assets/logos/foundry_local.svg | 40 +++ .../components/provider_info_helpers.test.tsx | 12 + .../src/components/provider_info_helpers.tsx | 4 + 17 files changed, 562 insertions(+) create mode 100644 docs/my-website/docs/providers/foundry_local.md create mode 100644 litellm/llms/foundry_local/__init__.py create mode 100644 litellm/llms/foundry_local/chat/__init__.py create mode 100644 litellm/llms/foundry_local/chat/transformation.py create mode 100644 litellm/proxy/_experimental/out/assets/logos/foundry_local.svg create mode 100644 tests/unit/llms/foundry_local/chat/test_transformation.py create mode 100644 ui/litellm-dashboard/public/assets/logos/foundry_local.svg diff --git a/docs/my-website/docs/providers/foundry_local.md b/docs/my-website/docs/providers/foundry_local.md new file mode 100644 index 00000000000..acb9af8d86b --- /dev/null +++ b/docs/my-website/docs/providers/foundry_local.md @@ -0,0 +1,240 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Foundry Local + +https://github.com/microsoft/Foundry-Local + +:::tip + +**We support ALL Foundry Local models, just set `model=foundry_local/` as a prefix when sending litellm requests** + +::: + + +| Property | Details | +|-------|-------| +| Description | Run AI models on-device with Microsoft Foundry Local. | +| Provider Route on LiteLLM | `foundry_local/` | +| Provider Doc | [Foundry Local ↗](https://learn.microsoft.com/en-us/azure/foundry-local/) | +| Supported OpenAI Endpoints | `/chat/completions` | + +## Quick Start + +Use [Foundry Local Python SDK 2.2.0](https://pypi.org/project/foundry-local-sdk/2.2.0/) to run models on-device. Foundry Local supports in-process inference, but LiteLLM uses its optional OpenAI-compatible HTTP service. Start that service and point `FOUNDRY_LOCAL_API_BASE` at its `/v1` URL. + + + + +```bash +pip install foundry-local-sdk==2.2.0 +``` + +```python +import os +from foundry_local_sdk import Configuration, FoundryLocalManager +from litellm import completion + +FoundryLocalManager.initialize(Configuration(app_name="litellm-foundry-local")) +manager = FoundryLocalManager.instance + +ep_result = manager.download_and_register_eps() +if not ep_result.success: + raise RuntimeError(f"Execution provider setup failed: {ep_result.failed_eps}") + +model = manager.catalog.get_model("qwen2.5-0.5b") +model.download() +model.load() +manager.start_web_service() + +os.environ["FOUNDRY_LOCAL_API_BASE"] = f"{manager.urls[0].rstrip('/')}/v1" +try: + response = completion( + model=f"foundry_local/{model.id}", + messages=[{"role": "user", "content": "What is the golden ratio?"}], + ) + print(response.choices[0].message.content) +finally: + manager.stop_web_service() + model.unload() + manager.close() +``` + + + + +```bash +npm install foundry-local-sdk +``` + +```javascript +import { FoundryLocalManager } from "foundry-local-sdk"; + +const manager = FoundryLocalManager.create({ + appName: "litellm-foundry-local", +}); + +const epResult = await manager.downloadAndRegisterEps(); +if (!epResult.success) { + throw new Error(`Execution provider setup failed: ${epResult.failedEps}`); +} + +const model = await manager.catalog.getModel("qwen2.5-0.5b"); +await model.download(); +await model.load(); +manager.startWebService(); + +const apiBase = `${manager.urls[0].replace(/\/$/, "")}/v1`; +console.log(apiBase); +console.log(`Use model: foundry_local/${model.id}`); +``` + +Keep this process running while LiteLLM uses the service. Configure the printed URL in the Python or proxy process, since JavaScript cannot set another process's environment variables. When finished, stop the service with `manager.stopWebService()` and unload the model with `await model.unload()`. + + + + +### SDK 2.2.0 requirements + +The unified `foundry-local-sdk` Python wheel supports Windows x64, Linux x64/ARM64, and macOS Apple silicon. It requires Python 3.11 through 3.14. Windows wheels include the WinML runtime, so the separate `foundry-local-sdk-winml` package is no longer needed. + +Call `download_and_register_eps()` before selecting and loading a model to make hardware execution providers available. Catalog access no longer waits for execution-provider downloads. + +The service uses a dynamically assigned local port by default. Read `manager.urls` after starting it instead of assuming port 5272. For a separate proxy process that needs a fixed address, initialize Python with `Configuration(app_name="litellm-foundry-local", web=Configuration.WebService(urls="http://127.0.0.1:5272"))` and keep that service process running. + +LiteLLM does not import or install the Foundry Local SDK. The SDK's Python requirement applies only to the process hosting Foundry Local, and existing OpenAI-compatible Foundry Local endpoints continue to work. This provider uses chat completions for both streaming and non-streaming requests. + +## API Key +```python +# env variable +os.environ['FOUNDRY_LOCAL_API_BASE'] # e.g. http://127.0.0.1:5272/v1 +os.environ['FOUNDRY_LOCAL_API_KEY'] # optional, not required for local use +``` + +## Sample Usage +```python +from litellm import completion +import os + +os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1" + +response = completion( + model="foundry_local/qwen2.5-0.5b", + messages=[ + { + "role": "user", + "content": "What's the weather like in Boston today in Fahrenheit?", + } + ] +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion +import os + +os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1" + +response = completion( + model="foundry_local/qwen2.5-0.5b", + messages=[ + { + "role": "user", + "content": "What's the weather like in Boston today in Fahrenheit?", + } + ], + stream=True, +) + +for chunk in response: + print(chunk) +``` + +## Usage with DSPy + +Foundry Local works seamlessly with [DSPy](https://dspy.ai/) via LiteLLM: + +```python +import dspy +import os + +os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1" + +lm = dspy.LM("foundry_local/qwen2.5-0.5b") +dspy.configure(lm=lm) +``` + +## Usage with LiteLLM Proxy Server + +Here's how to call a Foundry Local model with the LiteLLM Proxy Server + +1. Modify the config.yaml + + ```yaml + model_list: + - model_name: my-model + litellm_params: + model: foundry_local/qwen2.5-0.5b + api_base: http://127.0.0.1:5272/v1 + ``` + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml + ``` + +3. Send Request to LiteLLM Proxy Server + + + + + + ```python + import openai + client = openai.OpenAI( + api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys + base_url="http://0.0.0.0:4000" # litellm-proxy-base url + ) + + response = client.chat.completions.create( + model="my-model", + messages = [ + { + "role": "user", + "content": "what llm are you" + } + ], + ) + + print(response) + ``` + + + + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "my-model", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + }' + ``` + + + + + +## Supported Parameters + +See [Supported Parameters](../completion/input.md#translated-openai-params) for supported parameters. diff --git a/litellm/__init__.py b/litellm/__init__.py index 1784deeb797..6436ca53737 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2058,6 +2058,9 @@ if TYPE_CHECKING: from .llms.lm_studio.chat.transformation import ( LMStudioChatConfig as _LMStudioChatConfig, ) + from .llms.foundry_local.chat.transformation import ( + FoundryLocalChatConfig as _FoundryLocalChatConfig, + ) from .llms.lm_studio.embed.transformation import ( LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig, ) @@ -2083,6 +2086,7 @@ if TYPE_CHECKING: DeepInfraConfig: Type[_DeepInfraConfig] LlamafileChatConfig: Type[_LlamafileChatConfig] LMStudioChatConfig: Type[_LMStudioChatConfig] + FoundryLocalChatConfig: Type[_FoundryLocalChatConfig] LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig] IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index e0cb7c680bc..ce31432da9f 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -319,6 +319,7 @@ LLM_CONFIG_NAMES: Final = ( "DeepSeekChatConfig", "TencentChatConfig", "LMStudioChatConfig", + "FoundryLocalChatConfig", "LmStudioEmbeddingConfig", "NscaleConfig", "PerplexityChatConfig", @@ -1211,6 +1212,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"), "TencentChatConfig": (".llms.tencent.chat.transformation", "TencentChatConfig"), "LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"), + "FoundryLocalChatConfig": ( + ".llms.foundry_local.chat.transformation", + "FoundryLocalChatConfig", + ), "LmStudioEmbeddingConfig": ( ".llms.lm_studio.embed.transformation", "LmStudioEmbeddingConfig", diff --git a/litellm/constants.py b/litellm/constants.py index 1c1dfb9aa1c..324ca34663d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -838,6 +838,7 @@ LITELLM_CHAT_PROVIDERS: Final = [ "hosted_vllm", "llamafile", "lm_studio", + "foundry_local", "galadriel", "gradient_ai", "github_copilot", # GitHub Copilot Chat API @@ -1087,6 +1088,7 @@ openai_compatible_providers: Final[list[str]] = [ "hosted_vllm", "llamafile", "lm_studio", + "foundry_local", "galadriel", "github_copilot", # GitHub Copilot Chat API "chatgpt", # ChatGPT subscription API diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index a67c1bb7df8..f36e1ab23a2 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -735,6 +735,11 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.LMStudioChatConfig().get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "foundry_local": + ( + api_base, + dynamic_api_key, + ) = litellm.FoundryLocalChatConfig().get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "deepseek": # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1 api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta" diff --git a/litellm/llms/foundry_local/__init__.py b/litellm/llms/foundry_local/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/foundry_local/chat/__init__.py b/litellm/llms/foundry_local/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/foundry_local/chat/transformation.py b/litellm/llms/foundry_local/chat/transformation.py new file mode 100644 index 00000000000..efe4fcc4f99 --- /dev/null +++ b/litellm/llms/foundry_local/chat/transformation.py @@ -0,0 +1,21 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Foundry Local's `/v1/chat/completions` + +Foundry Local (https://github.com/microsoft/Foundry-Local) runs LLMs on-device +and exposes an OpenAI-compatible REST API endpoint. +""" + +from typing import Final + +from litellm.secret_managers.main import get_secret_str + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +class FoundryLocalChatConfig(OpenAIGPTConfig): + def get_openai_compatible_provider_info( + self, api_base: str | None, api_key: str | None + ) -> tuple[str | None, str | None]: + resolved_api_base: Final = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE") + dynamic_api_key: Final = api_key or get_secret_str("FOUNDRY_LOCAL_API_KEY") or "fake-api-key" + return resolved_api_base, dynamic_api_key diff --git a/litellm/proxy/_experimental/out/assets/logos/foundry_local.svg b/litellm/proxy/_experimental/out/assets/logos/foundry_local.svg new file mode 100644 index 00000000000..412a6fb70d6 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/foundry_local.svg @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 4bec42a7868..67efd2814b3 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2163,6 +2163,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "FOUNDRY_LOCAL", + "provider_display_name": "Foundry Local", + "litellm_provider": "foundry_local", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "http://localhost:5272/v1", + "tooltip": "Foundry Local endpoint (from foundry-local-sdk or manually set)", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": "Optional - Foundry Local does not require authentication", + "required": false, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "phi-3.5-mini" + }, { "provider": "MARITALK", "provider_display_name": "Maritalk", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5939a350970..1e1104489a3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4484,6 +4484,7 @@ class LlmProviders(str, Enum): TENCENT = "tencent" LLAMAFILE = "llamafile" LM_STUDIO = "lm_studio" + FOUNDRY_LOCAL = "foundry_local" GALADRIEL = "galadriel" NEBIUS = "nebius" INFINITY = "infinity" diff --git a/litellm/utils.py b/litellm/utils.py index 897138a940c..f851e4b8635 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9133,6 +9133,7 @@ class ProviderConfigManager: LlmProviders.HOSTED_VLLM: (lambda: litellm.HostedVLLMChatConfig(), False), LlmProviders.LLAMAFILE: (lambda: litellm.LlamafileChatConfig(), False), LlmProviders.LM_STUDIO: (lambda: litellm.LMStudioChatConfig(), False), + LlmProviders.FOUNDRY_LOCAL: (lambda: litellm.FoundryLocalChatConfig(), False), LlmProviders.GALADRIEL: (lambda: litellm.GaladrielChatConfig(), False), LlmProviders.REPLICATE: (lambda: litellm.ReplicateConfig(), False), LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False), diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index d65da3f41fd..e676438d87e 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1622,6 +1622,24 @@ "interactions": true } }, + "foundry_local": { + "display_name": "Foundry Local (`foundry_local`)", + "url": "https://docs.litellm.ai/docs/providers/foundry_local", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, "maritalk": { "display_name": "Maritalk (`maritalk`)", "url": "https://docs.litellm.ai/docs/providers/maritalk", diff --git a/tests/unit/llms/foundry_local/chat/test_transformation.py b/tests/unit/llms/foundry_local/chat/test_transformation.py new file mode 100644 index 00000000000..35a1400134c --- /dev/null +++ b/tests/unit/llms/foundry_local/chat/test_transformation.py @@ -0,0 +1,141 @@ +import json +from typing import Final + +import httpx +import pytest +from openai import AsyncOpenAI, OpenAI + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.types.utils import ModelResponse + + +@pytest.mark.parametrize( + ("api_base", "api_key", "env_base", "env_key", "expected_base", "expected_key"), + [ + (None, None, None, None, None, "fake-api-key"), + (None, None, "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "fake-api-key"), + (None, None, "http://127.0.0.1:61234/v1", "env-key", "http://127.0.0.1:61234/v1", "env-key"), + ("http://localhost:5272/v1", "explicit-key", None, None, "http://localhost:5272/v1", "explicit-key"), + ( + "http://localhost:5272/v1", + "explicit-key", + "http://127.0.0.1:61234/v1", + "env-key", + "http://localhost:5272/v1", + "explicit-key", + ), + ], +) +def test_should_resolve_foundry_endpoint_and_credentials( + monkeypatch: pytest.MonkeyPatch, + api_base: str | None, + api_key: str | None, + env_base: str | None, + env_key: str | None, + expected_base: str | None, + expected_key: str, +) -> None: + monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False) + monkeypatch.delenv("FOUNDRY_LOCAL_API_KEY", raising=False) + if env_base is not None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", env_base) + if env_key is not None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", env_key) + + assert get_llm_provider("foundry_local/local-model:1", api_base=api_base, api_key=api_key) == ( + "local-model:1", + "foundry_local", + expected_key, + expected_base, + ) + + +def _respond(request: httpx.Request, stream: bool) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/chat/completions" + payload: Final = json.loads(request.content) + assert payload["model"] == "local-model:1" + assert payload["messages"] == [{"role": "user", "content": "Hello"}] + assert payload["max_tokens"] == 16 + assert bool(payload.get("stream")) is stream + if stream: + chunk: Final = { + "id": "chatcmpl-local", + "object": "chat.completion.chunk", + "created": 0, + "model": "local-model:1", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello"}, "finish_reason": None}], + } + final_chunk: Final = { + **chunk, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(final_chunk)}\n\ndata: [DONE]\n\n", + ) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-local", + "object": "chat.completion", + "created": 0, + "model": "local-model:1", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + }, + ) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_should_complete_chat_over_foundry_http_service(stream: bool) -> None: + with ( + httpx.Client(transport=httpx.MockTransport(lambda request: _respond(request, stream))) as http_client, + OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = litellm.completion( + model="foundry_local/local-model:1", + api_base="http://127.0.0.1:61234/v1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=16, + stream=stream, + client=client, + ) + if stream: + assert not isinstance(response, ModelResponse) + chunks: Final = tuple(response) + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "Hello" + assert chunks[-1].choices[0].finish_reason == "stop" + else: + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "Hello" + assert response.choices[0].finish_reason == "stop" + assert response.usage.total_tokens == 3 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_should_complete_chat_asynchronously_over_foundry_http_service(stream: bool) -> None: + async with ( + httpx.AsyncClient(transport=httpx.MockTransport(lambda request: _respond(request, stream))) as http_client, + AsyncOpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = await litellm.acompletion( + model="foundry_local/local-model:1", + api_base="http://127.0.0.1:61234/v1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=16, + stream=stream, + client=client, + ) + if stream: + assert not isinstance(response, ModelResponse) + chunks: Final = [chunk async for chunk in response] + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "Hello" + assert chunks[-1].choices[0].finish_reason == "stop" + else: + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "Hello" + assert response.choices[0].finish_reason == "stop" + assert response.usage.total_tokens == 3 diff --git a/ui/litellm-dashboard/public/assets/logos/foundry_local.svg b/ui/litellm-dashboard/public/assets/logos/foundry_local.svg new file mode 100644 index 00000000000..412a6fb70d6 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/foundry_local.svg @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 0ebd2cdff17..186560f63d9 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -20,6 +20,18 @@ const BUNDLED_MODEL_MAP: Record = JSON.pa ); describe("provider_info_helpers", () => { + it("should resolve Foundry Local for the model form and provider display", () => { + expect(resolveLitellmProviderSlug("FOUNDRY_LOCAL")).toBe("foundry_local"); + expect(getProviderLogoAndName("foundry_local")).toEqual({ + displayName: "Foundry Local", + logo: providerLogoMap[Providers.FOUNDRY_LOCAL], + }); + expect(getProviderLogoAndName("foundry_local").logo).toContain("foundry_local"); + expect(PROVIDER_CREATE_FIELDS.find((provider) => provider.provider === "FOUNDRY_LOCAL")?.litellm_provider).toBe( + "foundry_local", + ); + }); + it("maps Microsoft 365 Copilot to its chat model provider and placeholder", () => { expect(provider_map.MICROSOFT_365_COPILOT).toBe("microsoft_365_copilot"); expect(getPlaceholder(Providers.MICROSOFT_365_COPILOT)).toBe("microsoft_365_copilot/chat"); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 5a163bce3b6..56788de2b61 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -20,6 +20,7 @@ import elevenlabsLogo from "../../public/assets/logos/elevenlabs.png"; import falAiLogo from "../../public/assets/logos/fal_ai.jpg"; import featherlessLogo from "../../public/assets/logos/featherless.svg"; import fireworksLogo from "../../public/assets/logos/fireworks.svg"; +import foundryLocalLogo from "../../public/assets/logos/foundry_local.svg"; import friendliLogo from "../../public/assets/logos/friendli.svg"; import gigachatLogo from "../../public/assets/logos/gigachat.svg"; import githubCopilotLogo from "../../public/assets/logos/github_copilot.svg"; @@ -134,6 +135,7 @@ export enum Providers { LEMONADE = "Lemonade", LLAMAFILE = "Llamafile", LM_STUDIO = "Lm Studio", + FOUNDRY_LOCAL = "Foundry Local", LLAMA = "Meta Llama", MARITALK = "Maritalk", MICROSOFT_365_COPILOT = "Microsoft 365 Copilot", @@ -261,6 +263,7 @@ export const provider_map: Record = { LLAMAFILE: "llamafile", LLAMA: "meta_llama", LM_STUDIO: "lm_studio", + FOUNDRY_LOCAL: "foundry_local", MICROSOFT_365_COPILOT: "microsoft_365_copilot", MARITALK: "maritalk", MiniMax: "minimax", @@ -364,6 +367,7 @@ export const providerLogoMap: Partial> = { [Providers.FalAI]: falAiLogo.src, [Providers.FEATHERLESS_AI]: featherlessLogo.src, [Providers.FireworksAI]: fireworksLogo.src, + [Providers.FOUNDRY_LOCAL]: foundryLocalLogo.src, [Providers.FRIENDLIAI]: friendliLogo.src, [Providers.GIGACHAT]: gigachatLogo.src, [Providers.GITHUB_COPILOT]: githubCopilotLogo.src, From 8d9cf7a407071fb34894b18948606af651ed0c37 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 21 Oct 2026 13:56:07 -0500 Subject: [PATCH 2/5] docs: use Foundry Local SDK 2.2.1 for current setup Keep setup instructions on the latest published patch release Files changed, docs/my-website/docs/providers/foundry_local.md Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cf44fcd-0425-4d84-8ee0-ef774b78ec80 --- docs/my-website/docs/providers/foundry_local.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/providers/foundry_local.md b/docs/my-website/docs/providers/foundry_local.md index acb9af8d86b..733e4ce94f6 100644 --- a/docs/my-website/docs/providers/foundry_local.md +++ b/docs/my-website/docs/providers/foundry_local.md @@ -21,13 +21,13 @@ https://github.com/microsoft/Foundry-Local ## Quick Start -Use [Foundry Local Python SDK 2.2.0](https://pypi.org/project/foundry-local-sdk/2.2.0/) to run models on-device. Foundry Local supports in-process inference, but LiteLLM uses its optional OpenAI-compatible HTTP service. Start that service and point `FOUNDRY_LOCAL_API_BASE` at its `/v1` URL. +Use [Foundry Local Python SDK 2.2.1](https://pypi.org/project/foundry-local-sdk/2.2.1/) to run models on-device. Foundry Local supports in-process inference, but LiteLLM uses its optional OpenAI-compatible HTTP service. Start that service and point `FOUNDRY_LOCAL_API_BASE` at its `/v1` URL. ```bash -pip install foundry-local-sdk==2.2.0 +pip install foundry-local-sdk==2.2.1 ``` ```python @@ -94,7 +94,7 @@ Keep this process running while LiteLLM uses the service. Configure the printed -### SDK 2.2.0 requirements +### SDK 2.2.1 requirements The unified `foundry-local-sdk` Python wheel supports Windows x64, Linux x64/ARM64, and macOS Apple silicon. It requires Python 3.11 through 3.14. Windows wheels include the WinML runtime, so the separate `foundry-local-sdk-winml` package is no longer needed. From 09a7438aa6d1c1834e57a8ebfa38812c529113dc Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 21 Oct 2026 16:18:06 -0500 Subject: [PATCH 3/5] fix: run Foundry Local tests in CI and preserve routing inputs Assign the provider tests to the existing shard, mark their directories as packages, and cover mixed endpoint and key precedence with both provider selection modes. Move the setup page to the separately published docs repository Files changed, .github/workflows/test-unit.yml, docs/my-website/docs/providers/foundry_local.md, litellm/litellm_core_utils/get_llm_provider_logic.py, tests/unit/llms/foundry_local/__init__.py, tests/unit/llms/foundry_local/chat/__init__.py, tests/unit/llms/foundry_local/chat/test_transformation.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cf44fcd-0425-4d84-8ee0-ef774b78ec80 --- .github/workflows/test-unit.yml | 1 + .../docs/providers/foundry_local.md | 240 ------------------ .../get_llm_provider_logic.py | 8 +- tests/unit/llms/foundry_local/__init__.py | 0 .../unit/llms/foundry_local/chat/__init__.py | 0 .../foundry_local/chat/test_transformation.py | 10 +- 6 files changed, 14 insertions(+), 245 deletions(-) delete mode 100644 docs/my-website/docs/providers/foundry_local.md create mode 100644 tests/unit/llms/foundry_local/__init__.py create mode 100644 tests/unit/llms/foundry_local/chat/__init__.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index c593acbe8fb..d08fc021631 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -267,6 +267,7 @@ jobs: tests/unit/llms/featherless_ai tests/unit/llms/firecrawl tests/unit/llms/fireworks_ai + tests/unit/llms/foundry_local tests/unit/llms/friendliai tests/unit/llms/gdc tests/unit/llms/gemini diff --git a/docs/my-website/docs/providers/foundry_local.md b/docs/my-website/docs/providers/foundry_local.md deleted file mode 100644 index 733e4ce94f6..00000000000 --- a/docs/my-website/docs/providers/foundry_local.md +++ /dev/null @@ -1,240 +0,0 @@ -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - -# Foundry Local - -https://github.com/microsoft/Foundry-Local - -:::tip - -**We support ALL Foundry Local models, just set `model=foundry_local/` as a prefix when sending litellm requests** - -::: - - -| Property | Details | -|-------|-------| -| Description | Run AI models on-device with Microsoft Foundry Local. | -| Provider Route on LiteLLM | `foundry_local/` | -| Provider Doc | [Foundry Local ↗](https://learn.microsoft.com/en-us/azure/foundry-local/) | -| Supported OpenAI Endpoints | `/chat/completions` | - -## Quick Start - -Use [Foundry Local Python SDK 2.2.1](https://pypi.org/project/foundry-local-sdk/2.2.1/) to run models on-device. Foundry Local supports in-process inference, but LiteLLM uses its optional OpenAI-compatible HTTP service. Start that service and point `FOUNDRY_LOCAL_API_BASE` at its `/v1` URL. - - - - -```bash -pip install foundry-local-sdk==2.2.1 -``` - -```python -import os -from foundry_local_sdk import Configuration, FoundryLocalManager -from litellm import completion - -FoundryLocalManager.initialize(Configuration(app_name="litellm-foundry-local")) -manager = FoundryLocalManager.instance - -ep_result = manager.download_and_register_eps() -if not ep_result.success: - raise RuntimeError(f"Execution provider setup failed: {ep_result.failed_eps}") - -model = manager.catalog.get_model("qwen2.5-0.5b") -model.download() -model.load() -manager.start_web_service() - -os.environ["FOUNDRY_LOCAL_API_BASE"] = f"{manager.urls[0].rstrip('/')}/v1" -try: - response = completion( - model=f"foundry_local/{model.id}", - messages=[{"role": "user", "content": "What is the golden ratio?"}], - ) - print(response.choices[0].message.content) -finally: - manager.stop_web_service() - model.unload() - manager.close() -``` - - - - -```bash -npm install foundry-local-sdk -``` - -```javascript -import { FoundryLocalManager } from "foundry-local-sdk"; - -const manager = FoundryLocalManager.create({ - appName: "litellm-foundry-local", -}); - -const epResult = await manager.downloadAndRegisterEps(); -if (!epResult.success) { - throw new Error(`Execution provider setup failed: ${epResult.failedEps}`); -} - -const model = await manager.catalog.getModel("qwen2.5-0.5b"); -await model.download(); -await model.load(); -manager.startWebService(); - -const apiBase = `${manager.urls[0].replace(/\/$/, "")}/v1`; -console.log(apiBase); -console.log(`Use model: foundry_local/${model.id}`); -``` - -Keep this process running while LiteLLM uses the service. Configure the printed URL in the Python or proxy process, since JavaScript cannot set another process's environment variables. When finished, stop the service with `manager.stopWebService()` and unload the model with `await model.unload()`. - - - - -### SDK 2.2.1 requirements - -The unified `foundry-local-sdk` Python wheel supports Windows x64, Linux x64/ARM64, and macOS Apple silicon. It requires Python 3.11 through 3.14. Windows wheels include the WinML runtime, so the separate `foundry-local-sdk-winml` package is no longer needed. - -Call `download_and_register_eps()` before selecting and loading a model to make hardware execution providers available. Catalog access no longer waits for execution-provider downloads. - -The service uses a dynamically assigned local port by default. Read `manager.urls` after starting it instead of assuming port 5272. For a separate proxy process that needs a fixed address, initialize Python with `Configuration(app_name="litellm-foundry-local", web=Configuration.WebService(urls="http://127.0.0.1:5272"))` and keep that service process running. - -LiteLLM does not import or install the Foundry Local SDK. The SDK's Python requirement applies only to the process hosting Foundry Local, and existing OpenAI-compatible Foundry Local endpoints continue to work. This provider uses chat completions for both streaming and non-streaming requests. - -## API Key -```python -# env variable -os.environ['FOUNDRY_LOCAL_API_BASE'] # e.g. http://127.0.0.1:5272/v1 -os.environ['FOUNDRY_LOCAL_API_KEY'] # optional, not required for local use -``` - -## Sample Usage -```python -from litellm import completion -import os - -os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1" - -response = completion( - model="foundry_local/qwen2.5-0.5b", - messages=[ - { - "role": "user", - "content": "What's the weather like in Boston today in Fahrenheit?", - } - ] -) -print(response) -``` - -## Sample Usage - Streaming -```python -from litellm import completion -import os - -os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1" - -response = completion( - model="foundry_local/qwen2.5-0.5b", - messages=[ - { - "role": "user", - "content": "What's the weather like in Boston today in Fahrenheit?", - } - ], - stream=True, -) - -for chunk in response: - print(chunk) -``` - -## Usage with DSPy - -Foundry Local works seamlessly with [DSPy](https://dspy.ai/) via LiteLLM: - -```python -import dspy -import os - -os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1" - -lm = dspy.LM("foundry_local/qwen2.5-0.5b") -dspy.configure(lm=lm) -``` - -## Usage with LiteLLM Proxy Server - -Here's how to call a Foundry Local model with the LiteLLM Proxy Server - -1. Modify the config.yaml - - ```yaml - model_list: - - model_name: my-model - litellm_params: - model: foundry_local/qwen2.5-0.5b - api_base: http://127.0.0.1:5272/v1 - ``` - - -2. Start the proxy - - ```bash - $ litellm --config /path/to/config.yaml - ``` - -3. Send Request to LiteLLM Proxy Server - - - - - - ```python - import openai - client = openai.OpenAI( - api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys - base_url="http://0.0.0.0:4000" # litellm-proxy-base url - ) - - response = client.chat.completions.create( - model="my-model", - messages = [ - { - "role": "user", - "content": "what llm are you" - } - ], - ) - - print(response) - ``` - - - - - ```shell - curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "my-model", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ] - }' - ``` - - - - - -## Supported Parameters - -See [Supported Parameters](../completion/input.md#translated-openai-params) for supported parameters. diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index f36e1ab23a2..c62252dbee9 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -736,10 +736,10 @@ def _get_openai_compatible_provider_info( dynamic_api_key, ) = litellm.LMStudioChatConfig().get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "foundry_local": - ( - api_base, - dynamic_api_key, - ) = litellm.FoundryLocalChatConfig().get_openai_compatible_provider_info(api_base, api_key) + foundry_api_base, foundry_api_key = litellm.FoundryLocalChatConfig().get_openai_compatible_provider_info( + api_base, api_key + ) + return model, custom_llm_provider, foundry_api_key, foundry_api_base elif custom_llm_provider == "deepseek": # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1 api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta" diff --git a/tests/unit/llms/foundry_local/__init__.py b/tests/unit/llms/foundry_local/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/foundry_local/chat/__init__.py b/tests/unit/llms/foundry_local/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/foundry_local/chat/test_transformation.py b/tests/unit/llms/foundry_local/chat/test_transformation.py index 35a1400134c..7630425ef15 100644 --- a/tests/unit/llms/foundry_local/chat/test_transformation.py +++ b/tests/unit/llms/foundry_local/chat/test_transformation.py @@ -10,12 +10,18 @@ from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.types.utils import ModelResponse +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [("foundry_local/local-model:1", None), ("local-model:1", "foundry_local")], +) @pytest.mark.parametrize( ("api_base", "api_key", "env_base", "env_key", "expected_base", "expected_key"), [ (None, None, None, None, None, "fake-api-key"), (None, None, "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "fake-api-key"), (None, None, "http://127.0.0.1:61234/v1", "env-key", "http://127.0.0.1:61234/v1", "env-key"), + ("http://localhost:5272/v1", None, None, "env-key", "http://localhost:5272/v1", "env-key"), + (None, "explicit-key", "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "explicit-key"), ("http://localhost:5272/v1", "explicit-key", None, None, "http://localhost:5272/v1", "explicit-key"), ( "http://localhost:5272/v1", @@ -29,6 +35,8 @@ from litellm.types.utils import ModelResponse ) def test_should_resolve_foundry_endpoint_and_credentials( monkeypatch: pytest.MonkeyPatch, + model: str, + custom_llm_provider: str | None, api_base: str | None, api_key: str | None, env_base: str | None, @@ -43,7 +51,7 @@ def test_should_resolve_foundry_endpoint_and_credentials( if env_key is not None: monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", env_key) - assert get_llm_provider("foundry_local/local-model:1", api_base=api_base, api_key=api_key) == ( + assert get_llm_provider(model, custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key) == ( "local-model:1", "foundry_local", expected_key, From 50854b47c6da0b980e67c6f3e41ef0634b8e821c Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 21 Oct 2026 18:13:48 -0500 Subject: [PATCH 4/5] fix: complete Foundry Local embedding and transcription support Route embeddings through the existing OpenAI transport and adapt audio uploads to Foundry's filename-based JSON contract. Keep temporary audio alive through sync/async requests and clean it up after errors or cancellation. Reject missing endpoints instead of falling back to a public service. Files changed: litellm/__init__.py, litellm/_lazy_imports_registry.py, litellm/main.py, litellm/utils.py, litellm/llms/base_llm/audio_transcription/transformation.py, litellm/llms/custom_httpx/llm_http_handler.py, litellm/llms/foundry_local/audio_transcription/__init__.py, litellm/llms/foundry_local/audio_transcription/transformation.py, litellm/llms/foundry_local/chat/transformation.py, provider_endpoints_support.json, tests/unit/llms/foundry_local/chat/test_transformation.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 837c7cf9-c279-4a20-b611-200f84ddf448 --- litellm/__init__.py | 4 + litellm/_lazy_imports_registry.py | 5 + .../audio_transcription/transformation.py | 4 + litellm/llms/custom_httpx/llm_http_handler.py | 174 ++++++++------- .../audio_transcription/__init__.py | 0 .../audio_transcription/transformation.py | 72 +++++++ .../llms/foundry_local/chat/transformation.py | 5 +- litellm/main.py | 1 + litellm/utils.py | 2 + provider_endpoints_support.json | 4 +- .../foundry_local/chat/test_transformation.py | 200 +++++++++++++++++- 11 files changed, 379 insertions(+), 92 deletions(-) create mode 100644 litellm/llms/foundry_local/audio_transcription/__init__.py create mode 100644 litellm/llms/foundry_local/audio_transcription/transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 6436ca53737..75198696d34 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2061,6 +2061,9 @@ if TYPE_CHECKING: from .llms.foundry_local.chat.transformation import ( FoundryLocalChatConfig as _FoundryLocalChatConfig, ) + from .llms.foundry_local.audio_transcription.transformation import ( + FoundryLocalAudioTranscriptionConfig as _FoundryLocalAudioTranscriptionConfig, + ) from .llms.lm_studio.embed.transformation import ( LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig, ) @@ -2087,6 +2090,7 @@ if TYPE_CHECKING: LlamafileChatConfig: Type[_LlamafileChatConfig] LMStudioChatConfig: Type[_LMStudioChatConfig] FoundryLocalChatConfig: Type[_FoundryLocalChatConfig] + FoundryLocalAudioTranscriptionConfig: Type[_FoundryLocalAudioTranscriptionConfig] LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig] IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index ce31432da9f..b36972e215d 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -320,6 +320,7 @@ LLM_CONFIG_NAMES: Final = ( "TencentChatConfig", "LMStudioChatConfig", "FoundryLocalChatConfig", + "FoundryLocalAudioTranscriptionConfig", "LmStudioEmbeddingConfig", "NscaleConfig", "PerplexityChatConfig", @@ -1216,6 +1217,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.foundry_local.chat.transformation", "FoundryLocalChatConfig", ), + "FoundryLocalAudioTranscriptionConfig": ( + ".llms.foundry_local.audio_transcription.transformation", + "FoundryLocalAudioTranscriptionConfig", + ), "LmStudioEmbeddingConfig": ( ".llms.lm_studio.embed.transformation", "LmStudioEmbeddingConfig", diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index e4bf148abf3..88d4728a28f 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final @@ -37,6 +38,9 @@ class AudioTranscriptionRequestData: class BaseAudioTranscriptionConfig(BaseConfig, ABC): + def audio_file_context(self, audio_file: FileTypes) -> AbstractContextManager[FileTypes]: + return nullcontext(audio_file) + @abstractmethod def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: pass diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ac2ac1fe31b..2b5a86521fb 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1950,50 +1950,48 @@ class BaseLLMHTTPHandler: shared_session=shared_session, ) - # Prepare the request - ( - headers, - complete_url, - data, - files, - ) = self._prepare_audio_transcription_request( - model=model, - audio_file=audio_file, - optional_params=optional_params, - litellm_params=litellm_params, - logging_obj=logging_obj, - api_key=api_key, - api_base=api_base, - headers=headers, - provider_config=provider_config, - ) - - if client is None or not isinstance(client, HTTPHandler): - client = get_httpx_client() - - json_data: Final = data if files is None and isinstance(data, dict) else None - - try: - response: Final = client.post( - url=complete_url, + with provider_config.audio_file_context(audio_file) as prepared_audio_file: + ( + prepared_headers, + complete_url, + data, + files, + ) = self._prepare_audio_transcription_request( + model=model, + audio_file=prepared_audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, headers=headers, - data=data if json_data is None else None, - files=files, - json=json_data, - timeout=timeout, + provider_config=provider_config, ) - except Exception as e: - raise self._handle_error(e=e, provider_config=provider_config) - return self._transform_audio_transcription_response( - provider_config=provider_config, - model=model, - response=response, - model_response=model_response, - logging_obj=logging_obj, - optional_params=optional_params, - api_key=api_key, - ) + sync_client: Final = client if isinstance(client, HTTPHandler) else get_httpx_client() + json_data: Final = data if files is None and isinstance(data, dict) else None + + try: + response: Final = sync_client.post( + url=complete_url, + headers=prepared_headers, + data=data if json_data is None else None, + files=files, + json=json_data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + return self._transform_audio_transcription_response( + provider_config=provider_config, + model=model, + response=response, + model_response=model_response, + logging_obj=logging_obj, + optional_params=optional_params, + api_key=api_key, + ) async def async_audio_transcriptions( self, @@ -2016,56 +2014,56 @@ class BaseLLMHTTPHandler: if provider_config is None: raise ValueError(f"No provider config found for model: {model} and provider: {custom_llm_provider}") - # Prepare the request - ( - headers, - complete_url, - data, - files, - ) = self._prepare_audio_transcription_request( - model=model, - audio_file=audio_file, - optional_params=optional_params, - litellm_params=litellm_params, - logging_obj=logging_obj, - api_key=api_key, - api_base=api_base, - headers=headers, - provider_config=provider_config, - ) - - if client is None or not isinstance(client, AsyncHTTPHandler): - async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), - params={"ssl_verify": litellm_params.get("ssl_verify", None)}, - shared_session=shared_session, - ) - else: - async_httpx_client = client - - json_data: Final = data if files is None and isinstance(data, dict) else None - - try: - response: Final = await async_httpx_client.post( - url=complete_url, + with provider_config.audio_file_context(audio_file) as prepared_audio_file: + ( + prepared_headers, + complete_url, + data, + files, + ) = self._prepare_audio_transcription_request( + model=model, + audio_file=prepared_audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, headers=headers, - data=data if json_data is None else None, - files=files, - json=json_data, - timeout=timeout, + provider_config=provider_config, ) - except Exception as e: - raise self._handle_error(e=e, provider_config=provider_config) - return self._transform_audio_transcription_response( - provider_config=provider_config, - model=model, - response=response, - model_response=model_response, - logging_obj=logging_obj, - optional_params=optional_params, - api_key=api_key, - ) + async_httpx_client: Final = ( + client + if isinstance(client, AsyncHTTPHandler) + else get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, + ) + ) + json_data: Final = data if files is None and isinstance(data, dict) else None + + try: + response: Final = await async_httpx_client.post( + url=complete_url, + headers=prepared_headers, + data=data if json_data is None else None, + files=files, + json=json_data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + return self._transform_audio_transcription_response( + provider_config=provider_config, + model=model, + response=response, + model_response=model_response, + logging_obj=logging_obj, + optional_params=optional_params, + api_key=api_key, + ) def search( self, diff --git a/litellm/llms/foundry_local/audio_transcription/__init__.py b/litellm/llms/foundry_local/audio_transcription/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/foundry_local/audio_transcription/transformation.py b/litellm/llms/foundry_local/audio_transcription/transformation.py new file mode 100644 index 00000000000..8463d7578a4 --- /dev/null +++ b/litellm/llms/foundry_local/audio_transcription/transformation.py @@ -0,0 +1,72 @@ +from collections.abc import Generator, Mapping, Sequence +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Final + +from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData +from litellm.llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig +from litellm.types.llms.openai import AllMessageValues, OpenAIAudioTranscriptionOptionalParams +from litellm.types.utils import FileTypes + + +class FoundryLocalAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): + @property + def has_native_transcription_endpoint(self) -> bool: + return True + + def get_supported_openai_params( + self, model: str + ) -> list[OpenAIAudioTranscriptionOptionalParams]: # mutable-ok: base class contract + return ["language", "prompt", "response_format", "temperature"] + + def validate_environment( + self, + headers: Mapping[str, object], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, object]: # mutable-ok: the shared HTTP handler adds the request Content-Type header + return {**headers, **({"Authorization": f"Bearer {api_key}"} if api_key else {})} + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + if not api_base: + raise ValueError("Foundry Local transcription requires api_base or FOUNDRY_LOCAL_API_BASE") + return f"{api_base.rstrip('/')}/audio/transcriptions" + + @contextmanager + def audio_file_context(self, audio_file: FileTypes) -> Generator[FileTypes, None, None]: + processed: Final = process_audio_file(audio_file) + # Foundry reads a filename rather than an upload. Close it before HTTP for Windows. + with TemporaryDirectory(prefix="litellm-foundry-audio-") as directory: + path: Final = Path(directory) / f"audio{Path(processed.filename).suffix or '.wav'}" + path.write_bytes(processed.file_content) + yield path + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> AudioTranscriptionRequestData: + if not isinstance(audio_file, Path): + raise ValueError("Foundry Local audio must be prepared using audio_file_context") + if optional_params.get("stream"): + raise ValueError("Streaming Foundry Local transcription is not supported by this integration") + return AudioTranscriptionRequestData( + data={**optional_params, "model": model, "filename": str(audio_file), "stream": False}, + content_type="application/json", + ) diff --git a/litellm/llms/foundry_local/chat/transformation.py b/litellm/llms/foundry_local/chat/transformation.py index efe4fcc4f99..410cb2a8359 100644 --- a/litellm/llms/foundry_local/chat/transformation.py +++ b/litellm/llms/foundry_local/chat/transformation.py @@ -7,6 +7,7 @@ and exposes an OpenAI-compatible REST API endpoint. from typing import Final +import litellm from litellm.secret_managers.main import get_secret_str from ...openai.chat.gpt_transformation import OpenAIGPTConfig @@ -16,6 +17,8 @@ class FoundryLocalChatConfig(OpenAIGPTConfig): def get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: - resolved_api_base: Final = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE") + resolved_api_base: Final = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE") or litellm.api_base + if not resolved_api_base: + raise ValueError("Foundry Local requires api_base or FOUNDRY_LOCAL_API_BASE") dynamic_api_key: Final = api_key or get_secret_str("FOUNDRY_LOCAL_API_KEY") or "fake-api-key" return resolved_api_base, dynamic_api_key diff --git a/litellm/main.py b/litellm/main.py index d647cab24e3..f01b78783b9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6810,6 +6810,7 @@ def embedding( or custom_llm_provider == "together_ai" or custom_llm_provider == "nvidia_nim" or custom_llm_provider == "litellm_proxy" + or custom_llm_provider == "foundry_local" or (model in litellm.open_ai_embedding_models and custom_llm_provider is None) ): api_base = ( diff --git a/litellm/utils.py b/litellm/utils.py index f851e4b8635..088070bb9ed 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9674,6 +9674,8 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAudioTranscriptionConfig | None: + if provider == LlmProviders.FOUNDRY_LOCAL: + return litellm.FoundryLocalAudioTranscriptionConfig() model_cost_entry: Final = _get_model_cost_entry_for_provider_config( model=model, provider=provider, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e676438d87e..939734c4958 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1629,9 +1629,9 @@ "chat_completions": true, "messages": true, "responses": true, - "embeddings": false, + "embeddings": true, "image_generations": false, - "audio_transcriptions": false, + "audio_transcriptions": true, "audio_speech": false, "moderations": false, "batches": false, diff --git a/tests/unit/llms/foundry_local/chat/test_transformation.py b/tests/unit/llms/foundry_local/chat/test_transformation.py index 7630425ef15..8123c763436 100644 --- a/tests/unit/llms/foundry_local/chat/test_transformation.py +++ b/tests/unit/llms/foundry_local/chat/test_transformation.py @@ -1,4 +1,7 @@ +import asyncio import json +from io import BytesIO +from pathlib import Path from typing import Final import httpx @@ -7,6 +10,9 @@ from openai import AsyncOpenAI, OpenAI import litellm from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.foundry_local.audio_transcription.transformation import FoundryLocalAudioTranscriptionConfig +from litellm.llms.openai.common_utils import OpenAIError from litellm.types.utils import ModelResponse @@ -17,7 +23,6 @@ from litellm.types.utils import ModelResponse @pytest.mark.parametrize( ("api_base", "api_key", "env_base", "env_key", "expected_base", "expected_key"), [ - (None, None, None, None, None, "fake-api-key"), (None, None, "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "fake-api-key"), (None, None, "http://127.0.0.1:61234/v1", "env-key", "http://127.0.0.1:61234/v1", "env-key"), ("http://localhost:5272/v1", None, None, "env-key", "http://localhost:5272/v1", "env-key"), @@ -96,6 +101,28 @@ def _respond(request: httpx.Request, stream: bool) -> httpx.Response: ) +@pytest.mark.parametrize("provider", [None, "foundry_local"]) +def test_should_require_a_foundry_endpoint(monkeypatch: pytest.MonkeyPatch, provider: str | None) -> None: + monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False) + with pytest.raises(litellm.BadRequestError, match="requires api_base"): + get_llm_provider( + "local-model:1" if provider else "foundry_local/local-model:1", + custom_llm_provider=provider, + ) + + +def test_should_preserve_global_api_base_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False) + monkeypatch.delenv("FOUNDRY_LOCAL_API_KEY", raising=False) + monkeypatch.setattr(litellm, "api_base", "http://127.0.0.1:61234/v1") + assert get_llm_provider("foundry_local/local-model:1") == ( + "local-model:1", + "foundry_local", + "fake-api-key", + "http://127.0.0.1:61234/v1", + ) + + @pytest.mark.parametrize("stream", [False, True]) def test_should_complete_chat_over_foundry_http_service(stream: bool) -> None: with ( @@ -122,6 +149,177 @@ def test_should_complete_chat_over_foundry_http_service(stream: bool) -> None: assert response.usage.total_tokens == 3 +def _embedding_response(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/embeddings" + assert request.headers["Authorization"] == "Bearer env-key" + payload: Final = json.loads(request.content) + assert payload["model"] == "embedding-model:1" + assert payload["input"] == ["Hello", "World"] + return httpx.Response( + 200, + json={ + "object": "list", + "model": "embedding-model:1", + "data": [ + {"object": "embedding", "index": 0, "embedding": [0.25, 0.75]}, + {"object": "embedding", "index": 1, "embedding": [0.5, 0.5]}, + ], + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + + +def test_should_embed_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key") + with ( + httpx.Client(transport=httpx.MockTransport(_embedding_response)) as http_client, + OpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = litellm.embedding( + model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client + ) + assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]] + assert response.usage.prompt_tokens == 2 + + +@pytest.mark.asyncio +async def test_should_embed_asynchronously_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key") + async with ( + httpx.AsyncClient(transport=httpx.MockTransport(_embedding_response)) as http_client, + AsyncOpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = await litellm.aembedding( + model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client + ) + assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]] + assert response.usage.prompt_tokens == 2 + + +@pytest.mark.parametrize("status_code", [200, 400]) +@pytest.mark.parametrize("input_type", ["bytes", "tuple", "stream", "path"]) +def test_should_transcribe_and_clean_up_audio( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status_code: int, input_type: str +) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key") + paths: Final = [] + original: Final = tmp_path / "input.wav" + original.write_bytes(b"audio-content") + inputs: Final = { + "bytes": b"audio-content", + "tuple": ("input.wav", b"audio-content", "audio/wav"), + "stream": BytesIO(b"audio-content"), + "path": original, + } + + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions" + assert request.headers["Content-Type"] == "application/json" + assert request.headers["Authorization"] == "Bearer env-key" + payload: Final = json.loads(request.content) + assert payload["model"] == "whisper-tiny" + assert payload["language"] == "en" + assert payload["prompt"] == "Greeting" + assert payload["stream"] is False + path: Final = Path(payload["filename"]) + paths.append(path) + assert path.read_bytes() == b"audio-content" + return httpx.Response( + status_code, json={"text": "Hello"} if status_code == 200 else {"error": {"message": "Invalid audio"}} + ) + + with httpx.Client(transport=httpx.MockTransport(respond)) as http_client: + client: Final = HTTPHandler(client=http_client) + if status_code == 200: + response: Final = litellm.transcription( + model="foundry_local/whisper-tiny", + file=inputs[input_type], + language="en", + prompt="Greeting", + client=client, + max_retries=0, + ) + assert response.text == "Hello" + else: + with pytest.raises(OpenAIError, match="Invalid audio") as error: + litellm.transcription( + model="foundry_local/whisper-tiny", + file=inputs[input_type], + language="en", + prompt="Greeting", + client=client, + max_retries=0, + ) + assert error.value.status_code == 400 + assert len(paths) == 1 + assert not paths[0].exists() + assert not paths[0].parent.exists() + assert original.read_bytes() == b"audio-content" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["success", "error", "cancel"]) +async def test_should_keep_async_audio_until_completion_and_clean_up( + monkeypatch: pytest.MonkeyPatch, outcome: str +) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + paths: Final = [] + + async def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions" + payload: Final = json.loads(request.content) + path: Final = Path(payload["filename"]) + paths.append(path) + await asyncio.sleep(0) + assert path.read_bytes() == b"audio-content" + if outcome == "cancel": + raise asyncio.CancelledError + return httpx.Response( + 400 if outcome == "error" else 200, + json={"error": {"message": "Invalid audio"}} if outcome == "error" else {"text": "Hello"}, + ) + + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + async with client.client: + if outcome == "success": + response: Final = await litellm.atranscription( + model="foundry_local/whisper-tiny", + file=("input.wav", b"audio-content"), + client=client, + max_retries=0, + ) + assert response.text == "Hello" + else: + with pytest.raises(asyncio.CancelledError if outcome == "cancel" else litellm.BadRequestError): + await litellm.atranscription( + model="foundry_local/whisper-tiny", + file=("input.wav", b"audio-content"), + client=client, + max_retries=0, + ) + assert len(paths) == 1 + assert not paths[0].exists() + assert not paths[0].parent.exists() + + +def test_should_reject_bare_audio_paths_and_unsupported_transcription_options(tmp_path: Path) -> None: + config: Final = FoundryLocalAudioTranscriptionConfig() + path: Final = tmp_path / "input.wav" + path.write_bytes(b"audio-content") + with pytest.raises(ValueError, match="bare str"): + with config.audio_file_context(str(path)): + pytest.fail("A bare string must not be opened as a server-side path") + with pytest.raises(litellm.UnsupportedParamsError): + litellm.utils.get_optional_params_transcription( + model="whisper-tiny", custom_llm_provider="foundry_local", timestamp_granularities=["word"] + ) + with pytest.raises(ValueError, match="requires api_base"): + config.get_complete_url(None, None, "whisper-tiny", {}, {}) + + @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) async def test_should_complete_chat_asynchronously_over_foundry_http_service(stream: bool) -> None: From 47944bbb581ed7a4282838de8827933e7861d539 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 22 Oct 2026 23:53:26 -0500 Subject: [PATCH 5/5] fix: use TypeError for unprepared Foundry Local audio Satisfy the strict TRY004 CI gate by distinguishing an invalid audio type from an invalid value. Add regression coverage for the rejected input. Files changed: litellm/llms/foundry_local/audio_transcription/transformation.py; tests/unit/llms/foundry_local/chat/test_transformation.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4aafcdf7-d0d2-4be7-843f-e52bb16c92da --- .../foundry_local/audio_transcription/transformation.py | 2 +- tests/unit/llms/foundry_local/chat/test_transformation.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/llms/foundry_local/audio_transcription/transformation.py b/litellm/llms/foundry_local/audio_transcription/transformation.py index 8463d7578a4..6c400fc33f5 100644 --- a/litellm/llms/foundry_local/audio_transcription/transformation.py +++ b/litellm/llms/foundry_local/audio_transcription/transformation.py @@ -63,7 +63,7 @@ class FoundryLocalAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig litellm_params: Mapping[str, object], ) -> AudioTranscriptionRequestData: if not isinstance(audio_file, Path): - raise ValueError("Foundry Local audio must be prepared using audio_file_context") + raise TypeError("Foundry Local audio must be prepared using audio_file_context") if optional_params.get("stream"): raise ValueError("Streaming Foundry Local transcription is not supported by this integration") return AudioTranscriptionRequestData( diff --git a/tests/unit/llms/foundry_local/chat/test_transformation.py b/tests/unit/llms/foundry_local/chat/test_transformation.py index 8123c763436..10ff927b5d7 100644 --- a/tests/unit/llms/foundry_local/chat/test_transformation.py +++ b/tests/unit/llms/foundry_local/chat/test_transformation.py @@ -320,6 +320,12 @@ def test_should_reject_bare_audio_paths_and_unsupported_transcription_options(tm config.get_complete_url(None, None, "whisper-tiny", {}, {}) +def test_should_require_prepared_audio_for_transcription() -> None: + config: Final = FoundryLocalAudioTranscriptionConfig() + with pytest.raises(TypeError, match="must be prepared using audio_file_context"): + config.transform_audio_transcription_request("whisper-tiny", b"audio-content", {}, {}) + + @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) async def test_should_complete_chat_asynchronously_over_foundry_http_service(stream: bool) -> None: