From 09a7438aa6d1c1834e57a8ebfa38812c529113dc Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 21 Oct 2026 16:18:06 -0500 Subject: [PATCH] fix: run Foundry Local tests in CI and preserve routing inputs Assign the provider tests to the existing shard, mark their directories as packages, and cover mixed endpoint and key precedence with both provider selection modes. Move the setup page to the separately published docs repository Files changed, .github/workflows/test-unit.yml, docs/my-website/docs/providers/foundry_local.md, litellm/litellm_core_utils/get_llm_provider_logic.py, tests/unit/llms/foundry_local/__init__.py, tests/unit/llms/foundry_local/chat/__init__.py, tests/unit/llms/foundry_local/chat/test_transformation.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2cf44fcd-0425-4d84-8ee0-ef774b78ec80 --- .github/workflows/test-unit.yml | 1 + .../docs/providers/foundry_local.md | 240 ------------------ .../get_llm_provider_logic.py | 8 +- tests/unit/llms/foundry_local/__init__.py | 0 .../unit/llms/foundry_local/chat/__init__.py | 0 .../foundry_local/chat/test_transformation.py | 10 +- 6 files changed, 14 insertions(+), 245 deletions(-) delete mode 100644 docs/my-website/docs/providers/foundry_local.md create mode 100644 tests/unit/llms/foundry_local/__init__.py create mode 100644 tests/unit/llms/foundry_local/chat/__init__.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index c593acbe8fb..d08fc021631 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -267,6 +267,7 @@ jobs: tests/unit/llms/featherless_ai tests/unit/llms/firecrawl tests/unit/llms/fireworks_ai + tests/unit/llms/foundry_local tests/unit/llms/friendliai tests/unit/llms/gdc tests/unit/llms/gemini diff --git a/docs/my-website/docs/providers/foundry_local.md b/docs/my-website/docs/providers/foundry_local.md deleted file mode 100644 index 733e4ce94f6..00000000000 --- a/docs/my-website/docs/providers/foundry_local.md +++ /dev/null @@ -1,240 +0,0 @@ -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - -# Foundry Local - -https://github.com/microsoft/Foundry-Local - -:::tip - -**We support ALL Foundry Local models, just set `model=foundry_local/` as a prefix when sending litellm requests** - -::: - - -| Property | Details | -|-------|-------| -| Description | Run AI models on-device with Microsoft Foundry Local. | -| Provider Route on LiteLLM | `foundry_local/` | -| Provider Doc | [Foundry Local ↗](https://learn.microsoft.com/en-us/azure/foundry-local/) | -| Supported OpenAI Endpoints | `/chat/completions` | - -## Quick Start - -Use [Foundry Local Python SDK 2.2.1](https://pypi.org/project/foundry-local-sdk/2.2.1/) to run models on-device. Foundry Local supports in-process inference, but LiteLLM uses its optional OpenAI-compatible HTTP service. Start that service and point `FOUNDRY_LOCAL_API_BASE` at its `/v1` URL. - - - - -```bash -pip install foundry-local-sdk==2.2.1 -``` - -```python -import os -from foundry_local_sdk import Configuration, FoundryLocalManager -from litellm import completion - -FoundryLocalManager.initialize(Configuration(app_name="litellm-foundry-local")) -manager = FoundryLocalManager.instance - -ep_result = manager.download_and_register_eps() -if not ep_result.success: - raise RuntimeError(f"Execution provider setup failed: {ep_result.failed_eps}") - -model = manager.catalog.get_model("qwen2.5-0.5b") -model.download() -model.load() -manager.start_web_service() - -os.environ["FOUNDRY_LOCAL_API_BASE"] = f"{manager.urls[0].rstrip('/')}/v1" -try: - response = completion( - model=f"foundry_local/{model.id}", - messages=[{"role": "user", "content": "What is the golden ratio?"}], - ) - print(response.choices[0].message.content) -finally: - manager.stop_web_service() - model.unload() - manager.close() -``` - - - - -```bash -npm install foundry-local-sdk -``` - -```javascript -import { FoundryLocalManager } from "foundry-local-sdk"; - -const manager = FoundryLocalManager.create({ - appName: "litellm-foundry-local", -}); - -const epResult = await manager.downloadAndRegisterEps(); -if (!epResult.success) { - throw new Error(`Execution provider setup failed: ${epResult.failedEps}`); -} - -const model = await manager.catalog.getModel("qwen2.5-0.5b"); -await model.download(); -await model.load(); -manager.startWebService(); - -const apiBase = `${manager.urls[0].replace(/\/$/, "")}/v1`; -console.log(apiBase); -console.log(`Use model: foundry_local/${model.id}`); -``` - -Keep this process running while LiteLLM uses the service. Configure the printed URL in the Python or proxy process, since JavaScript cannot set another process's environment variables. When finished, stop the service with `manager.stopWebService()` and unload the model with `await model.unload()`. - - - - -### SDK 2.2.1 requirements - -The unified `foundry-local-sdk` Python wheel supports Windows x64, Linux x64/ARM64, and macOS Apple silicon. It requires Python 3.11 through 3.14. Windows wheels include the WinML runtime, so the separate `foundry-local-sdk-winml` package is no longer needed. - -Call `download_and_register_eps()` before selecting and loading a model to make hardware execution providers available. Catalog access no longer waits for execution-provider downloads. - -The service uses a dynamically assigned local port by default. Read `manager.urls` after starting it instead of assuming port 5272. For a separate proxy process that needs a fixed address, initialize Python with `Configuration(app_name="litellm-foundry-local", web=Configuration.WebService(urls="http://127.0.0.1:5272"))` and keep that service process running. - -LiteLLM does not import or install the Foundry Local SDK. The SDK's Python requirement applies only to the process hosting Foundry Local, and existing OpenAI-compatible Foundry Local endpoints continue to work. This provider uses chat completions for both streaming and non-streaming requests. - -## API Key -```python -# env variable -os.environ['FOUNDRY_LOCAL_API_BASE'] # e.g. http://127.0.0.1:5272/v1 -os.environ['FOUNDRY_LOCAL_API_KEY'] # optional, not required for local use -``` - -## Sample Usage -```python -from litellm import completion -import os - -os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1" - -response = completion( - model="foundry_local/qwen2.5-0.5b", - messages=[ - { - "role": "user", - "content": "What's the weather like in Boston today in Fahrenheit?", - } - ] -) -print(response) -``` - -## Sample Usage - Streaming -```python -from litellm import completion -import os - -os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1" - -response = completion( - model="foundry_local/qwen2.5-0.5b", - messages=[ - { - "role": "user", - "content": "What's the weather like in Boston today in Fahrenheit?", - } - ], - stream=True, -) - -for chunk in response: - print(chunk) -``` - -## Usage with DSPy - -Foundry Local works seamlessly with [DSPy](https://dspy.ai/) via LiteLLM: - -```python -import dspy -import os - -os.environ['FOUNDRY_LOCAL_API_BASE'] = "http://127.0.0.1:5272/v1" - -lm = dspy.LM("foundry_local/qwen2.5-0.5b") -dspy.configure(lm=lm) -``` - -## Usage with LiteLLM Proxy Server - -Here's how to call a Foundry Local model with the LiteLLM Proxy Server - -1. Modify the config.yaml - - ```yaml - model_list: - - model_name: my-model - litellm_params: - model: foundry_local/qwen2.5-0.5b - api_base: http://127.0.0.1:5272/v1 - ``` - - -2. Start the proxy - - ```bash - $ litellm --config /path/to/config.yaml - ``` - -3. Send Request to LiteLLM Proxy Server - - - - - - ```python - import openai - client = openai.OpenAI( - api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys - base_url="http://0.0.0.0:4000" # litellm-proxy-base url - ) - - response = client.chat.completions.create( - model="my-model", - messages = [ - { - "role": "user", - "content": "what llm are you" - } - ], - ) - - print(response) - ``` - - - - - ```shell - curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "my-model", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ] - }' - ``` - - - - - -## Supported Parameters - -See [Supported Parameters](../completion/input.md#translated-openai-params) for supported parameters. diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index f36e1ab23a2..c62252dbee9 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -736,10 +736,10 @@ def _get_openai_compatible_provider_info( dynamic_api_key, ) = litellm.LMStudioChatConfig().get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "foundry_local": - ( - api_base, - dynamic_api_key, - ) = litellm.FoundryLocalChatConfig().get_openai_compatible_provider_info(api_base, api_key) + foundry_api_base, foundry_api_key = litellm.FoundryLocalChatConfig().get_openai_compatible_provider_info( + api_base, api_key + ) + return model, custom_llm_provider, foundry_api_key, foundry_api_base elif custom_llm_provider == "deepseek": # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1 api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta" diff --git a/tests/unit/llms/foundry_local/__init__.py b/tests/unit/llms/foundry_local/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/foundry_local/chat/__init__.py b/tests/unit/llms/foundry_local/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/foundry_local/chat/test_transformation.py b/tests/unit/llms/foundry_local/chat/test_transformation.py index 35a1400134c..7630425ef15 100644 --- a/tests/unit/llms/foundry_local/chat/test_transformation.py +++ b/tests/unit/llms/foundry_local/chat/test_transformation.py @@ -10,12 +10,18 @@ from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.types.utils import ModelResponse +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [("foundry_local/local-model:1", None), ("local-model:1", "foundry_local")], +) @pytest.mark.parametrize( ("api_base", "api_key", "env_base", "env_key", "expected_base", "expected_key"), [ (None, None, None, None, None, "fake-api-key"), (None, None, "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "fake-api-key"), (None, None, "http://127.0.0.1:61234/v1", "env-key", "http://127.0.0.1:61234/v1", "env-key"), + ("http://localhost:5272/v1", None, None, "env-key", "http://localhost:5272/v1", "env-key"), + (None, "explicit-key", "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "explicit-key"), ("http://localhost:5272/v1", "explicit-key", None, None, "http://localhost:5272/v1", "explicit-key"), ( "http://localhost:5272/v1", @@ -29,6 +35,8 @@ from litellm.types.utils import ModelResponse ) def test_should_resolve_foundry_endpoint_and_credentials( monkeypatch: pytest.MonkeyPatch, + model: str, + custom_llm_provider: str | None, api_base: str | None, api_key: str | None, env_base: str | None, @@ -43,7 +51,7 @@ def test_should_resolve_foundry_endpoint_and_credentials( if env_key is not None: monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", env_key) - assert get_llm_provider("foundry_local/local-model:1", api_base=api_base, api_key=api_key) == ( + assert get_llm_provider(model, custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key) == ( "local-model:1", "foundry_local", expected_key,