Merge f96dad092d into e8f572b39b

.github/workflows/test-unit.yml

@@ -286,6 +286,7 @@ jobs:
               tests/unit/llms/featherless_ai
               tests/unit/llms/firecrawl
               tests/unit/llms/fireworks_ai
+              tests/unit/llms/foundry_local
               tests/unit/llms/friendliai
               tests/unit/llms/gdc
               tests/unit/llms/gemini

litellm/__init__.py

@@ -2123,6 +2123,12 @@ if TYPE_CHECKING:
     from .llms.lm_studio.chat.transformation import (
         LMStudioChatConfig as _LMStudioChatConfig,
     )
+    from .llms.foundry_local.chat.transformation import (
+        FoundryLocalChatConfig as _FoundryLocalChatConfig,
+    )
+    from .llms.foundry_local.audio_transcription.transformation import (
+        FoundryLocalAudioTranscriptionConfig as _FoundryLocalAudioTranscriptionConfig,
+    )
     from .llms.lm_studio.embed.transformation import (
         LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig,
     )
@@ -2149,6 +2155,8 @@ if TYPE_CHECKING:
     ClfAiGatewayConfig: type[_ClfAiGatewayConfig]
     LlamafileChatConfig: Type[_LlamafileChatConfig]
     LMStudioChatConfig: Type[_LMStudioChatConfig]
+    FoundryLocalChatConfig: Type[_FoundryLocalChatConfig]
+    FoundryLocalAudioTranscriptionConfig: Type[_FoundryLocalAudioTranscriptionConfig]
     LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]
     IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig]
     VertexAIConfig: Type[_VertexGeminiConfig]  # Alias for VertexGeminiConfig

litellm/_lazy_imports_registry.py

@@ -324,6 +324,8 @@ LLM_CONFIG_NAMES: Final = (
     "DeepSeekChatConfig",
     "TencentChatConfig",
     "LMStudioChatConfig",
+    "FoundryLocalChatConfig",
+    "FoundryLocalAudioTranscriptionConfig",
     "LmStudioEmbeddingConfig",
     "NscaleConfig",
     "PerplexityChatConfig",
@@ -1224,6 +1226,14 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"),
     "TencentChatConfig": (".llms.tencent.chat.transformation", "TencentChatConfig"),
     "LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"),
+    "FoundryLocalChatConfig": (
+        ".llms.foundry_local.chat.transformation",
+        "FoundryLocalChatConfig",
+    ),
+    "FoundryLocalAudioTranscriptionConfig": (
+        ".llms.foundry_local.audio_transcription.transformation",
+        "FoundryLocalAudioTranscriptionConfig",
+    ),
     "LmStudioEmbeddingConfig": (
         ".llms.lm_studio.embed.transformation",
         "LmStudioEmbeddingConfig",

litellm/constants.py

@@ -859,6 +859,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
     "hosted_vllm",
     "llamafile",
     "lm_studio",
+    "foundry_local",
     "galadriel",
     "gradient_ai",
     "github_copilot",  # GitHub Copilot Chat API
@@ -1114,6 +1115,7 @@ openai_compatible_providers: Final[list[str]] = [
     "hosted_vllm",
     "llamafile",
     "lm_studio",
+    "foundry_local",
     "galadriel",
     "zerogpu",
     "github_copilot",  # GitHub Copilot Chat API

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -751,6 +751,11 @@ def _get_openai_compatible_provider_info(
             api_base,
             dynamic_api_key,
         ) = litellm.LMStudioChatConfig().get_openai_compatible_provider_info(api_base, api_key)
+    elif custom_llm_provider == "foundry_local":
+        foundry_api_base, foundry_api_key = litellm.FoundryLocalChatConfig().get_openai_compatible_provider_info(
+            api_base, api_key
+        )
+        return model, custom_llm_provider, foundry_api_key, foundry_api_base
     elif custom_llm_provider == "deepseek":
         # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1
         api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta"

litellm/llms/base_llm/audio_transcription/transformation.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from contextlib import AbstractContextManager, nullcontext
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Final
 
@@ -37,6 +38,9 @@ class AudioTranscriptionRequestData:
 
 
 class BaseAudioTranscriptionConfig(BaseConfig, ABC):
+    def audio_file_context(self, audio_file: FileTypes) -> AbstractContextManager[FileTypes]:
+        return nullcontext(audio_file)
+
     @abstractmethod
     def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]:
         pass

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2073,50 +2073,48 @@ class BaseLLMHTTPHandler:
                 shared_session=shared_session,
             )
 
-        # Prepare the request
-        (
-            headers,
-            complete_url,
-            data,
-            files,
-        ) = self._prepare_audio_transcription_request(
-            model=model,
-            audio_file=audio_file,
-            optional_params=optional_params,
-            litellm_params=litellm_params,
-            logging_obj=logging_obj,
-            api_key=api_key,
-            api_base=api_base,
-            headers=headers,
-            provider_config=provider_config,
-        )
-
-        if client is None or not isinstance(client, HTTPHandler):
-            client = get_httpx_client()
-
-        json_data: Final = data if files is None and isinstance(data, dict) else None
-
-        try:
-            response: Final = client.post(
-                url=complete_url,
+        with provider_config.audio_file_context(audio_file) as prepared_audio_file:
+            (
+                prepared_headers,
+                complete_url,
+                data,
+                files,
+            ) = self._prepare_audio_transcription_request(
+                model=model,
+                audio_file=prepared_audio_file,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                logging_obj=logging_obj,
+                api_key=api_key,
+                api_base=api_base,
                 headers=headers,
-                data=data if json_data is None else None,
-                files=files,
-                json=json_data,
-                timeout=timeout,
+                provider_config=provider_config,
             )
-        except Exception as e:
-            raise self._handle_error(e=e, provider_config=provider_config)
 
-        return self._transform_audio_transcription_response(
-            provider_config=provider_config,
-            model=model,
-            response=response,
-            model_response=model_response,
-            logging_obj=logging_obj,
-            optional_params=optional_params,
-            api_key=api_key,
-        )
+            sync_client: Final = client if isinstance(client, HTTPHandler) else get_httpx_client()
+            json_data: Final = data if files is None and isinstance(data, dict) else None
+
+            try:
+                response: Final = sync_client.post(
+                    url=complete_url,
+                    headers=prepared_headers,
+                    data=data if json_data is None else None,
+                    files=files,
+                    json=json_data,
+                    timeout=timeout,
+                )
+            except Exception as e:
+                raise self._handle_error(e=e, provider_config=provider_config)
+
+            return self._transform_audio_transcription_response(
+                provider_config=provider_config,
+                model=model,
+                response=response,
+                model_response=model_response,
+                logging_obj=logging_obj,
+                optional_params=optional_params,
+                api_key=api_key,
+            )
 
     async def async_audio_transcriptions(
         self,
@@ -2139,56 +2137,56 @@ class BaseLLMHTTPHandler:
         if provider_config is None:
             raise ValueError(f"No provider config found for model: {model} and provider: {custom_llm_provider}")
 
-        # Prepare the request
-        (
-            headers,
-            complete_url,
-            data,
-            files,
-        ) = self._prepare_audio_transcription_request(
-            model=model,
-            audio_file=audio_file,
-            optional_params=optional_params,
-            litellm_params=litellm_params,
-            logging_obj=logging_obj,
-            api_key=api_key,
-            api_base=api_base,
-            headers=headers,
-            provider_config=provider_config,
-        )
-
-        if client is None or not isinstance(client, AsyncHTTPHandler):
-            async_httpx_client = get_async_httpx_client(
-                llm_provider=litellm.LlmProviders(custom_llm_provider),
-                params={"ssl_verify": litellm_params.get("ssl_verify", None)},
-                shared_session=shared_session,
-            )
-        else:
-            async_httpx_client = client
-
-        json_data: Final = data if files is None and isinstance(data, dict) else None
-
-        try:
-            response: Final = await async_httpx_client.post(
-                url=complete_url,
+        with provider_config.audio_file_context(audio_file) as prepared_audio_file:
+            (
+                prepared_headers,
+                complete_url,
+                data,
+                files,
+            ) = self._prepare_audio_transcription_request(
+                model=model,
+                audio_file=prepared_audio_file,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                logging_obj=logging_obj,
+                api_key=api_key,
+                api_base=api_base,
                 headers=headers,
-                data=data if json_data is None else None,
-                files=files,
-                json=json_data,
-                timeout=timeout,
+                provider_config=provider_config,
             )
-        except Exception as e:
-            raise self._handle_error(e=e, provider_config=provider_config)
 
-        return self._transform_audio_transcription_response(
-            provider_config=provider_config,
-            model=model,
-            response=response,
-            model_response=model_response,
-            logging_obj=logging_obj,
-            optional_params=optional_params,
-            api_key=api_key,
-        )
+            async_httpx_client: Final = (
+                client
+                if isinstance(client, AsyncHTTPHandler)
+                else get_async_httpx_client(
+                    llm_provider=litellm.LlmProviders(custom_llm_provider),
+                    params={"ssl_verify": litellm_params.get("ssl_verify", None)},
+                    shared_session=shared_session,
+                )
+            )
+            json_data: Final = data if files is None and isinstance(data, dict) else None
+
+            try:
+                response: Final = await async_httpx_client.post(
+                    url=complete_url,
+                    headers=prepared_headers,
+                    data=data if json_data is None else None,
+                    files=files,
+                    json=json_data,
+                    timeout=timeout,
+                )
+            except Exception as e:
+                raise self._handle_error(e=e, provider_config=provider_config)
+
+            return self._transform_audio_transcription_response(
+                provider_config=provider_config,
+                model=model,
+                response=response,
+                model_response=model_response,
+                logging_obj=logging_obj,
+                optional_params=optional_params,
+                api_key=api_key,
+            )
 
     def search(
         self,

litellm/llms/foundry_local/audio_transcription/transformation.py

@@ -0,0 +1,72 @@
+from collections.abc import Generator, Mapping, Sequence
+from contextlib import contextmanager
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Final
+
+from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
+from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData
+from litellm.llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig
+from litellm.types.llms.openai import AllMessageValues, OpenAIAudioTranscriptionOptionalParams
+from litellm.types.utils import FileTypes
+
+
+class FoundryLocalAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
+    @property
+    def has_native_transcription_endpoint(self) -> bool:
+        return True
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list[OpenAIAudioTranscriptionOptionalParams]:  # mutable-ok: base class contract
+        return ["language", "prompt", "response_format", "temperature"]
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, object],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, object]:  # mutable-ok: the shared HTTP handler adds the request Content-Type header
+        return {**headers, **({"Authorization": f"Bearer {api_key}"} if api_key else {})}
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        if not api_base:
+            raise ValueError("Foundry Local transcription requires api_base or FOUNDRY_LOCAL_API_BASE")
+        return f"{api_base.rstrip('/')}/audio/transcriptions"
+
+    @contextmanager
+    def audio_file_context(self, audio_file: FileTypes) -> Generator[FileTypes, None, None]:
+        processed: Final = process_audio_file(audio_file)
+        # Foundry reads a filename rather than an upload. Close it before HTTP for Windows.
+        with TemporaryDirectory(prefix="litellm-foundry-audio-") as directory:
+            path: Final = Path(directory) / f"audio{Path(processed.filename).suffix or '.wav'}"
+            path.write_bytes(processed.file_content)
+            yield path
+
+    def transform_audio_transcription_request(
+        self,
+        model: str,
+        audio_file: FileTypes,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> AudioTranscriptionRequestData:
+        if not isinstance(audio_file, Path):
+            raise TypeError("Foundry Local audio must be prepared using audio_file_context")
+        if optional_params.get("stream"):
+            raise ValueError("Streaming Foundry Local transcription is not supported by this integration")
+        return AudioTranscriptionRequestData(
+            data={**optional_params, "model": model, "filename": str(audio_file), "stream": False},
+            content_type="application/json",
+        )

litellm/llms/foundry_local/chat/transformation.py

@@ -0,0 +1,24 @@
+"""
+Translate from OpenAI's `/v1/chat/completions` to Foundry Local's `/v1/chat/completions`
+
+Foundry Local (https://github.com/microsoft/Foundry-Local) runs LLMs on-device
+and exposes an OpenAI-compatible REST API endpoint.
+"""
+
+from typing import Final
+
+import litellm
+from litellm.secret_managers.main import get_secret_str
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class FoundryLocalChatConfig(OpenAIGPTConfig):
+    def get_openai_compatible_provider_info(
+        self, api_base: str | None, api_key: str | None
+    ) -> tuple[str | None, str | None]:
+        resolved_api_base: Final = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE") or litellm.api_base
+        if not resolved_api_base:
+            raise ValueError("Foundry Local requires api_base or FOUNDRY_LOCAL_API_BASE")
+        dynamic_api_key: Final = api_key or get_secret_str("FOUNDRY_LOCAL_API_KEY") or "fake-api-key"
+        return resolved_api_base, dynamic_api_key

litellm/main.py

@@ -7074,6 +7074,7 @@ def embedding(
             or custom_llm_provider == "together_ai"
             or custom_llm_provider == "nvidia_nim"
             or custom_llm_provider == "litellm_proxy"
+            or custom_llm_provider == "foundry_local"
             or (model in litellm.open_ai_embedding_models and custom_llm_provider is None)
         ):
             api_base = (

litellm/proxy/_experimental/out/assets/logos/foundry_local.svg

@@ -0,0 +1,40 @@
+<svg width="256" height="256" viewBox="0 0 256 256" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_1197_17983)">
+<rect width="256" height="256" fill="white"/>
+<path d="M154.903 33.0322H35.785C34.2618 33.0322 33.0323 34.2618 33.0323 35.7849V63.3118C33.0323 64.835 34.2618 66.0645 35.785 66.0645H181.677C186.238 66.0645 189.936 69.7623 189.936 74.3225V70.9C189.936 51.3101 175.319 33.0322 154.903 33.0322Z" fill="url(#paint0_linear_1197_17983)"/>
+<path d="M178.255 42.7031C185.733 50.1996 189.936 60.3112 189.936 70.8998V220.215C189.936 221.738 191.165 222.967 192.688 222.967H220.215C221.738 222.967 222.968 221.738 222.968 220.215V101.106C222.968 92.3433 219.49 83.9476 213.297 77.7449L178.255 42.7031Z" fill="url(#paint1_linear_1197_17983)"/>
+<path d="M113.613 74.3228H35.785C34.2618 74.3228 33.0323 75.5523 33.0323 77.0754V104.602C33.0323 106.125 34.2618 107.355 35.785 107.355H140.387C144.947 107.355 148.645 111.053 148.645 115.613V112.191C148.645 92.6006 134.028 74.3228 113.613 74.3228Z" fill="url(#paint2_linear_1197_17983)"/>
+<path d="M136.965 83.9937C144.443 91.4901 148.645 101.602 148.645 112.19V220.215C148.645 221.738 149.875 222.968 151.398 222.968H178.925C180.448 222.968 181.677 221.738 181.677 220.215V142.397C181.677 133.634 178.2 125.238 172.006 119.035L136.965 83.9937Z" fill="url(#paint3_linear_1197_17983)"/>
+<path d="M72.3223 115.613H35.785C34.2618 115.613 33.0323 116.842 33.0323 118.365V145.892C33.0323 147.416 34.2618 148.645 35.785 148.645H99.0968C103.657 148.645 107.355 152.343 107.355 156.903V153.481C107.355 133.891 92.7381 115.613 72.3223 115.613Z" fill="url(#paint4_linear_1197_17983)"/>
+<path d="M95.6743 125.284C103.152 132.781 107.355 142.892 107.355 153.481V220.215C107.355 221.738 108.584 222.968 110.108 222.968H137.634C139.158 222.968 140.387 221.738 140.387 220.215V183.687C140.387 174.924 136.91 166.529 130.716 160.326L95.6743 125.284Z" fill="url(#paint5_linear_1197_17983)"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear_1197_17983" x1="189.936" y1="70.4672" x2="33.0323" y2="70.4672" gradientUnits="userSpaceOnUse">
+<stop stop-color="#2C08AC"/>
+<stop offset="0.8" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint1_linear_1197_17983" x1="201.96" y1="42.7031" x2="311.288" y2="175.174" gradientUnits="userSpaceOnUse">
+<stop offset="0.3" stop-color="#7274FF"/>
+<stop offset="1" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint2_linear_1197_17983" x1="148.645" y1="111.758" x2="33.0323" y2="111.758" gradientUnits="userSpaceOnUse">
+<stop stop-color="#2C08AC"/>
+<stop offset="0.8" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint3_linear_1197_17983" x1="160.67" y1="83.9937" x2="238.429" y2="206.207" gradientUnits="userSpaceOnUse">
+<stop offset="0.3" stop-color="#7274FF"/>
+<stop offset="1" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint4_linear_1197_17983" x1="107.355" y1="150.688" x2="33.0323" y2="150.688" gradientUnits="userSpaceOnUse">
+<stop stop-color="#2C08AC"/>
+<stop offset="0.8" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint5_linear_1197_17983" x1="119.38" y1="125.284" x2="164.354" y2="225.849" gradientUnits="userSpaceOnUse">
+<stop offset="0.3" stop-color="#7274FF"/>
+<stop offset="1" stop-color="#4F42FD"/>
+</linearGradient>
+<clipPath id="clip0_1197_17983">
+<rect width="256" height="256" fill="white"/>
+</clipPath>
+</defs>
+</svg>

litellm/proxy/public_endpoints/provider_create_fields.json

@@ -2321,6 +2321,34 @@
     ],
     "default_model_placeholder": "gpt-3.5-turbo"
   },
+  {
+    "provider": "FOUNDRY_LOCAL",
+    "provider_display_name": "Foundry Local",
+    "litellm_provider": "foundry_local",
+    "credential_fields": [
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": "http://localhost:5272/v1",
+        "tooltip": "Foundry Local endpoint (from foundry-local-sdk or manually set)",
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_key",
+        "label": "API Key",
+        "placeholder": null,
+        "tooltip": "Optional - Foundry Local does not require authentication",
+        "required": false,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      }
+    ],
+    "default_model_placeholder": "phi-3.5-mini"
+  },
   {
     "provider": "MARITALK",
     "provider_display_name": "Maritalk",

litellm/types/utils.py

@@ -4613,6 +4613,7 @@ class LlmProviders(str, Enum):
     TENCENT = "tencent"
     LLAMAFILE = "llamafile"
     LM_STUDIO = "lm_studio"
+    FOUNDRY_LOCAL = "foundry_local"
     GALADRIEL = "galadriel"
     NEBIUS = "nebius"
     INFINITY = "infinity"

litellm/utils.py

@@ -9586,6 +9586,7 @@ class ProviderConfigManager:
             LlmProviders.HOSTED_VLLM: (lambda: litellm.HostedVLLMChatConfig(), False),
             LlmProviders.LLAMAFILE: (lambda: litellm.LlamafileChatConfig(), False),
             LlmProviders.LM_STUDIO: (lambda: litellm.LMStudioChatConfig(), False),
+            LlmProviders.FOUNDRY_LOCAL: (lambda: litellm.FoundryLocalChatConfig(), False),
             LlmProviders.GALADRIEL: (lambda: litellm.GaladrielChatConfig(), False),
             LlmProviders.REPLICATE: (lambda: litellm.ReplicateConfig(), False),
             LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False),
@@ -10167,6 +10168,8 @@ class ProviderConfigManager:
         model: str,
         provider: LlmProviders,
     ) -> BaseAudioTranscriptionConfig | None:
+        if provider == LlmProviders.FOUNDRY_LOCAL:
+            return litellm.FoundryLocalAudioTranscriptionConfig()
         model_cost_entry: Final = _get_model_cost_entry_for_provider_config(
             model=model,
             provider=provider,

provider_endpoints_support.json

@@ -2475,6 +2475,24 @@
         "generateContent": false
       }
     },
+    "foundry_local": {
+      "display_name": "Foundry Local (`foundry_local`)",
+      "url": "https://docs.litellm.ai/docs/providers/foundry_local",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": true,
+        "responses": true,
+        "embeddings": true,
+        "image_generations": false,
+        "audio_transcriptions": true,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": true,
+        "interactions": true
+      }
+    },
     "maritalk": {
       "display_name": "Maritalk (`maritalk`)",
       "url": "https://docs.litellm.ai/docs/providers/maritalk",

tests/unit/llms/foundry_local/chat/test_transformation.py

@@ -0,0 +1,353 @@
+import asyncio
+import json
+from io import BytesIO
+from pathlib import Path
+from typing import Final
+
+import httpx
+import pytest
+from openai import AsyncOpenAI, OpenAI
+
+import litellm
+from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.foundry_local.audio_transcription.transformation import FoundryLocalAudioTranscriptionConfig
+from litellm.llms.openai.common_utils import OpenAIError
+from litellm.types.utils import ModelResponse
+
+
+@pytest.mark.parametrize(
+    ("model", "custom_llm_provider"),
+    [("foundry_local/local-model:1", None), ("local-model:1", "foundry_local")],
+)
+@pytest.mark.parametrize(
+    ("api_base", "api_key", "env_base", "env_key", "expected_base", "expected_key"),
+    [
+        (None, None, "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "fake-api-key"),
+        (None, None, "http://127.0.0.1:61234/v1", "env-key", "http://127.0.0.1:61234/v1", "env-key"),
+        ("http://localhost:5272/v1", None, None, "env-key", "http://localhost:5272/v1", "env-key"),
+        (None, "explicit-key", "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "explicit-key"),
+        ("http://localhost:5272/v1", "explicit-key", None, None, "http://localhost:5272/v1", "explicit-key"),
+        (
+            "http://localhost:5272/v1",
+            "explicit-key",
+            "http://127.0.0.1:61234/v1",
+            "env-key",
+            "http://localhost:5272/v1",
+            "explicit-key",
+        ),
+    ],
+)
+def test_should_resolve_foundry_endpoint_and_credentials(
+    monkeypatch: pytest.MonkeyPatch,
+    model: str,
+    custom_llm_provider: str | None,
+    api_base: str | None,
+    api_key: str | None,
+    env_base: str | None,
+    env_key: str | None,
+    expected_base: str | None,
+    expected_key: str,
+) -> None:
+    monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False)
+    monkeypatch.delenv("FOUNDRY_LOCAL_API_KEY", raising=False)
+    if env_base is not None:
+        monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", env_base)
+    if env_key is not None:
+        monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", env_key)
+
+    assert get_llm_provider(model, custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key) == (
+        "local-model:1",
+        "foundry_local",
+        expected_key,
+        expected_base,
+    )
+
+
+def _respond(request: httpx.Request, stream: bool) -> httpx.Response:
+    assert str(request.url) == "http://127.0.0.1:61234/v1/chat/completions"
+    payload: Final = json.loads(request.content)
+    assert payload["model"] == "local-model:1"
+    assert payload["messages"] == [{"role": "user", "content": "Hello"}]
+    assert payload["max_tokens"] == 16
+    assert bool(payload.get("stream")) is stream
+    if stream:
+        chunk: Final = {
+            "id": "chatcmpl-local",
+            "object": "chat.completion.chunk",
+            "created": 0,
+            "model": "local-model:1",
+            "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello"}, "finish_reason": None}],
+        }
+        final_chunk: Final = {
+            **chunk,
+            "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+        }
+        return httpx.Response(
+            200,
+            headers={"content-type": "text/event-stream"},
+            text=f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(final_chunk)}\n\ndata: [DONE]\n\n",
+        )
+    return httpx.Response(
+        200,
+        json={
+            "id": "chatcmpl-local",
+            "object": "chat.completion",
+            "created": 0,
+            "model": "local-model:1",
+            "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}],
+            "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
+        },
+    )
+
+
+@pytest.mark.parametrize("provider", [None, "foundry_local"])
+def test_should_require_a_foundry_endpoint(monkeypatch: pytest.MonkeyPatch, provider: str | None) -> None:
+    monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False)
+    with pytest.raises(litellm.BadRequestError, match="requires api_base"):
+        get_llm_provider(
+            "local-model:1" if provider else "foundry_local/local-model:1",
+            custom_llm_provider=provider,
+        )
+
+
+def test_should_preserve_global_api_base_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False)
+    monkeypatch.delenv("FOUNDRY_LOCAL_API_KEY", raising=False)
+    monkeypatch.setattr(litellm, "api_base", "http://127.0.0.1:61234/v1")
+    assert get_llm_provider("foundry_local/local-model:1") == (
+        "local-model:1",
+        "foundry_local",
+        "fake-api-key",
+        "http://127.0.0.1:61234/v1",
+    )
+
+
+@pytest.mark.parametrize("stream", [False, True])
+def test_should_complete_chat_over_foundry_http_service(stream: bool) -> None:
+    with (
+        httpx.Client(transport=httpx.MockTransport(lambda request: _respond(request, stream))) as http_client,
+        OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client,
+    ):
+        response: Final = litellm.completion(
+            model="foundry_local/local-model:1",
+            api_base="http://127.0.0.1:61234/v1",
+            messages=[{"role": "user", "content": "Hello"}],
+            max_tokens=16,
+            stream=stream,
+            client=client,
+        )
+        if stream:
+            assert not isinstance(response, ModelResponse)
+            chunks: Final = tuple(response)
+            assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "Hello"
+            assert chunks[-1].choices[0].finish_reason == "stop"
+        else:
+            assert isinstance(response, ModelResponse)
+            assert response.choices[0].message.content == "Hello"
+            assert response.choices[0].finish_reason == "stop"
+            assert response.usage.total_tokens == 3
+
+
+def _embedding_response(request: httpx.Request) -> httpx.Response:
+    assert str(request.url) == "http://127.0.0.1:61234/v1/embeddings"
+    assert request.headers["Authorization"] == "Bearer env-key"
+    payload: Final = json.loads(request.content)
+    assert payload["model"] == "embedding-model:1"
+    assert payload["input"] == ["Hello", "World"]
+    return httpx.Response(
+        200,
+        json={
+            "object": "list",
+            "model": "embedding-model:1",
+            "data": [
+                {"object": "embedding", "index": 0, "embedding": [0.25, 0.75]},
+                {"object": "embedding", "index": 1, "embedding": [0.5, 0.5]},
+            ],
+            "usage": {"prompt_tokens": 2, "total_tokens": 2},
+        },
+    )
+
+
+def test_should_embed_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1")
+    monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key")
+    with (
+        httpx.Client(transport=httpx.MockTransport(_embedding_response)) as http_client,
+        OpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client,
+    ):
+        response: Final = litellm.embedding(
+            model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client
+        )
+        assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]]
+        assert response.usage.prompt_tokens == 2
+
+
+@pytest.mark.asyncio
+async def test_should_embed_asynchronously_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1")
+    monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key")
+    async with (
+        httpx.AsyncClient(transport=httpx.MockTransport(_embedding_response)) as http_client,
+        AsyncOpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client,
+    ):
+        response: Final = await litellm.aembedding(
+            model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client
+        )
+        assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]]
+        assert response.usage.prompt_tokens == 2
+
+
+@pytest.mark.parametrize("status_code", [200, 400])
+@pytest.mark.parametrize("input_type", ["bytes", "tuple", "stream", "path"])
+def test_should_transcribe_and_clean_up_audio(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status_code: int, input_type: str
+) -> None:
+    monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1")
+    monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key")
+    paths: Final = []
+    original: Final = tmp_path / "input.wav"
+    original.write_bytes(b"audio-content")
+    inputs: Final = {
+        "bytes": b"audio-content",
+        "tuple": ("input.wav", b"audio-content", "audio/wav"),
+        "stream": BytesIO(b"audio-content"),
+        "path": original,
+    }
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions"
+        assert request.headers["Content-Type"] == "application/json"
+        assert request.headers["Authorization"] == "Bearer env-key"
+        payload: Final = json.loads(request.content)
+        assert payload["model"] == "whisper-tiny"
+        assert payload["language"] == "en"
+        assert payload["prompt"] == "Greeting"
+        assert payload["stream"] is False
+        path: Final = Path(payload["filename"])
+        paths.append(path)
+        assert path.read_bytes() == b"audio-content"
+        return httpx.Response(
+            status_code, json={"text": "Hello"} if status_code == 200 else {"error": {"message": "Invalid audio"}}
+        )
+
+    with httpx.Client(transport=httpx.MockTransport(respond)) as http_client:
+        client: Final = HTTPHandler(client=http_client)
+        if status_code == 200:
+            response: Final = litellm.transcription(
+                model="foundry_local/whisper-tiny",
+                file=inputs[input_type],
+                language="en",
+                prompt="Greeting",
+                client=client,
+                max_retries=0,
+            )
+            assert response.text == "Hello"
+        else:
+            with pytest.raises(OpenAIError, match="Invalid audio") as error:
+                litellm.transcription(
+                    model="foundry_local/whisper-tiny",
+                    file=inputs[input_type],
+                    language="en",
+                    prompt="Greeting",
+                    client=client,
+                    max_retries=0,
+                )
+            assert error.value.status_code == 400
+    assert len(paths) == 1
+    assert not paths[0].exists()
+    assert not paths[0].parent.exists()
+    assert original.read_bytes() == b"audio-content"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("outcome", ["success", "error", "cancel"])
+async def test_should_keep_async_audio_until_completion_and_clean_up(
+    monkeypatch: pytest.MonkeyPatch, outcome: str
+) -> None:
+    monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1")
+    paths: Final = []
+
+    async def respond(request: httpx.Request) -> httpx.Response:
+        assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions"
+        payload: Final = json.loads(request.content)
+        path: Final = Path(payload["filename"])
+        paths.append(path)
+        await asyncio.sleep(0)
+        assert path.read_bytes() == b"audio-content"
+        if outcome == "cancel":
+            raise asyncio.CancelledError
+        return httpx.Response(
+            400 if outcome == "error" else 200,
+            json={"error": {"message": "Invalid audio"}} if outcome == "error" else {"text": "Hello"},
+        )
+
+    client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond))
+    async with client.client:
+        if outcome == "success":
+            response: Final = await litellm.atranscription(
+                model="foundry_local/whisper-tiny",
+                file=("input.wav", b"audio-content"),
+                client=client,
+                max_retries=0,
+            )
+            assert response.text == "Hello"
+        else:
+            with pytest.raises(asyncio.CancelledError if outcome == "cancel" else litellm.BadRequestError):
+                await litellm.atranscription(
+                    model="foundry_local/whisper-tiny",
+                    file=("input.wav", b"audio-content"),
+                    client=client,
+                    max_retries=0,
+                )
+    assert len(paths) == 1
+    assert not paths[0].exists()
+    assert not paths[0].parent.exists()
+
+
+def test_should_reject_bare_audio_paths_and_unsupported_transcription_options(tmp_path: Path) -> None:
+    config: Final = FoundryLocalAudioTranscriptionConfig()
+    path: Final = tmp_path / "input.wav"
+    path.write_bytes(b"audio-content")
+    with pytest.raises(ValueError, match="bare str"):
+        with config.audio_file_context(str(path)):
+            pytest.fail("A bare string must not be opened as a server-side path")
+    with pytest.raises(litellm.UnsupportedParamsError):
+        litellm.utils.get_optional_params_transcription(
+            model="whisper-tiny", custom_llm_provider="foundry_local", timestamp_granularities=["word"]
+        )
+    with pytest.raises(ValueError, match="requires api_base"):
+        config.get_complete_url(None, None, "whisper-tiny", {}, {})
+
+
+def test_should_require_prepared_audio_for_transcription() -> None:
+    config: Final = FoundryLocalAudioTranscriptionConfig()
+    with pytest.raises(TypeError, match="must be prepared using audio_file_context"):
+        config.transform_audio_transcription_request("whisper-tiny", b"audio-content", {}, {})
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stream", [False, True])
+async def test_should_complete_chat_asynchronously_over_foundry_http_service(stream: bool) -> None:
+    async with (
+        httpx.AsyncClient(transport=httpx.MockTransport(lambda request: _respond(request, stream))) as http_client,
+        AsyncOpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client,
+    ):
+        response: Final = await litellm.acompletion(
+            model="foundry_local/local-model:1",
+            api_base="http://127.0.0.1:61234/v1",
+            messages=[{"role": "user", "content": "Hello"}],
+            max_tokens=16,
+            stream=stream,
+            client=client,
+        )
+        if stream:
+            assert not isinstance(response, ModelResponse)
+            chunks: Final = [chunk async for chunk in response]
+            assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "Hello"
+            assert chunks[-1].choices[0].finish_reason == "stop"
+        else:
+            assert isinstance(response, ModelResponse)
+            assert response.choices[0].message.content == "Hello"
+            assert response.choices[0].finish_reason == "stop"
+            assert response.usage.total_tokens == 3

ui/litellm-dashboard/public/assets/logos/foundry_local.svg

@@ -0,0 +1,40 @@
+<svg width="256" height="256" viewBox="0 0 256 256" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_1197_17983)">
+<rect width="256" height="256" fill="white"/>
+<path d="M154.903 33.0322H35.785C34.2618 33.0322 33.0323 34.2618 33.0323 35.7849V63.3118C33.0323 64.835 34.2618 66.0645 35.785 66.0645H181.677C186.238 66.0645 189.936 69.7623 189.936 74.3225V70.9C189.936 51.3101 175.319 33.0322 154.903 33.0322Z" fill="url(#paint0_linear_1197_17983)"/>
+<path d="M178.255 42.7031C185.733 50.1996 189.936 60.3112 189.936 70.8998V220.215C189.936 221.738 191.165 222.967 192.688 222.967H220.215C221.738 222.967 222.968 221.738 222.968 220.215V101.106C222.968 92.3433 219.49 83.9476 213.297 77.7449L178.255 42.7031Z" fill="url(#paint1_linear_1197_17983)"/>
+<path d="M113.613 74.3228H35.785C34.2618 74.3228 33.0323 75.5523 33.0323 77.0754V104.602C33.0323 106.125 34.2618 107.355 35.785 107.355H140.387C144.947 107.355 148.645 111.053 148.645 115.613V112.191C148.645 92.6006 134.028 74.3228 113.613 74.3228Z" fill="url(#paint2_linear_1197_17983)"/>
+<path d="M136.965 83.9937C144.443 91.4901 148.645 101.602 148.645 112.19V220.215C148.645 221.738 149.875 222.968 151.398 222.968H178.925C180.448 222.968 181.677 221.738 181.677 220.215V142.397C181.677 133.634 178.2 125.238 172.006 119.035L136.965 83.9937Z" fill="url(#paint3_linear_1197_17983)"/>
+<path d="M72.3223 115.613H35.785C34.2618 115.613 33.0323 116.842 33.0323 118.365V145.892C33.0323 147.416 34.2618 148.645 35.785 148.645H99.0968C103.657 148.645 107.355 152.343 107.355 156.903V153.481C107.355 133.891 92.7381 115.613 72.3223 115.613Z" fill="url(#paint4_linear_1197_17983)"/>
+<path d="M95.6743 125.284C103.152 132.781 107.355 142.892 107.355 153.481V220.215C107.355 221.738 108.584 222.968 110.108 222.968H137.634C139.158 222.968 140.387 221.738 140.387 220.215V183.687C140.387 174.924 136.91 166.529 130.716 160.326L95.6743 125.284Z" fill="url(#paint5_linear_1197_17983)"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear_1197_17983" x1="189.936" y1="70.4672" x2="33.0323" y2="70.4672" gradientUnits="userSpaceOnUse">
+<stop stop-color="#2C08AC"/>
+<stop offset="0.8" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint1_linear_1197_17983" x1="201.96" y1="42.7031" x2="311.288" y2="175.174" gradientUnits="userSpaceOnUse">
+<stop offset="0.3" stop-color="#7274FF"/>
+<stop offset="1" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint2_linear_1197_17983" x1="148.645" y1="111.758" x2="33.0323" y2="111.758" gradientUnits="userSpaceOnUse">
+<stop stop-color="#2C08AC"/>
+<stop offset="0.8" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint3_linear_1197_17983" x1="160.67" y1="83.9937" x2="238.429" y2="206.207" gradientUnits="userSpaceOnUse">
+<stop offset="0.3" stop-color="#7274FF"/>
+<stop offset="1" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint4_linear_1197_17983" x1="107.355" y1="150.688" x2="33.0323" y2="150.688" gradientUnits="userSpaceOnUse">
+<stop stop-color="#2C08AC"/>
+<stop offset="0.8" stop-color="#4F42FD"/>
+</linearGradient>
+<linearGradient id="paint5_linear_1197_17983" x1="119.38" y1="125.284" x2="164.354" y2="225.849" gradientUnits="userSpaceOnUse">
+<stop offset="0.3" stop-color="#7274FF"/>
+<stop offset="1" stop-color="#4F42FD"/>
+</linearGradient>
+<clipPath id="clip0_1197_17983">
+<rect width="256" height="256" fill="white"/>
+</clipPath>
+</defs>
+</svg>

ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx

@@ -20,6 +20,18 @@ const BUNDLED_MODEL_MAP: Record<string, { litellm_provider?: string }> = JSON.pa
 );
 
 describe("provider_info_helpers", () => {
+  it("should resolve Foundry Local for the model form and provider display", () => {
+    expect(resolveLitellmProviderSlug("FOUNDRY_LOCAL")).toBe("foundry_local");
+    expect(getProviderLogoAndName("foundry_local")).toEqual({
+      displayName: "Foundry Local",
+      logo: providerLogoMap[Providers.FOUNDRY_LOCAL],
+    });
+    expect(getProviderLogoAndName("foundry_local").logo).toContain("foundry_local");
+    expect(PROVIDER_CREATE_FIELDS.find((provider) => provider.provider === "FOUNDRY_LOCAL")?.litellm_provider).toBe(
+      "foundry_local",
+    );
+  });
+
   it("resolves Meta Model API separately from Meta Llama and isolates their bundled models", () => {
     expect(resolveLitellmProviderSlug("META")).toBe("meta");
     expect(getPlaceholder(Providers.META)).toBe("meta/muse-spark-1.1");

ui/litellm-dashboard/src/components/provider_info_helpers.tsx

@@ -23,6 +23,7 @@ import elevenlabsLogo from "../../public/assets/logos/elevenlabs.png";
 import falAiLogo from "../../public/assets/logos/fal_ai.jpg";
 import featherlessLogo from "../../public/assets/logos/featherless.svg";
 import fireworksLogo from "../../public/assets/logos/fireworks.svg";
+import foundryLocalLogo from "../../public/assets/logos/foundry_local.svg";
 import friendliLogo from "../../public/assets/logos/friendli.svg";
 import gigachatLogo from "../../public/assets/logos/gigachat.svg";
 import githubCopilotLogo from "../../public/assets/logos/github_copilot.svg";
@@ -139,6 +140,7 @@ export enum Providers {
   LEMONADE = "Lemonade",
   LLAMAFILE = "Llamafile",
   LM_STUDIO = "Lm Studio",
+  FOUNDRY_LOCAL = "Foundry Local",
   LLAMA = "Meta Llama",
   META = "Meta Model API",
   MARITALK = "Maritalk",
@@ -273,6 +275,7 @@ export const provider_map: Record<string, string> = {
   LLAMA: "meta_llama",
   META: "meta",
   LM_STUDIO: "lm_studio",
+  FOUNDRY_LOCAL: "foundry_local",
   MICROSOFT_365_COPILOT: "microsoft_365_copilot",
   MARITALK: "maritalk",
   MiniMax: "minimax",
@@ -381,6 +384,7 @@ export const providerLogoMap: Partial<Record<Providers, string>> = {
   [Providers.FalAI]: falAiLogo.src,
   [Providers.FEATHERLESS_AI]: featherlessLogo.src,
   [Providers.FireworksAI]: fireworksLogo.src,
+  [Providers.FOUNDRY_LOCAL]: foundryLocalLogo.src,
   [Providers.FRIENDLIAI]: friendliLogo.src,
   [Providers.GIGACHAT]: gigachatLogo.src,
   [Providers.GITHUB_COPILOT]: githubCopilotLogo.src,