diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 5ec5975d83b..41b14bd175c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -286,6 +286,7 @@ jobs: tests/unit/llms/featherless_ai tests/unit/llms/firecrawl tests/unit/llms/fireworks_ai + tests/unit/llms/foundry_local tests/unit/llms/friendliai tests/unit/llms/gdc tests/unit/llms/gemini diff --git a/litellm/__init__.py b/litellm/__init__.py index f64531f86fb..e990b55e318 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2123,6 +2123,12 @@ if TYPE_CHECKING: from .llms.lm_studio.chat.transformation import ( LMStudioChatConfig as _LMStudioChatConfig, ) + from .llms.foundry_local.chat.transformation import ( + FoundryLocalChatConfig as _FoundryLocalChatConfig, + ) + from .llms.foundry_local.audio_transcription.transformation import ( + FoundryLocalAudioTranscriptionConfig as _FoundryLocalAudioTranscriptionConfig, + ) from .llms.lm_studio.embed.transformation import ( LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig, ) @@ -2149,6 +2155,8 @@ if TYPE_CHECKING: ClfAiGatewayConfig: type[_ClfAiGatewayConfig] LlamafileChatConfig: Type[_LlamafileChatConfig] LMStudioChatConfig: Type[_LMStudioChatConfig] + FoundryLocalChatConfig: Type[_FoundryLocalChatConfig] + FoundryLocalAudioTranscriptionConfig: Type[_FoundryLocalAudioTranscriptionConfig] LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig] IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index c1f484f8d64..fc7ff57dfb5 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -324,6 +324,8 @@ LLM_CONFIG_NAMES: Final = ( "DeepSeekChatConfig", "TencentChatConfig", "LMStudioChatConfig", + "FoundryLocalChatConfig", + "FoundryLocalAudioTranscriptionConfig", "LmStudioEmbeddingConfig", "NscaleConfig", "PerplexityChatConfig", @@ -1224,6 +1226,14 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"), "TencentChatConfig": (".llms.tencent.chat.transformation", "TencentChatConfig"), "LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"), + "FoundryLocalChatConfig": ( + ".llms.foundry_local.chat.transformation", + "FoundryLocalChatConfig", + ), + "FoundryLocalAudioTranscriptionConfig": ( + ".llms.foundry_local.audio_transcription.transformation", + "FoundryLocalAudioTranscriptionConfig", + ), "LmStudioEmbeddingConfig": ( ".llms.lm_studio.embed.transformation", "LmStudioEmbeddingConfig", diff --git a/litellm/constants.py b/litellm/constants.py index c586a5f767a..eba214aed09 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -859,6 +859,7 @@ LITELLM_CHAT_PROVIDERS: Final = [ "hosted_vllm", "llamafile", "lm_studio", + "foundry_local", "galadriel", "gradient_ai", "github_copilot", # GitHub Copilot Chat API @@ -1114,6 +1115,7 @@ openai_compatible_providers: Final[list[str]] = [ "hosted_vllm", "llamafile", "lm_studio", + "foundry_local", "galadriel", "zerogpu", "github_copilot", # GitHub Copilot Chat API diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 38242291b71..0eb12f9be40 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -751,6 +751,11 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.LMStudioChatConfig().get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "foundry_local": + foundry_api_base, foundry_api_key = litellm.FoundryLocalChatConfig().get_openai_compatible_provider_info( + api_base, api_key + ) + return model, custom_llm_provider, foundry_api_key, foundry_api_base elif custom_llm_provider == "deepseek": # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1 api_base = api_base or get_secret("DEEPSEEK_API_BASE") or "https://api.deepseek.com/beta" diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index e4bf148abf3..88d4728a28f 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final @@ -37,6 +38,9 @@ class AudioTranscriptionRequestData: class BaseAudioTranscriptionConfig(BaseConfig, ABC): + def audio_file_context(self, audio_file: FileTypes) -> AbstractContextManager[FileTypes]: + return nullcontext(audio_file) + @abstractmethod def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: pass diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9a2b352dd86..7fb104f7cc3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2073,50 +2073,48 @@ class BaseLLMHTTPHandler: shared_session=shared_session, ) - # Prepare the request - ( - headers, - complete_url, - data, - files, - ) = self._prepare_audio_transcription_request( - model=model, - audio_file=audio_file, - optional_params=optional_params, - litellm_params=litellm_params, - logging_obj=logging_obj, - api_key=api_key, - api_base=api_base, - headers=headers, - provider_config=provider_config, - ) - - if client is None or not isinstance(client, HTTPHandler): - client = get_httpx_client() - - json_data: Final = data if files is None and isinstance(data, dict) else None - - try: - response: Final = client.post( - url=complete_url, + with provider_config.audio_file_context(audio_file) as prepared_audio_file: + ( + prepared_headers, + complete_url, + data, + files, + ) = self._prepare_audio_transcription_request( + model=model, + audio_file=prepared_audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, headers=headers, - data=data if json_data is None else None, - files=files, - json=json_data, - timeout=timeout, + provider_config=provider_config, ) - except Exception as e: - raise self._handle_error(e=e, provider_config=provider_config) - return self._transform_audio_transcription_response( - provider_config=provider_config, - model=model, - response=response, - model_response=model_response, - logging_obj=logging_obj, - optional_params=optional_params, - api_key=api_key, - ) + sync_client: Final = client if isinstance(client, HTTPHandler) else get_httpx_client() + json_data: Final = data if files is None and isinstance(data, dict) else None + + try: + response: Final = sync_client.post( + url=complete_url, + headers=prepared_headers, + data=data if json_data is None else None, + files=files, + json=json_data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + return self._transform_audio_transcription_response( + provider_config=provider_config, + model=model, + response=response, + model_response=model_response, + logging_obj=logging_obj, + optional_params=optional_params, + api_key=api_key, + ) async def async_audio_transcriptions( self, @@ -2139,56 +2137,56 @@ class BaseLLMHTTPHandler: if provider_config is None: raise ValueError(f"No provider config found for model: {model} and provider: {custom_llm_provider}") - # Prepare the request - ( - headers, - complete_url, - data, - files, - ) = self._prepare_audio_transcription_request( - model=model, - audio_file=audio_file, - optional_params=optional_params, - litellm_params=litellm_params, - logging_obj=logging_obj, - api_key=api_key, - api_base=api_base, - headers=headers, - provider_config=provider_config, - ) - - if client is None or not isinstance(client, AsyncHTTPHandler): - async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), - params={"ssl_verify": litellm_params.get("ssl_verify", None)}, - shared_session=shared_session, - ) - else: - async_httpx_client = client - - json_data: Final = data if files is None and isinstance(data, dict) else None - - try: - response: Final = await async_httpx_client.post( - url=complete_url, + with provider_config.audio_file_context(audio_file) as prepared_audio_file: + ( + prepared_headers, + complete_url, + data, + files, + ) = self._prepare_audio_transcription_request( + model=model, + audio_file=prepared_audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, headers=headers, - data=data if json_data is None else None, - files=files, - json=json_data, - timeout=timeout, + provider_config=provider_config, ) - except Exception as e: - raise self._handle_error(e=e, provider_config=provider_config) - return self._transform_audio_transcription_response( - provider_config=provider_config, - model=model, - response=response, - model_response=model_response, - logging_obj=logging_obj, - optional_params=optional_params, - api_key=api_key, - ) + async_httpx_client: Final = ( + client + if isinstance(client, AsyncHTTPHandler) + else get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, + ) + ) + json_data: Final = data if files is None and isinstance(data, dict) else None + + try: + response: Final = await async_httpx_client.post( + url=complete_url, + headers=prepared_headers, + data=data if json_data is None else None, + files=files, + json=json_data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + return self._transform_audio_transcription_response( + provider_config=provider_config, + model=model, + response=response, + model_response=model_response, + logging_obj=logging_obj, + optional_params=optional_params, + api_key=api_key, + ) def search( self, diff --git a/litellm/llms/foundry_local/__init__.py b/litellm/llms/foundry_local/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/foundry_local/audio_transcription/__init__.py b/litellm/llms/foundry_local/audio_transcription/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/foundry_local/audio_transcription/transformation.py b/litellm/llms/foundry_local/audio_transcription/transformation.py new file mode 100644 index 00000000000..6c400fc33f5 --- /dev/null +++ b/litellm/llms/foundry_local/audio_transcription/transformation.py @@ -0,0 +1,72 @@ +from collections.abc import Generator, Mapping, Sequence +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Final + +from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData +from litellm.llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig +from litellm.types.llms.openai import AllMessageValues, OpenAIAudioTranscriptionOptionalParams +from litellm.types.utils import FileTypes + + +class FoundryLocalAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): + @property + def has_native_transcription_endpoint(self) -> bool: + return True + + def get_supported_openai_params( + self, model: str + ) -> list[OpenAIAudioTranscriptionOptionalParams]: # mutable-ok: base class contract + return ["language", "prompt", "response_format", "temperature"] + + def validate_environment( + self, + headers: Mapping[str, object], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, object]: # mutable-ok: the shared HTTP handler adds the request Content-Type header + return {**headers, **({"Authorization": f"Bearer {api_key}"} if api_key else {})} + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + if not api_base: + raise ValueError("Foundry Local transcription requires api_base or FOUNDRY_LOCAL_API_BASE") + return f"{api_base.rstrip('/')}/audio/transcriptions" + + @contextmanager + def audio_file_context(self, audio_file: FileTypes) -> Generator[FileTypes, None, None]: + processed: Final = process_audio_file(audio_file) + # Foundry reads a filename rather than an upload. Close it before HTTP for Windows. + with TemporaryDirectory(prefix="litellm-foundry-audio-") as directory: + path: Final = Path(directory) / f"audio{Path(processed.filename).suffix or '.wav'}" + path.write_bytes(processed.file_content) + yield path + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> AudioTranscriptionRequestData: + if not isinstance(audio_file, Path): + raise TypeError("Foundry Local audio must be prepared using audio_file_context") + if optional_params.get("stream"): + raise ValueError("Streaming Foundry Local transcription is not supported by this integration") + return AudioTranscriptionRequestData( + data={**optional_params, "model": model, "filename": str(audio_file), "stream": False}, + content_type="application/json", + ) diff --git a/litellm/llms/foundry_local/chat/__init__.py b/litellm/llms/foundry_local/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/foundry_local/chat/transformation.py b/litellm/llms/foundry_local/chat/transformation.py new file mode 100644 index 00000000000..410cb2a8359 --- /dev/null +++ b/litellm/llms/foundry_local/chat/transformation.py @@ -0,0 +1,24 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Foundry Local's `/v1/chat/completions` + +Foundry Local (https://github.com/microsoft/Foundry-Local) runs LLMs on-device +and exposes an OpenAI-compatible REST API endpoint. +""" + +from typing import Final + +import litellm +from litellm.secret_managers.main import get_secret_str + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +class FoundryLocalChatConfig(OpenAIGPTConfig): + def get_openai_compatible_provider_info( + self, api_base: str | None, api_key: str | None + ) -> tuple[str | None, str | None]: + resolved_api_base: Final = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE") or litellm.api_base + if not resolved_api_base: + raise ValueError("Foundry Local requires api_base or FOUNDRY_LOCAL_API_BASE") + dynamic_api_key: Final = api_key or get_secret_str("FOUNDRY_LOCAL_API_KEY") or "fake-api-key" + return resolved_api_base, dynamic_api_key diff --git a/litellm/main.py b/litellm/main.py index 5346741e4d1..386f2f64c74 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7074,6 +7074,7 @@ def embedding( or custom_llm_provider == "together_ai" or custom_llm_provider == "nvidia_nim" or custom_llm_provider == "litellm_proxy" + or custom_llm_provider == "foundry_local" or (model in litellm.open_ai_embedding_models and custom_llm_provider is None) ): api_base = ( diff --git a/litellm/proxy/_experimental/out/assets/logos/foundry_local.svg b/litellm/proxy/_experimental/out/assets/logos/foundry_local.svg new file mode 100644 index 00000000000..412a6fb70d6 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/foundry_local.svg @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 01a822c5d93..9b2e1233726 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2321,6 +2321,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "FOUNDRY_LOCAL", + "provider_display_name": "Foundry Local", + "litellm_provider": "foundry_local", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "http://localhost:5272/v1", + "tooltip": "Foundry Local endpoint (from foundry-local-sdk or manually set)", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": "Optional - Foundry Local does not require authentication", + "required": false, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "phi-3.5-mini" + }, { "provider": "MARITALK", "provider_display_name": "Maritalk", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1a8f5adf154..0dcf22b905f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4613,6 +4613,7 @@ class LlmProviders(str, Enum): TENCENT = "tencent" LLAMAFILE = "llamafile" LM_STUDIO = "lm_studio" + FOUNDRY_LOCAL = "foundry_local" GALADRIEL = "galadriel" NEBIUS = "nebius" INFINITY = "infinity" diff --git a/litellm/utils.py b/litellm/utils.py index 560d0c31e4a..ad3bbaf459f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9586,6 +9586,7 @@ class ProviderConfigManager: LlmProviders.HOSTED_VLLM: (lambda: litellm.HostedVLLMChatConfig(), False), LlmProviders.LLAMAFILE: (lambda: litellm.LlamafileChatConfig(), False), LlmProviders.LM_STUDIO: (lambda: litellm.LMStudioChatConfig(), False), + LlmProviders.FOUNDRY_LOCAL: (lambda: litellm.FoundryLocalChatConfig(), False), LlmProviders.GALADRIEL: (lambda: litellm.GaladrielChatConfig(), False), LlmProviders.REPLICATE: (lambda: litellm.ReplicateConfig(), False), LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False), @@ -10167,6 +10168,8 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAudioTranscriptionConfig | None: + if provider == LlmProviders.FOUNDRY_LOCAL: + return litellm.FoundryLocalAudioTranscriptionConfig() model_cost_entry: Final = _get_model_cost_entry_for_provider_config( model=model, provider=provider, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index d33e61026df..5049a09a967 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2475,6 +2475,24 @@ "generateContent": false } }, + "foundry_local": { + "display_name": "Foundry Local (`foundry_local`)", + "url": "https://docs.litellm.ai/docs/providers/foundry_local", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, "maritalk": { "display_name": "Maritalk (`maritalk`)", "url": "https://docs.litellm.ai/docs/providers/maritalk", diff --git a/tests/unit/llms/foundry_local/__init__.py b/tests/unit/llms/foundry_local/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/foundry_local/chat/__init__.py b/tests/unit/llms/foundry_local/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/foundry_local/chat/test_transformation.py b/tests/unit/llms/foundry_local/chat/test_transformation.py new file mode 100644 index 00000000000..10ff927b5d7 --- /dev/null +++ b/tests/unit/llms/foundry_local/chat/test_transformation.py @@ -0,0 +1,353 @@ +import asyncio +import json +from io import BytesIO +from pathlib import Path +from typing import Final + +import httpx +import pytest +from openai import AsyncOpenAI, OpenAI + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.foundry_local.audio_transcription.transformation import FoundryLocalAudioTranscriptionConfig +from litellm.llms.openai.common_utils import OpenAIError +from litellm.types.utils import ModelResponse + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [("foundry_local/local-model:1", None), ("local-model:1", "foundry_local")], +) +@pytest.mark.parametrize( + ("api_base", "api_key", "env_base", "env_key", "expected_base", "expected_key"), + [ + (None, None, "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "fake-api-key"), + (None, None, "http://127.0.0.1:61234/v1", "env-key", "http://127.0.0.1:61234/v1", "env-key"), + ("http://localhost:5272/v1", None, None, "env-key", "http://localhost:5272/v1", "env-key"), + (None, "explicit-key", "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "explicit-key"), + ("http://localhost:5272/v1", "explicit-key", None, None, "http://localhost:5272/v1", "explicit-key"), + ( + "http://localhost:5272/v1", + "explicit-key", + "http://127.0.0.1:61234/v1", + "env-key", + "http://localhost:5272/v1", + "explicit-key", + ), + ], +) +def test_should_resolve_foundry_endpoint_and_credentials( + monkeypatch: pytest.MonkeyPatch, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + api_key: str | None, + env_base: str | None, + env_key: str | None, + expected_base: str | None, + expected_key: str, +) -> None: + monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False) + monkeypatch.delenv("FOUNDRY_LOCAL_API_KEY", raising=False) + if env_base is not None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", env_base) + if env_key is not None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", env_key) + + assert get_llm_provider(model, custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key) == ( + "local-model:1", + "foundry_local", + expected_key, + expected_base, + ) + + +def _respond(request: httpx.Request, stream: bool) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/chat/completions" + payload: Final = json.loads(request.content) + assert payload["model"] == "local-model:1" + assert payload["messages"] == [{"role": "user", "content": "Hello"}] + assert payload["max_tokens"] == 16 + assert bool(payload.get("stream")) is stream + if stream: + chunk: Final = { + "id": "chatcmpl-local", + "object": "chat.completion.chunk", + "created": 0, + "model": "local-model:1", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello"}, "finish_reason": None}], + } + final_chunk: Final = { + **chunk, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(final_chunk)}\n\ndata: [DONE]\n\n", + ) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-local", + "object": "chat.completion", + "created": 0, + "model": "local-model:1", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + }, + ) + + +@pytest.mark.parametrize("provider", [None, "foundry_local"]) +def test_should_require_a_foundry_endpoint(monkeypatch: pytest.MonkeyPatch, provider: str | None) -> None: + monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False) + with pytest.raises(litellm.BadRequestError, match="requires api_base"): + get_llm_provider( + "local-model:1" if provider else "foundry_local/local-model:1", + custom_llm_provider=provider, + ) + + +def test_should_preserve_global_api_base_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False) + monkeypatch.delenv("FOUNDRY_LOCAL_API_KEY", raising=False) + monkeypatch.setattr(litellm, "api_base", "http://127.0.0.1:61234/v1") + assert get_llm_provider("foundry_local/local-model:1") == ( + "local-model:1", + "foundry_local", + "fake-api-key", + "http://127.0.0.1:61234/v1", + ) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_should_complete_chat_over_foundry_http_service(stream: bool) -> None: + with ( + httpx.Client(transport=httpx.MockTransport(lambda request: _respond(request, stream))) as http_client, + OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = litellm.completion( + model="foundry_local/local-model:1", + api_base="http://127.0.0.1:61234/v1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=16, + stream=stream, + client=client, + ) + if stream: + assert not isinstance(response, ModelResponse) + chunks: Final = tuple(response) + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "Hello" + assert chunks[-1].choices[0].finish_reason == "stop" + else: + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "Hello" + assert response.choices[0].finish_reason == "stop" + assert response.usage.total_tokens == 3 + + +def _embedding_response(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/embeddings" + assert request.headers["Authorization"] == "Bearer env-key" + payload: Final = json.loads(request.content) + assert payload["model"] == "embedding-model:1" + assert payload["input"] == ["Hello", "World"] + return httpx.Response( + 200, + json={ + "object": "list", + "model": "embedding-model:1", + "data": [ + {"object": "embedding", "index": 0, "embedding": [0.25, 0.75]}, + {"object": "embedding", "index": 1, "embedding": [0.5, 0.5]}, + ], + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + + +def test_should_embed_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key") + with ( + httpx.Client(transport=httpx.MockTransport(_embedding_response)) as http_client, + OpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = litellm.embedding( + model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client + ) + assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]] + assert response.usage.prompt_tokens == 2 + + +@pytest.mark.asyncio +async def test_should_embed_asynchronously_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key") + async with ( + httpx.AsyncClient(transport=httpx.MockTransport(_embedding_response)) as http_client, + AsyncOpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = await litellm.aembedding( + model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client + ) + assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]] + assert response.usage.prompt_tokens == 2 + + +@pytest.mark.parametrize("status_code", [200, 400]) +@pytest.mark.parametrize("input_type", ["bytes", "tuple", "stream", "path"]) +def test_should_transcribe_and_clean_up_audio( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status_code: int, input_type: str +) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key") + paths: Final = [] + original: Final = tmp_path / "input.wav" + original.write_bytes(b"audio-content") + inputs: Final = { + "bytes": b"audio-content", + "tuple": ("input.wav", b"audio-content", "audio/wav"), + "stream": BytesIO(b"audio-content"), + "path": original, + } + + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions" + assert request.headers["Content-Type"] == "application/json" + assert request.headers["Authorization"] == "Bearer env-key" + payload: Final = json.loads(request.content) + assert payload["model"] == "whisper-tiny" + assert payload["language"] == "en" + assert payload["prompt"] == "Greeting" + assert payload["stream"] is False + path: Final = Path(payload["filename"]) + paths.append(path) + assert path.read_bytes() == b"audio-content" + return httpx.Response( + status_code, json={"text": "Hello"} if status_code == 200 else {"error": {"message": "Invalid audio"}} + ) + + with httpx.Client(transport=httpx.MockTransport(respond)) as http_client: + client: Final = HTTPHandler(client=http_client) + if status_code == 200: + response: Final = litellm.transcription( + model="foundry_local/whisper-tiny", + file=inputs[input_type], + language="en", + prompt="Greeting", + client=client, + max_retries=0, + ) + assert response.text == "Hello" + else: + with pytest.raises(OpenAIError, match="Invalid audio") as error: + litellm.transcription( + model="foundry_local/whisper-tiny", + file=inputs[input_type], + language="en", + prompt="Greeting", + client=client, + max_retries=0, + ) + assert error.value.status_code == 400 + assert len(paths) == 1 + assert not paths[0].exists() + assert not paths[0].parent.exists() + assert original.read_bytes() == b"audio-content" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["success", "error", "cancel"]) +async def test_should_keep_async_audio_until_completion_and_clean_up( + monkeypatch: pytest.MonkeyPatch, outcome: str +) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + paths: Final = [] + + async def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions" + payload: Final = json.loads(request.content) + path: Final = Path(payload["filename"]) + paths.append(path) + await asyncio.sleep(0) + assert path.read_bytes() == b"audio-content" + if outcome == "cancel": + raise asyncio.CancelledError + return httpx.Response( + 400 if outcome == "error" else 200, + json={"error": {"message": "Invalid audio"}} if outcome == "error" else {"text": "Hello"}, + ) + + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + async with client.client: + if outcome == "success": + response: Final = await litellm.atranscription( + model="foundry_local/whisper-tiny", + file=("input.wav", b"audio-content"), + client=client, + max_retries=0, + ) + assert response.text == "Hello" + else: + with pytest.raises(asyncio.CancelledError if outcome == "cancel" else litellm.BadRequestError): + await litellm.atranscription( + model="foundry_local/whisper-tiny", + file=("input.wav", b"audio-content"), + client=client, + max_retries=0, + ) + assert len(paths) == 1 + assert not paths[0].exists() + assert not paths[0].parent.exists() + + +def test_should_reject_bare_audio_paths_and_unsupported_transcription_options(tmp_path: Path) -> None: + config: Final = FoundryLocalAudioTranscriptionConfig() + path: Final = tmp_path / "input.wav" + path.write_bytes(b"audio-content") + with pytest.raises(ValueError, match="bare str"): + with config.audio_file_context(str(path)): + pytest.fail("A bare string must not be opened as a server-side path") + with pytest.raises(litellm.UnsupportedParamsError): + litellm.utils.get_optional_params_transcription( + model="whisper-tiny", custom_llm_provider="foundry_local", timestamp_granularities=["word"] + ) + with pytest.raises(ValueError, match="requires api_base"): + config.get_complete_url(None, None, "whisper-tiny", {}, {}) + + +def test_should_require_prepared_audio_for_transcription() -> None: + config: Final = FoundryLocalAudioTranscriptionConfig() + with pytest.raises(TypeError, match="must be prepared using audio_file_context"): + config.transform_audio_transcription_request("whisper-tiny", b"audio-content", {}, {}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_should_complete_chat_asynchronously_over_foundry_http_service(stream: bool) -> None: + async with ( + httpx.AsyncClient(transport=httpx.MockTransport(lambda request: _respond(request, stream))) as http_client, + AsyncOpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = await litellm.acompletion( + model="foundry_local/local-model:1", + api_base="http://127.0.0.1:61234/v1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=16, + stream=stream, + client=client, + ) + if stream: + assert not isinstance(response, ModelResponse) + chunks: Final = [chunk async for chunk in response] + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "Hello" + assert chunks[-1].choices[0].finish_reason == "stop" + else: + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "Hello" + assert response.choices[0].finish_reason == "stop" + assert response.usage.total_tokens == 3 diff --git a/ui/litellm-dashboard/public/assets/logos/foundry_local.svg b/ui/litellm-dashboard/public/assets/logos/foundry_local.svg new file mode 100644 index 00000000000..412a6fb70d6 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/foundry_local.svg @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 4c483434831..d0a60b3c5f3 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -20,6 +20,18 @@ const BUNDLED_MODEL_MAP: Record = JSON.pa ); describe("provider_info_helpers", () => { + it("should resolve Foundry Local for the model form and provider display", () => { + expect(resolveLitellmProviderSlug("FOUNDRY_LOCAL")).toBe("foundry_local"); + expect(getProviderLogoAndName("foundry_local")).toEqual({ + displayName: "Foundry Local", + logo: providerLogoMap[Providers.FOUNDRY_LOCAL], + }); + expect(getProviderLogoAndName("foundry_local").logo).toContain("foundry_local"); + expect(PROVIDER_CREATE_FIELDS.find((provider) => provider.provider === "FOUNDRY_LOCAL")?.litellm_provider).toBe( + "foundry_local", + ); + }); + it("resolves Meta Model API separately from Meta Llama and isolates their bundled models", () => { expect(resolveLitellmProviderSlug("META")).toBe("meta"); expect(getPlaceholder(Providers.META)).toBe("meta/muse-spark-1.1"); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index a8491f656a0..d2f3f0a83f8 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -23,6 +23,7 @@ import elevenlabsLogo from "../../public/assets/logos/elevenlabs.png"; import falAiLogo from "../../public/assets/logos/fal_ai.jpg"; import featherlessLogo from "../../public/assets/logos/featherless.svg"; import fireworksLogo from "../../public/assets/logos/fireworks.svg"; +import foundryLocalLogo from "../../public/assets/logos/foundry_local.svg"; import friendliLogo from "../../public/assets/logos/friendli.svg"; import gigachatLogo from "../../public/assets/logos/gigachat.svg"; import githubCopilotLogo from "../../public/assets/logos/github_copilot.svg"; @@ -139,6 +140,7 @@ export enum Providers { LEMONADE = "Lemonade", LLAMAFILE = "Llamafile", LM_STUDIO = "Lm Studio", + FOUNDRY_LOCAL = "Foundry Local", LLAMA = "Meta Llama", META = "Meta Model API", MARITALK = "Maritalk", @@ -273,6 +275,7 @@ export const provider_map: Record = { LLAMA: "meta_llama", META: "meta", LM_STUDIO: "lm_studio", + FOUNDRY_LOCAL: "foundry_local", MICROSOFT_365_COPILOT: "microsoft_365_copilot", MARITALK: "maritalk", MiniMax: "minimax", @@ -381,6 +384,7 @@ export const providerLogoMap: Partial> = { [Providers.FalAI]: falAiLogo.src, [Providers.FEATHERLESS_AI]: featherlessLogo.src, [Providers.FireworksAI]: fireworksLogo.src, + [Providers.FOUNDRY_LOCAL]: foundryLocalLogo.src, [Providers.FRIENDLIAI]: friendliLogo.src, [Providers.GIGACHAT]: gigachatLogo.src, [Providers.GITHUB_COPILOT]: githubCopilotLogo.src,