diff --git a/litellm/__init__.py b/litellm/__init__.py index 6436ca53737..75198696d34 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2061,6 +2061,9 @@ if TYPE_CHECKING: from .llms.foundry_local.chat.transformation import ( FoundryLocalChatConfig as _FoundryLocalChatConfig, ) + from .llms.foundry_local.audio_transcription.transformation import ( + FoundryLocalAudioTranscriptionConfig as _FoundryLocalAudioTranscriptionConfig, + ) from .llms.lm_studio.embed.transformation import ( LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig, ) @@ -2087,6 +2090,7 @@ if TYPE_CHECKING: LlamafileChatConfig: Type[_LlamafileChatConfig] LMStudioChatConfig: Type[_LMStudioChatConfig] FoundryLocalChatConfig: Type[_FoundryLocalChatConfig] + FoundryLocalAudioTranscriptionConfig: Type[_FoundryLocalAudioTranscriptionConfig] LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig] IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index ce31432da9f..b36972e215d 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -320,6 +320,7 @@ LLM_CONFIG_NAMES: Final = ( "TencentChatConfig", "LMStudioChatConfig", "FoundryLocalChatConfig", + "FoundryLocalAudioTranscriptionConfig", "LmStudioEmbeddingConfig", "NscaleConfig", "PerplexityChatConfig", @@ -1216,6 +1217,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.foundry_local.chat.transformation", "FoundryLocalChatConfig", ), + "FoundryLocalAudioTranscriptionConfig": ( + ".llms.foundry_local.audio_transcription.transformation", + "FoundryLocalAudioTranscriptionConfig", + ), "LmStudioEmbeddingConfig": ( ".llms.lm_studio.embed.transformation", "LmStudioEmbeddingConfig", diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index e4bf148abf3..88d4728a28f 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final @@ -37,6 +38,9 @@ class AudioTranscriptionRequestData: class BaseAudioTranscriptionConfig(BaseConfig, ABC): + def audio_file_context(self, audio_file: FileTypes) -> AbstractContextManager[FileTypes]: + return nullcontext(audio_file) + @abstractmethod def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: pass diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ac2ac1fe31b..2b5a86521fb 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1950,50 +1950,48 @@ class BaseLLMHTTPHandler: shared_session=shared_session, ) - # Prepare the request - ( - headers, - complete_url, - data, - files, - ) = self._prepare_audio_transcription_request( - model=model, - audio_file=audio_file, - optional_params=optional_params, - litellm_params=litellm_params, - logging_obj=logging_obj, - api_key=api_key, - api_base=api_base, - headers=headers, - provider_config=provider_config, - ) - - if client is None or not isinstance(client, HTTPHandler): - client = get_httpx_client() - - json_data: Final = data if files is None and isinstance(data, dict) else None - - try: - response: Final = client.post( - url=complete_url, + with provider_config.audio_file_context(audio_file) as prepared_audio_file: + ( + prepared_headers, + complete_url, + data, + files, + ) = self._prepare_audio_transcription_request( + model=model, + audio_file=prepared_audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, headers=headers, - data=data if json_data is None else None, - files=files, - json=json_data, - timeout=timeout, + provider_config=provider_config, ) - except Exception as e: - raise self._handle_error(e=e, provider_config=provider_config) - return self._transform_audio_transcription_response( - provider_config=provider_config, - model=model, - response=response, - model_response=model_response, - logging_obj=logging_obj, - optional_params=optional_params, - api_key=api_key, - ) + sync_client: Final = client if isinstance(client, HTTPHandler) else get_httpx_client() + json_data: Final = data if files is None and isinstance(data, dict) else None + + try: + response: Final = sync_client.post( + url=complete_url, + headers=prepared_headers, + data=data if json_data is None else None, + files=files, + json=json_data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + return self._transform_audio_transcription_response( + provider_config=provider_config, + model=model, + response=response, + model_response=model_response, + logging_obj=logging_obj, + optional_params=optional_params, + api_key=api_key, + ) async def async_audio_transcriptions( self, @@ -2016,56 +2014,56 @@ class BaseLLMHTTPHandler: if provider_config is None: raise ValueError(f"No provider config found for model: {model} and provider: {custom_llm_provider}") - # Prepare the request - ( - headers, - complete_url, - data, - files, - ) = self._prepare_audio_transcription_request( - model=model, - audio_file=audio_file, - optional_params=optional_params, - litellm_params=litellm_params, - logging_obj=logging_obj, - api_key=api_key, - api_base=api_base, - headers=headers, - provider_config=provider_config, - ) - - if client is None or not isinstance(client, AsyncHTTPHandler): - async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), - params={"ssl_verify": litellm_params.get("ssl_verify", None)}, - shared_session=shared_session, - ) - else: - async_httpx_client = client - - json_data: Final = data if files is None and isinstance(data, dict) else None - - try: - response: Final = await async_httpx_client.post( - url=complete_url, + with provider_config.audio_file_context(audio_file) as prepared_audio_file: + ( + prepared_headers, + complete_url, + data, + files, + ) = self._prepare_audio_transcription_request( + model=model, + audio_file=prepared_audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, headers=headers, - data=data if json_data is None else None, - files=files, - json=json_data, - timeout=timeout, + provider_config=provider_config, ) - except Exception as e: - raise self._handle_error(e=e, provider_config=provider_config) - return self._transform_audio_transcription_response( - provider_config=provider_config, - model=model, - response=response, - model_response=model_response, - logging_obj=logging_obj, - optional_params=optional_params, - api_key=api_key, - ) + async_httpx_client: Final = ( + client + if isinstance(client, AsyncHTTPHandler) + else get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, + ) + ) + json_data: Final = data if files is None and isinstance(data, dict) else None + + try: + response: Final = await async_httpx_client.post( + url=complete_url, + headers=prepared_headers, + data=data if json_data is None else None, + files=files, + json=json_data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + return self._transform_audio_transcription_response( + provider_config=provider_config, + model=model, + response=response, + model_response=model_response, + logging_obj=logging_obj, + optional_params=optional_params, + api_key=api_key, + ) def search( self, diff --git a/litellm/llms/foundry_local/audio_transcription/__init__.py b/litellm/llms/foundry_local/audio_transcription/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/foundry_local/audio_transcription/transformation.py b/litellm/llms/foundry_local/audio_transcription/transformation.py new file mode 100644 index 00000000000..8463d7578a4 --- /dev/null +++ b/litellm/llms/foundry_local/audio_transcription/transformation.py @@ -0,0 +1,72 @@ +from collections.abc import Generator, Mapping, Sequence +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Final + +from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData +from litellm.llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig +from litellm.types.llms.openai import AllMessageValues, OpenAIAudioTranscriptionOptionalParams +from litellm.types.utils import FileTypes + + +class FoundryLocalAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): + @property + def has_native_transcription_endpoint(self) -> bool: + return True + + def get_supported_openai_params( + self, model: str + ) -> list[OpenAIAudioTranscriptionOptionalParams]: # mutable-ok: base class contract + return ["language", "prompt", "response_format", "temperature"] + + def validate_environment( + self, + headers: Mapping[str, object], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, object]: # mutable-ok: the shared HTTP handler adds the request Content-Type header + return {**headers, **({"Authorization": f"Bearer {api_key}"} if api_key else {})} + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + if not api_base: + raise ValueError("Foundry Local transcription requires api_base or FOUNDRY_LOCAL_API_BASE") + return f"{api_base.rstrip('/')}/audio/transcriptions" + + @contextmanager + def audio_file_context(self, audio_file: FileTypes) -> Generator[FileTypes, None, None]: + processed: Final = process_audio_file(audio_file) + # Foundry reads a filename rather than an upload. Close it before HTTP for Windows. + with TemporaryDirectory(prefix="litellm-foundry-audio-") as directory: + path: Final = Path(directory) / f"audio{Path(processed.filename).suffix or '.wav'}" + path.write_bytes(processed.file_content) + yield path + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> AudioTranscriptionRequestData: + if not isinstance(audio_file, Path): + raise ValueError("Foundry Local audio must be prepared using audio_file_context") + if optional_params.get("stream"): + raise ValueError("Streaming Foundry Local transcription is not supported by this integration") + return AudioTranscriptionRequestData( + data={**optional_params, "model": model, "filename": str(audio_file), "stream": False}, + content_type="application/json", + ) diff --git a/litellm/llms/foundry_local/chat/transformation.py b/litellm/llms/foundry_local/chat/transformation.py index efe4fcc4f99..410cb2a8359 100644 --- a/litellm/llms/foundry_local/chat/transformation.py +++ b/litellm/llms/foundry_local/chat/transformation.py @@ -7,6 +7,7 @@ and exposes an OpenAI-compatible REST API endpoint. from typing import Final +import litellm from litellm.secret_managers.main import get_secret_str from ...openai.chat.gpt_transformation import OpenAIGPTConfig @@ -16,6 +17,8 @@ class FoundryLocalChatConfig(OpenAIGPTConfig): def get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: - resolved_api_base: Final = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE") + resolved_api_base: Final = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE") or litellm.api_base + if not resolved_api_base: + raise ValueError("Foundry Local requires api_base or FOUNDRY_LOCAL_API_BASE") dynamic_api_key: Final = api_key or get_secret_str("FOUNDRY_LOCAL_API_KEY") or "fake-api-key" return resolved_api_base, dynamic_api_key diff --git a/litellm/main.py b/litellm/main.py index d647cab24e3..f01b78783b9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6810,6 +6810,7 @@ def embedding( or custom_llm_provider == "together_ai" or custom_llm_provider == "nvidia_nim" or custom_llm_provider == "litellm_proxy" + or custom_llm_provider == "foundry_local" or (model in litellm.open_ai_embedding_models and custom_llm_provider is None) ): api_base = ( diff --git a/litellm/utils.py b/litellm/utils.py index f851e4b8635..088070bb9ed 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9674,6 +9674,8 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAudioTranscriptionConfig | None: + if provider == LlmProviders.FOUNDRY_LOCAL: + return litellm.FoundryLocalAudioTranscriptionConfig() model_cost_entry: Final = _get_model_cost_entry_for_provider_config( model=model, provider=provider, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e676438d87e..939734c4958 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1629,9 +1629,9 @@ "chat_completions": true, "messages": true, "responses": true, - "embeddings": false, + "embeddings": true, "image_generations": false, - "audio_transcriptions": false, + "audio_transcriptions": true, "audio_speech": false, "moderations": false, "batches": false, diff --git a/tests/unit/llms/foundry_local/chat/test_transformation.py b/tests/unit/llms/foundry_local/chat/test_transformation.py index 7630425ef15..8123c763436 100644 --- a/tests/unit/llms/foundry_local/chat/test_transformation.py +++ b/tests/unit/llms/foundry_local/chat/test_transformation.py @@ -1,4 +1,7 @@ +import asyncio import json +from io import BytesIO +from pathlib import Path from typing import Final import httpx @@ -7,6 +10,9 @@ from openai import AsyncOpenAI, OpenAI import litellm from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.foundry_local.audio_transcription.transformation import FoundryLocalAudioTranscriptionConfig +from litellm.llms.openai.common_utils import OpenAIError from litellm.types.utils import ModelResponse @@ -17,7 +23,6 @@ from litellm.types.utils import ModelResponse @pytest.mark.parametrize( ("api_base", "api_key", "env_base", "env_key", "expected_base", "expected_key"), [ - (None, None, None, None, None, "fake-api-key"), (None, None, "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "fake-api-key"), (None, None, "http://127.0.0.1:61234/v1", "env-key", "http://127.0.0.1:61234/v1", "env-key"), ("http://localhost:5272/v1", None, None, "env-key", "http://localhost:5272/v1", "env-key"), @@ -96,6 +101,28 @@ def _respond(request: httpx.Request, stream: bool) -> httpx.Response: ) +@pytest.mark.parametrize("provider", [None, "foundry_local"]) +def test_should_require_a_foundry_endpoint(monkeypatch: pytest.MonkeyPatch, provider: str | None) -> None: + monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False) + with pytest.raises(litellm.BadRequestError, match="requires api_base"): + get_llm_provider( + "local-model:1" if provider else "foundry_local/local-model:1", + custom_llm_provider=provider, + ) + + +def test_should_preserve_global_api_base_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False) + monkeypatch.delenv("FOUNDRY_LOCAL_API_KEY", raising=False) + monkeypatch.setattr(litellm, "api_base", "http://127.0.0.1:61234/v1") + assert get_llm_provider("foundry_local/local-model:1") == ( + "local-model:1", + "foundry_local", + "fake-api-key", + "http://127.0.0.1:61234/v1", + ) + + @pytest.mark.parametrize("stream", [False, True]) def test_should_complete_chat_over_foundry_http_service(stream: bool) -> None: with ( @@ -122,6 +149,177 @@ def test_should_complete_chat_over_foundry_http_service(stream: bool) -> None: assert response.usage.total_tokens == 3 +def _embedding_response(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/embeddings" + assert request.headers["Authorization"] == "Bearer env-key" + payload: Final = json.loads(request.content) + assert payload["model"] == "embedding-model:1" + assert payload["input"] == ["Hello", "World"] + return httpx.Response( + 200, + json={ + "object": "list", + "model": "embedding-model:1", + "data": [ + {"object": "embedding", "index": 0, "embedding": [0.25, 0.75]}, + {"object": "embedding", "index": 1, "embedding": [0.5, 0.5]}, + ], + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + + +def test_should_embed_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key") + with ( + httpx.Client(transport=httpx.MockTransport(_embedding_response)) as http_client, + OpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = litellm.embedding( + model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client + ) + assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]] + assert response.usage.prompt_tokens == 2 + + +@pytest.mark.asyncio +async def test_should_embed_asynchronously_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key") + async with ( + httpx.AsyncClient(transport=httpx.MockTransport(_embedding_response)) as http_client, + AsyncOpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client, + ): + response: Final = await litellm.aembedding( + model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client + ) + assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]] + assert response.usage.prompt_tokens == 2 + + +@pytest.mark.parametrize("status_code", [200, 400]) +@pytest.mark.parametrize("input_type", ["bytes", "tuple", "stream", "path"]) +def test_should_transcribe_and_clean_up_audio( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status_code: int, input_type: str +) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key") + paths: Final = [] + original: Final = tmp_path / "input.wav" + original.write_bytes(b"audio-content") + inputs: Final = { + "bytes": b"audio-content", + "tuple": ("input.wav", b"audio-content", "audio/wav"), + "stream": BytesIO(b"audio-content"), + "path": original, + } + + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions" + assert request.headers["Content-Type"] == "application/json" + assert request.headers["Authorization"] == "Bearer env-key" + payload: Final = json.loads(request.content) + assert payload["model"] == "whisper-tiny" + assert payload["language"] == "en" + assert payload["prompt"] == "Greeting" + assert payload["stream"] is False + path: Final = Path(payload["filename"]) + paths.append(path) + assert path.read_bytes() == b"audio-content" + return httpx.Response( + status_code, json={"text": "Hello"} if status_code == 200 else {"error": {"message": "Invalid audio"}} + ) + + with httpx.Client(transport=httpx.MockTransport(respond)) as http_client: + client: Final = HTTPHandler(client=http_client) + if status_code == 200: + response: Final = litellm.transcription( + model="foundry_local/whisper-tiny", + file=inputs[input_type], + language="en", + prompt="Greeting", + client=client, + max_retries=0, + ) + assert response.text == "Hello" + else: + with pytest.raises(OpenAIError, match="Invalid audio") as error: + litellm.transcription( + model="foundry_local/whisper-tiny", + file=inputs[input_type], + language="en", + prompt="Greeting", + client=client, + max_retries=0, + ) + assert error.value.status_code == 400 + assert len(paths) == 1 + assert not paths[0].exists() + assert not paths[0].parent.exists() + assert original.read_bytes() == b"audio-content" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["success", "error", "cancel"]) +async def test_should_keep_async_audio_until_completion_and_clean_up( + monkeypatch: pytest.MonkeyPatch, outcome: str +) -> None: + monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1") + paths: Final = [] + + async def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions" + payload: Final = json.loads(request.content) + path: Final = Path(payload["filename"]) + paths.append(path) + await asyncio.sleep(0) + assert path.read_bytes() == b"audio-content" + if outcome == "cancel": + raise asyncio.CancelledError + return httpx.Response( + 400 if outcome == "error" else 200, + json={"error": {"message": "Invalid audio"}} if outcome == "error" else {"text": "Hello"}, + ) + + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + async with client.client: + if outcome == "success": + response: Final = await litellm.atranscription( + model="foundry_local/whisper-tiny", + file=("input.wav", b"audio-content"), + client=client, + max_retries=0, + ) + assert response.text == "Hello" + else: + with pytest.raises(asyncio.CancelledError if outcome == "cancel" else litellm.BadRequestError): + await litellm.atranscription( + model="foundry_local/whisper-tiny", + file=("input.wav", b"audio-content"), + client=client, + max_retries=0, + ) + assert len(paths) == 1 + assert not paths[0].exists() + assert not paths[0].parent.exists() + + +def test_should_reject_bare_audio_paths_and_unsupported_transcription_options(tmp_path: Path) -> None: + config: Final = FoundryLocalAudioTranscriptionConfig() + path: Final = tmp_path / "input.wav" + path.write_bytes(b"audio-content") + with pytest.raises(ValueError, match="bare str"): + with config.audio_file_context(str(path)): + pytest.fail("A bare string must not be opened as a server-side path") + with pytest.raises(litellm.UnsupportedParamsError): + litellm.utils.get_optional_params_transcription( + model="whisper-tiny", custom_llm_provider="foundry_local", timestamp_granularities=["word"] + ) + with pytest.raises(ValueError, match="requires api_base"): + config.get_complete_url(None, None, "whisper-tiny", {}, {}) + + @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) async def test_should_complete_chat_asynchronously_over_foundry_http_service(stream: bool) -> None: