scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 359

fix: complete Foundry Local embedding and transcription support

Browse source 
Route embeddings through the existing OpenAI transport and adapt audio uploads to Foundry's filename-based JSON contract. Keep temporary audio alive through sync/async requests and clean it up after errors or cancellation. Reject missing endpoints instead of falling back to a public service.

Files changed: litellm/__init__.py, litellm/_lazy_imports_registry.py, litellm/main.py, litellm/utils.py, litellm/llms/base_llm/audio_transcription/transformation.py, litellm/llms/custom_httpx/llm_http_handler.py, litellm/llms/foundry_local/audio_transcription/__init__.py, litellm/llms/foundry_local/audio_transcription/transformation.py, litellm/llms/foundry_local/chat/transformation.py, provider_endpoints_support.json, tests/unit/llms/foundry_local/chat/test_transformation.py

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 837c7cf9-c279-4a20-b611-200f84ddf448
This commit is contained in:
Bhagirath Mehta 2026-10-21 18:13:48 -05:00
parent 09a7438aa6
commit 50854b47c6
No known key found for this signature in database
11 changed files with 379 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/__init__.py
View file

@ -2061,6 +2061,9 @@ if TYPE_CHECKING:
from .llms.foundry_local.chat.transformation import (
FoundryLocalChatConfig as _FoundryLocalChatConfig,
)
from .llms.foundry_local.audio_transcription.transformation import (
FoundryLocalAudioTranscriptionConfig as _FoundryLocalAudioTranscriptionConfig,
)
from .llms.lm_studio.embed.transformation import (
LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig,
)
@ -2087,6 +2090,7 @@ if TYPE_CHECKING:
LlamafileChatConfig: Type[_LlamafileChatConfig]
LMStudioChatConfig: Type[_LMStudioChatConfig]
FoundryLocalChatConfig: Type[_FoundryLocalChatConfig]
FoundryLocalAudioTranscriptionConfig: Type[_FoundryLocalAudioTranscriptionConfig]
LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]
IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig]
VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig

5
litellm/_lazy_imports_registry.py
View file

@ -320,6 +320,7 @@ LLM_CONFIG_NAMES: Final = (
"TencentChatConfig",
"LMStudioChatConfig",
"FoundryLocalChatConfig",
"FoundryLocalAudioTranscriptionConfig",
"LmStudioEmbeddingConfig",
"NscaleConfig",
"PerplexityChatConfig",
@ -1216,6 +1217,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
".llms.foundry_local.chat.transformation",
"FoundryLocalChatConfig",
),
"FoundryLocalAudioTranscriptionConfig": (
".llms.foundry_local.audio_transcription.transformation",
"FoundryLocalAudioTranscriptionConfig",
),
"LmStudioEmbeddingConfig": (
".llms.lm_studio.embed.transformation",
"LmStudioEmbeddingConfig",

4
litellm/llms/base_llm/audio_transcription/transformation.py
View file

@ -1,4 +1,5 @@
from abc import ABC, abstractmethod
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final
@ -37,6 +38,9 @@ class AudioTranscriptionRequestData:
class BaseAudioTranscriptionConfig(BaseConfig, ABC):
def audio_file_context(self, audio_file: FileTypes) -> AbstractContextManager[FileTypes]:
return nullcontext(audio_file)
@abstractmethod
def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]:
pass

174
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -1950,50 +1950,48 @@ class BaseLLMHTTPHandler:
shared_session=shared_session,
)
# Prepare the request
(
headers,
complete_url,
data,
files,
) = self._prepare_audio_transcription_request(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
headers=headers,
provider_config=provider_config,
)
if client is None or not isinstance(client, HTTPHandler):
client = get_httpx_client()
json_data: Final = data if files is None and isinstance(data, dict) else None
try:
response: Final = client.post(
url=complete_url,
with provider_config.audio_file_context(audio_file) as prepared_audio_file:
(
prepared_headers,
complete_url,
data,
files,
) = self._prepare_audio_transcription_request(
model=model,
audio_file=prepared_audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
headers=headers,
data=data if json_data is None else None,
files=files,
json=json_data,
timeout=timeout,
provider_config=provider_config,
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return self._transform_audio_transcription_response(
provider_config=provider_config,
model=model,
response=response,
model_response=model_response,
logging_obj=logging_obj,
optional_params=optional_params,
api_key=api_key,
)
sync_client: Final = client if isinstance(client, HTTPHandler) else get_httpx_client()
json_data: Final = data if files is None and isinstance(data, dict) else None
try:
response: Final = sync_client.post(
url=complete_url,
headers=prepared_headers,
data=data if json_data is None else None,
files=files,
json=json_data,
timeout=timeout,
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return self._transform_audio_transcription_response(
provider_config=provider_config,
model=model,
response=response,
model_response=model_response,
logging_obj=logging_obj,
optional_params=optional_params,
api_key=api_key,
)
async def async_audio_transcriptions(
self,
@ -2016,56 +2014,56 @@ class BaseLLMHTTPHandler:
if provider_config is None:
raise ValueError(f"No provider config found for model: {model} and provider: {custom_llm_provider}")
# Prepare the request
(
headers,
complete_url,
data,
files,
) = self._prepare_audio_transcription_request(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
headers=headers,
provider_config=provider_config,
)
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
shared_session=shared_session,
)
else:
async_httpx_client = client
json_data: Final = data if files is None and isinstance(data, dict) else None
try:
response: Final = await async_httpx_client.post(
url=complete_url,
with provider_config.audio_file_context(audio_file) as prepared_audio_file:
(
prepared_headers,
complete_url,
data,
files,
) = self._prepare_audio_transcription_request(
model=model,
audio_file=prepared_audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
headers=headers,
data=data if json_data is None else None,
files=files,
json=json_data,
timeout=timeout,
provider_config=provider_config,
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return self._transform_audio_transcription_response(
provider_config=provider_config,
model=model,
response=response,
model_response=model_response,
logging_obj=logging_obj,
optional_params=optional_params,
api_key=api_key,
)
async_httpx_client: Final = (
client
if isinstance(client, AsyncHTTPHandler)
else get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
shared_session=shared_session,
)
)
json_data: Final = data if files is None and isinstance(data, dict) else None
try:
response: Final = await async_httpx_client.post(
url=complete_url,
headers=prepared_headers,
data=data if json_data is None else None,
files=files,
json=json_data,
timeout=timeout,
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return self._transform_audio_transcription_response(
provider_config=provider_config,
model=model,
response=response,
model_response=model_response,
logging_obj=logging_obj,
optional_params=optional_params,
api_key=api_key,
)
def search(
self,

0
litellm/llms/foundry_local/audio_transcription/__init__.py Normal file
View file

72
litellm/llms/foundry_local/audio_transcription/transformation.py Normal file
View file

@ -0,0 +1,72 @@
from collections.abc import Generator, Mapping, Sequence
from contextlib import contextmanager
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Final
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData
from litellm.llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig
from litellm.types.llms.openai import AllMessageValues, OpenAIAudioTranscriptionOptionalParams
from litellm.types.utils import FileTypes
class FoundryLocalAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
@property
def has_native_transcription_endpoint(self) -> bool:
return True
def get_supported_openai_params(
self, model: str
) -> list[OpenAIAudioTranscriptionOptionalParams]: # mutable-ok: base class contract
return ["language", "prompt", "response_format", "temperature"]
def validate_environment(
self,
headers: Mapping[str, object],
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, object]: # mutable-ok: the shared HTTP handler adds the request Content-Type header
return {**headers, **({"Authorization": f"Bearer {api_key}"} if api_key else {})}
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
stream: bool | None = None,
) -> str:
if not api_base:
raise ValueError("Foundry Local transcription requires api_base or FOUNDRY_LOCAL_API_BASE")
return f"{api_base.rstrip('/')}/audio/transcriptions"
@contextmanager
def audio_file_context(self, audio_file: FileTypes) -> Generator[FileTypes, None, None]:
processed: Final = process_audio_file(audio_file)
# Foundry reads a filename rather than an upload. Close it before HTTP for Windows.
with TemporaryDirectory(prefix="litellm-foundry-audio-") as directory:
path: Final = Path(directory) / f"audio{Path(processed.filename).suffix or '.wav'}"
path.write_bytes(processed.file_content)
yield path
def transform_audio_transcription_request(
self,
model: str,
audio_file: FileTypes,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> AudioTranscriptionRequestData:
if not isinstance(audio_file, Path):
raise ValueError("Foundry Local audio must be prepared using audio_file_context")
if optional_params.get("stream"):
raise ValueError("Streaming Foundry Local transcription is not supported by this integration")
return AudioTranscriptionRequestData(
data={**optional_params, "model": model, "filename": str(audio_file), "stream": False},
content_type="application/json",
)

5
litellm/llms/foundry_local/chat/transformation.py
View file

@ -7,6 +7,7 @@ and exposes an OpenAI-compatible REST API endpoint.
from typing import Final
import litellm
from litellm.secret_managers.main import get_secret_str
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@ -16,6 +17,8 @@ class FoundryLocalChatConfig(OpenAIGPTConfig):
def get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None
) -> tuple[str | None, str | None]:
resolved_api_base: Final = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE")
resolved_api_base: Final = api_base or get_secret_str("FOUNDRY_LOCAL_API_BASE") or litellm.api_base
if not resolved_api_base:
raise ValueError("Foundry Local requires api_base or FOUNDRY_LOCAL_API_BASE")
dynamic_api_key: Final = api_key or get_secret_str("FOUNDRY_LOCAL_API_KEY") or "fake-api-key"
return resolved_api_base, dynamic_api_key

1
litellm/main.py
View file

@ -6810,6 +6810,7 @@ def embedding(
or custom_llm_provider == "together_ai"
or custom_llm_provider == "nvidia_nim"
or custom_llm_provider == "litellm_proxy"
or custom_llm_provider == "foundry_local"
or (model in litellm.open_ai_embedding_models and custom_llm_provider is None)
):
api_base = (

2
litellm/utils.py
View file

@ -9674,6 +9674,8 @@ class ProviderConfigManager:
model: str,
provider: LlmProviders,
) -> BaseAudioTranscriptionConfig | None:
if provider == LlmProviders.FOUNDRY_LOCAL:
return litellm.FoundryLocalAudioTranscriptionConfig()
model_cost_entry: Final = _get_model_cost_entry_for_provider_config(
model=model,
provider=provider,

4
provider_endpoints_support.json
View file

@ -1629,9 +1629,9 @@
"chat_completions": true,
"messages": true,
"responses": true,
"embeddings": false,
"embeddings": true,
"image_generations": false,
"audio_transcriptions": false,
"audio_transcriptions": true,
"audio_speech": false,
"moderations": false,
"batches": false,

200
tests/unit/llms/foundry_local/chat/test_transformation.py
View file

@ -1,4 +1,7 @@
import asyncio
import json
from io import BytesIO
from pathlib import Path
from typing import Final
import httpx
@ -7,6 +10,9 @@ from openai import AsyncOpenAI, OpenAI
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.foundry_local.audio_transcription.transformation import FoundryLocalAudioTranscriptionConfig
from litellm.llms.openai.common_utils import OpenAIError
from litellm.types.utils import ModelResponse
@ -17,7 +23,6 @@ from litellm.types.utils import ModelResponse
@pytest.mark.parametrize(
("api_base", "api_key", "env_base", "env_key", "expected_base", "expected_key"),
[
(None, None, None, None, None, "fake-api-key"),
(None, None, "http://127.0.0.1:61234/v1", None, "http://127.0.0.1:61234/v1", "fake-api-key"),
(None, None, "http://127.0.0.1:61234/v1", "env-key", "http://127.0.0.1:61234/v1", "env-key"),
("http://localhost:5272/v1", None, None, "env-key", "http://localhost:5272/v1", "env-key"),
@ -96,6 +101,28 @@ def _respond(request: httpx.Request, stream: bool) -> httpx.Response:
)
@pytest.mark.parametrize("provider", [None, "foundry_local"])
def test_should_require_a_foundry_endpoint(monkeypatch: pytest.MonkeyPatch, provider: str | None) -> None:
monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False)
with pytest.raises(litellm.BadRequestError, match="requires api_base"):
get_llm_provider(
"local-model:1" if provider else "foundry_local/local-model:1",
custom_llm_provider=provider,
)
def test_should_preserve_global_api_base_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("FOUNDRY_LOCAL_API_BASE", raising=False)
monkeypatch.delenv("FOUNDRY_LOCAL_API_KEY", raising=False)
monkeypatch.setattr(litellm, "api_base", "http://127.0.0.1:61234/v1")
assert get_llm_provider("foundry_local/local-model:1") == (
"local-model:1",
"foundry_local",
"fake-api-key",
"http://127.0.0.1:61234/v1",
)
@pytest.mark.parametrize("stream", [False, True])
def test_should_complete_chat_over_foundry_http_service(stream: bool) -> None:
with (
@ -122,6 +149,177 @@ def test_should_complete_chat_over_foundry_http_service(stream: bool) -> None:
assert response.usage.total_tokens == 3
def _embedding_response(request: httpx.Request) -> httpx.Response:
assert str(request.url) == "http://127.0.0.1:61234/v1/embeddings"
assert request.headers["Authorization"] == "Bearer env-key"
payload: Final = json.loads(request.content)
assert payload["model"] == "embedding-model:1"
assert payload["input"] == ["Hello", "World"]
return httpx.Response(
200,
json={
"object": "list",
"model": "embedding-model:1",
"data": [
{"object": "embedding", "index": 0, "embedding": [0.25, 0.75]},
{"object": "embedding", "index": 1, "embedding": [0.5, 0.5]},
],
"usage": {"prompt_tokens": 2, "total_tokens": 2},
},
)
def test_should_embed_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1")
monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key")
with (
httpx.Client(transport=httpx.MockTransport(_embedding_response)) as http_client,
OpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client,
):
response: Final = litellm.embedding(
model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client
)
assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]]
assert response.usage.prompt_tokens == 2
@pytest.mark.asyncio
async def test_should_embed_asynchronously_through_foundry_http_service(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1")
monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key")
async with (
httpx.AsyncClient(transport=httpx.MockTransport(_embedding_response)) as http_client,
AsyncOpenAI(api_key="env-key", base_url="http://127.0.0.1:61234/v1", http_client=http_client) as client,
):
response: Final = await litellm.aembedding(
model="foundry_local/embedding-model:1", input=["Hello", "World"], client=client
)
assert [item["embedding"] for item in response.data] == [[0.25, 0.75], [0.5, 0.5]]
assert response.usage.prompt_tokens == 2
@pytest.mark.parametrize("status_code", [200, 400])
@pytest.mark.parametrize("input_type", ["bytes", "tuple", "stream", "path"])
def test_should_transcribe_and_clean_up_audio(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status_code: int, input_type: str
) -> None:
monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1")
monkeypatch.setenv("FOUNDRY_LOCAL_API_KEY", "env-key")
paths: Final = []
original: Final = tmp_path / "input.wav"
original.write_bytes(b"audio-content")
inputs: Final = {
"bytes": b"audio-content",
"tuple": ("input.wav", b"audio-content", "audio/wav"),
"stream": BytesIO(b"audio-content"),
"path": original,
}
def respond(request: httpx.Request) -> httpx.Response:
assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions"
assert request.headers["Content-Type"] == "application/json"
assert request.headers["Authorization"] == "Bearer env-key"
payload: Final = json.loads(request.content)
assert payload["model"] == "whisper-tiny"
assert payload["language"] == "en"
assert payload["prompt"] == "Greeting"
assert payload["stream"] is False
path: Final = Path(payload["filename"])
paths.append(path)
assert path.read_bytes() == b"audio-content"
return httpx.Response(
status_code, json={"text": "Hello"} if status_code == 200 else {"error": {"message": "Invalid audio"}}
)
with httpx.Client(transport=httpx.MockTransport(respond)) as http_client:
client: Final = HTTPHandler(client=http_client)
if status_code == 200:
response: Final = litellm.transcription(
model="foundry_local/whisper-tiny",
file=inputs[input_type],
language="en",
prompt="Greeting",
client=client,
max_retries=0,
)
assert response.text == "Hello"
else:
with pytest.raises(OpenAIError, match="Invalid audio") as error:
litellm.transcription(
model="foundry_local/whisper-tiny",
file=inputs[input_type],
language="en",
prompt="Greeting",
client=client,
max_retries=0,
)
assert error.value.status_code == 400
assert len(paths) == 1
assert not paths[0].exists()
assert not paths[0].parent.exists()
assert original.read_bytes() == b"audio-content"
@pytest.mark.asyncio
@pytest.mark.parametrize("outcome", ["success", "error", "cancel"])
async def test_should_keep_async_audio_until_completion_and_clean_up(
monkeypatch: pytest.MonkeyPatch, outcome: str
) -> None:
monkeypatch.setenv("FOUNDRY_LOCAL_API_BASE", "http://127.0.0.1:61234/v1")
paths: Final = []
async def respond(request: httpx.Request) -> httpx.Response:
assert str(request.url) == "http://127.0.0.1:61234/v1/audio/transcriptions"
payload: Final = json.loads(request.content)
path: Final = Path(payload["filename"])
paths.append(path)
await asyncio.sleep(0)
assert path.read_bytes() == b"audio-content"
if outcome == "cancel":
raise asyncio.CancelledError
return httpx.Response(
400 if outcome == "error" else 200,
json={"error": {"message": "Invalid audio"}} if outcome == "error" else {"text": "Hello"},
)
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond))
async with client.client:
if outcome == "success":
response: Final = await litellm.atranscription(
model="foundry_local/whisper-tiny",
file=("input.wav", b"audio-content"),
client=client,
max_retries=0,
)
assert response.text == "Hello"
else:
with pytest.raises(asyncio.CancelledError if outcome == "cancel" else litellm.BadRequestError):
await litellm.atranscription(
model="foundry_local/whisper-tiny",
file=("input.wav", b"audio-content"),
client=client,
max_retries=0,
)
assert len(paths) == 1
assert not paths[0].exists()
assert not paths[0].parent.exists()
def test_should_reject_bare_audio_paths_and_unsupported_transcription_options(tmp_path: Path) -> None:
config: Final = FoundryLocalAudioTranscriptionConfig()
path: Final = tmp_path / "input.wav"
path.write_bytes(b"audio-content")
with pytest.raises(ValueError, match="bare str"):
with config.audio_file_context(str(path)):
pytest.fail("A bare string must not be opened as a server-side path")
with pytest.raises(litellm.UnsupportedParamsError):
litellm.utils.get_optional_params_transcription(
model="whisper-tiny", custom_llm_provider="foundry_local", timestamp_granularities=["word"]
)
with pytest.raises(ValueError, match="requires api_base"):
config.get_complete_url(None, None, "whisper-tiny", {}, {})
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_should_complete_chat_asynchronously_over_foundry_http_service(stream: bool) -> None: