From 3ceaf793ac2a8547911bf40dd6181df31e1d6311 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:33:18 -0700 Subject: [PATCH 1/5] fix(speech): use the provider api key get_llm_provider resolves speech() called get_llm_provider and then dropped its dynamic_api_key everywhere except the logging params, so a TTS deployment whose credential lives in a provider-specific env var fell through to OPENAI_API_KEY on the openai-compatible branch and to AZURE_OPENAI_API_KEY / AZURE_API_KEY on the azure branch, failing with a missing-credentials error naming variables the admin never configured. Fold dynamic_api_key into both branch key chains, right after an explicitly passed api_key, so a provider-scoped key is used while an explicit one stays authoritative. --- litellm/main.py | 4 +- tests/test_litellm/test_main.py | 73 +++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 0bca4a7350e..229520bebf1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8073,7 +8073,8 @@ def speech( # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or dynamic_api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") ) @@ -8157,6 +8158,7 @@ def speech( api_key = ( api_key + or dynamic_api_key or litellm.api_key or litellm.azure_key or get_secret("AZURE_OPENAI_API_KEY") diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 3c8bf142835..7e2bc3dbe30 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -1,6 +1,8 @@ import asyncio import base64 +import threading from datetime import datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import contextlib import copy import json @@ -3259,3 +3261,74 @@ def test_stream_chunk_builder_leaves_xai_reported_cost_to_the_calculator(monkeyp assert getattr(response.usage, "cost", None) == pytest.approx(0.42) assert response._hidden_params.get("response_cost") is None assert logging_obj._response_cost_calculator(result=response) == pytest.approx(0.63) + + +@contextlib.contextmanager +def _recording_tts_server(): + received: Final[list[Mapping[str, str]]] = [] + + class _Handler(BaseHTTPRequestHandler): + def do_POST(self): + received.append({key.lower(): value for key, value in self.headers.items()}) + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + body = b"ID3\x04\x00\x00\x00fake-mp3-payload" + self.send_response(200) + self.send_header("Content-Type", "audio/mpeg") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + return + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", received + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_speech_openai_compatible_sends_the_provider_scoped_key(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "openai_key", None) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-scoped-key") + monkeypatch.setenv("OPENAI_API_KEY", "generic-openai-key") + + with _recording_tts_server() as (api_base, received): + response: Final = litellm.speech( + model="hosted_vllm/tts-1", + input="which key goes out", + voice="alloy", + api_base=api_base, + ) + + assert response.content == b"ID3\x04\x00\x00\x00fake-mp3-payload" + assert len(received) == 1 + assert received[0]["authorization"] == "Bearer hosted-vllm-scoped-key" + + +def test_speech_azure_sends_the_provider_scoped_key(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "azure_key", None) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("AZURE_AI_API_KEY", "azure-ai-scoped-key") + monkeypatch.setenv("AZURE_OPENAI_API_KEY", "legacy-azure-openai-key") + monkeypatch.setenv("AZURE_API_KEY", "legacy-azure-key") + + with _recording_tts_server() as (api_base, received): + response: Final = litellm.speech( + model="azure_ai/tts-1", + input="which key goes out", + voice="alloy", + api_base=api_base, + api_version="2024-05-01-preview", + ) + + assert response.content == b"ID3\x04\x00\x00\x00fake-mp3-payload" + assert len(received) == 1 + assert received[0]["api-key"] == "azure-ai-scoped-key" From bc0c5b5ba73401f1a00708449ac46bb0c3fe7830 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:40:53 -0700 Subject: [PATCH 2/5] test(main): assert an explicit api_key still outranks the provider-scoped one in speech() --- tests/test_litellm/test_main.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 7e2bc3dbe30..4e40903a3bb 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -3332,3 +3332,22 @@ def test_speech_azure_sends_the_provider_scoped_key(monkeypatch: pytest.MonkeyPa assert response.content == b"ID3\x04\x00\x00\x00fake-mp3-payload" assert len(received) == 1 assert received[0]["api-key"] == "azure-ai-scoped-key" + + +def test_speech_keeps_an_explicitly_passed_api_key_over_the_provider_scoped_one(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "openai_key", None) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-scoped-key") + + with _recording_tts_server() as (api_base, received): + litellm.speech( + model="hosted_vllm/tts-1", + input="which key goes out", + voice="alloy", + api_base=api_base, + api_key="caller-supplied-key", + ) + + assert len(received) == 1 + assert received[0]["authorization"] == "Bearer caller-supplied-key" From 33f71357d22493ccc85f0c521e74ea79bd6c4e84 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:49:24 -0700 Subject: [PATCH 3/5] test(main): hand recorded TTS headers across threads through a queue --- tests/test_litellm/test_main.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 4e40903a3bb..7fd1e37bce1 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -1,5 +1,6 @@ import asyncio import base64 +import queue import threading from datetime import datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -3265,11 +3266,11 @@ def test_stream_chunk_builder_leaves_xai_reported_cost_to_the_calculator(monkeyp @contextlib.contextmanager def _recording_tts_server(): - received: Final[list[Mapping[str, str]]] = [] + received: Final[queue.Queue[Mapping[str, str]]] = queue.Queue() class _Handler(BaseHTTPRequestHandler): def do_POST(self): - received.append({key.lower(): value for key, value in self.headers.items()}) + received.put({key.lower(): value for key, value in self.headers.items()}) self.rfile.read(int(self.headers.get("Content-Length") or 0)) body = b"ID3\x04\x00\x00\x00fake-mp3-payload" self.send_response(200) @@ -3308,8 +3309,8 @@ def test_speech_openai_compatible_sends_the_provider_scoped_key(monkeypatch: pyt ) assert response.content == b"ID3\x04\x00\x00\x00fake-mp3-payload" - assert len(received) == 1 - assert received[0]["authorization"] == "Bearer hosted-vllm-scoped-key" + assert received.get(timeout=5)["authorization"] == "Bearer hosted-vllm-scoped-key" + assert received.empty() def test_speech_azure_sends_the_provider_scoped_key(monkeypatch: pytest.MonkeyPatch): @@ -3330,8 +3331,8 @@ def test_speech_azure_sends_the_provider_scoped_key(monkeypatch: pytest.MonkeyPa ) assert response.content == b"ID3\x04\x00\x00\x00fake-mp3-payload" - assert len(received) == 1 - assert received[0]["api-key"] == "azure-ai-scoped-key" + assert received.get(timeout=5)["api-key"] == "azure-ai-scoped-key" + assert received.empty() def test_speech_keeps_an_explicitly_passed_api_key_over_the_provider_scoped_one(monkeypatch: pytest.MonkeyPatch): @@ -3349,5 +3350,5 @@ def test_speech_keeps_an_explicitly_passed_api_key_over_the_provider_scoped_one( api_key="caller-supplied-key", ) - assert len(received) == 1 - assert received[0]["authorization"] == "Bearer caller-supplied-key" + assert received.get(timeout=5)["authorization"] == "Bearer caller-supplied-key" + assert received.empty() From feef600f64d422a447204f5d429e5de19a5c3acc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:29:45 -0700 Subject: [PATCH 4/5] fix(speech): try the provider-scoped key last so working global keys keep priority Putting get_llm_provider's key ahead of litellm.api_key, OPENAI_API_KEY and AZURE_API_KEY broke TTS setups that already worked through a global key, and let local-inference placeholders like fake-api-key outrank a real OPENAI_API_KEY. The provider-scoped key is now the last fallback in both speech key chains, so only callers that had no key at all change behavior --- litellm/main.py | 4 ++-- tests/unit/test_main.py | 48 +++++++++++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 22199da259f..864906c1439 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8677,10 +8677,10 @@ def speech( # set API KEY api_key = ( api_key - or dynamic_api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") + or dynamic_api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there ) organization = ( @@ -8763,11 +8763,11 @@ def speech( api_key = ( api_key - or dynamic_api_key or litellm.api_key or litellm.azure_key or get_secret("AZURE_OPENAI_API_KEY") or get_secret("AZURE_API_KEY") + or dynamic_api_key ) azure_ad_token: Final[str | None] = optional_params.get("extra_body", {}).pop( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 17433f7c5ee..74cb80d348b 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -4999,11 +4999,33 @@ def _clear_global_tts_keys(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, attribute, None) -def test_speech_openai_compatible_sends_the_provider_scoped_key( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +def _set_or_clear_env(monkeypatch: pytest.MonkeyPatch, values: dict[str, str | None]) -> None: + for name, value in values.items(): + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) + + +@pytest.mark.parametrize( + ("provider_key", "openai_key", "sent_authorization"), + [ + ("hosted-vllm-scoped-key", None, "Bearer hosted-vllm-scoped-key"), + ("hosted-vllm-scoped-key", "generic-openai-key", "Bearer generic-openai-key"), + (None, "generic-openai-key", "Bearer generic-openai-key"), + (None, None, "Bearer fake-api-key"), + ], + ids=["provider-key-alone", "generic-key-still-wins", "generic-key-beats-placeholder", "placeholder-when-unset"], +) +def test_speech_openai_compatible_falls_back_to_the_provider_scoped_key( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + provider_key: str | None, + openai_key: str | None, + sent_authorization: str, ): _clear_global_tts_keys(monkeypatch) - monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-scoped-key") + _set_or_clear_env(monkeypatch, {"HOSTED_VLLM_API_KEY": provider_key, "OPENAI_API_KEY": openai_key}) route: Final = respx_mock.post("https://vllm.internal/v1/audio/speech").mock( return_value=httpx.Response(200, content=TTS_AUDIO, headers={"content-type": "audio/mpeg"}) ) @@ -5014,14 +5036,22 @@ def test_speech_openai_compatible_sends_the_provider_scoped_key( assert response.content == TTS_AUDIO assert route.call_count == 1 - assert route.calls.last.request.headers["authorization"] == "Bearer hosted-vllm-scoped-key" + assert route.calls.last.request.headers["authorization"] == sent_authorization -def test_speech_azure_sends_the_provider_scoped_key(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): +@pytest.mark.parametrize( + ("generic_azure_key", "sent_api_key"), + [(None, "azure-ai-scoped-key"), ("generic-azure-key", "generic-azure-key")], + ids=["provider-key-alone", "generic-key-still-wins"], +) +def test_speech_azure_falls_back_to_the_provider_scoped_key( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, generic_azure_key: str | None, sent_api_key: str +): _clear_global_tts_keys(monkeypatch) - monkeypatch.setenv("AZURE_AI_API_KEY", "azure-ai-scoped-key") - monkeypatch.setenv("AZURE_OPENAI_API_KEY", "legacy-azure-openai-key") - monkeypatch.setenv("AZURE_API_KEY", "legacy-azure-key") + _set_or_clear_env( + monkeypatch, + {"AZURE_AI_API_KEY": "azure-ai-scoped-key", "AZURE_OPENAI_API_KEY": None, "AZURE_API_KEY": generic_azure_key}, + ) route: Final = respx_mock.post( url__regex=r"https://foundry\.internal/openai/deployments/tts-1/audio/speech\?api-version=.+" ).mock(return_value=httpx.Response(200, content=TTS_AUDIO, headers={"content-type": "audio/mpeg"})) @@ -5036,7 +5066,7 @@ def test_speech_azure_sends_the_provider_scoped_key(respx_mock: respx.MockRouter assert response.content == TTS_AUDIO assert route.call_count == 1 - assert route.calls.last.request.headers["api-key"] == "azure-ai-scoped-key" + assert route.calls.last.request.headers["api-key"] == sent_api_key def test_speech_keeps_an_explicitly_passed_api_key_over_the_provider_scoped_one( From d30be01aca5826954ae8a120f49ded188317d073 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:08:26 -0700 Subject: [PATCH 5/5] fix(speech): keep the provider key first but skip local-inference placeholders Trying the provider key last let OPENAI_API_KEY or AZURE_API_KEY go to a hosted_vllm, deepinfra or azure_ai endpoint whose own key was configured, which leaks the global credential and still fails the reported setup. The provider key goes first again, matching completion() and transcription(), and the fake-api-key / dummy-key placeholders get_llm_provider returns for local-inference providers are skipped there, so a real OPENAI_API_KEY still wins over them --- litellm/constants.py | 1 + litellm/main.py | 7 +++++-- tests/unit/test_main.py | 28 ++++++++++++++++++++++------ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 6a503463a8c..238d60e9dde 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -342,6 +342,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = get_env_int( # Provider-specific API base URLs XAI_API_BASE: Final = "https://api.x.ai/v1" PLACEHOLDER_API_KEY: Final = "fake-api-key" +PLACEHOLDER_API_KEYS: Final = frozenset({PLACEHOLDER_API_KEY, "dummy-key"}) OPEN_SANDBOX_API_BASE_ENV_VAR: Final = "OPEN_SANDBOX_API_BASE" OPEN_SANDBOX_API_KEY_ENV_VAR: Final = "OPEN_SANDBOX_API_KEY" OPEN_SANDBOX_DEFAULT_TEMPLATE: Final = "opensandbox/code-interpreter:v1.1.0" diff --git a/litellm/main.py b/litellm/main.py index 864906c1439..f42c2b104b7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -66,6 +66,7 @@ from litellm.constants import ( DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, NADIR_DEFAULT_API_BASE, OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS, + PLACEHOLDER_API_KEYS, ) from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger @@ -8588,6 +8589,7 @@ def speech( model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url ) + provider_api_key: Final = None if dynamic_api_key in PLACEHOLDER_API_KEYS else dynamic_api_key kwargs.pop("tags", []) optional_params = {} @@ -8677,10 +8679,11 @@ def speech( # set API KEY api_key = ( api_key + or provider_api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") - or dynamic_api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or dynamic_api_key ) organization = ( @@ -8763,11 +8766,11 @@ def speech( api_key = ( api_key + or provider_api_key or litellm.api_key or litellm.azure_key or get_secret("AZURE_OPENAI_API_KEY") or get_secret("AZURE_API_KEY") - or dynamic_api_key ) azure_ad_token: Final[str | None] = optional_params.get("extra_body", {}).pop( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 74cb80d348b..72c2ec0678d 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5011,13 +5011,13 @@ def _set_or_clear_env(monkeypatch: pytest.MonkeyPatch, values: dict[str, str | N ("provider_key", "openai_key", "sent_authorization"), [ ("hosted-vllm-scoped-key", None, "Bearer hosted-vllm-scoped-key"), - ("hosted-vllm-scoped-key", "generic-openai-key", "Bearer generic-openai-key"), + ("hosted-vllm-scoped-key", "generic-openai-key", "Bearer hosted-vllm-scoped-key"), (None, "generic-openai-key", "Bearer generic-openai-key"), (None, None, "Bearer fake-api-key"), ], - ids=["provider-key-alone", "generic-key-still-wins", "generic-key-beats-placeholder", "placeholder-when-unset"], + ids=["provider-key-alone", "provider-key-beats-generic", "generic-key-beats-placeholder", "placeholder-when-unset"], ) -def test_speech_openai_compatible_falls_back_to_the_provider_scoped_key( +def test_speech_openai_compatible_sends_the_provider_scoped_key( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, provider_key: str | None, @@ -5041,10 +5041,10 @@ def test_speech_openai_compatible_falls_back_to_the_provider_scoped_key( @pytest.mark.parametrize( ("generic_azure_key", "sent_api_key"), - [(None, "azure-ai-scoped-key"), ("generic-azure-key", "generic-azure-key")], - ids=["provider-key-alone", "generic-key-still-wins"], + [(None, "azure-ai-scoped-key"), ("generic-azure-key", "azure-ai-scoped-key")], + ids=["provider-key-alone", "provider-key-beats-generic"], ) -def test_speech_azure_falls_back_to_the_provider_scoped_key( +def test_speech_azure_sends_the_provider_scoped_key( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, generic_azure_key: str | None, sent_api_key: str ): _clear_global_tts_keys(monkeypatch) @@ -5069,6 +5069,22 @@ def test_speech_azure_falls_back_to_the_provider_scoped_key( assert route.calls.last.request.headers["api-key"] == sent_api_key +def test_speech_docker_model_runner_placeholder_does_not_beat_a_real_key( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _clear_global_tts_keys(monkeypatch) + _set_or_clear_env(monkeypatch, {"DOCKER_MODEL_RUNNER_API_KEY": None, "OPENAI_API_KEY": "generic-openai-key"}) + route: Final = respx_mock.post("https://dmr.internal/v1/audio/speech").mock( + return_value=httpx.Response(200, content=TTS_AUDIO, headers={"content-type": "audio/mpeg"}) + ) + + litellm.speech( + model="docker_model_runner/tts-1", input="which key goes out", voice="alloy", api_base="https://dmr.internal/v1" + ) + + assert route.calls.last.request.headers["authorization"] == "Bearer generic-openai-key" + + def test_speech_keeps_an_explicitly_passed_api_key_over_the_provider_scoped_one( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ):