From 4918d3eccfc76ae046c190f0e822ec2c153fc5a6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:17:26 +0000 Subject: [PATCH] fix(proxy): initialize the secret manager before resolving os.environ config refs The split gateway/backend images uvicorn the app directly, so proxy_cli.py never runs and the secret manager was still unset when get_config() replaced every os.environ/... value; keys that only live in the secret manager resolved to None Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 31 +++++++ .../proxy/proxy_server/test_proxy_config.py | 90 +++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4486cd7de59..3b1739c1d38 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4207,12 +4207,43 @@ class ProxyConfig: printed_yaml = copy.deepcopy(config) printed_yaml.pop("environment_variables", None) + self._ensure_secret_manager_initialized(config=config, config_file_path=config_file_path) + config = self._check_for_os_environ_vars(config=config) self.update_config_state(config=config) return config + def _ensure_secret_manager_initialized(self, config: Mapping[str, object], config_file_path: str | None) -> None: + """ + Initialize the secret manager, if configured, before `os.environ/...` config references are resolved. + + `proxy_cli.py` does this for the monolithic image; entrypoints that uvicorn the app directly + (the split gateway/backend images) never run it, so secret-manager-only keys resolve to None. + """ + if litellm.secret_manager_client is not None: + return + + general_settings = config.get("general_settings") + if not isinstance(general_settings, dict): + return + + key_management_system = general_settings.get("key_management_system") + if not isinstance(key_management_system, str): + return + + key_management_settings = general_settings.get("key_management_settings") + if isinstance(key_management_settings, dict): + litellm._key_management_settings = KeyManagementSettings( + **self._check_for_os_environ_vars(config=copy.deepcopy(key_management_settings)) + ) + + self.initialize_secret_manager( + key_management_system=key_management_system, + config_file_path=config_file_path, + ) + def update_config_state(self, config: dict): self.config = config diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 8d1d8185e4d..fcc05dd47dd 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -26,6 +26,8 @@ from litellm.proxy.proxy_server import ( resolve_routing_plugins, ) +from litellm.types.secret_managers.main import KeyManagementSettings + from .conftest import normalize # --------------------------------------------------------------------------- @@ -687,6 +689,94 @@ async def test_ProxyConfig_get_config_loads_from_file(tmp_path, monkeypatch): } +CUSTOM_SECRET_MANAGER_MODULE = """ +from typing import Optional, Union + +import httpx + +from litellm.integrations.custom_secret_manager import CustomSecretManager + + +class InMemorySecretManager(CustomSecretManager): + def __init__(self): + super().__init__(secret_manager_name="in_memory") + self.secrets = { + "LITELLM_MASTER_KEY": "sk-from-secret-manager", + "MY_AZURE_KEY": "azure-key-from-secret-manager", + } + + async def async_read_secret( + self, + secret_name: str, + optional_params: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + ) -> Optional[str]: + return self.secrets.get(secret_name) + + def sync_read_secret( + self, + secret_name: str, + optional_params: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + ) -> Optional[str]: + return self.secrets.get(secret_name) + + async def async_write_secret(self, *args, **kwargs): + raise NotImplementedError + + async def async_delete_secret(self, *args, **kwargs): + raise NotImplementedError + + async def async_rotate_secret(self, *args, **kwargs): + raise NotImplementedError +""" + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_resolves_secret_manager_keys_without_the_cli(tmp_path, monkeypatch): + """get_config() must initialize the configured secret manager before resolving `os.environ/...`. + + Entrypoints that uvicorn the app directly (the split gateway/backend images) never run + proxy_cli.py, so without this the keys that only live in the secret manager resolve to None. + """ + (tmp_path / "my_secret_manager.py").write_text(CUSTOM_SECRET_MANAGER_MODULE) + config_file = tmp_path / "config.yaml" + config_file.write_text( + "model_list:\n" + " - model_name: gpt-4.1\n" + " litellm_params:\n" + " model: azure/gpt-4.1\n" + " api_key: os.environ/MY_AZURE_KEY\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " key_management_system: custom\n" + " key_management_settings:\n" + " access_mode: read_only\n" + " custom_secret_manager: my_secret_manager.InMemorySecretManager\n" + " hosted_keys:\n" + " - LITELLM_MASTER_KEY\n" + " - MY_AZURE_KEY\n" + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + monkeypatch.delenv("LITELLM_MASTER_KEY", raising=False) + monkeypatch.delenv("MY_AZURE_KEY", raising=False) + monkeypatch.setattr(litellm, "secret_manager_client", None) + monkeypatch.setattr(litellm, "_key_management_system", None) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings()) + + cfg = await ProxyConfig().get_config(config_file_path=str(config_file)) + + assert { + "master_key": cfg["general_settings"]["master_key"], + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + } == { + "master_key": "sk-from-secret-manager", + "api_key": "azure-key-from-secret-manager", + } + + @pytest.mark.asyncio async def test_ProxyConfig_get_config_missing_file_raises(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)