diff --git a/litellm/_logging.py b/litellm/_logging.py index 34ae62154a8..a10d87c9792 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -629,6 +629,11 @@ def _terminal_colors(stdout: TextIO | None, stderr: TextIO | None) -> bool: return _color_enabled(stdout) and _color_enabled(stderr) +def terminal_text(message: str) -> str: + """The message with its ANSI colors kept only on a color terminal, by the rule the plain log format uses.""" + return message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message) + + def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: """The plain-text log format, colorized only when both streams are an interactive terminal.""" return _COLOR_LOG_FORMAT if _terminal_colors(stdout, stderr) else _PLAIN_LOG_FORMAT @@ -950,8 +955,7 @@ def print_startup_warning(message: str) -> None: def _write_startup_message(message: str, caller: FrameType | None) -> None: if not litellm.json_logs: - printed: Final = message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message) - print(printed, flush=True) # noqa: T201 # plain mode: this is the operator-terminal print it replaces + print(terminal_text(message), flush=True) # noqa: T201 # plain mode: this is the operator-terminal print it replaces return text: Final = _ANSI_ESCAPE.sub("", message).strip("\n") if not text: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index f376c8c1098..9fb9404bcdd 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -20,7 +20,7 @@ from pydantic import ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict import litellm -from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning +from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning, terminal_text from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY from litellm.proxy.db.pgbouncer import ( PgBouncerError, @@ -235,19 +235,23 @@ class _ConfigStartupOutputSettings(TypedDict, total=False): _CONFIG_STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(_ConfigStartupOutputSettings) +def _apply_config_suppress_startup_output(litellm_settings: StartupOutputSettings | None) -> None: + if litellm_settings is None or litellm_settings.get("suppress_startup_output") is not True: + return + litellm.suppress_startup_output = True + os.environ["LITELLM_SUPPRESS_STARTUP_OUTPUT"] = "true" + + def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | None) -> None: """Turn on the config's json_logs and suppress_startup_output before any worker starts. Workers that uvicorn or granian spawn import litellm afresh and read both from the env at import, so they log JSON and stay quiet from their first line instead of from when load_config reaches the key.""" - if litellm_settings is None: + _apply_config_suppress_startup_output(litellm_settings) + if litellm_settings is None or litellm_settings.get("json_logs") is not True: return - if litellm_settings.get("json_logs") is True: - litellm.json_logs = True - os.environ["JSON_LOGS"] = "true" - litellm.turn_on_json() - if litellm_settings.get("suppress_startup_output") is True: - litellm.suppress_startup_output = True - os.environ["LITELLM_SUPPRESS_STARTUP_OUTPUT"] = "true" + litellm.json_logs = True + os.environ["JSON_LOGS"] = "true" + litellm.turn_on_json() class ProxyInitializationHelpers: @@ -1224,14 +1228,9 @@ def run_server( if validate_config is True: ProxyInitializationHelpers.run_config_validation(config) return - env_config_file: Final = os.environ.get("CONFIG_FILE_PATH") if config is None else None - env_config_file_path: Final = ( - env_config_file if env_config_file and ProxyConfig().is_yaml(config_file_path=env_config_file) else None - ) held_prisma_notice: Final = ( _print_or_hold_enforce_prisma_migration_check_notice( - reads_config_file=config is not None or env_config_file_path is not None, - returns_before_config=health is True or test is True, + reads_config_file=config is not None, returns_before_config=health is True or test is True ) if enforce_prisma_migration_check else None @@ -1326,6 +1325,19 @@ def run_server( os.environ[k] = v litellm_settings = None + env_config_file: Final = os.environ.get("CONFIG_FILE_PATH") + if ( + env_config_file + and os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is None + and ProxyConfig().is_yaml(config_file_path=env_config_file) + ): + import asyncio + + _apply_config_suppress_startup_output( + _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python( + asyncio.run(ProxyConfig().read_config_file(config_file_path=env_config_file)) + ).get("litellm_settings") + ) if config is not None: """ Allow user to pass in db url via config @@ -1409,16 +1421,6 @@ def run_server( os.chdir(original_dir) if database_url is not None and isinstance(database_url, str): os.environ["DATABASE_URL"] = database_url - elif env_config_file_path is not None: - import asyncio - - _apply_config_logging_settings( - _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python( - asyncio.run(ProxyConfig().get_config(config_file_path=env_config_file_path)) - ).get("litellm_settings") - ) - if held_prisma_notice is not None: - held_prisma_notice() # Handle database URL construction when no config file is used if config is None and os.getenv("DATABASE_URL") is None: @@ -1453,7 +1455,9 @@ def run_server( _bad_scheme = unsupported_db_scheme(_candidate_url) if _bad_scheme is not None: print( - f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m" + ), file=sys.stderr, flush=True, ) @@ -1586,22 +1590,26 @@ def run_server( # issues, and any `prisma db push` against a # partitioned LiteLLM_SpendLogs. print( - f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m", + terminal_text(f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m"), file=sys.stderr, flush=True, ) sys.exit(2) if not setup_ok: print( - "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " - "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " + "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" + ) ) sys.exit(1) else: print( - "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " - "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " - "or run a shipped LiteLLM image.\033[0m" + terminal_text( + "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable " + "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` " + "or run a shipped LiteLLM image.\033[0m" + ) ) sys.exit(1) pgbouncer_settings: Final = PgBouncerSettings() @@ -1612,8 +1620,10 @@ def run_server( ) if isinstance(pooled_database_url, PgBouncerError): print( - f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " - f"could not start: {pooled_database_url.reason}\033[0m", + terminal_text( + f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer " + f"could not start: {pooled_database_url.reason}\033[0m" + ), file=sys.stderr, flush=True, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 980866ccaab..abf9aafad9c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5755,6 +5755,12 @@ class ProxyConfig: # verbose_proxy_logger.debug(f"loaded config={json.dumps(config, indent=4)}") return config + async def read_config_file(self, config_file_path: str) -> Mapping[str, object]: + """The config file with its includes merged, without the bucket, database and secret steps of get_config.""" + return await self._process_includes( + config=self._load_yaml_file(config_file_path), config_file_path=os.path.abspath(config_file_path) + ) + async def _process_includes(self, config: dict, config_file_path: str) -> dict: """ Process includes by appending their contents to the main config diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 03f57fcbc86..4876c075314 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -2647,6 +2647,7 @@ class TestRunServerDbSetup: assert exit_info.value.code == 1 assert "a database URL is set but the prisma CLI is neither on PATH nor importable" in out assert "pip install 'litellm[extra_proxy]'" in out + assert "\x1b[" not in out else: assert "prisma CLI" not in out assert "Setup complete" in out @@ -2725,6 +2726,7 @@ class TestRunServerDbSetup: mock_setup_database, mock_atexit_register, mock_subprocess_run, + capsys, ): """A sqlite DATABASE_URL must exit immediately, before any prisma call, instead of stalling on a migration against the postgresql-only schema.""" @@ -2763,6 +2765,9 @@ class TestRunServerDbSetup: ) assert exc_info.value.code == 1 mock_setup_database.assert_not_called() + err: Final = capsys.readouterr().err + assert "sqlite" in err + assert "\x1b[" not in err @patch("subprocess.run") @patch("atexit.register") @@ -3956,6 +3961,50 @@ class TestValidateConfigFlag: mock_popen.assert_not_called() +COUNTING_SECRET_MANAGER_MODULE: Final = """ +import os + +from litellm.integrations.custom_secret_manager import CustomSecretManager + + +class CountingSecretManager(CustomSecretManager): + def __init__(self): + super().__init__() + self._log("constructed") + + @staticmethod + def _log(event): + with open(os.environ["SECRET_READ_LOG"], "a") as f: + f.write(event + "\\n") + + def sync_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + self._log(secret_name) + return "from-secret-manager" + + async def async_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + self._log(secret_name) + return "from-secret-manager" +""" + +SECRET_MANAGER_BACKED_CONFIG: Final = """ +model_list: + - model_name: my-model + litellm_params: + model: openai/gpt-5.4-mini + api_key: os.environ/MY_PROVIDER_KEY + +litellm_settings: + suppress_startup_output: true + +general_settings: + key_management_system: custom + key_management_settings: + custom_secret_manager: counting_secret_manager.CountingSecretManager + hosted_keys: + - MY_PROVIDER_KEY +""" + + class TestJsonLogsConfigReachesSpawnedWorkers: """`litellm_settings.json_logs: true` turns JSON on inside the CLI process, but the workers that uvicorn or granian spawn are fresh interpreters that pick plain or JSON from the JSON_LOGS @@ -3963,36 +4012,51 @@ class TestJsonLogsConfigReachesSpawnedWorkers: its own load_config reaches the key. `suppress_startup_output` has the same problem with the banner each worker prints before its load_config.""" + @staticmethod + def _write_config(path: Path, litellm_settings: dict[str, object]) -> Path: + path.write_text(yaml.dump({"model_list": [], "litellm_settings": litellm_settings})) + return path + @staticmethod def _env_after_boot( - config_path: Path, - litellm_settings: dict[str, object], - name: str = "JSON_LOGS", + name: str, + config: Path | None = None, config_file_path_env: Path | None = None, + extra_env: dict[str, str] | None = None, + proxy_config_class: object = None, ) -> str | None: from litellm.proxy.proxy_server import ProxyConfig - config_path.write_text(yaml.dump({"model_list": [], "litellm_settings": litellm_settings})) + def load(config_file_path: str) -> object: + return yaml.safe_load(Path(config_file_path).read_text()) + mock_proxy_config: Final = MagicMock() mock_proxy_config.return_value.is_yaml = ProxyConfig().is_yaml - mock_proxy_config.return_value.get_config = AsyncMock( - side_effect=lambda config_file_path: yaml.safe_load(Path(config_file_path).read_text()) - ) + mock_proxy_config.return_value.get_config = AsyncMock(side_effect=load) + mock_proxy_config.return_value.read_config_file = AsyncMock(side_effect=load) mock_proxy_module: Final = MagicMock( app=MagicMock(), - ProxyConfig=mock_proxy_config, + ProxyConfig=proxy_config_class or mock_proxy_config, KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) clean_env: Final = { k: v for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL", "JSON_LOGS", "LITELLM_SUPPRESS_STARTUP_OUTPUT", "CONFIG_FILE_PATH") + if k + not in ( + "DATABASE_URL", + "DIRECT_URL", + "JSON_LOGS", + "LITELLM_SUPPRESS_STARTUP_OUTPUT", + "CONFIG_FILE_PATH", + "LITELLM_CONFIG_BUCKET_NAME", + ) } config_env: Final = {} if config_file_path_env is None else {"CONFIG_FILE_PATH": str(config_file_path_env)} - config_args: Final = ["--config", str(config_path)] if config_file_path_env is None else [] + config_args: Final = [] if config is None else ["--config", str(config)] with ( - patch.dict(os.environ, {**clean_env, **config_env}, clear=True), + patch.dict(os.environ, {**clean_env, **config_env, **(extra_env or {})}, clear=True), patch.dict( "sys.modules", {"proxy_server": mock_proxy_module, "litellm.proxy.proxy_server": mock_proxy_module}, @@ -4005,39 +4069,111 @@ class TestJsonLogsConfigReachesSpawnedWorkers: def test_json_logs_true_exports_json_logs_to_spawned_workers(self, tmp_path: Path, monkeypatch) -> None: monkeypatch.setattr(litellm, "json_logs", False) # run_server flips it on; undone at teardown - assert self._env_after_boot(tmp_path / "config.yaml", {"json_logs": True}) == "true" + config: Final = self._write_config(tmp_path / "config.yaml", {"json_logs": True}) + assert self._env_after_boot("JSON_LOGS", config=config) == "true" def test_plain_logs_config_leaves_json_logs_unset(self, tmp_path: Path, monkeypatch) -> None: monkeypatch.setattr(litellm, "json_logs", False) - assert self._env_after_boot(tmp_path / "config.yaml", {}) is None + assert self._env_after_boot("JSON_LOGS", config=self._write_config(tmp_path / "config.yaml", {})) is None @pytest.mark.parametrize(("setting", "exported"), [(True, "true"), ("true", None), (False, None)]) def test_only_a_boolean_suppress_startup_output_reaches_spawned_workers( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, setting: object, exported: str | None ) -> None: monkeypatch.setattr(litellm, "suppress_startup_output", False) - settings: Final = {"suppress_startup_output": setting} - assert self._env_after_boot(tmp_path / "config.yaml", settings, "LITELLM_SUPPRESS_STARTUP_OUTPUT") == exported + config: Final = self._write_config(tmp_path / "config.yaml", {"suppress_startup_output": setting}) + assert self._env_after_boot("LITELLM_SUPPRESS_STARTUP_OUTPUT", config=config) == exported - @pytest.mark.parametrize("name", ["JSON_LOGS", "LITELLM_SUPPRESS_STARTUP_OUTPUT"]) - def test_a_config_named_by_CONFIG_FILE_PATH_reaches_spawned_workers_too( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str + @pytest.mark.parametrize(("name", "exported"), [("LITELLM_SUPPRESS_STARTUP_OUTPUT", "true"), ("JSON_LOGS", None)]) + def test_a_config_named_by_CONFIG_FILE_PATH_passes_only_suppress_startup_output_to_spawned_workers( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str, exported: str | None ) -> None: + """Its json_logs stays the workers' own to read, as before, so uvicorn's lines keep their plain format.""" monkeypatch.setattr(litellm, "json_logs", False) monkeypatch.setattr(litellm, "suppress_startup_output", False) settings: Final[dict[str, object]] = {"json_logs": True, "suppress_startup_output": True} - config_path: Final = tmp_path / "config.yaml" - assert self._env_after_boot(config_path, settings, name, config_file_path_env=config_path) == "true" + config_file: Final = self._write_config(tmp_path / "config.yaml", settings) + assert self._env_after_boot(name, config_file_path_env=config_file) == exported - @pytest.mark.parametrize(("config_file", "env_file"), [("config.yaml", "missing.yaml"), ("config.json", "config.json")]) - def test_a_CONFIG_FILE_PATH_workers_skip_is_skipped_by_the_cli_too( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config_file: str, env_file: str + def test_a_CONFIG_FILE_PATH_quiet_config_wins_over_a_cli_config_too( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Workers load CONFIG_FILE_PATH only when it names a YAML file, and a boot with any other value starts, - so the CLI reads nothing from it either instead of failing the boot.""" - monkeypatch.setattr(litellm, "json_logs", False) - settings: Final[dict[str, object]] = {"json_logs": True} - assert self._env_after_boot(tmp_path / config_file, settings, config_file_path_env=tmp_path / env_file) is None + """Workers load CONFIG_FILE_PATH ahead of --config, so its key quiets the CLI when both are set.""" + monkeypatch.setattr(litellm, "suppress_startup_output", False) + cli_config: Final = self._write_config(tmp_path / "cli.yaml", {}) + env_config: Final = self._write_config(tmp_path / "env.yaml", {"suppress_startup_output": True}) + exported: Final = self._env_after_boot( + "LITELLM_SUPPRESS_STARTUP_OUTPUT", config=cli_config, config_file_path_env=env_config + ) + assert exported == "true" + + @pytest.mark.parametrize( + ("config_file", "env_file", "extra_env"), + [ + ("config.yaml", "missing.yaml", {}), + ("config.json", "config.json", {}), + ("config.yaml", "config.yaml", {"LITELLM_CONFIG_BUCKET_NAME": "litellm-configs"}), + ], + ids=["missing-file", "not-yaml", "bucket-config"], + ) + def test_a_CONFIG_FILE_PATH_workers_skip_is_skipped_by_the_cli_too( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + config_file: str, + env_file: str, + extra_env: dict[str, str], + ) -> None: + """Workers load CONFIG_FILE_PATH only when it names a YAML file and no config bucket is set, and a boot + with any other value starts, so the CLI reads nothing from it either instead of failing the boot.""" + monkeypatch.setattr(litellm, "suppress_startup_output", False) + self._write_config(tmp_path / config_file, {"suppress_startup_output": True}) + exported: Final = self._env_after_boot( + "LITELLM_SUPPRESS_STARTUP_OUTPUT", config_file_path_env=tmp_path / env_file, extra_env=extra_env + ) + assert exported is None + + def test_a_CONFIG_FILE_PATH_boot_reads_no_secret_before_the_workers_do( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The CLI takes only the suppress key from CONFIG_FILE_PATH, so a config backed by a secret manager is + read from it once per worker, as before, not once more by the CLI.""" + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setattr(litellm, "suppress_startup_output", False) + monkeypatch.setattr(litellm, "secret_manager_client", None) + monkeypatch.setattr(litellm, "_key_management_system", None) + monkeypatch.setattr(litellm, "_key_management_settings", litellm._key_management_settings) + reads: Final = tmp_path / "secret_reads.log" + (tmp_path / "counting_secret_manager.py").write_text(COUNTING_SECRET_MANAGER_MODULE) + config_file: Final = tmp_path / "config.yaml" + config_file.write_text(SECRET_MANAGER_BACKED_CONFIG) + + exported: Final = self._env_after_boot( + "LITELLM_SUPPRESS_STARTUP_OUTPUT", + config_file_path_env=config_file, + extra_env={"SECRET_READ_LOG": str(reads)}, + proxy_config_class=ProxyConfig, + ) + + assert exported == "true" + assert not reads.exists(), reads.read_text() + + def test_a_CONFIG_FILE_PATH_key_set_in_an_included_file_reaches_spawned_workers( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setattr(litellm, "suppress_startup_output", False) + self._write_config(tmp_path / "quiet.yaml", {"suppress_startup_output": True}) + config_file: Final = tmp_path / "config.yaml" + config_file.write_text(yaml.dump({"include": ["quiet.yaml"], "model_list": []})) + + exported: Final = self._env_after_boot( + "LITELLM_SUPPRESS_STARTUP_OUTPUT", config_file_path_env=config_file, proxy_config_class=ProxyConfig + ) + + assert exported == "true" class TestEnforcePrismaMigrationCheckNoticeAroundTheConfigRead: