From eab4a6ba2606f03eeae0e69c2ab9c1df8aa5862f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 6 Jul 2026 19:32:48 +0000 Subject: [PATCH] feat(proxy): warn on startup when running in orchestrator without Redis At startup, detect container orchestrator env vars (KUBERNETES_SERVICE_HOST, ECS_CONTAINER_METADATA_URI, NOMAD_ALLOC_ID, FLY_APP_NAME) and emit a prominent warning if Redis is not configured. Lists the features that break silently across replicas without Redis: rate limiting, spend tracking, SSO login, health-check coordination, cron-job deduplication, cooldown sharing. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 64 +++++++++++++ .../proxy/proxy_server/test_lifecycle.py | 96 +++++++++++++++---- 2 files changed, 144 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1474c15e778..cdcca13dbcb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -977,6 +977,11 @@ async def proxy_startup_event(app: FastAPI): redis_usage_cache=transaction_buffer_redis_cache, ) + ## Warn if running in an orchestrator without Redis ## + ProxyStartupEvent._warn_no_redis_multi_instance( + redis_usage_cache=redis_usage_cache, + ) + ## SEMANTIC TOOL FILTER ## # Read litellm_settings from config for semantic filter initialization try: @@ -7238,6 +7243,65 @@ class ProxyStartupEvent: proxy_logging_obj.startup_event(llm_router=llm_router, redis_usage_cache=redis_usage_cache) + @staticmethod + def _warn_no_redis_multi_instance( + redis_usage_cache: Optional[RedisCache], + ) -> None: + """ + Emit a startup warning when the proxy is running inside a container + orchestrator (Kubernetes, ECS, etc.) without Redis configured. + + Without Redis, features like rate limiting, spend tracking, SSO + login, health-check coordination, and cron-job deduplication only + work within a single process. Multiple replicas will each maintain + independent in-memory state, leading to silent correctness issues. + """ + if redis_usage_cache is not None: + return + + _orchestrator_env_vars = ( + "KUBERNETES_SERVICE_HOST", + "ECS_CONTAINER_METADATA_URI", + "ECS_CONTAINER_METADATA_URI_V4", + "NOMAD_ALLOC_ID", + "FLY_APP_NAME", + ) + detected = next( + (var for var in _orchestrator_env_vars if os.environ.get(var)), + None, + ) + if detected is None: + return + + verbose_proxy_logger.warning( + "\n" + "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" + "WARNING: Redis is not configured but this instance appears\n" + "to be running inside a container orchestrator (%s is set).\n" + "If you are running multiple replicas, the following features\n" + "will NOT work correctly without Redis:\n" + " - rate limiting (TPM/RPM limits enforced per-instance, not globally)\n" + " - spend tracking (each replica tracks independently)\n" + " - SSO / UI login (auth codes not shared across replicas)\n" + " - health-check coordination (duplicate checks across replicas)\n" + " - cron-job deduplication (every replica runs every job)\n" + " - cooldown sharing (model cooldowns not propagated)\n" + "\n" + "To fix this, add a Redis cache in your proxy config:\n" + "\n" + " litellm_settings:\n" + " cache: true\n" + " cache_params:\n" + " type: redis\n" + " host: os.environ/REDIS_HOST\n" + " port: os.environ/REDIS_PORT\n" + " password: os.environ/REDIS_PASSWORD\n" + "\n" + "or set REDIS_HOST / REDIS_PORT / REDIS_PASSWORD env vars.\n" + "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", + detected, + ) + @staticmethod def _validate_redis_transaction_buffer_config( general_settings: dict, diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index a3f5049ef1d..db4bbe69a7a 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -31,6 +31,7 @@ from typing_extensions import TypedDict import litellm.proxy.proxy_server as ps from litellm.proxy.proxy_server import ( + ProxyStartupEvent, _initialize_shared_aiohttp_session, _resolve_pydantic_type, _resolve_typed_dict_type, @@ -335,9 +336,7 @@ def test__redact_worker_config_for_logging_masks_nested_secret_fields(): "database_url": nested_db_url, "database_extra_connection_params": {"password": nested_extra_pw}, "alert_to_webhook_url": {"budget_alerts": nested_webhook}, - "pass_through_endpoints": [ - {"path": "/up", "headers": {"Authorization": nested_bearer}} - ], + "pass_through_endpoints": [{"path": "/up", "headers": {"Authorization": nested_bearer}}], } } } @@ -389,16 +388,13 @@ def test_load_from_azure_key_vault_disabled_no_side_effect(monkeypatch): import litellm sentinel_secret_mgr = object() - monkeypatch.setattr( - litellm, "secret_manager_client", sentinel_secret_mgr, raising=False - ) + monkeypatch.setattr(litellm, "secret_manager_client", sentinel_secret_mgr, raising=False) result = load_from_azure_key_vault(use_azure_key_vault=False) observed = { "return_value": result, - "secret_manager_unchanged": litellm.secret_manager_client - is sentinel_secret_mgr, + "secret_manager_unchanged": litellm.secret_manager_client is sentinel_secret_mgr, "called_with": False, } assert normalize(observed) == { @@ -552,9 +548,7 @@ def test_get_litellm_model_info_uses_base_model_for_lookup(monkeypatch): observed = { "called_arg": ( - fake_get.call_args.args[0] - if fake_get.call_args.args - else fake_get.call_args.kwargs.get("model") + fake_get.call_args.args[0] if fake_get.call_args.args else fake_get.call_args.kwargs.get("model") ), "returned_max_tokens": result.get("max_tokens"), "returned_cost": result.get("input_cost_per_token"), @@ -601,9 +595,7 @@ def test_run_ollama_serve_invokes_subprocess_popen(monkeypatch): def test_run_ollama_serve_popen_failure_is_swallowed(monkeypatch): """Popen raising OSError must NOT propagate — function logs and returns.""" - monkeypatch.setattr( - ps.subprocess, "Popen", MagicMock(side_effect=OSError("no ollama binary")) - ) + monkeypatch.setattr(ps.subprocess, "Popen", MagicMock(side_effect=OSError("no ollama binary"))) result = run_ollama_serve() assert result is None @@ -623,8 +615,7 @@ async def test_proxy_startup_event_is_async_context_manager_with_expected_signat observed = { "param_count": len(sig.parameters), "has_app_param": "app" in sig.parameters, - "wrapped_is_async": inspect.iscoroutinefunction(wrapped) - or inspect.isasyncgenfunction(wrapped), + "wrapped_is_async": inspect.iscoroutinefunction(wrapped) or inspect.isasyncgenfunction(wrapped), "has_asynccontextmanager_wrapper": wrapped is not None, } assert normalize(observed) == { @@ -668,3 +659,76 @@ def test_otel_global_provider_published_after_callback_init(): "preset logger will not exist yet and a second generic logger will own " "the global provider, orphaning gen-ai spans" ) + + +# --------------------------------------------------------------------------- +# _warn_no_redis_multi_instance +# --------------------------------------------------------------------------- + +_ORCHESTRATOR_ENV_VARS = ( + "KUBERNETES_SERVICE_HOST", + "ECS_CONTAINER_METADATA_URI", + "ECS_CONTAINER_METADATA_URI_V4", + "NOMAD_ALLOC_ID", + "FLY_APP_NAME", +) + + +@pytest.mark.parametrize("env_var", _ORCHESTRATOR_ENV_VARS) +def test_warn_no_redis_multi_instance_warns_for_orchestrator(monkeypatch, caplog, env_var): + """When Redis is None and an orchestrator env var is set, a warning must + be emitted listing the affected features.""" + for var in _ORCHESTRATOR_ENV_VARS: + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv(env_var, "10.0.0.1") + + import logging + + with caplog.at_level(logging.WARNING): + ProxyStartupEvent._warn_no_redis_multi_instance(redis_usage_cache=None) + + combined = "\n".join(caplog.messages) + assert "Redis is not configured" in combined + assert env_var in combined + assert "rate limiting" in combined + + +def test_warn_no_redis_multi_instance_silent_when_redis_configured(monkeypatch, caplog): + """When Redis IS configured, no warning is emitted even inside k8s.""" + monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1") + + import logging + + with caplog.at_level(logging.WARNING): + ProxyStartupEvent._warn_no_redis_multi_instance( + redis_usage_cache=MagicMock(), + ) + + assert not any("Redis is not configured" in m for m in caplog.messages) + + +def test_warn_no_redis_multi_instance_silent_outside_orchestrator(monkeypatch, caplog): + """When no orchestrator env var is set, no warning is emitted even without + Redis -- the user may be running a single instance on bare metal.""" + for var in _ORCHESTRATOR_ENV_VARS: + monkeypatch.delenv(var, raising=False) + + import logging + + with caplog.at_level(logging.WARNING): + ProxyStartupEvent._warn_no_redis_multi_instance(redis_usage_cache=None) + + assert not any("Redis is not configured" in m for m in caplog.messages) + + +def test_warn_no_redis_multi_instance_called_during_startup(): + """The warning call must appear in proxy_startup_event source, after + Redis setup and the transaction buffer validation.""" + wrapped = getattr(proxy_startup_event, "__wrapped__", proxy_startup_event) + source = inspect.getsource(wrapped) + redis_buffer_pos = source.find("_validate_redis_transaction_buffer_config(") + warn_pos = source.find("_warn_no_redis_multi_instance(") + assert warn_pos != -1, "_warn_no_redis_multi_instance call missing from proxy_startup_event" + assert redis_buffer_pos < warn_pos, ( + "_warn_no_redis_multi_instance must run after _validate_redis_transaction_buffer_config" + )