scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

feat(proxy): warn on startup when running in orchestrator without Redis

Browse source 
At startup, detect container orchestrator env vars (KUBERNETES_SERVICE_HOST,
ECS_CONTAINER_METADATA_URI, NOMAD_ALLOC_ID, FLY_APP_NAME) and emit a
prominent warning if Redis is not configured. Lists the features that break
silently across replicas without Redis: rate limiting, spend tracking, SSO
login, health-check coordination, cron-job deduplication, cooldown sharing.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia 2026-07-06 19:32:48 +00:00
parent 29035c4a99
commit eab4a6ba26
2 changed files with 144 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
litellm/proxy/proxy_server.py
View file

@ -977,6 +977,11 @@ async def proxy_startup_event(app: FastAPI):
redis_usage_cache=transaction_buffer_redis_cache,
)
## Warn if running in an orchestrator without Redis ##
ProxyStartupEvent._warn_no_redis_multi_instance(
redis_usage_cache=redis_usage_cache,
)
## SEMANTIC TOOL FILTER ##
# Read litellm_settings from config for semantic filter initialization
try:
@ -7238,6 +7243,65 @@ class ProxyStartupEvent:
proxy_logging_obj.startup_event(llm_router=llm_router, redis_usage_cache=redis_usage_cache)
@staticmethod
def _warn_no_redis_multi_instance(
redis_usage_cache: Optional[RedisCache],
) -> None:
"""
Emit a startup warning when the proxy is running inside a container
orchestrator (Kubernetes, ECS, etc.) without Redis configured.
Without Redis, features like rate limiting, spend tracking, SSO
login, health-check coordination, and cron-job deduplication only
work within a single process. Multiple replicas will each maintain
independent in-memory state, leading to silent correctness issues.
"""
if redis_usage_cache is not None:
return
_orchestrator_env_vars = (
"KUBERNETES_SERVICE_HOST",
"ECS_CONTAINER_METADATA_URI",
"ECS_CONTAINER_METADATA_URI_V4",
"NOMAD_ALLOC_ID",
"FLY_APP_NAME",
)
detected = next(
(var for var in _orchestrator_env_vars if os.environ.get(var)),
None,
)
if detected is None:
return
verbose_proxy_logger.warning(
"\n"
"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
"WARNING: Redis is not configured but this instance appears\n"
"to be running inside a container orchestrator (%s is set).\n"
"If you are running multiple replicas, the following features\n"
"will NOT work correctly without Redis:\n"
" - rate limiting (TPM/RPM limits enforced per-instance, not globally)\n"
" - spend tracking (each replica tracks independently)\n"
" - SSO / UI login (auth codes not shared across replicas)\n"
" - health-check coordination (duplicate checks across replicas)\n"
" - cron-job deduplication (every replica runs every job)\n"
" - cooldown sharing (model cooldowns not propagated)\n"
"\n"
"To fix this, add a Redis cache in your proxy config:\n"
"\n"
" litellm_settings:\n"
" cache: true\n"
" cache_params:\n"
" type: redis\n"
" host: os.environ/REDIS_HOST\n"
" port: os.environ/REDIS_PORT\n"
" password: os.environ/REDIS_PASSWORD\n"
"\n"
"or set REDIS_HOST / REDIS_PORT / REDIS_PASSWORD env vars.\n"
"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
detected,
)
@staticmethod
def _validate_redis_transaction_buffer_config(
general_settings: dict,

96
tests/test_litellm/proxy/proxy_server/test_lifecycle.py
View file

@ -31,6 +31,7 @@ from typing_extensions import TypedDict
import litellm.proxy.proxy_server as ps
from litellm.proxy.proxy_server import (
ProxyStartupEvent,
_initialize_shared_aiohttp_session,
_resolve_pydantic_type,
_resolve_typed_dict_type,
@ -335,9 +336,7 @@ def test__redact_worker_config_for_logging_masks_nested_secret_fields():
"database_url": nested_db_url,
"database_extra_connection_params": {"password": nested_extra_pw},
"alert_to_webhook_url": {"budget_alerts": nested_webhook},
"pass_through_endpoints": [
{"path": "/up", "headers": {"Authorization": nested_bearer}}
],
"pass_through_endpoints": [{"path": "/up", "headers": {"Authorization": nested_bearer}}],
}
}
}
@ -389,16 +388,13 @@ def test_load_from_azure_key_vault_disabled_no_side_effect(monkeypatch):
import litellm
sentinel_secret_mgr = object()
monkeypatch.setattr(
litellm, "secret_manager_client", sentinel_secret_mgr, raising=False
)
monkeypatch.setattr(litellm, "secret_manager_client", sentinel_secret_mgr, raising=False)
result = load_from_azure_key_vault(use_azure_key_vault=False)
observed = {
"return_value": result,
"secret_manager_unchanged": litellm.secret_manager_client
is sentinel_secret_mgr,
"secret_manager_unchanged": litellm.secret_manager_client is sentinel_secret_mgr,
"called_with": False,
}
assert normalize(observed) == {
@ -552,9 +548,7 @@ def test_get_litellm_model_info_uses_base_model_for_lookup(monkeypatch):
observed = {
"called_arg": (
fake_get.call_args.args[0]
if fake_get.call_args.args
else fake_get.call_args.kwargs.get("model")
fake_get.call_args.args[0] if fake_get.call_args.args else fake_get.call_args.kwargs.get("model")
),
"returned_max_tokens": result.get("max_tokens"),
"returned_cost": result.get("input_cost_per_token"),
@ -601,9 +595,7 @@ def test_run_ollama_serve_invokes_subprocess_popen(monkeypatch):
def test_run_ollama_serve_popen_failure_is_swallowed(monkeypatch):
"""Popen raising OSError must NOT propagate — function logs and returns."""
monkeypatch.setattr(
ps.subprocess, "Popen", MagicMock(side_effect=OSError("no ollama binary"))
)
monkeypatch.setattr(ps.subprocess, "Popen", MagicMock(side_effect=OSError("no ollama binary")))
result = run_ollama_serve()
assert result is None
@ -623,8 +615,7 @@ async def test_proxy_startup_event_is_async_context_manager_with_expected_signat
observed = {
"param_count": len(sig.parameters),
"has_app_param": "app" in sig.parameters,
"wrapped_is_async": inspect.iscoroutinefunction(wrapped)
or inspect.isasyncgenfunction(wrapped),
"wrapped_is_async": inspect.iscoroutinefunction(wrapped) or inspect.isasyncgenfunction(wrapped),
"has_asynccontextmanager_wrapper": wrapped is not None,
}
assert normalize(observed) == {
@ -668,3 +659,76 @@ def test_otel_global_provider_published_after_callback_init():
"preset logger will not exist yet and a second generic logger will own "
"the global provider, orphaning gen-ai spans"
)
# ---------------------------------------------------------------------------
# _warn_no_redis_multi_instance
# ---------------------------------------------------------------------------
_ORCHESTRATOR_ENV_VARS = (
"KUBERNETES_SERVICE_HOST",
"ECS_CONTAINER_METADATA_URI",
"ECS_CONTAINER_METADATA_URI_V4",
"NOMAD_ALLOC_ID",
"FLY_APP_NAME",
)
@pytest.mark.parametrize("env_var", _ORCHESTRATOR_ENV_VARS)
def test_warn_no_redis_multi_instance_warns_for_orchestrator(monkeypatch, caplog, env_var):
"""When Redis is None and an orchestrator env var is set, a warning must
be emitted listing the affected features."""
for var in _ORCHESTRATOR_ENV_VARS:
monkeypatch.delenv(var, raising=False)
monkeypatch.setenv(env_var, "10.0.0.1")
import logging
with caplog.at_level(logging.WARNING):
ProxyStartupEvent._warn_no_redis_multi_instance(redis_usage_cache=None)
combined = "\n".join(caplog.messages)
assert "Redis is not configured" in combined
assert env_var in combined
assert "rate limiting" in combined
def test_warn_no_redis_multi_instance_silent_when_redis_configured(monkeypatch, caplog):
"""When Redis IS configured, no warning is emitted even inside k8s."""
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1")
import logging
with caplog.at_level(logging.WARNING):
ProxyStartupEvent._warn_no_redis_multi_instance(
redis_usage_cache=MagicMock(),
)
assert not any("Redis is not configured" in m for m in caplog.messages)
def test_warn_no_redis_multi_instance_silent_outside_orchestrator(monkeypatch, caplog):
"""When no orchestrator env var is set, no warning is emitted even without
Redis -- the user may be running a single instance on bare metal."""
for var in _ORCHESTRATOR_ENV_VARS:
monkeypatch.delenv(var, raising=False)
import logging
with caplog.at_level(logging.WARNING):
ProxyStartupEvent._warn_no_redis_multi_instance(redis_usage_cache=None)
assert not any("Redis is not configured" in m for m in caplog.messages)
def test_warn_no_redis_multi_instance_called_during_startup():
"""The warning call must appear in proxy_startup_event source, after
Redis setup and the transaction buffer validation."""
wrapped = getattr(proxy_startup_event, "__wrapped__", proxy_startup_event)
source = inspect.getsource(wrapped)
redis_buffer_pos = source.find("_validate_redis_transaction_buffer_config(")
warn_pos = source.find("_warn_no_redis_multi_instance(")
assert warn_pos != -1, "_warn_no_redis_multi_instance call missing from proxy_startup_event"
assert redis_buffer_pos < warn_pos, (
"_warn_no_redis_multi_instance must run after _validate_redis_transaction_buffer_config"
)