refactor(prometheus): make input length flag config/env only

Drop the Admin UI General Settings row, the safe DB override entry, the startup reorder that loaded DB litellm_settings before Prometheus callbacks, and the alerting-only Prometheus path. The flag now behaves like prometheus_emit_stream_label: litellm_settings in config.yaml or an os.environ reference, applied on restart.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-10 05:58:42 +00:00
parent 4420619579
commit 1b19ceead6
5 changed files with 32 additions and 454 deletions

View file

@ -1762,7 +1762,6 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"anthropic_prompt_caching_ttl",
"max_ui_session_budget",
"budget_rollover",
"prometheus_emit_input_sequence_length_label",
"mcp_tool_search",
]
SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]

View file

@ -270,7 +270,6 @@ from litellm.constants import (
from litellm.exceptions import RejectedRequestError
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus import PrometheusLogger
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
@ -1201,16 +1200,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
general_settings=general_settings
)
should_load_db_litellm_settings: Final = await ProxyStartupEvent.resolve_store_model_in_db(
prisma_client=prisma_client, configured=store_model_in_db
)
await ProxyStartupEvent._initialize_startup_logging(
ProxyStartupEvent._initialize_startup_logging(
llm_router=llm_router,
proxy_logging_obj=proxy_logging_obj,
redis_usage_cache=transaction_buffer_redis_cache,
prisma_client=prisma_client,
should_load_db_litellm_settings=should_load_db_litellm_settings,
proxy_config_obj=proxy_config,
)
## V2 OTEL: publish the chosen V2 logger's TracerProvider as the OTel global.
@ -1301,7 +1294,6 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
proxy_budget_rescheduler_max_time=proxy_budget_rescheduler_max_time,
proxy_batch_write_at=proxy_batch_write_at,
proxy_logging_obj=proxy_logging_obj,
resolved_store_model_in_db=should_load_db_litellm_settings,
)
if prisma_client is not None
else None
@ -5528,8 +5520,6 @@ class ProxyConfig:
parse_budget_reset_time(value)
setattr(litellm, key, value)
elif key == "prometheus_emit_input_sequence_length_label":
self._apply_safe_litellm_setting_override(key, value)
elif key == "drop_params":
litellm.drop_params = drop_params_flag(value, "litellm_settings.drop_params", verbose_proxy_logger)
else:
@ -6065,9 +6055,6 @@ class ProxyConfig:
if _alert == "slack":
# [OLD] v0 implementation - already handled by update_values above
pass
elif _alert == "prometheus":
if PrometheusLogger.get_instance() is None:
litellm.logging_callback_manager.add_litellm_callback("prometheus")
else:
# [NEW] v1 implementation - init as a custom logger
if _alert in litellm._known_custom_logger_compatible_callbacks:
@ -6993,7 +6980,8 @@ class ProxyConfig:
return current_config
elif param_name == "litellm_settings" and isinstance(db_param_value, dict):
for key, value in db_param_value.items():
self._apply_safe_litellm_setting_override(key, value)
if key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES: # params that are safe to override with db values
setattr(litellm, key, value)
# If param doesn't exist in config, add it
if param_name not in current_config:
@ -7274,9 +7262,9 @@ class ProxyConfig:
await self._init_hashicorp_vault_config_override(prisma_client=prisma_client)
await self._init_cyberark_config_override(prisma_client=prisma_client)
await self.apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client)
await self._apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client)
async def apply_safe_litellm_settings_overrides_from_db(self, prisma_client: PrismaClient) -> None:
async def _apply_safe_litellm_settings_overrides_from_db(self, prisma_client: PrismaClient) -> None:
config_record: Final = await get_config_param(prisma_client, "litellm_settings")
if config_record is None or config_record.param_value is None:
return
@ -7285,19 +7273,8 @@ class ProxyConfig:
if not isinstance(litellm_settings, dict):
return
for key, value in litellm_settings.items():
self._apply_safe_litellm_setting_override(key, value)
@staticmethod
def _apply_safe_litellm_setting_override(key: str, value: object) -> None:
if key not in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
return
if key == "prometheus_emit_input_sequence_length_label":
if isinstance(value, bool):
if key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
setattr(litellm, key, value)
elif isinstance(value, str) and (normalized_value := str_to_bool(value)) is not None:
setattr(litellm, key, normalized_value)
return
setattr(litellm, key, value)
async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient):
"""
@ -9048,45 +9025,14 @@ class ProxyStartupEvent:
max_budget,
)
@staticmethod
async def resolve_store_model_in_db(prisma_client: PrismaClient | None, configured: bool | str) -> bool:
default: Final = str_to_bool(configured) if isinstance(configured, str) else configured is True
if (get_secret_bool("STORE_MODEL_IN_DB", default) or configured) is True:
return True
if prisma_client is None:
return False
try:
db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
where={"param_name": "general_settings"}
)
except Exception as e: # noqa: BLE001 # a config-row read failure must not block proxy startup
verbose_proxy_logger.debug("Failed to check DB for store_model_in_db: %s", str(e))
return False
if db_general_settings is None or not isinstance(db_general_settings.param_value, dict):
return False
db_value: Final = db_general_settings.param_value.get("store_model_in_db")
if db_value is True or (isinstance(db_value, str) and db_value.lower() == "true"):
verbose_proxy_logger.info("store_model_in_db=True loaded from DB, overriding config/env")
return True
return False
@classmethod
async def _initialize_startup_logging(
def _initialize_startup_logging(
cls,
llm_router: Router | None,
proxy_logging_obj: ProxyLogging,
redis_usage_cache: RedisCache | None,
prisma_client: PrismaClient | None = None,
should_load_db_litellm_settings: bool = False,
proxy_config_obj: ProxyConfig | None = None,
) -> None:
):
"""Initialize logging and alerting on startup"""
if should_load_db_litellm_settings and prisma_client is not None and proxy_config_obj is not None:
try:
await proxy_config_obj.apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client)
except Exception as e: # noqa: BLE001 # a config-row read failure must not block proxy startup
verbose_proxy_logger.warning("Could not read litellm_settings from the database: %s", e)
## COST TRACKING ##
cost_tracking()
@ -9485,7 +9431,6 @@ class ProxyStartupEvent:
proxy_budget_rescheduler_max_time: int,
proxy_batch_write_at: int,
proxy_logging_obj: ProxyLogging,
resolved_store_model_in_db: bool = False,
) -> ProxyWorkerHeartbeat:
"""Initializes scheduled background jobs"""
global heuristic_v1_tuning_baselines, store_model_in_db, scheduler # rebind-ok: startup publishes the one read-only baseline snapshot
@ -9618,11 +9563,23 @@ class ProxyStartupEvent:
prisma_client.spend_logs_queue_monitor_task = monitor_task # rebind-ok: the client owns its monitor handle
### ADD NEW MODELS ###
store_model_in_db = ( # rebind-ok: preserve legacy YAML values unless env or DB explicitly enables storage
resolved_store_model_in_db
or await cls.resolve_store_model_in_db(prisma_client=prisma_client, configured=store_model_in_db)
or store_model_in_db
)
store_model_in_db = get_secret_bool("STORE_MODEL_IN_DB", store_model_in_db) or store_model_in_db
# If store_model_in_db is still False, check DB for override.
# This breaks the chicken-and-egg where DB has store_model_in_db=True
# but YAML config has False.
if store_model_in_db is not True and prisma_client is not None:
try:
_db_gs_record: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
where={"param_name": "general_settings"}
)
if _db_gs_record is not None and isinstance(_db_gs_record.param_value, dict):
_db_val: Final = _db_gs_record.param_value.get("store_model_in_db")
if _db_val is True or (isinstance(_db_val, str) and _db_val.lower() == "true"):
store_model_in_db = True
verbose_proxy_logger.info("store_model_in_db=True loaded from DB, overriding config/env")
except Exception as e:
verbose_proxy_logger.debug("Failed to check DB for store_model_in_db: %s", str(e))
config_reload_interval_seconds = proxy_config_reload_interval_seconds
if not isinstance(config_reload_interval_seconds, int) or config_reload_interval_seconds <= 0:
@ -17168,13 +17125,6 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie
"forgiving it. Applies to key, user, team, team member, org, tag and end-user budgets."
),
},
"prometheus_emit_input_sequence_length_label": { # mutable-ok: frozen with the registry below
"type": "Boolean",
"description": (
"Break latency and time-to-first-token metrics into input token length buckets. "
"Takes effect on the next proxy restart."
),
},
"max_ui_session_budget": {
"type": "Dollar",
"default": 1.0,

View file

@ -17,26 +17,22 @@ Pins covered:
from __future__ import annotations
import datetime
import asyncio
import inspect
import json
import logging
import os
import subprocess
from collections.abc import Awaitable, Callable
from typing import Final, Optional, Union
from unittest.mock import AsyncMock, MagicMock, call, patch
from typing import List, Optional, Union
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from prometheus_client import REGISTRY
from fastapi import FastAPI
from pydantic import BaseModel
from typing_extensions import TypedDict
import litellm
import litellm.litellm_core_utils.litellm_logging as logging_module
import litellm.proxy.proxy_server as ps
import litellm.proxy.utils as proxy_utils
from litellm.integrations.prometheus import PrometheusLogger
from litellm.proxy.proxy_server import (
ProxyStartupEvent,
_initialize_shared_aiohttp_session,
@ -754,221 +750,6 @@ async def test_proxy_startup_event_invalid_missing_app_arg_raises():
pass
@pytest.mark.asyncio
@pytest.mark.parametrize(
("configured", "db_value", "expected"),
(
(True, None, True),
(False, True, True),
(False, "true", True),
(False, False, False),
(False, None, False),
),
)
async def test_resolve_store_model_in_db_uses_config_or_db(
monkeypatch: pytest.MonkeyPatch, configured: bool, db_value: object, expected: bool
):
monkeypatch.setattr(ps, "get_secret_bool", lambda name, default: default)
db_record: Final = None if db_value is None else MagicMock(param_value={"store_model_in_db": db_value})
prisma_client: Final = MagicMock()
prisma_client.db.litellm_config.find_first = AsyncMock(return_value=db_record)
result: Final = await ProxyStartupEvent.resolve_store_model_in_db(
prisma_client=prisma_client, configured=configured
)
assert result is expected
assert prisma_client.db.litellm_config.find_first.await_count == (0 if configured else 1)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("configured", "environment", "db_value", "expected"),
(
("false", None, False, False),
("true", None, False, False),
("false", "false", False, False),
("false", "true", False, True),
(True, "false", False, True),
(False, "false", True, True),
("false", None, True, True),
),
)
async def test_resolve_store_model_in_db_preserves_legacy_config_and_env_precedence(
monkeypatch: pytest.MonkeyPatch,
configured: bool | str,
environment: str | None,
db_value: bool,
expected: bool,
):
if environment is None:
monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
else:
monkeypatch.setenv("STORE_MODEL_IN_DB", environment)
prisma_client: Final = MagicMock()
prisma_client.db.litellm_config.find_first = AsyncMock(
return_value=MagicMock(param_value={"store_model_in_db": db_value})
)
assert await ProxyStartupEvent.resolve_store_model_in_db(prisma_client, configured) is expected
assert prisma_client.db.litellm_config.find_first.await_count == (
0 if configured is True or environment == "true" else 1
)
@pytest.mark.asyncio
async def test_resolve_store_model_in_db_continues_after_database_failure(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
prisma_client: Final = MagicMock()
prisma_client.db.litellm_config.find_first = AsyncMock(side_effect=RuntimeError("database unavailable"))
assert await ProxyStartupEvent.resolve_store_model_in_db(prisma_client, configured=False) is False
@pytest.mark.asyncio
@pytest.mark.parametrize("param_value", ("legacy", ["store_model_in_db"], 7))
async def test_resolve_store_model_in_db_ignores_non_mapping_row(monkeypatch: pytest.MonkeyPatch, param_value: object):
monkeypatch.setattr(ps, "get_secret_bool", lambda name, default: default)
prisma_client: Final = MagicMock()
prisma_client.db.litellm_config.find_first = AsyncMock(return_value=MagicMock(param_value=param_value))
assert await ProxyStartupEvent.resolve_store_model_in_db(prisma_client=prisma_client, configured=False) is False
@pytest.mark.asyncio
async def test_startup_logging_applies_db_settings_before_callback_init(monkeypatch: pytest.MonkeyPatch):
events: Final = MagicMock()
proxy_config: Final = MagicMock()
async def apply_db_settings(prisma_client: object) -> None:
events.db_settings()
proxy_config.apply_safe_litellm_settings_overrides_from_db = apply_db_settings
proxy_logging: Final = MagicMock()
proxy_logging.startup_event.side_effect = lambda **kwargs: events.callback_init()
prisma_client: Final = MagicMock()
prisma_client.db.litellm_config.find_first = AsyncMock(
return_value=MagicMock(param_value={"store_model_in_db": True})
)
monkeypatch.setattr(ps, "cost_tracking", MagicMock())
monkeypatch.setattr(ps, "get_secret_bool", lambda name, default: default)
await ProxyStartupEvent._initialize_startup_logging(
llm_router=None,
proxy_logging_obj=proxy_logging,
redis_usage_cache=None,
prisma_client=prisma_client,
should_load_db_litellm_settings=True,
proxy_config_obj=proxy_config,
)
assert events.method_calls == [call.db_settings(), call.callback_init()]
@pytest.mark.asyncio
@pytest.mark.parametrize(("yaml_value", "db_value"), ((False, True), (True, False)))
@pytest.mark.parametrize("also_callback", (False, True))
async def test_prometheus_alerting_uses_persisted_settings_without_duplicate_callbacks(
monkeypatch: pytest.MonkeyPatch, yaml_value: bool, db_value: bool, also_callback: bool
):
collectors: Final = tuple(REGISTRY._collector_to_names) # pyright: ignore[reportPrivateUsage] # isolate global registry
for collector in collectors:
REGISTRY.unregister(collector)
monkeypatch.setattr(logging_module, "_in_memory_loggers", [])
monkeypatch.setattr(litellm, "prometheus_emit_input_sequence_length_label", yaml_value)
monkeypatch.setattr(litellm, "callbacks", ["prometheus"] if also_callback else [])
monkeypatch.setattr(proxy_utils, "PROXY_HOOKS", ())
monkeypatch.setattr(ps, "cost_tracking", MagicMock())
proxy_logging: Final = ps.ProxyLogging(user_api_key_cache=ps.user_api_key_cache)
proxy_logging.deprecation_check_started = True
monkeypatch.setattr(ps, "proxy_logging_obj", proxy_logging)
config: Final = ps.ProxyConfig()
settings: Final = {"alerting": ["prometheus"], "alert_types": []}
prisma_client: Final = MagicMock()
monkeypatch.setattr(proxy_utils, "litellm_config_cache", proxy_utils.DualCache())
prisma_client.get_generic_data = AsyncMock(
return_value=MagicMock(param_value={"prometheus_emit_input_sequence_length_label": db_value})
)
try:
config._load_alerting_settings(settings)
config._load_alerting_settings(settings)
await ProxyStartupEvent._initialize_startup_logging(
llm_router=None,
proxy_logging_obj=proxy_logging,
redis_usage_cache=None,
prisma_client=prisma_client,
should_load_db_litellm_settings=True,
proxy_config_obj=config,
)
logger: Final = PrometheusLogger.get_instance()
assert logger is not None
assert litellm.prometheus_emit_input_sequence_length_label is db_value
for metric in (
"litellm_llm_api_latency_metric",
"litellm_request_total_latency_metric",
"litellm_llm_api_time_to_first_token_metric",
):
assert ("input_sequence_length" in logger.get_labels_for_metric(metric)) is db_value
config._load_alerting_settings(settings)
config._load_alerting_settings(settings)
assert "prometheus" not in litellm.callbacks
assert sum(isinstance(callback, PrometheusLogger) for callback in litellm.callbacks) == 1
assert sum(isinstance(callback, PrometheusLogger) for callback in litellm._async_success_callback) == 1
now: Final = datetime.datetime.now()
await logger.async_log_success_event(
{
"model": "test-model",
"litellm_params": {"metadata": {}},
"start_time": now,
"end_time": now,
"standard_logging_object": {
"id": "alerting-startup",
"call_type": "completion",
"status": "success",
"model": "test-model",
"model_group": "test-model",
"model_id": "test-model",
"api_base": "https://api.openai.com",
"custom_llm_provider": "openai",
"request_tags": [],
"prompt_tokens": 4_000,
"completion_tokens": 20,
"total_tokens": 4_020,
"response_cost": 0,
"startTime": now,
"endTime": now,
"metadata": {
"user_api_key_user_id": None,
"user_api_key_hash": None,
"user_api_key_alias": None,
"user_api_key_team_id": None,
"user_api_key_team_alias": None,
"user_api_key_user_email": None,
},
"hidden_params": {},
},
},
None,
now,
now,
)
samples: Final = tuple(
sample
for metric in REGISTRY.collect()
for sample in metric.samples
if sample.name == "litellm_request_total_latency_metric_count"
)
assert len(samples) == 1
assert samples[0].value == 1
assert samples[0].labels.get("input_sequence_length") == ("4k-16k" if db_value else None)
finally:
for collector in tuple(REGISTRY._collector_to_names): # pyright: ignore[reportPrivateUsage] # restore test registry
REGISTRY.unregister(collector)
for collector in collectors:
REGISTRY.register(collector)
@pytest.mark.asyncio
async def test_proxy_startup_event_prunes_dead_workers_live_gauges(tmp_path):
"""With PROMETHEUS_MULTIPROC_DIR set, a booting worker drops the live-gauge files of pids that no longer

View file

@ -12,16 +12,13 @@ import json
import logging
import os
import re
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Dict, Final
from typing import Any, Dict
from unittest.mock import AsyncMock, MagicMock
import pytest
import litellm
import litellm.proxy.proxy_server as ps
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import CommonProxyErrors
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.proxy_server import (
@ -1163,28 +1160,6 @@ async def test_ProxyConfig_load_config_builds_the_secret_manager_exactly_once(tm
} == {"constructions": 1, "master_key": "master-from-vault"}
@pytest.mark.asyncio
@pytest.mark.parametrize(("raw_value", "expected"), (("true", True), ("false", False)))
async def test_load_config_normalizes_hosted_input_sequence_length_flag(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, raw_value: str, expected: bool
) -> None:
monkeypatch.setattr(litellm, "prometheus_emit_input_sequence_length_label", not expected)
monkeypatch.delenv("PROM_FLAG", raising=False)
config_yaml: Final = (
VAULT_BACKED_CONFIG.replace("master_key: os.environ/LITELLM_MASTER_KEY", "master_key: null")
.replace(" - MY_PROVIDER_KEY", " - MY_PROVIDER_KEY\n - PROM_FLAG")
+ "\nlitellm_settings:\n prometheus_emit_input_sequence_length_label: os.environ/PROM_FLAG\n"
)
config_file_path: Final = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml)
(tmp_path / "vault_secret_manager.py").write_text(
VAULT_SECRET_MANAGER_MODULE + f"\nVAULT['PROM_FLAG'] = {raw_value!r}\n"
)
await ProxyConfig().load_config(router=None, config_file_path=config_file_path)
assert litellm.prometheus_emit_input_sequence_length_label is expected
@pytest.mark.asyncio
async def test_ProxyConfig_get_config_reuses_an_already_initialized_secret_manager(tmp_path, monkeypatch):
"""get_config() also runs on management-endpoint request paths.
@ -2023,32 +1998,6 @@ def test_ProxyConfig__load_alerting_settings_noop_when_no_alerting():
}
def test_ProxyConfig__load_alerting_settings_preserves_other_logger_arguments(monkeypatch: pytest.MonkeyPatch):
logger: Final = CustomLogger()
factory: Final = MagicMock(return_value=logger)
proxy_logging: Final = MagicMock()
monkeypatch.setattr(ps, "_init_custom_logger_compatible_class", factory)
monkeypatch.setattr(ps, "proxy_logging_obj", proxy_logging)
settings: Final = {
"alerting": ["slack", "pagerduty", "prometheus"],
"alerting_args": {"routing_key": "test-routing-key"},
"alerting_threshold": 15,
}
ProxyConfig()._load_alerting_settings(settings)
factory.assert_called_once_with(
logging_integration="pagerduty",
internal_usage_cache=None,
llm_router=None,
custom_logger_init_args={"alerting_args": {"routing_key": "test-routing-key"}},
)
assert logger in ps.litellm.callbacks
assert "prometheus" in ps.litellm.callbacks
assert proxy_logging.update_values.call_args.kwargs["alerting"] == settings["alerting"]
assert proxy_logging.update_values.call_args.kwargs["alerting_threshold"] == 15
def test_ProxyConfig__load_alerting_settings_invalid_alerting_raises():
pc = ProxyConfig()
with pytest.raises(RuntimeError):
@ -3096,31 +3045,6 @@ async def test_ProxyConfig_add_deployment_applies_db_router_settings(monkeypatch
fake_router.update_settings.assert_called_once_with(routing_strategy="latency-based-routing")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("param_value", "expected"),
(
({"prometheus_emit_input_sequence_length_label": "true"}, True),
('{"prometheus_emit_input_sequence_length_label": "true"}', True),
('{"prometheus_emit_input_sequence_length_label": false}', False),
(None, False),
('["prometheus_emit_input_sequence_length_label"]', False),
),
)
async def test_apply_safe_litellm_settings_overrides_parses_input_sequence_length_record(
monkeypatch: pytest.MonkeyPatch, param_value: object, expected: bool
):
from litellm.proxy import proxy_server
config_record: Final = None if param_value is None else SimpleNamespace(param_value=param_value)
monkeypatch.setattr(proxy_server, "get_config_param", AsyncMock(return_value=config_record))
monkeypatch.setattr(litellm, "prometheus_emit_input_sequence_length_label", False)
await ProxyConfig().apply_safe_litellm_settings_overrides_from_db(prisma_client=MagicMock())
assert litellm.prometheus_emit_input_sequence_length_label is expected
def _stub_add_deployment_collaborators(
monkeypatch: pytest.MonkeyPatch, pc: ProxyConfig, fake_prisma: MagicMock
) -> None:
@ -3444,24 +3368,6 @@ def test_ProxyConfig__update_config_fields_merges_dict():
assert out == {"general_settings": {"a": 1, "b": 3, "c": 4, "d": 5}}
@pytest.mark.parametrize(
("db_value", "expected"),
((True, True), (False, False), ("true", True), ("false", False), ("invalid", False)),
)
def test_update_config_fields_normalizes_input_sequence_length_flag(
monkeypatch: pytest.MonkeyPatch, db_value: object, expected: bool
):
monkeypatch.setattr(litellm, "prometheus_emit_input_sequence_length_label", False)
ProxyConfig()._update_config_fields(
current_config={},
param_name="litellm_settings",
db_param_value={"prometheus_emit_input_sequence_length_label": db_value},
)
assert litellm.prometheus_emit_input_sequence_length_label is expected
def test_ProxyConfig__update_config_fields_invalid_param_raises():
pc = ProxyConfig()
with pytest.raises(TypeError):

View file

@ -7448,8 +7448,7 @@ async def test_batch_cost_poller_is_confirmed_before_serving(monkeypatch):
@pytest.mark.asyncio
@pytest.mark.parametrize("resolve_before_logging", (False, True))
async def test_store_model_in_db_db_override_when_config_false(resolve_before_logging: bool):
async def test_store_model_in_db_db_override_when_config_false():
"""
Verify the early DB check in initialize_scheduled_background_jobs
overrides store_model_in_db=False when DB has True.
@ -7472,11 +7471,8 @@ async def test_store_model_in_db_db_override_when_config_false(resolve_before_lo
with (
patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
patch("litellm.proxy.proxy_server.store_model_in_db", False),
patch("litellm.proxy.proxy_server.get_secret_bool", return_value=False) as secret_lookup,
patch("litellm.proxy.proxy_server.get_secret_bool", return_value=False),
):
resolved: Final = resolve_before_logging and await ProxyStartupEvent.resolve_store_model_in_db(
prisma_client=mock_prisma_client, configured=False
)
await ProxyStartupEvent.initialize_scheduled_background_jobs(
general_settings={},
prisma_client=mock_prisma_client,
@ -7484,13 +7480,8 @@ async def test_store_model_in_db_db_override_when_config_false(resolve_before_lo
proxy_budget_rescheduler_max_time=2,
proxy_batch_write_at=5,
proxy_logging_obj=mock_proxy_logging,
resolved_store_model_in_db=resolved,
)
mock_prisma_client.db.litellm_config.find_first.assert_awaited_once_with(
where={"param_name": "general_settings"}
)
assert sum(args.args[0] == "STORE_MODEL_IN_DB" for args in secret_lookup.call_args_list) == 1
import litellm.proxy.proxy_server as ps
# store_model_in_db should now be True (overridden by DB)
@ -7500,54 +7491,6 @@ async def test_store_model_in_db_db_override_when_config_false(resolve_before_lo
mock_proxy_config.get_credentials.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize("configured", ("true", "false"))
async def test_scheduled_startup_preserves_quoted_config_persistence(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, configured: str
):
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from litellm.proxy.proxy_server import ProxyConfig, ProxyStartupEvent
from litellm.proxy.utils import ProxyLogging
prisma: Final = MagicMock(insert_data=AsyncMock())
prisma.db.litellm_config.find_first = AsyncMock(return_value=None)
proxy_logging: Final = MagicMock(spec=ProxyLogging)
proxy_logging.slack_alerting_instance = MagicMock()
proxy_logging.db_spend_update_writer = MagicMock()
config: Final = _mock_scheduled_proxy_config()
scheduler: Final = AsyncIOScheduler()
config_file: Final = tmp_path / "config.yaml"
config_file.write_text("general_settings: {}\n")
monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
monkeypatch.setattr(proxy_server_module, "prisma_client", prisma)
monkeypatch.setattr(proxy_server_module, "general_settings", {"store_model_in_db": configured})
monkeypatch.setattr(proxy_server_module, "store_model_in_db", configured)
monkeypatch.setattr(proxy_server_module, "proxy_config", config)
monkeypatch.setattr(proxy_server_module, "user_config_file_path", str(config_file))
monkeypatch.setattr(proxy_server_module, "AsyncIOScheduler", MagicMock(return_value=scheduler))
try:
await ProxyStartupEvent.initialize_scheduled_background_jobs(
general_settings={},
prisma_client=prisma,
proxy_budget_rescheduler_min_time=1,
proxy_budget_rescheduler_max_time=2,
proxy_batch_write_at=5,
proxy_logging_obj=proxy_logging,
)
await ProxyConfig().save_config(new_config={"litellm_settings": {"enable_anthropic_prompt_caching": True}})
assert proxy_server_module.store_model_in_db == configured
assert scheduler.get_job("add_deployment_job") is None
assert config_file.read_text() == "general_settings: {}\n"
prisma.insert_data.assert_awaited_once_with(
data={"litellm_settings": {"enable_anthropic_prompt_caching": True}}, table_name="config"
)
finally:
scheduler.shutdown(wait=False)
@pytest.mark.asyncio
async def test_store_model_in_db_db_check_skipped_when_already_true(monkeypatch):
"""
@ -10338,7 +10281,6 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch):
assert fields["enable_anthropic_prompt_caching"]["field_tab"] == "prompt_caching"
assert fields["anthropic_prompt_caching_ttl"]["field_tab"] == "prompt_caching"
assert fields["budget_exceeded_throttle_percentage"]["field_tab"] is None
assert fields["prometheus_emit_input_sequence_length_label"]["field_type"] == "Boolean"
finally:
app.dependency_overrides.clear()