mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
refactor(prometheus): make input length flag config/env only
Drop the Admin UI General Settings row, the safe DB override entry, the startup reorder that loaded DB litellm_settings before Prometheus callbacks, and the alerting-only Prometheus path. The flag now behaves like prometheus_emit_stream_label: litellm_settings in config.yaml or an os.environ reference, applied on restart. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
4420619579
commit
1b19ceead6
5 changed files with 32 additions and 454 deletions
|
|
@ -1762,7 +1762,6 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
|
|||
"anthropic_prompt_caching_ttl",
|
||||
"max_ui_session_budget",
|
||||
"budget_rollover",
|
||||
"prometheus_emit_input_sequence_length_label",
|
||||
"mcp_tool_search",
|
||||
]
|
||||
SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
|
||||
|
|
|
|||
|
|
@ -270,7 +270,6 @@ from litellm.constants import (
|
|||
from litellm.exceptions import RejectedRequestError
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
validated_max_agentic_loops,
|
||||
|
|
@ -1201,16 +1200,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
|
|||
general_settings=general_settings
|
||||
)
|
||||
|
||||
should_load_db_litellm_settings: Final = await ProxyStartupEvent.resolve_store_model_in_db(
|
||||
prisma_client=prisma_client, configured=store_model_in_db
|
||||
)
|
||||
await ProxyStartupEvent._initialize_startup_logging(
|
||||
ProxyStartupEvent._initialize_startup_logging(
|
||||
llm_router=llm_router,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
redis_usage_cache=transaction_buffer_redis_cache,
|
||||
prisma_client=prisma_client,
|
||||
should_load_db_litellm_settings=should_load_db_litellm_settings,
|
||||
proxy_config_obj=proxy_config,
|
||||
)
|
||||
|
||||
## V2 OTEL: publish the chosen V2 logger's TracerProvider as the OTel global.
|
||||
|
|
@ -1301,7 +1294,6 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
|
|||
proxy_budget_rescheduler_max_time=proxy_budget_rescheduler_max_time,
|
||||
proxy_batch_write_at=proxy_batch_write_at,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
resolved_store_model_in_db=should_load_db_litellm_settings,
|
||||
)
|
||||
if prisma_client is not None
|
||||
else None
|
||||
|
|
@ -5528,8 +5520,6 @@ class ProxyConfig:
|
|||
|
||||
parse_budget_reset_time(value)
|
||||
setattr(litellm, key, value)
|
||||
elif key == "prometheus_emit_input_sequence_length_label":
|
||||
self._apply_safe_litellm_setting_override(key, value)
|
||||
elif key == "drop_params":
|
||||
litellm.drop_params = drop_params_flag(value, "litellm_settings.drop_params", verbose_proxy_logger)
|
||||
else:
|
||||
|
|
@ -6065,9 +6055,6 @@ class ProxyConfig:
|
|||
if _alert == "slack":
|
||||
# [OLD] v0 implementation - already handled by update_values above
|
||||
pass
|
||||
elif _alert == "prometheus":
|
||||
if PrometheusLogger.get_instance() is None:
|
||||
litellm.logging_callback_manager.add_litellm_callback("prometheus")
|
||||
else:
|
||||
# [NEW] v1 implementation - init as a custom logger
|
||||
if _alert in litellm._known_custom_logger_compatible_callbacks:
|
||||
|
|
@ -6993,7 +6980,8 @@ class ProxyConfig:
|
|||
return current_config
|
||||
elif param_name == "litellm_settings" and isinstance(db_param_value, dict):
|
||||
for key, value in db_param_value.items():
|
||||
self._apply_safe_litellm_setting_override(key, value)
|
||||
if key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES: # params that are safe to override with db values
|
||||
setattr(litellm, key, value)
|
||||
|
||||
# If param doesn't exist in config, add it
|
||||
if param_name not in current_config:
|
||||
|
|
@ -7274,9 +7262,9 @@ class ProxyConfig:
|
|||
await self._init_hashicorp_vault_config_override(prisma_client=prisma_client)
|
||||
await self._init_cyberark_config_override(prisma_client=prisma_client)
|
||||
|
||||
await self.apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client)
|
||||
await self._apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client)
|
||||
|
||||
async def apply_safe_litellm_settings_overrides_from_db(self, prisma_client: PrismaClient) -> None:
|
||||
async def _apply_safe_litellm_settings_overrides_from_db(self, prisma_client: PrismaClient) -> None:
|
||||
config_record: Final = await get_config_param(prisma_client, "litellm_settings")
|
||||
if config_record is None or config_record.param_value is None:
|
||||
return
|
||||
|
|
@ -7285,19 +7273,8 @@ class ProxyConfig:
|
|||
if not isinstance(litellm_settings, dict):
|
||||
return
|
||||
for key, value in litellm_settings.items():
|
||||
self._apply_safe_litellm_setting_override(key, value)
|
||||
|
||||
@staticmethod
|
||||
def _apply_safe_litellm_setting_override(key: str, value: object) -> None:
|
||||
if key not in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
|
||||
return
|
||||
if key == "prometheus_emit_input_sequence_length_label":
|
||||
if isinstance(value, bool):
|
||||
if key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
|
||||
setattr(litellm, key, value)
|
||||
elif isinstance(value, str) and (normalized_value := str_to_bool(value)) is not None:
|
||||
setattr(litellm, key, normalized_value)
|
||||
return
|
||||
setattr(litellm, key, value)
|
||||
|
||||
async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient):
|
||||
"""
|
||||
|
|
@ -9048,45 +9025,14 @@ class ProxyStartupEvent:
|
|||
max_budget,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def resolve_store_model_in_db(prisma_client: PrismaClient | None, configured: bool | str) -> bool:
|
||||
default: Final = str_to_bool(configured) if isinstance(configured, str) else configured is True
|
||||
if (get_secret_bool("STORE_MODEL_IN_DB", default) or configured) is True:
|
||||
return True
|
||||
if prisma_client is None:
|
||||
return False
|
||||
try:
|
||||
db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
|
||||
where={"param_name": "general_settings"}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # a config-row read failure must not block proxy startup
|
||||
verbose_proxy_logger.debug("Failed to check DB for store_model_in_db: %s", str(e))
|
||||
return False
|
||||
if db_general_settings is None or not isinstance(db_general_settings.param_value, dict):
|
||||
return False
|
||||
db_value: Final = db_general_settings.param_value.get("store_model_in_db")
|
||||
if db_value is True or (isinstance(db_value, str) and db_value.lower() == "true"):
|
||||
verbose_proxy_logger.info("store_model_in_db=True loaded from DB, overriding config/env")
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
async def _initialize_startup_logging(
|
||||
def _initialize_startup_logging(
|
||||
cls,
|
||||
llm_router: Router | None,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
redis_usage_cache: RedisCache | None,
|
||||
prisma_client: PrismaClient | None = None,
|
||||
should_load_db_litellm_settings: bool = False,
|
||||
proxy_config_obj: ProxyConfig | None = None,
|
||||
) -> None:
|
||||
):
|
||||
"""Initialize logging and alerting on startup"""
|
||||
if should_load_db_litellm_settings and prisma_client is not None and proxy_config_obj is not None:
|
||||
try:
|
||||
await proxy_config_obj.apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client)
|
||||
except Exception as e: # noqa: BLE001 # a config-row read failure must not block proxy startup
|
||||
verbose_proxy_logger.warning("Could not read litellm_settings from the database: %s", e)
|
||||
|
||||
## COST TRACKING ##
|
||||
cost_tracking()
|
||||
|
||||
|
|
@ -9485,7 +9431,6 @@ class ProxyStartupEvent:
|
|||
proxy_budget_rescheduler_max_time: int,
|
||||
proxy_batch_write_at: int,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
resolved_store_model_in_db: bool = False,
|
||||
) -> ProxyWorkerHeartbeat:
|
||||
"""Initializes scheduled background jobs"""
|
||||
global heuristic_v1_tuning_baselines, store_model_in_db, scheduler # rebind-ok: startup publishes the one read-only baseline snapshot
|
||||
|
|
@ -9618,11 +9563,23 @@ class ProxyStartupEvent:
|
|||
prisma_client.spend_logs_queue_monitor_task = monitor_task # rebind-ok: the client owns its monitor handle
|
||||
|
||||
### ADD NEW MODELS ###
|
||||
store_model_in_db = ( # rebind-ok: preserve legacy YAML values unless env or DB explicitly enables storage
|
||||
resolved_store_model_in_db
|
||||
or await cls.resolve_store_model_in_db(prisma_client=prisma_client, configured=store_model_in_db)
|
||||
or store_model_in_db
|
||||
)
|
||||
store_model_in_db = get_secret_bool("STORE_MODEL_IN_DB", store_model_in_db) or store_model_in_db
|
||||
|
||||
# If store_model_in_db is still False, check DB for override.
|
||||
# This breaks the chicken-and-egg where DB has store_model_in_db=True
|
||||
# but YAML config has False.
|
||||
if store_model_in_db is not True and prisma_client is not None:
|
||||
try:
|
||||
_db_gs_record: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
|
||||
where={"param_name": "general_settings"}
|
||||
)
|
||||
if _db_gs_record is not None and isinstance(_db_gs_record.param_value, dict):
|
||||
_db_val: Final = _db_gs_record.param_value.get("store_model_in_db")
|
||||
if _db_val is True or (isinstance(_db_val, str) and _db_val.lower() == "true"):
|
||||
store_model_in_db = True
|
||||
verbose_proxy_logger.info("store_model_in_db=True loaded from DB, overriding config/env")
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug("Failed to check DB for store_model_in_db: %s", str(e))
|
||||
|
||||
config_reload_interval_seconds = proxy_config_reload_interval_seconds
|
||||
if not isinstance(config_reload_interval_seconds, int) or config_reload_interval_seconds <= 0:
|
||||
|
|
@ -17168,13 +17125,6 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie
|
|||
"forgiving it. Applies to key, user, team, team member, org, tag and end-user budgets."
|
||||
),
|
||||
},
|
||||
"prometheus_emit_input_sequence_length_label": { # mutable-ok: frozen with the registry below
|
||||
"type": "Boolean",
|
||||
"description": (
|
||||
"Break latency and time-to-first-token metrics into input token length buckets. "
|
||||
"Takes effect on the next proxy restart."
|
||||
),
|
||||
},
|
||||
"max_ui_session_budget": {
|
||||
"type": "Dollar",
|
||||
"default": 1.0,
|
||||
|
|
|
|||
|
|
@ -17,26 +17,22 @@ Pins covered:
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Final, Optional, Union
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
from typing import List, Optional, Union
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from prometheus_client import REGISTRY
|
||||
from fastapi import FastAPI
|
||||
from pydantic import BaseModel
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
import litellm
|
||||
import litellm.litellm_core_utils.litellm_logging as logging_module
|
||||
import litellm.proxy.proxy_server as ps
|
||||
import litellm.proxy.utils as proxy_utils
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.proxy.proxy_server import (
|
||||
ProxyStartupEvent,
|
||||
_initialize_shared_aiohttp_session,
|
||||
|
|
@ -754,221 +750,6 @@ async def test_proxy_startup_event_invalid_missing_app_arg_raises():
|
|||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "db_value", "expected"),
|
||||
(
|
||||
(True, None, True),
|
||||
(False, True, True),
|
||||
(False, "true", True),
|
||||
(False, False, False),
|
||||
(False, None, False),
|
||||
),
|
||||
)
|
||||
async def test_resolve_store_model_in_db_uses_config_or_db(
|
||||
monkeypatch: pytest.MonkeyPatch, configured: bool, db_value: object, expected: bool
|
||||
):
|
||||
monkeypatch.setattr(ps, "get_secret_bool", lambda name, default: default)
|
||||
db_record: Final = None if db_value is None else MagicMock(param_value={"store_model_in_db": db_value})
|
||||
prisma_client: Final = MagicMock()
|
||||
prisma_client.db.litellm_config.find_first = AsyncMock(return_value=db_record)
|
||||
|
||||
result: Final = await ProxyStartupEvent.resolve_store_model_in_db(
|
||||
prisma_client=prisma_client, configured=configured
|
||||
)
|
||||
|
||||
assert result is expected
|
||||
assert prisma_client.db.litellm_config.find_first.await_count == (0 if configured else 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "environment", "db_value", "expected"),
|
||||
(
|
||||
("false", None, False, False),
|
||||
("true", None, False, False),
|
||||
("false", "false", False, False),
|
||||
("false", "true", False, True),
|
||||
(True, "false", False, True),
|
||||
(False, "false", True, True),
|
||||
("false", None, True, True),
|
||||
),
|
||||
)
|
||||
async def test_resolve_store_model_in_db_preserves_legacy_config_and_env_precedence(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configured: bool | str,
|
||||
environment: str | None,
|
||||
db_value: bool,
|
||||
expected: bool,
|
||||
):
|
||||
if environment is None:
|
||||
monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("STORE_MODEL_IN_DB", environment)
|
||||
prisma_client: Final = MagicMock()
|
||||
prisma_client.db.litellm_config.find_first = AsyncMock(
|
||||
return_value=MagicMock(param_value={"store_model_in_db": db_value})
|
||||
)
|
||||
|
||||
assert await ProxyStartupEvent.resolve_store_model_in_db(prisma_client, configured) is expected
|
||||
assert prisma_client.db.litellm_config.find_first.await_count == (
|
||||
0 if configured is True or environment == "true" else 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_store_model_in_db_continues_after_database_failure(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
|
||||
prisma_client: Final = MagicMock()
|
||||
prisma_client.db.litellm_config.find_first = AsyncMock(side_effect=RuntimeError("database unavailable"))
|
||||
|
||||
assert await ProxyStartupEvent.resolve_store_model_in_db(prisma_client, configured=False) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("param_value", ("legacy", ["store_model_in_db"], 7))
|
||||
async def test_resolve_store_model_in_db_ignores_non_mapping_row(monkeypatch: pytest.MonkeyPatch, param_value: object):
|
||||
monkeypatch.setattr(ps, "get_secret_bool", lambda name, default: default)
|
||||
prisma_client: Final = MagicMock()
|
||||
prisma_client.db.litellm_config.find_first = AsyncMock(return_value=MagicMock(param_value=param_value))
|
||||
|
||||
assert await ProxyStartupEvent.resolve_store_model_in_db(prisma_client=prisma_client, configured=False) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_startup_logging_applies_db_settings_before_callback_init(monkeypatch: pytest.MonkeyPatch):
|
||||
events: Final = MagicMock()
|
||||
proxy_config: Final = MagicMock()
|
||||
|
||||
async def apply_db_settings(prisma_client: object) -> None:
|
||||
events.db_settings()
|
||||
|
||||
proxy_config.apply_safe_litellm_settings_overrides_from_db = apply_db_settings
|
||||
proxy_logging: Final = MagicMock()
|
||||
proxy_logging.startup_event.side_effect = lambda **kwargs: events.callback_init()
|
||||
prisma_client: Final = MagicMock()
|
||||
prisma_client.db.litellm_config.find_first = AsyncMock(
|
||||
return_value=MagicMock(param_value={"store_model_in_db": True})
|
||||
)
|
||||
monkeypatch.setattr(ps, "cost_tracking", MagicMock())
|
||||
monkeypatch.setattr(ps, "get_secret_bool", lambda name, default: default)
|
||||
|
||||
await ProxyStartupEvent._initialize_startup_logging(
|
||||
llm_router=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
redis_usage_cache=None,
|
||||
prisma_client=prisma_client,
|
||||
should_load_db_litellm_settings=True,
|
||||
proxy_config_obj=proxy_config,
|
||||
)
|
||||
|
||||
assert events.method_calls == [call.db_settings(), call.callback_init()]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("yaml_value", "db_value"), ((False, True), (True, False)))
|
||||
@pytest.mark.parametrize("also_callback", (False, True))
|
||||
async def test_prometheus_alerting_uses_persisted_settings_without_duplicate_callbacks(
|
||||
monkeypatch: pytest.MonkeyPatch, yaml_value: bool, db_value: bool, also_callback: bool
|
||||
):
|
||||
collectors: Final = tuple(REGISTRY._collector_to_names) # pyright: ignore[reportPrivateUsage] # isolate global registry
|
||||
for collector in collectors:
|
||||
REGISTRY.unregister(collector)
|
||||
monkeypatch.setattr(logging_module, "_in_memory_loggers", [])
|
||||
monkeypatch.setattr(litellm, "prometheus_emit_input_sequence_length_label", yaml_value)
|
||||
monkeypatch.setattr(litellm, "callbacks", ["prometheus"] if also_callback else [])
|
||||
monkeypatch.setattr(proxy_utils, "PROXY_HOOKS", ())
|
||||
monkeypatch.setattr(ps, "cost_tracking", MagicMock())
|
||||
proxy_logging: Final = ps.ProxyLogging(user_api_key_cache=ps.user_api_key_cache)
|
||||
proxy_logging.deprecation_check_started = True
|
||||
monkeypatch.setattr(ps, "proxy_logging_obj", proxy_logging)
|
||||
config: Final = ps.ProxyConfig()
|
||||
settings: Final = {"alerting": ["prometheus"], "alert_types": []}
|
||||
prisma_client: Final = MagicMock()
|
||||
monkeypatch.setattr(proxy_utils, "litellm_config_cache", proxy_utils.DualCache())
|
||||
prisma_client.get_generic_data = AsyncMock(
|
||||
return_value=MagicMock(param_value={"prometheus_emit_input_sequence_length_label": db_value})
|
||||
)
|
||||
try:
|
||||
config._load_alerting_settings(settings)
|
||||
config._load_alerting_settings(settings)
|
||||
await ProxyStartupEvent._initialize_startup_logging(
|
||||
llm_router=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
redis_usage_cache=None,
|
||||
prisma_client=prisma_client,
|
||||
should_load_db_litellm_settings=True,
|
||||
proxy_config_obj=config,
|
||||
)
|
||||
logger: Final = PrometheusLogger.get_instance()
|
||||
assert logger is not None
|
||||
assert litellm.prometheus_emit_input_sequence_length_label is db_value
|
||||
for metric in (
|
||||
"litellm_llm_api_latency_metric",
|
||||
"litellm_request_total_latency_metric",
|
||||
"litellm_llm_api_time_to_first_token_metric",
|
||||
):
|
||||
assert ("input_sequence_length" in logger.get_labels_for_metric(metric)) is db_value
|
||||
|
||||
config._load_alerting_settings(settings)
|
||||
config._load_alerting_settings(settings)
|
||||
assert "prometheus" not in litellm.callbacks
|
||||
assert sum(isinstance(callback, PrometheusLogger) for callback in litellm.callbacks) == 1
|
||||
assert sum(isinstance(callback, PrometheusLogger) for callback in litellm._async_success_callback) == 1
|
||||
now: Final = datetime.datetime.now()
|
||||
await logger.async_log_success_event(
|
||||
{
|
||||
"model": "test-model",
|
||||
"litellm_params": {"metadata": {}},
|
||||
"start_time": now,
|
||||
"end_time": now,
|
||||
"standard_logging_object": {
|
||||
"id": "alerting-startup",
|
||||
"call_type": "completion",
|
||||
"status": "success",
|
||||
"model": "test-model",
|
||||
"model_group": "test-model",
|
||||
"model_id": "test-model",
|
||||
"api_base": "https://api.openai.com",
|
||||
"custom_llm_provider": "openai",
|
||||
"request_tags": [],
|
||||
"prompt_tokens": 4_000,
|
||||
"completion_tokens": 20,
|
||||
"total_tokens": 4_020,
|
||||
"response_cost": 0,
|
||||
"startTime": now,
|
||||
"endTime": now,
|
||||
"metadata": {
|
||||
"user_api_key_user_id": None,
|
||||
"user_api_key_hash": None,
|
||||
"user_api_key_alias": None,
|
||||
"user_api_key_team_id": None,
|
||||
"user_api_key_team_alias": None,
|
||||
"user_api_key_user_email": None,
|
||||
},
|
||||
"hidden_params": {},
|
||||
},
|
||||
},
|
||||
None,
|
||||
now,
|
||||
now,
|
||||
)
|
||||
samples: Final = tuple(
|
||||
sample
|
||||
for metric in REGISTRY.collect()
|
||||
for sample in metric.samples
|
||||
if sample.name == "litellm_request_total_latency_metric_count"
|
||||
)
|
||||
assert len(samples) == 1
|
||||
assert samples[0].value == 1
|
||||
assert samples[0].labels.get("input_sequence_length") == ("4k-16k" if db_value else None)
|
||||
finally:
|
||||
for collector in tuple(REGISTRY._collector_to_names): # pyright: ignore[reportPrivateUsage] # restore test registry
|
||||
REGISTRY.unregister(collector)
|
||||
for collector in collectors:
|
||||
REGISTRY.register(collector)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_startup_event_prunes_dead_workers_live_gauges(tmp_path):
|
||||
"""With PROMETHEUS_MULTIPROC_DIR set, a booting worker drops the live-gauge files of pids that no longer
|
||||
|
|
|
|||
|
|
@ -12,16 +12,13 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, Dict, Final
|
||||
from typing import Any, Dict
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import CommonProxyErrors
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
|
||||
from litellm.proxy.proxy_server import (
|
||||
|
|
@ -1163,28 +1160,6 @@ async def test_ProxyConfig_load_config_builds_the_secret_manager_exactly_once(tm
|
|||
} == {"constructions": 1, "master_key": "master-from-vault"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("raw_value", "expected"), (("true", True), ("false", False)))
|
||||
async def test_load_config_normalizes_hosted_input_sequence_length_flag(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, raw_value: str, expected: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "prometheus_emit_input_sequence_length_label", not expected)
|
||||
monkeypatch.delenv("PROM_FLAG", raising=False)
|
||||
config_yaml: Final = (
|
||||
VAULT_BACKED_CONFIG.replace("master_key: os.environ/LITELLM_MASTER_KEY", "master_key: null")
|
||||
.replace(" - MY_PROVIDER_KEY", " - MY_PROVIDER_KEY\n - PROM_FLAG")
|
||||
+ "\nlitellm_settings:\n prometheus_emit_input_sequence_length_label: os.environ/PROM_FLAG\n"
|
||||
)
|
||||
config_file_path: Final = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml)
|
||||
(tmp_path / "vault_secret_manager.py").write_text(
|
||||
VAULT_SECRET_MANAGER_MODULE + f"\nVAULT['PROM_FLAG'] = {raw_value!r}\n"
|
||||
)
|
||||
|
||||
await ProxyConfig().load_config(router=None, config_file_path=config_file_path)
|
||||
|
||||
assert litellm.prometheus_emit_input_sequence_length_label is expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ProxyConfig_get_config_reuses_an_already_initialized_secret_manager(tmp_path, monkeypatch):
|
||||
"""get_config() also runs on management-endpoint request paths.
|
||||
|
|
@ -2023,32 +1998,6 @@ def test_ProxyConfig__load_alerting_settings_noop_when_no_alerting():
|
|||
}
|
||||
|
||||
|
||||
def test_ProxyConfig__load_alerting_settings_preserves_other_logger_arguments(monkeypatch: pytest.MonkeyPatch):
|
||||
logger: Final = CustomLogger()
|
||||
factory: Final = MagicMock(return_value=logger)
|
||||
proxy_logging: Final = MagicMock()
|
||||
monkeypatch.setattr(ps, "_init_custom_logger_compatible_class", factory)
|
||||
monkeypatch.setattr(ps, "proxy_logging_obj", proxy_logging)
|
||||
settings: Final = {
|
||||
"alerting": ["slack", "pagerduty", "prometheus"],
|
||||
"alerting_args": {"routing_key": "test-routing-key"},
|
||||
"alerting_threshold": 15,
|
||||
}
|
||||
|
||||
ProxyConfig()._load_alerting_settings(settings)
|
||||
|
||||
factory.assert_called_once_with(
|
||||
logging_integration="pagerduty",
|
||||
internal_usage_cache=None,
|
||||
llm_router=None,
|
||||
custom_logger_init_args={"alerting_args": {"routing_key": "test-routing-key"}},
|
||||
)
|
||||
assert logger in ps.litellm.callbacks
|
||||
assert "prometheus" in ps.litellm.callbacks
|
||||
assert proxy_logging.update_values.call_args.kwargs["alerting"] == settings["alerting"]
|
||||
assert proxy_logging.update_values.call_args.kwargs["alerting_threshold"] == 15
|
||||
|
||||
|
||||
def test_ProxyConfig__load_alerting_settings_invalid_alerting_raises():
|
||||
pc = ProxyConfig()
|
||||
with pytest.raises(RuntimeError):
|
||||
|
|
@ -3096,31 +3045,6 @@ async def test_ProxyConfig_add_deployment_applies_db_router_settings(monkeypatch
|
|||
fake_router.update_settings.assert_called_once_with(routing_strategy="latency-based-routing")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("param_value", "expected"),
|
||||
(
|
||||
({"prometheus_emit_input_sequence_length_label": "true"}, True),
|
||||
('{"prometheus_emit_input_sequence_length_label": "true"}', True),
|
||||
('{"prometheus_emit_input_sequence_length_label": false}', False),
|
||||
(None, False),
|
||||
('["prometheus_emit_input_sequence_length_label"]', False),
|
||||
),
|
||||
)
|
||||
async def test_apply_safe_litellm_settings_overrides_parses_input_sequence_length_record(
|
||||
monkeypatch: pytest.MonkeyPatch, param_value: object, expected: bool
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
config_record: Final = None if param_value is None else SimpleNamespace(param_value=param_value)
|
||||
monkeypatch.setattr(proxy_server, "get_config_param", AsyncMock(return_value=config_record))
|
||||
monkeypatch.setattr(litellm, "prometheus_emit_input_sequence_length_label", False)
|
||||
|
||||
await ProxyConfig().apply_safe_litellm_settings_overrides_from_db(prisma_client=MagicMock())
|
||||
|
||||
assert litellm.prometheus_emit_input_sequence_length_label is expected
|
||||
|
||||
|
||||
def _stub_add_deployment_collaborators(
|
||||
monkeypatch: pytest.MonkeyPatch, pc: ProxyConfig, fake_prisma: MagicMock
|
||||
) -> None:
|
||||
|
|
@ -3444,24 +3368,6 @@ def test_ProxyConfig__update_config_fields_merges_dict():
|
|||
assert out == {"general_settings": {"a": 1, "b": 3, "c": 4, "d": 5}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("db_value", "expected"),
|
||||
((True, True), (False, False), ("true", True), ("false", False), ("invalid", False)),
|
||||
)
|
||||
def test_update_config_fields_normalizes_input_sequence_length_flag(
|
||||
monkeypatch: pytest.MonkeyPatch, db_value: object, expected: bool
|
||||
):
|
||||
monkeypatch.setattr(litellm, "prometheus_emit_input_sequence_length_label", False)
|
||||
|
||||
ProxyConfig()._update_config_fields(
|
||||
current_config={},
|
||||
param_name="litellm_settings",
|
||||
db_param_value={"prometheus_emit_input_sequence_length_label": db_value},
|
||||
)
|
||||
|
||||
assert litellm.prometheus_emit_input_sequence_length_label is expected
|
||||
|
||||
|
||||
def test_ProxyConfig__update_config_fields_invalid_param_raises():
|
||||
pc = ProxyConfig()
|
||||
with pytest.raises(TypeError):
|
||||
|
|
|
|||
|
|
@ -7448,8 +7448,7 @@ async def test_batch_cost_poller_is_confirmed_before_serving(monkeypatch):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("resolve_before_logging", (False, True))
|
||||
async def test_store_model_in_db_db_override_when_config_false(resolve_before_logging: bool):
|
||||
async def test_store_model_in_db_db_override_when_config_false():
|
||||
"""
|
||||
Verify the early DB check in initialize_scheduled_background_jobs
|
||||
overrides store_model_in_db=False when DB has True.
|
||||
|
|
@ -7472,11 +7471,8 @@ async def test_store_model_in_db_db_override_when_config_false(resolve_before_lo
|
|||
with (
|
||||
patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", False),
|
||||
patch("litellm.proxy.proxy_server.get_secret_bool", return_value=False) as secret_lookup,
|
||||
patch("litellm.proxy.proxy_server.get_secret_bool", return_value=False),
|
||||
):
|
||||
resolved: Final = resolve_before_logging and await ProxyStartupEvent.resolve_store_model_in_db(
|
||||
prisma_client=mock_prisma_client, configured=False
|
||||
)
|
||||
await ProxyStartupEvent.initialize_scheduled_background_jobs(
|
||||
general_settings={},
|
||||
prisma_client=mock_prisma_client,
|
||||
|
|
@ -7484,13 +7480,8 @@ async def test_store_model_in_db_db_override_when_config_false(resolve_before_lo
|
|||
proxy_budget_rescheduler_max_time=2,
|
||||
proxy_batch_write_at=5,
|
||||
proxy_logging_obj=mock_proxy_logging,
|
||||
resolved_store_model_in_db=resolved,
|
||||
)
|
||||
|
||||
mock_prisma_client.db.litellm_config.find_first.assert_awaited_once_with(
|
||||
where={"param_name": "general_settings"}
|
||||
)
|
||||
assert sum(args.args[0] == "STORE_MODEL_IN_DB" for args in secret_lookup.call_args_list) == 1
|
||||
import litellm.proxy.proxy_server as ps
|
||||
|
||||
# store_model_in_db should now be True (overridden by DB)
|
||||
|
|
@ -7500,54 +7491,6 @@ async def test_store_model_in_db_db_override_when_config_false(resolve_before_lo
|
|||
mock_proxy_config.get_credentials.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("configured", ("true", "false"))
|
||||
async def test_scheduled_startup_preserves_quoted_config_persistence(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, configured: str
|
||||
):
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
|
||||
from litellm.proxy.proxy_server import ProxyConfig, ProxyStartupEvent
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
prisma: Final = MagicMock(insert_data=AsyncMock())
|
||||
prisma.db.litellm_config.find_first = AsyncMock(return_value=None)
|
||||
proxy_logging: Final = MagicMock(spec=ProxyLogging)
|
||||
proxy_logging.slack_alerting_instance = MagicMock()
|
||||
proxy_logging.db_spend_update_writer = MagicMock()
|
||||
config: Final = _mock_scheduled_proxy_config()
|
||||
scheduler: Final = AsyncIOScheduler()
|
||||
config_file: Final = tmp_path / "config.yaml"
|
||||
config_file.write_text("general_settings: {}\n")
|
||||
monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
|
||||
monkeypatch.setattr(proxy_server_module, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server_module, "general_settings", {"store_model_in_db": configured})
|
||||
monkeypatch.setattr(proxy_server_module, "store_model_in_db", configured)
|
||||
monkeypatch.setattr(proxy_server_module, "proxy_config", config)
|
||||
monkeypatch.setattr(proxy_server_module, "user_config_file_path", str(config_file))
|
||||
monkeypatch.setattr(proxy_server_module, "AsyncIOScheduler", MagicMock(return_value=scheduler))
|
||||
|
||||
try:
|
||||
await ProxyStartupEvent.initialize_scheduled_background_jobs(
|
||||
general_settings={},
|
||||
prisma_client=prisma,
|
||||
proxy_budget_rescheduler_min_time=1,
|
||||
proxy_budget_rescheduler_max_time=2,
|
||||
proxy_batch_write_at=5,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
)
|
||||
await ProxyConfig().save_config(new_config={"litellm_settings": {"enable_anthropic_prompt_caching": True}})
|
||||
|
||||
assert proxy_server_module.store_model_in_db == configured
|
||||
assert scheduler.get_job("add_deployment_job") is None
|
||||
assert config_file.read_text() == "general_settings: {}\n"
|
||||
prisma.insert_data.assert_awaited_once_with(
|
||||
data={"litellm_settings": {"enable_anthropic_prompt_caching": True}}, table_name="config"
|
||||
)
|
||||
finally:
|
||||
scheduler.shutdown(wait=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_model_in_db_db_check_skipped_when_already_true(monkeypatch):
|
||||
"""
|
||||
|
|
@ -10338,7 +10281,6 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch):
|
|||
assert fields["enable_anthropic_prompt_caching"]["field_tab"] == "prompt_caching"
|
||||
assert fields["anthropic_prompt_caching_ttl"]["field_tab"] == "prompt_caching"
|
||||
assert fields["budget_exceeded_throttle_percentage"]["field_tab"] is None
|
||||
assert fields["prometheus_emit_input_sequence_length_label"]["field_type"] == "Boolean"
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue