diff --git a/litellm/proxy/config_resolvers/__init__.py b/litellm/proxy/config_resolvers/__init__.py index 88b4c3961f0..73fc03a60a0 100644 --- a/litellm/proxy/config_resolvers/__init__.py +++ b/litellm/proxy/config_resolvers/__init__.py @@ -5,5 +5,6 @@ from litellm.proxy.config_resolvers._descriptors import ( FieldSource, resolve_fields, ) +from litellm.proxy.config_resolvers.settings_store import SettingsStore -__all__ = ["FieldDescriptor", "FieldSource", "resolve_fields"] +__all__ = ["FieldDescriptor", "FieldSource", "SettingsStore", "resolve_fields"] diff --git a/litellm/proxy/config_resolvers/_descriptors.py b/litellm/proxy/config_resolvers/_descriptors.py index edc0eeb1cf6..8903d1d9e00 100644 --- a/litellm/proxy/config_resolvers/_descriptors.py +++ b/litellm/proxy/config_resolvers/_descriptors.py @@ -13,7 +13,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Final, Literal -FieldSource = Literal["db", "env", "default", "unset"] +FieldSource = Literal["config", "db", "env", "default", "unset"] @dataclass(frozen=True, slots=True) @@ -69,5 +69,5 @@ def resolve_fields( """ resolved: Final = tuple(_resolve_one(descriptor, db_values, env, empty_db_is_set) for descriptor in descriptors) values: Final = {field_name: value for field_name, value, _ in resolved} - provenance: Final = {field_name: source for field_name, _, source in resolved} + provenance: Final[dict[str, FieldSource]] = dict((field_name, source) for field_name, _, source in resolved) return values, provenance diff --git a/litellm/proxy/config_resolvers/settings_rules.py b/litellm/proxy/config_resolvers/settings_rules.py new file mode 100644 index 00000000000..58516ee4795 --- /dev/null +++ b/litellm/proxy/config_resolvers/settings_rules.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from typing_extensions import ( + assert_never, +) + +from litellm.proxy.config_resolvers._descriptors import FieldSource + +JsonValue: TypeAlias = None | bool | int | float | str | list["JsonValue"] | dict[str, "JsonValue"] +Section: TypeAlias = Literal[ + "general_settings", + "router_settings", + "litellm_settings", + "environment_variables", + "ui_settings", +] +DbRow: TypeAlias = Section +RuleKind: TypeAlias = Literal[ + "db_wins", + "config_wins", + "db_fallback_to_config", + "list_union", + "merge_by_path", + "db_overlay", +] + + +@dataclass(frozen=True, slots=True) +class Absent: + pass + + +ABSENT: Final = Absent() +SettingValue: TypeAlias = JsonValue | Absent + + +@dataclass(frozen=True, slots=True) +class KeyRule: + db_row: DbRow + kind: RuleKind + + +@dataclass(frozen=True, slots=True) +class Resolved: + value: SettingValue + source: FieldSource + + +_DB_GENERAL_SETTINGS: Final[tuple[str, ...]] = ( + "max_parallel_requests", + "global_max_parallel_requests", + "alerting_args", + "ui_access_mode", + "disable_auto_add_proxy_admin_to_teams", + "store_model_in_db", + "maximum_spend_logs_retention_period", + "maximum_autorouter_session_retention_period", + "maximum_health_check_retention_period", + "user_url_validation", + "user_url_allowed_hosts", + "provider_url_destination_allowed_hosts", +) +_CONFIG_GENERAL_SETTINGS: Final[tuple[str, ...]] = ( + "max_batch_file_size_mb", + "max_file_size_mb", + "allowed_file_extensions", + "blocked_file_extensions", + "store_prompts_in_spend_logs", + "apply_user_budget_to_team_keys", + "enable_openai_websocket_passthrough", + "user_api_key_cache_max_size", +) +_CLEANUP_BOUNDS: Final[tuple[str, ...]] = ( + "maximum_spend_logs_cleanup_batch_size", + "maximum_spend_logs_cleanup_max_batches", + "maximum_spend_logs_cleanup_run_budget", + "maximum_spend_logs_cleanup_batch_timeout", +) +_UI_SETTINGS_FIELDS: Final[tuple[str, ...]] = ( + "allow_public_health_readiness_details", + "forward_client_headers_to_llm_api", + "forward_llm_provider_auth_headers", + "disable_agents_for_internal_users", + "allow_agents_for_team_admins", + "disable_vector_stores_for_internal_users", + "allow_vector_stores_for_team_admins", + "disable_key_generate_for_org_admin", + "team_admin_editable_team_fields", +) + + +DUAL_SOURCE_KEYS: Final[Mapping[tuple[Section, str], KeyRule]] = MappingProxyType( + { + **{ + ("general_settings", key): KeyRule(db_row="general_settings", kind="db_wins") + for key in _DB_GENERAL_SETTINGS + }, + **{ + ("general_settings", key): KeyRule(db_row="general_settings", kind="config_wins") + for key in _CONFIG_GENERAL_SETTINGS + }, + **{ + ("general_settings", key): KeyRule(db_row="general_settings", kind="db_fallback_to_config") + for key in _CLEANUP_BOUNDS + }, + ("general_settings", "alerting"): KeyRule(db_row="general_settings", kind="list_union"), + ("general_settings", "pass_through_endpoints"): KeyRule(db_row="general_settings", kind="merge_by_path"), + ("general_settings", "*"): KeyRule(db_row="general_settings", kind="db_overlay"), + ("router_settings", "*"): KeyRule(db_row="router_settings", kind="db_overlay"), + ("litellm_settings", "*"): KeyRule(db_row="litellm_settings", kind="db_overlay"), + ("environment_variables", "*"): KeyRule(db_row="environment_variables", kind="db_overlay"), + **{("general_settings", key): KeyRule(db_row="ui_settings", kind="db_wins") for key in _UI_SETTINGS_FIELDS}, + } +) + + +def rule_for(section: Section, key: str) -> KeyRule: + return DUAL_SOURCE_KEYS.get((section, key), DUAL_SOURCE_KEYS[(section, "*")]) + + +def coerce_bool(value: JsonValue) -> JsonValue: + if value is None or isinstance(value, bool): + return value + if isinstance(value, str): + return value.lower() == "true" + return bool(value) + + +def resolve(rule: KeyRule, yaml_value: SettingValue, db_value: SettingValue) -> Resolved: + match rule.kind: + case "db_wins" | "db_fallback_to_config": + return _db_wins(yaml_value, db_value) + case "config_wins": + return _config_wins(yaml_value, db_value) + case "list_union": + return _list_union(yaml_value, db_value) + case "merge_by_path": + return _merge_by_path(yaml_value, db_value) + case "db_overlay": + return _db_overlay(yaml_value, db_value) + case _: + assert_never(rule.kind) + + +def _db_wins(yaml_value: SettingValue, db_value: SettingValue) -> Resolved: + if _db_is_present(db_value): + return Resolved(value=db_value, source="db") + if yaml_value is not ABSENT: + return Resolved(value=yaml_value, source="config") + return Resolved(value=ABSENT, source="unset") + + +def _config_wins(yaml_value: SettingValue, db_value: SettingValue) -> Resolved: + if yaml_value is not ABSENT: + return Resolved(value=yaml_value, source="config") + if _db_is_present(db_value): + return Resolved(value=db_value, source="db") + return Resolved(value=ABSENT, source="unset") + + +def _list_union(yaml_value: SettingValue, db_value: SettingValue) -> Resolved: + if not _db_is_present(db_value): + return _db_wins(yaml_value, db_value) + if not isinstance(yaml_value, list) or not isinstance(db_value, list): + return _db_wins(yaml_value, db_value) + return Resolved(value=[*yaml_value, *(value for value in db_value if value not in yaml_value)], source="db") + + +def _merge_by_path(yaml_value: SettingValue, db_value: SettingValue) -> Resolved: + if not _db_is_present(db_value): + return _db_wins(yaml_value, db_value) + if not isinstance(yaml_value, list) or not isinstance(db_value, list): + return _db_wins(yaml_value, db_value) + db_paths: Final = frozenset(_endpoint_path(value) for value in db_value if _endpoint_path(value) is not None) + return Resolved( + value=[*db_value, *(value for value in yaml_value if _endpoint_path(value) not in db_paths)], source="db" + ) + + +def _db_overlay(yaml_value: SettingValue, db_value: SettingValue) -> Resolved: + if not _db_is_present(db_value): + return _db_wins(yaml_value, db_value) + if not isinstance(yaml_value, dict) or not isinstance(db_value, dict): + return _db_wins(yaml_value, db_value) + overlay: Final = _overlay_mapping(yaml_value, db_value) + source: Final[FieldSource] = "db" if overlay != yaml_value else "config" + return Resolved(value=overlay, source=source) + + +def _overlay_mapping(yaml_value: dict[str, JsonValue], db_value: dict[str, JsonValue]) -> dict[str, JsonValue]: + retained: Final = { + key: _overlay_value(value, db_value[key]) if key in db_value else value for key, value in yaml_value.items() + } + additions: Final = { + key: value for key, value in db_value.items() if key not in yaml_value and not _db_overlay_defers(value) + } + return {**retained, **additions} + + +def _overlay_value(yaml_value: JsonValue, db_value: JsonValue) -> JsonValue: + if isinstance(yaml_value, dict) and isinstance(db_value, dict): + return _overlay_mapping(yaml_value, db_value) + return yaml_value if _db_overlay_defers(db_value) else db_value + + +def _db_overlay_defers(value: JsonValue) -> bool: + return value is None or (isinstance(value, list) and not value) + + +def is_absent(value: SettingValue) -> bool: + return value is ABSENT + + +def _db_is_present(value: SettingValue) -> bool: + return not is_absent(value) and value is not None + + +def _endpoint_path(value: JsonValue) -> str | None: + if not isinstance(value, dict): + return None + path: Final = value.get("path") + return path if isinstance(path, str) else None diff --git a/litellm/proxy/config_resolvers/settings_store.py b/litellm/proxy/config_resolvers/settings_store.py new file mode 100644 index 00000000000..2fbbf6705ea --- /dev/null +++ b/litellm/proxy/config_resolvers/settings_store.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from collections.abc import Iterator, Mapping, MutableMapping +from types import MappingProxyType +from typing import Final + +from litellm.proxy.config_resolvers._descriptors import FieldSource +from litellm.proxy.config_resolvers.settings_rules import ( + ABSENT, + Absent, + DbRow, + JsonValue, + Resolved, + Section, + SettingValue, + resolve, + rule_for, +) + +_EMPTY_VALUES: Final[Mapping[str, JsonValue]] = MappingProxyType({}) +_EMPTY_ROWS: Final[Mapping[DbRow, Mapping[str, JsonValue]]] = MappingProxyType({}) + + +class SettingsStore(MutableMapping[str, JsonValue]): + def __init__(self, section: Section) -> None: + self._section: Final = section + self._yaml_values: Mapping[str, JsonValue] = _EMPTY_VALUES + self._database_rows: Mapping[DbRow, Mapping[str, JsonValue]] = _EMPTY_ROWS + self._runtime_values: Mapping[str, JsonValue] = _EMPTY_VALUES + self._deleted_runtime_keys: frozenset[str] = frozenset() + + def load_yaml(self, mapping: Mapping[str, JsonValue]) -> None: + self._yaml_values = MappingProxyType(dict(mapping)) + self._clear_runtime() + + def apply_db_row(self, row: DbRow, db_row: Mapping[str, JsonValue]) -> None: + self._database_rows = MappingProxyType({**self._database_rows, row: MappingProxyType(dict(db_row))}) + self._clear_runtime() + + def resolved(self) -> Mapping[str, JsonValue]: + return MappingProxyType(dict(self)) + + def source(self, key: str) -> FieldSource: + return self._resolution_for(key).source + + def __getitem__(self, key: str) -> JsonValue: + if key in self._deleted_runtime_keys: + raise KeyError(key) + if key in self._runtime_values: + return self._runtime_values[key] + resolved: Final = self._resolution_for(key) + if isinstance(resolved.value, Absent): + raise KeyError(key) + return resolved.value + + def __setitem__(self, key: str, value: JsonValue) -> None: + self._runtime_values = MappingProxyType({**self._runtime_values, key: value}) + self._deleted_runtime_keys = self._deleted_runtime_keys - {key} + + def __delitem__(self, key: str) -> None: + if key not in self: + raise KeyError(key) + self._runtime_values = MappingProxyType( + {key_: value for key_, value in self._runtime_values.items() if key_ != key} + ) + self._deleted_runtime_keys = self._deleted_runtime_keys | {key} + + def __iter__(self) -> Iterator[str]: + return iter(key for key in self._keys() if key not in self._deleted_runtime_keys) + + def __len__(self) -> int: + return sum(1 for _ in self) + + def _clear_runtime(self) -> None: + self._runtime_values = _EMPTY_VALUES + self._deleted_runtime_keys = frozenset() + + def _keys(self) -> tuple[str, ...]: + return tuple( + dict.fromkeys( + ( + *self._yaml_values, + *(key for row in self._database_rows.values() for key in row), + *self._runtime_values, + ) + ) + ) + + def _resolution_for(self, key: str) -> Resolved: + rule: Final = rule_for(self._section, key) + yaml_value: Final[SettingValue] = self._yaml_values.get(key, ABSENT) + db_value: Final[SettingValue] = self._database_rows.get(rule.db_row, _EMPTY_VALUES).get(key, ABSENT) + return resolve(rule, yaml_value, db_value) diff --git a/tests/test_litellm/proxy/config_resolvers/test_settings_rules.py b/tests/test_litellm/proxy/config_resolvers/test_settings_rules.py new file mode 100644 index 00000000000..1f8c9fee1b2 --- /dev/null +++ b/tests/test_litellm/proxy/config_resolvers/test_settings_rules.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from litellm.proxy.config_resolvers._descriptors import FieldSource +from litellm.proxy.config_resolvers.settings_rules import ABSENT, DUAL_SOURCE_KEYS, KeyRule, Resolved, resolve + + +@pytest.mark.parametrize( + ("rule", "yaml_value", "db_value", "expected"), + ( + ( + KeyRule(db_row="general_settings", kind="db_wins"), + "from-config", + "from-db", + Resolved(value="from-db", source="db"), + ), + ( + KeyRule(db_row="general_settings", kind="config_wins"), + "from-config", + "from-db", + Resolved(value="from-config", source="config"), + ), + ( + KeyRule(db_row="general_settings", kind="db_fallback_to_config"), + "from-config", + None, + Resolved(value="from-config", source="config"), + ), + ( + KeyRule(db_row="general_settings", kind="list_union"), + ["config", "shared"], + ["db", "shared"], + Resolved(value=["config", "shared", "db"], source="db"), + ), + ( + KeyRule(db_row="general_settings", kind="merge_by_path"), + [{"path": "/config"}, {"path": "/shared", "source": "config"}], + [{"path": "/db"}, {"path": "/shared", "source": "db"}], + Resolved( + value=[ + {"path": "/db"}, + {"path": "/shared", "source": "db"}, + {"path": "/config"}, + ], + source="db", + ), + ), + ( + KeyRule(db_row="router_settings", kind="db_overlay"), + {"config": 1, "nested": {"config": True, "shared": "config"}, "fallbacks": ["config"]}, + {"db": 2, "nested": {"shared": "db", "db": True}, "fallbacks": []}, + Resolved( + value={ + "config": 1, + "db": 2, + "nested": {"config": True, "shared": "db", "db": True}, + "fallbacks": ["config"], + }, + source="db", + ), + ), + ), +) +def test_resolve_matches_the_config_and_db_precedence_rules( + rule: KeyRule, + yaml_value: object, + db_value: object, + expected: Resolved, +) -> None: + assert resolve(rule, yaml_value, db_value) == expected + + +@pytest.mark.parametrize("rule", tuple(DUAL_SOURCE_KEYS.values())) +def test_resolve_treats_none_from_the_database_as_absent(rule: KeyRule) -> None: + resolved: Final = resolve(rule, "from-config", None) + + assert resolved == Resolved(value="from-config", source="config") + + +def test_resolve_distinguishes_an_absent_config_value_from_a_configured_null() -> None: + absent: Final = resolve(KeyRule(db_row="general_settings", kind="db_wins"), ABSENT, None) + configured_null: Final = resolve(KeyRule(db_row="general_settings", kind="db_wins"), None, None) + + assert absent == Resolved(value=ABSENT, source="unset") + assert configured_null == Resolved(value=None, source="config") + + +def test_resolve_reports_config_db_and_unset_sources() -> None: + rule: Final = KeyRule(db_row="general_settings", kind="db_wins") + sources: Final[tuple[FieldSource, ...]] = ( + resolve(rule, "from-config", None).source, + resolve(rule, "from-config", "from-db").source, + resolve(rule, ABSENT, None).source, + ) + + assert sources == ("config", "db", "unset") diff --git a/tests/test_litellm/proxy/config_resolvers/test_settings_store.py b/tests/test_litellm/proxy/config_resolvers/test_settings_store.py new file mode 100644 index 00000000000..ef94463cc1d --- /dev/null +++ b/tests/test_litellm/proxy/config_resolvers/test_settings_store.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from litellm.proxy.config_resolvers.settings_store import SettingsStore + + +def test_settings_store_matches_plain_dict_mapping_operations() -> None: + store: Final = SettingsStore("general_settings") + + store["none"] = None + store["false"] = False + store["zero"] = 0 + store["empty_list"] = [] + store["empty_string"] = "" + store.update({"updated": "value"}) + defaulted: Final = store.setdefault("defaulted", "default") + existing: Final = store.setdefault("updated", "other") + popped: Final = store.pop("updated") + + assert defaulted == "default" + assert existing == "value" + assert popped == "value" + assert store.get("missing") is None + assert store["none"] is None + assert "false" in store + assert tuple(store) == ("none", "false", "zero", "empty_list", "empty_string", "defaulted") + assert len(store) == 6 + assert dict(store) == { + "none": None, + "false": False, + "zero": 0, + "empty_list": [], + "empty_string": "", + "defaulted": "default", + } + + +def test_settings_store_preserves_falsy_config_values_and_provenance() -> None: + store: Final = SettingsStore("general_settings") + yaml_values: Final = {"none": None, "false": False, "zero": 0, "empty_list": [], "empty_string": ""} + + store.load_yaml(yaml_values) + + assert dict(store) == yaml_values + assert tuple(store.source(key) for key in yaml_values) == ("config",) * len(yaml_values) + + +@pytest.mark.parametrize( + ("yaml_value", "db_value", "expected_value", "expected_source"), + ( + ("from-config", "from-db", "from-db", "db"), + ("from-config", None, "from-config", "config"), + (None, "from-db", "from-db", "db"), + (None, None, None, "config"), + ), +) +def test_settings_store_resolves_a_db_row_with_provenance( + yaml_value: object, + db_value: object, + expected_value: object, + expected_source: str, +) -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"ordinary": yaml_value}) + store.apply_db_row("general_settings", {"ordinary": db_value}) + + assert store["ordinary"] == expected_value + assert store.source("ordinary") == expected_source + + +def test_settings_store_applies_the_registered_config_precedence_rule() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"max_file_size_mb": 7, "max_parallel_requests": 3}) + store.apply_db_row("general_settings", {"max_file_size_mb": 9, "max_parallel_requests": 11}) + + assert dict(store) == {"max_file_size_mb": 7, "max_parallel_requests": 11} + assert store.source("max_file_size_mb") == "config" + assert store.source("max_parallel_requests") == "db" + + +def test_settings_store_resolved_view_is_read_only() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"configured": "value"}) + resolved: Final = store.resolved() + + with pytest.raises(TypeError): + resolved["configured"] = "changed" + + assert store["configured"] == "value" + + +def test_settings_store_starts_with_an_unset_source() -> None: + store: Final = SettingsStore("general_settings") + + assert store.source("unknown") == "unset"