diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index a6ef7de07ae..643f8d69300 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -190,6 +190,8 @@ class LitellmTableNames(str, enum.Enum): CACHE_CONFIG_TABLE_NAME = "LiteLLM_CacheConfig" CONFIG_OVERRIDES_TABLE_NAME = "LiteLLM_ConfigOverrides" CONFIG_TABLE_NAME = "LiteLLM_Config" + SSO_CONFIG_TABLE_NAME = "LiteLLM_SSOConfig" + UI_SETTINGS_TABLE_NAME = "LiteLLM_UISettings" class Litellm_EntityType(enum.Enum): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0158f601d32..2f6c48a751b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13962,6 +13962,7 @@ async def update_config( # effect of auto-enabling slack alerting. if config_info.general_settings is not None: existing = await _read_section("general_settings") + before_general_settings = copy.deepcopy(existing) updates = config_info.general_settings.dict(exclude_none=True) for k, v in updates.items(): if k == "alert_to_webhook_url": @@ -13971,6 +13972,11 @@ async def update_config( existing["alerting"].append("slack") existing[k] = v await _upsert_section("general_settings", existing) + asyncio.create_task( + create_config_audit_log( + "general_settings", "updated", before_general_settings, existing, user_api_key_dict + ) + ) # environment_variables: idempotently encrypt the request values # (plaintext on first write, OR ciphertext the UI read back via @@ -13979,10 +13985,16 @@ async def update_config( # their stored ciphertext byte-for-byte. if config_info.environment_variables is not None: existing = await _read_section("environment_variables") + before_environment_variables = copy.deepcopy(existing) existing.update( proxy_config._encrypt_env_variables_for_db(environment_variables=config_info.environment_variables) ) await _upsert_section("environment_variables", existing) + asyncio.create_task( + create_config_audit_log( + "environment_variables", "updated", before_environment_variables, existing, user_api_key_dict + ) + ) # litellm_settings: merge existing + request, request wins (matching # router_settings semantics — the caller's value for any given key is @@ -13994,6 +14006,7 @@ async def update_config( # entries that delete_callback (lowercase lookup) cannot find. if config_info.litellm_settings is not None: existing = await _read_section("litellm_settings") + before_litellm_settings = copy.deepcopy(existing) updated_litellm_settings = dict(config_info.litellm_settings) incoming_cb = updated_litellm_settings.get("success_callback") @@ -14015,12 +14028,24 @@ async def update_config( merged["success_callback"] = list(set(incoming_cb)) await _upsert_section("litellm_settings", merged) + asyncio.create_task( + create_config_audit_log( + "litellm_settings", "updated", before_litellm_settings, merged, user_api_key_dict + ) + ) # router_settings: merge existing + request, request wins. if config_info.router_settings is not None: existing = await _read_section("router_settings") + before_router_settings = copy.deepcopy(existing) updates = config_info.router_settings.dict(exclude_none=True) - await _upsert_section("router_settings", {**existing, **updates}) + new_router_settings = {**existing, **updates} + await _upsert_section("router_settings", new_router_settings) + asyncio.create_task( + create_config_audit_log( + "router_settings", "updated", before_router_settings, new_router_settings, user_api_key_dict + ) + ) await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) @@ -14152,6 +14177,8 @@ async def update_config_general_settings( else: general_settings = dict(db_general_settings.param_value) + before_general_settings = copy.deepcopy(general_settings) + ## update db field_value = data.field_value @@ -14171,6 +14198,11 @@ async def update_config_general_settings( }, ) await invalidate_config_param("general_settings") + asyncio.create_task( + create_config_audit_log( + "general_settings", "updated", before_general_settings, general_settings, user_api_key_dict + ) + ) if data.field_name == "plugins": register_plugins_from_config(general_settings) @@ -14555,6 +14587,8 @@ async def delete_config_general_settings( else: general_settings = dict(db_general_settings.param_value) + before_general_settings = copy.deepcopy(general_settings) + ## update db general_settings.pop(data.field_name, None) @@ -14570,6 +14604,11 @@ async def delete_config_general_settings( }, ) await invalidate_config_param("general_settings") + asyncio.create_task( + create_config_audit_log( + "general_settings", "deleted", before_general_settings, general_settings, user_api_key_dict + ) + ) return response @@ -14627,6 +14666,8 @@ async def delete_callback( detail={"error": f"Callback '{callback_name}' not found in active configuration"}, ) + before_success_callbacks = list(success_callbacks) + # Remove callback from success_callback list success_callbacks.remove(callback_name) config.setdefault("litellm_settings", {})["success_callback"] = success_callbacks @@ -14634,6 +14675,16 @@ async def delete_callback( # Save the updated configuration await proxy_config.save_config(new_config=config) + asyncio.create_task( + create_config_audit_log( + "litellm_settings", + "deleted", + {"success_callback": before_success_callbacks}, + {"success_callback": success_callbacks}, + user_api_key_dict, + ) + ) + # Restart the proxy to apply changes await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 1be17c86123..0fc303737b4 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -323,8 +323,12 @@ async def get_allowed_ips(): tags=["Budget & Spend Tracking"], dependencies=[Depends(user_api_key_auth)], ) -async def add_allowed_ip(ip_address: IPAddress): +async def add_allowed_ip( + ip_address: IPAddress, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): from litellm.proxy.proxy_server import ( + create_config_audit_log, general_settings, prisma_client, proxy_config, @@ -356,11 +360,22 @@ async def add_allowed_ip(ip_address: IPAddress): if "allowed_ips" not in config["general_settings"]: config["general_settings"]["allowed_ips"] = [] + before_allowed_ips = list(config["general_settings"]["allowed_ips"]) if ip_address.ip not in config["general_settings"]["allowed_ips"]: config["general_settings"]["allowed_ips"].append(ip_address.ip) await proxy_config.save_config(new_config=config) + asyncio.create_task( + create_config_audit_log( + param_name="general_settings", + action="updated", + before_value={"allowed_ips": before_allowed_ips}, + after_value={"allowed_ips": config["general_settings"]["allowed_ips"]}, + user_api_key_dict=user_api_key_dict, + ) + ) + return { "message": f"IP {ip_address.ip} address added successfully", "status": "success", @@ -372,8 +387,15 @@ async def add_allowed_ip(ip_address: IPAddress): tags=["Budget & Spend Tracking"], dependencies=[Depends(user_api_key_auth)], ) -async def delete_allowed_ip(ip_address: IPAddress): - from litellm.proxy.proxy_server import general_settings, proxy_config +async def delete_allowed_ip( + ip_address: IPAddress, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + from litellm.proxy.proxy_server import ( + create_config_audit_log, + general_settings, + proxy_config, + ) _allowed_ips: List = general_settings.get("allowed_ips", []) if ip_address.ip in _allowed_ips: @@ -391,11 +413,22 @@ async def delete_allowed_ip(ip_address: IPAddress): if "allowed_ips" not in config["general_settings"]: config["general_settings"]["allowed_ips"] = [] + before_allowed_ips = list(config["general_settings"]["allowed_ips"]) if ip_address.ip in config["general_settings"]["allowed_ips"]: config["general_settings"]["allowed_ips"].remove(ip_address.ip) await proxy_config.save_config(new_config=config) + asyncio.create_task( + create_config_audit_log( + param_name="general_settings", + action="deleted", + before_value={"allowed_ips": before_allowed_ips}, + after_value={"allowed_ips": config["general_settings"]["allowed_ips"]}, + user_api_key_dict=user_api_key_dict, + ) + ) + return {"message": f"IP {ip_address.ip} deleted successfully", "status": "success"} @@ -554,7 +587,7 @@ async def _update_litellm_setting( settings: Union[DefaultInternalUserParams, DefaultTeamSSOParams, MCPSemanticFilterSettings], settings_key: str, success_message: str, - user_api_key_dict: Optional[UserAPIKeyAuth] = None, + user_api_key_dict: UserAPIKeyAuth, ): """ Common utility function to update `litellm_settings` in both memory and config. @@ -564,8 +597,6 @@ async def _update_litellm_setting( settings_key: The key in litellm_settings to update success_message: Message to return on success user_api_key_dict: The acting admin, recorded as the audit-log actor. - Optional today so callers that have not been wired for auditing - keep working; the audit row is only written when an actor is passed. """ from litellm.proxy.proxy_server import ( create_config_audit_log, @@ -599,20 +630,19 @@ async def _update_litellm_setting( # Save the updated config await proxy_config.save_config(new_config=config) - if user_api_key_dict is not None: - # Fire-and-forget so an audit-log failure (transient DB blip, etc.) - # never surfaces as a 500 after save_config has already committed, - # matching the create_object_audit_log pattern used elsewhere - # (e.g. model_management_endpoints). - asyncio.create_task( - create_config_audit_log( - param_name=settings_key, - action="updated", - before_value=before_value, - after_value=in_memory_var, - user_api_key_dict=user_api_key_dict, - ) + # Fire-and-forget so an audit-log failure (transient DB blip, etc.) + # never surfaces as a 500 after save_config has already committed, + # matching the create_object_audit_log pattern used elsewhere + # (e.g. model_management_endpoints). + asyncio.create_task( + create_config_audit_log( + param_name=settings_key, + action="updated", + before_value=before_value, + after_value=in_memory_var, + user_api_key_dict=user_api_key_dict, ) + ) return { "message": success_message, @@ -653,7 +683,10 @@ async def update_internal_user_settings( tags=["SSO Settings"], dependencies=[Depends(user_api_key_auth)], ) -async def update_default_team_settings(settings: DefaultTeamSSOParams): +async def update_default_team_settings( + settings: DefaultTeamSSOParams, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Update the default team parameters for SSO users. These settings will be applied to new teams created from SSO. @@ -662,6 +695,7 @@ async def update_default_team_settings(settings: DefaultTeamSSOParams): settings=settings, settings_key="default_team_params", success_message="Default team settings updated successfully", + user_api_key_dict=user_api_key_dict, ) @@ -772,7 +806,10 @@ async def get_sso_settings(): tags=["SSO Settings"], dependencies=[Depends(user_api_key_auth)], ) -async def update_sso_settings(sso_config: SSOConfig): +async def update_sso_settings( + sso_config: SSOConfig, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Update SSO configuration by saving to the dedicated SSO table. """ @@ -780,6 +817,7 @@ async def update_sso_settings(sso_config: SSOConfig): import os from litellm.proxy.proxy_server import ( + create_config_audit_log, prisma_client, proxy_config, store_model_in_db, @@ -812,6 +850,20 @@ async def update_sso_settings(sso_config: SSOConfig): "proxy_base_url": "PROXY_BASE_URL", } + # Read the existing SSO row first so the audit log captures a real + # before/after diff. Stored values are encrypted; decrypt them so the + # before-snapshot has the same shape as after_value, and rely on + # create_config_audit_log's secret-name redaction to mask the + # *_client_secret fields before the audit row is written. + existing_sso_record = await SSOConfigRepository(prisma_client).table.find_unique(where={"id": "sso_config"}) + before_sso_data: Optional[Dict[str, Any]] = None + if existing_sso_record and existing_sso_record.sso_settings: + stored = existing_sso_record.sso_settings + if isinstance(stored, str): + stored = json.loads(stored) + if isinstance(stored, dict): + before_sso_data = proxy_config._decrypt_db_variables(stored) + # Load existing config config = await proxy_config.get_config() @@ -850,6 +902,17 @@ async def update_sso_settings(sso_config: SSOConfig): }, ) + asyncio.create_task( + create_config_audit_log( + param_name="sso_config", + action="updated", + before_value=before_sso_data, + after_value=sso_data, + user_api_key_dict=user_api_key_dict, + table_name=LitellmTableNames.SSO_CONFIG_TABLE_NAME, + ) + ) + # Remove SSO-related env vars from config.environment_variables try: env_var_entry = await ConfigRepository(prisma_client).table.find_unique( @@ -943,14 +1006,21 @@ def _validate_public_image_url(value: Optional[str], field_name: str) -> None: tags=["UI Theme Settings"], dependencies=[Depends(user_api_key_auth)], ) -async def update_ui_theme_settings(theme_config: UIThemeConfig): +async def update_ui_theme_settings( + theme_config: UIThemeConfig, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Update UI theme configuration. Updates logo settings for the admin UI. """ import os - from litellm.proxy.proxy_server import proxy_config, store_model_in_db + from litellm.proxy.proxy_server import ( + create_config_audit_log, + proxy_config, + store_model_in_db, + ) _validate_public_image_url(theme_config.logo_url, "logo_url") _validate_public_image_url(theme_config.favicon_url, "favicon_url") @@ -963,6 +1033,7 @@ async def update_ui_theme_settings(theme_config: UIThemeConfig): # Load existing config config = await proxy_config.get_config() + before_theme = config.get("litellm_settings", {}).get("ui_theme_config") # Update config with UI theme settings if "general_settings" not in config: @@ -1029,6 +1100,16 @@ async def update_ui_theme_settings(theme_config: UIThemeConfig): # Save the updated config await proxy_config.save_config(new_config=stored_config) + asyncio.create_task( + create_config_audit_log( + param_name="ui_theme_config", + action="updated", + before_value=before_theme, + after_value=theme_data, + user_api_key_dict=user_api_key_dict, + ) + ) + return { "message": "UI theme settings updated successfully.", "status": "success", @@ -1083,6 +1164,7 @@ async def update_mcp_semantic_filter_settings( settings=settings, settings_key="mcp_semantic_tool_filter", success_message="MCP Semantic Filter settings updated successfully. Changes will be applied across all pods within 10 seconds.", + user_api_key_dict=user_api_key_dict, ) try: from litellm.proxy.proxy_server import prisma_client, proxy_config @@ -1200,7 +1282,11 @@ async def update_ui_settings( Update UI-specific configuration flags. Only proxy admins are allowed to modify these settings. """ - from litellm.proxy.proxy_server import prisma_client, store_model_in_db + from litellm.proxy.proxy_server import ( + create_config_audit_log, + prisma_client, + store_model_in_db, + ) if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException(status_code=403, detail="Only proxy admins can update UI settings.") @@ -1282,6 +1368,17 @@ async def update_ui_settings( sanitized = {k: v for k, v in ui_settings.items() if k in ALLOWED_UI_SETTINGS_FIELDS} await user_api_key_cache.async_set_cache(key=UI_SETTINGS_CACHE_KEY, value=sanitized, ttl=UI_SETTINGS_CACHE_TTL) + asyncio.create_task( + create_config_audit_log( + param_name="ui_settings", + action="updated", + before_value=existing, + after_value=ui_settings, + user_api_key_dict=user_api_key_dict, + table_name=LitellmTableNames.UI_SETTINGS_TABLE_NAME, + ) + ) + return { "message": "UI settings updated successfully", "status": "success", diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index 7f5aee51f51..f39ff93cee7 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -258,6 +258,7 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp ) import litellm + from litellm.proxy._types import UserAPIKeyAuth settings = DefaultInternalUserParams( user_role=LitellmUserRoles.INTERNAL_USER, @@ -266,6 +267,7 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp settings=settings, settings_key="default_internal_user_params", success_message="ok", + user_api_key_dict=UserAPIKeyAuth(user_id="test-admin"), ) # Verify the in-memory variable was actually updated diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py index 443089b5f01..e0b90332ca0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py @@ -426,6 +426,7 @@ class TestUpdateLitellmSettingOrdering: settings=new_settings, settings_key="default_team_params", success_message="Updated", + user_api_key_dict=UserAPIKeyAuth(user_id="test-admin"), ) # In-memory value should be the NEW value, not the stale one @@ -459,6 +460,7 @@ class TestUpdateLitellmSettingOrdering: settings=DefaultTeamSSOParams(max_budget=100.0), settings_key="default_team_params", success_message="Updated", + user_api_key_dict=UserAPIKeyAuth(user_id="test-admin"), ) assert exc_info.value.status_code == 500 diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index dc35d71ccbd..88d9ad0d968 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -8764,3 +8764,278 @@ def test_dump_redacted_config_serializes_non_json_native_values(): restored = json.loads(out) assert "2026-06-30" in restored["updated_at"] + +@pytest.mark.asyncio +async def test_update_config_general_settings_emits_audit_log(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.proxy_server import update_config_general_settings + + existing = {"max_parallel_requests": 5, "some_api_key": "sk-stored-secret"} + fake = _fake_prisma_with_config(existing) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + admin = UserAPIKeyAuth( + api_key="hashed-admin", + user_id="admin-1", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + await update_config_general_settings( + data=ConfigFieldUpdate( + field_name="max_parallel_requests", + field_value=42, + config_type="general_settings", + ), + user_api_key_dict=admin, + ) + # Audit is scheduled via asyncio.create_task; yield so it runs. + await asyncio.sleep(0) + + fake.db.litellm_auditlog.create.assert_awaited_once() + written = fake.db.litellm_auditlog.create.call_args.kwargs["data"] + assert written["table_name"] == "LiteLLM_Config" + assert written["object_id"] == "general_settings" + assert written["action"] == "updated" + assert written["changed_by"] == "admin-1" + + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert before["max_parallel_requests"] == 5 + assert after["max_parallel_requests"] == 42 + assert "sk-stored-secret" not in written["before_value"] + assert "sk-stored-secret" not in written["updated_values"] + assert before["some_api_key"] != "sk-stored-secret" + + +@pytest.mark.asyncio +async def test_delete_config_general_settings_emits_deleted_audit_log(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import ConfigFieldDelete + from litellm.proxy.proxy_server import delete_config_general_settings + + existing = {"max_parallel_requests": 5} + fake = _fake_prisma_with_config(existing) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + admin = UserAPIKeyAuth( + api_key="hashed-admin", + user_id="admin-1", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + await delete_config_general_settings( + data=ConfigFieldDelete( + field_name="max_parallel_requests", config_type="general_settings" + ), + user_api_key_dict=admin, + ) + # Audit is scheduled via asyncio.create_task; yield so it runs. + await asyncio.sleep(0) + + fake.db.litellm_auditlog.create.assert_awaited_once() + written = fake.db.litellm_auditlog.create.call_args.kwargs["data"] + assert written["object_id"] == "general_settings" + assert written["action"] == "deleted" + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert before["max_parallel_requests"] == 5 + assert "max_parallel_requests" not in after + + +def test_update_config_audits_every_written_section(_update_config_setup, monkeypatch): + """/config/update must emit one audit row per section it writes, so each + of the four call sites (general_settings, environment_variables, + litellm_settings, router_settings) is mutation-protected. litellm_settings + is the row that holds default_internal_user_params ("default user settings").""" + import litellm.proxy.proxy_server as proxy_server_module + + client, prisma, restore = _update_config_setup( + initial_rows={"litellm_settings": {"drop_params": True}} + ) + audit_create = AsyncMock() + prisma.db.litellm_auditlog.create = audit_create + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + try: + resp = client.post( + "/config/update", + json={ + "general_settings": {"store_prompts_in_spend_logs": True}, + "environment_variables": {"FOO": "bar"}, + "litellm_settings": { + "default_internal_user_params": {"max_budget": 10} + }, + "router_settings": {"routing_strategy": "latency-based-routing"}, + }, + ) + assert resp.status_code == 200, resp.text + + audited = { + call.kwargs["data"]["object_id"]: call.kwargs["data"]["action"] + for call in audit_create.await_args_list + } + assert audited == { + "general_settings": "updated", + "environment_variables": "updated", + "litellm_settings": "updated", + "router_settings": "updated", + } + for call in audit_create.await_args_list: + assert call.kwargs["data"]["table_name"] == "LiteLLM_Config" + assert call.kwargs["data"]["changed_by"] == "test_admin" + + ls_call = next( + c + for c in audit_create.await_args_list + if c.kwargs["data"]["object_id"] == "litellm_settings" + ) + after = json.loads(ls_call.kwargs["data"]["updated_values"]) + assert after["default_internal_user_params"] == {"max_budget": 10} + finally: + restore() + + +def test_delete_callback_audits_litellm_settings_deletion( + _update_config_setup, monkeypatch +): + """/config/callback/delete must emit a deleted audit row for litellm_settings + capturing the success_callback list before and after removal.""" + import litellm.proxy.proxy_server as proxy_server_module + + client, prisma, restore = _update_config_setup() + audit_create = AsyncMock() + prisma.db.litellm_auditlog.create = audit_create + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + from litellm.proxy.proxy_server import proxy_config as real_proxy_config + + monkeypatch.setattr( + real_proxy_config, + "get_config", + AsyncMock( + return_value={ + "litellm_settings": {"success_callback": ["langfuse", "datadog"]} + } + ), + ) + monkeypatch.setattr( + real_proxy_config, "save_config", AsyncMock(return_value=None) + ) + try: + resp = client.post( + "/config/callback/delete", json={"callback_name": "datadog"} + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "litellm_settings" + assert written["action"] == "deleted" + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert before["success_callback"] == ["langfuse", "datadog"] + assert after["success_callback"] == ["langfuse"] + finally: + restore() + + +def test_delete_callback_audits_before_reload_failure(_update_config_setup, monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + + client, prisma, restore = _update_config_setup() + audit_create = AsyncMock() + prisma.db.litellm_auditlog.create = audit_create + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + from litellm.proxy.proxy_server import proxy_config as real_proxy_config + + monkeypatch.setattr( + real_proxy_config, + "get_config", + AsyncMock( + return_value={ + "litellm_settings": {"success_callback": ["langfuse", "datadog"]} + } + ), + ) + monkeypatch.setattr( + real_proxy_config, "save_config", AsyncMock(return_value=None) + ) + monkeypatch.setattr( + real_proxy_config, + "add_deployment", + AsyncMock(side_effect=RuntimeError("reload failed")), + ) + try: + resp = client.post( + "/config/callback/delete", json={"callback_name": "datadog"} + ) + assert resp.status_code == 500, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "litellm_settings" + assert written["action"] == "deleted" + finally: + restore() + + +def test_update_config_redacts_all_environment_variable_values( + _update_config_setup, monkeypatch +): + """environment_variables hold credentials under arbitrary uppercase keys + (DATABASE_URL) that key-name secret matching misses, so every value in the + section must be redacted before the audit row is written; a plaintext + secret must never reach LiteLLM_AuditLog.""" + import litellm.proxy.proxy_server as proxy_server_module + + # DATABASE_URL is the bug class: an uppercase env key that key-name secret + # matching does NOT flag, so only whole-section value redaction protects it. + client, prisma, restore = _update_config_setup( + initial_rows={ + "environment_variables": { + "DATABASE_URL": "enc:postgresql://OLDsecret@old.host:5432/db" + } + } + ) + audit_create = AsyncMock() + prisma.db.litellm_auditlog.create = audit_create + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + try: + resp = client.post( + "/config/update", + json={ + "environment_variables": { + "DATABASE_URL": "postgresql://u:p@db.internal:5432/litellm", + "LOG_LEVEL": "debug", + } + }, + ) + assert resp.status_code == 200, resp.text + + env_call = next( + c + for c in audit_create.await_args_list + if c.kwargs["data"]["object_id"] == "environment_variables" + ) + data = env_call.kwargs["data"] + + # the pre-existing secret must be redacted in the before snapshot + before = json.loads(data["before_value"]) + assert before == {"DATABASE_URL": "REDACTED"} + assert "OLDsecret" not in data["before_value"] + assert "old.host" not in data["before_value"] + + # the newly-written values must be redacted in the after snapshot + after = json.loads(data["updated_values"]) + assert after == {"DATABASE_URL": "REDACTED", "LOG_LEVEL": "REDACTED"} + assert "postgresql://" not in data["updated_values"] + assert "db.internal" not in data["updated_values"] + finally: + restore() diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 7a586f758f4..cb77c42fe9b 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -396,6 +396,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -466,6 +467,62 @@ class TestProxySettingEndpoints: create_sso_settings = json.loads(create_data["sso_settings"]) assert create_sso_settings["google_client_id"] == "new_google_client_id" + def test_update_sso_settings_audits_when_env_cleanup_fails( + self, mock_proxy_config, mock_auth, monkeypatch + ): + import json + from unittest.mock import AsyncMock, MagicMock + + monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key") + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) + mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() + mock_prisma.db.litellm_config = MagicMock() + mock_prisma.db.litellm_config.find_unique = AsyncMock( + side_effect=ValueError("cleanup failed") + ) + mock_prisma.db.litellm_config.update = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + from litellm.proxy.proxy_server import proxy_config + + monkeypatch.setattr( + proxy_config, + "_encrypt_env_variables", + lambda environment_variables: environment_variables, + ) + + create_config_audit_log = AsyncMock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.create_config_audit_log", + create_config_audit_log, + ) + + response = client.patch( + "/update/sso_settings", + json={"google_client_id": "new_google_client_id"}, + ) + + assert response.status_code == 500 + assert mock_prisma.db.litellm_ssoconfig.upsert.called + create_config_audit_log.assert_awaited_once() + audit_log_kwargs = create_config_audit_log.await_args.kwargs + assert audit_log_kwargs["param_name"] == "sso_config" + assert ( + audit_log_kwargs["after_value"]["google_client_id"] + == "new_google_client_id" + ) + assert ( + json.loads( + mock_prisma.db.litellm_ssoconfig.upsert.call_args.kwargs["data"][ + "create" + ]["sso_settings"] + )["google_client_id"] + == "new_google_client_id" + ) + def test_update_sso_settings_with_null_values_clears_env_vars( self, mock_proxy_config, mock_auth, monkeypatch ): @@ -478,6 +535,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() @@ -557,6 +615,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() env_var_entry = MagicMock() @@ -627,6 +686,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() @@ -704,6 +764,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -1350,6 +1411,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() upsert_mock = AsyncMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = upsert_mock mock_prisma.db.litellm_config = MagicMock() mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -1429,6 +1491,7 @@ class TestProxySettingEndpoints: mock_prisma = MagicMock() mock_prisma.db = MagicMock() mock_prisma.db.litellm_ssoconfig = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() env_var_entry = MagicMock() @@ -1480,6 +1543,7 @@ class TestProxySettingEndpoints: mock_prisma = MagicMock() mock_prisma.db = MagicMock() mock_prisma.db.litellm_ssoconfig = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() env_var_entry = MagicMock() @@ -1651,6 +1715,7 @@ class TestProxySettingEndpoints: # Mock the prisma client mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() mock_prisma.db.litellm_config = MagicMock() mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) @@ -1960,3 +2025,345 @@ def test_update_internal_user_settings_returns_200_when_audit_write_raises( assert resp.json()["status"] == "success" finally: app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_update_sso_settings_writes_redacted_audit_log(mock_proxy_config, monkeypatch): + """Updating SSO settings must write an audit row to the SSO config table + with the client secret redacted.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + fake_prisma.db.litellm_ssoconfig.upsert = AsyncMock() + # No prior SSO row, so before_value resolves to None. + fake_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) + fake_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr( + proxy_server_module.proxy_config, + "_encrypt_env_variables", + lambda environment_variables: environment_variables, + ) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/sso_settings", + json={ + "google_client_id": "client-id-123", + "google_client_secret": "super-secret-xyz", + }, + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "sso_config" + assert written["table_name"] == "LiteLLM_SSOConfig" + assert written["changed_by"] == "audit-admin" + + after = json.loads(written["updated_values"]) + assert after["google_client_id"] == "client-id-123" + assert after["google_client_secret"] == "REDACTED" + assert "super-secret-xyz" not in written["updated_values"] + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_update_sso_settings_audit_captures_redacted_before_snapshot( + mock_proxy_config, monkeypatch +): + """An auditor reviewing an SSO secret rotation needs to see a real + before/after diff in the audit row, not before_value=None. The endpoint + reads the existing (encrypted) SSO row, decrypts it, and lets the audit + helper redact the *_client_secret fields before persistence so neither + the old nor the new plaintext secret is recorded.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + fake_prisma.db.litellm_ssoconfig.upsert = AsyncMock() + + # Pre-existing SSO row contains the *prior* secret (would be ciphertext in + # production; the test patches _decrypt_db_variables to pass through). + existing_record = MagicMock() + existing_record.sso_settings = { + "google_client_id": "old-client-id", + "google_client_secret": "OLD-SUPER-SECRET", + } + fake_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=existing_record) + fake_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr( + proxy_server_module.proxy_config, + "_encrypt_env_variables", + lambda environment_variables: environment_variables, + ) + # Pretend the stored value is already plaintext for the test (production + # decrypts via Fernet); the audit helper still has to redact it. + monkeypatch.setattr( + proxy_server_module.proxy_config, + "_decrypt_db_variables", + lambda variables_dict: dict(variables_dict), + ) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/sso_settings", + json={ + "google_client_id": "new-client-id", + "google_client_secret": "NEW-SUPER-SECRET", + }, + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + + # Non-secret field shows the diff + assert before["google_client_id"] == "old-client-id" + assert after["google_client_id"] == "new-client-id" + + # Secret field is redacted in BOTH snapshots — auditor sees the + # rotation event without ever seeing either plaintext secret. + assert before["google_client_secret"] == "REDACTED" + assert after["google_client_secret"] == "REDACTED" + assert "OLD-SUPER-SECRET" not in written["before_value"] + assert "NEW-SUPER-SECRET" not in written["updated_values"] + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_add_allowed_ip_writes_audit_log(mock_proxy_config, monkeypatch): + """Adding an allowed IP is a system-wide security setting change and must + be audited with the before and after IP list.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(proxy_server_module, "general_settings", {}) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.post("/add/allowed_ip", json={"ip": "203.0.113.77"}) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "general_settings" + assert written["action"] == "updated" + assert written["changed_by"] == "audit-admin" + + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert "203.0.113.77" not in before["allowed_ips"] + assert "203.0.113.77" in after["allowed_ips"] + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_delete_allowed_ip_writes_deleted_audit_log(monkeypatch): + """Removing an allowed IP must be audited as a deletion, symmetric with the + add path.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + + config = {"general_settings": {"allowed_ips": ["203.0.113.77", "198.51.100.1"]}} + + async def _get_config(): + return config + + async def _save_config(new_config=None): + nonlocal config + if new_config is not None: + config = new_config + return config + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr( + proxy_server_module, "general_settings", {"allowed_ips": ["203.0.113.77"]} + ) + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr(proxy_server_module.proxy_config, "get_config", _get_config) + monkeypatch.setattr(proxy_server_module.proxy_config, "save_config", _save_config) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.post("/delete/allowed_ip", json={"ip": "203.0.113.77"}) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "general_settings" + assert written["action"] == "deleted" + before = json.loads(written["before_value"]) + after = json.loads(written["updated_values"]) + assert "203.0.113.77" in before["allowed_ips"] + assert "203.0.113.77" not in after["allowed_ips"] + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_update_ui_theme_settings_writes_audit_log(mock_proxy_config, monkeypatch): + """Updating the UI theme must be audited under ui_theme_config.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr( + proxy_server_module.proxy_config, + "_encrypt_env_variables", + lambda environment_variables: environment_variables, + ) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/ui_theme_settings", + json={"logo_url": "https://example.com/logo.png"}, + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "ui_theme_config" + assert written["action"] == "updated" + assert written["changed_by"] == "audit-admin" + after = json.loads(written["updated_values"]) + assert after["logo_url"] == "https://example.com/logo.png" + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_update_ui_settings_writes_audit_log(monkeypatch): + """Updating UI settings must be audited under the UI settings table.""" + from unittest.mock import AsyncMock, MagicMock + + import litellm + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + audit_create = AsyncMock() + fake_prisma = MagicMock() + fake_prisma.db.litellm_auditlog.create = audit_create + fake_prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=None) + fake_prisma.db.litellm_uisettings.upsert = AsyncMock() + + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "store_audit_logs", True) + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="audit-admin", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.patch( + "/update/ui_settings", json={"disable_custom_api_keys": True} + ) + assert resp.status_code == 200, resp.text + + audit_create.assert_awaited_once() + written = audit_create.await_args.kwargs["data"] + assert written["object_id"] == "ui_settings" + assert written["table_name"] == "LiteLLM_UISettings" + assert written["changed_by"] == "audit-admin" + after = json.loads(written["updated_values"]) + assert after["disable_custom_api_keys"] is True + finally: + app.dependency_overrides.pop(user_api_key_auth, None)