mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
feat(guardrails): write guardrail changes to the audit log (#51118)
* feat(guardrails): write guardrail changes to the audit log Creating, updating, patching or deleting a guardrail, and registering, approving or rejecting a team guardrail submission, now each write one row to LiteLLM_AuditLog with the guardrail before and after the change. Credentials the guardrails table encrypts at rest are redacted from those rows, and the routes take the litellm-changed-by header like key, team and model routes do. The Admin UI audit page gets a Guardrails table filter. Resolves LIT-10639 * fix(guardrails): audit an allowlisted guardrail snapshot and drop litellm-changed-by Guardrail audit rows now keep the guardrail id, name, info, status, team and the provider, mode and default_on settings, and name (never show) any other provider setting an update changed. Provider settings can hold credentials under any name, for example an apikey header, so no other value is written. The changer always comes from the caller's API key, so the litellm-changed-by header is gone. PUT audits the saved guardrail instead of the copy the live sync adds headers to. Resolves LIT-10639 * fix(guardrails): never let the audit snapshot fail a saved guardrail change The audit snapshot now reads guardrail settings stored as a JSON string and records empty settings when it cannot read them, so a saved approve or reject can no longer turn into a 500. Only the provider, mode and default_on settings are ever copied, so this cannot leak a credential. The create entry also records the active status the new guardrail is saved with. Resolves LIT-10639 * test(guardrails): run the guardrail audit tests through the real registry and in-memory handler The audit tests no longer replace GUARDRAIL_REGISTRY or the in-memory handler, and no test edits its inputs. Only the Prisma guardrails table is faked, returning exactly what the registry writes, so the PUT test now checks the real live sync adding x-api-key to the saved guardrail's headers. Resolves LIT-10639
This commit is contained in:
mrinal-berri
GitHub
parent
82471d4603
commit
43daa2598e
6 changed files with 421 additions and 10 deletions
|
|
@ -211,6 +211,7 @@ class LitellmTableNames(str, enum.Enum):
|
|||
SSO_CONFIG_TABLE_NAME = "LiteLLM_SSOConfig"
|
||||
UI_SETTINGS_TABLE_NAME = "LiteLLM_UISettings"
|
||||
AGENT_TABLE_NAME = "LiteLLM_AgentsTable"
|
||||
GUARDRAILS_TABLE_NAME = "LiteLLM_GuardrailsTable"
|
||||
|
||||
|
||||
from litellm.types.integrations.slack_alerting import (
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ CRUD ENDPOINTS FOR GUARDRAILS
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
|
|
@ -19,7 +20,7 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy._types import AUDIT_ACTIONS, LitellmTableNames, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX
|
||||
from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
|
||||
|
|
@ -53,6 +54,7 @@ from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401 # l
|
|||
_user_has_admin_view, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
user_api_key_has_admin_view,
|
||||
)
|
||||
from litellm.proxy.management_helpers.audit_logs import create_object_audit_log, is_audit_logging_enabled
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.table_repositories import GuardrailsRepository
|
||||
from litellm.types.guardrails import (
|
||||
|
|
@ -112,6 +114,84 @@ def _reject_encrypted_litellm_params(litellm_params: object) -> None:
|
|||
)
|
||||
|
||||
|
||||
_AUDIT_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
_AUDITED_GUARDRAIL_FIELDS: Final = ("guardrail_id", "guardrail_name", "guardrail_info", "status", "team_id")
|
||||
_AUDITED_LITELLM_PARAMS: Final = ("guardrail", "mode", "default_on")
|
||||
_NEW_GUARDRAIL_STATUS: Final = "active"
|
||||
|
||||
|
||||
def _as_audit_mapping(value: object) -> Mapping[str, object]:
|
||||
"""Rows can hold JSON columns as strings, and an audit snapshot must never fail a change that is already saved."""
|
||||
candidate: Final = (
|
||||
value.model_dump(exclude_none=True)
|
||||
if isinstance(value, BaseModel)
|
||||
else _parse_json_field(value)
|
||||
if isinstance(value, str)
|
||||
else value
|
||||
)
|
||||
try:
|
||||
return _AUDIT_JSON_OBJECT.validate_python(candidate or {})
|
||||
except ValidationError:
|
||||
return {}
|
||||
|
||||
|
||||
def _litellm_params_of(guardrail: object) -> Mapping[str, object]:
|
||||
return _as_audit_mapping(_as_audit_mapping(guardrail).get("litellm_params"))
|
||||
|
||||
|
||||
def _guardrail_audit_snapshot(guardrail: object) -> Mapping[str, object]:
|
||||
"""A guardrail as written to the audit log. Provider settings can hold credentials under any name, so only the
|
||||
provider, mode and default_on values are kept."""
|
||||
fields: Final = _as_audit_mapping(guardrail)
|
||||
params: Final = _litellm_params_of(guardrail)
|
||||
return {
|
||||
**{key: fields[key] for key in _AUDITED_GUARDRAIL_FIELDS if fields.get(key) is not None},
|
||||
"litellm_params": {key: params[key] for key in _AUDITED_LITELLM_PARAMS if key in params},
|
||||
}
|
||||
|
||||
|
||||
def _changed_litellm_param_names(before: object, after: object) -> tuple[str, ...]:
|
||||
"""Names, never values, of the provider settings an update changed beyond the ones the snapshot shows."""
|
||||
old: Final = _litellm_params_of(before)
|
||||
new: Final = _litellm_params_of(after)
|
||||
return tuple(
|
||||
sorted(
|
||||
key
|
||||
for key in old.keys() | new.keys()
|
||||
if key not in _AUDITED_LITELLM_PARAMS and old.get(key) != new.get(key)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _audit_guardrail_change(
|
||||
*,
|
||||
guardrail_id: str,
|
||||
action: AUDIT_ACTIONS,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
before: object = None,
|
||||
after: object = None,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import litellm_proxy_admin_name
|
||||
|
||||
if not is_audit_logging_enabled():
|
||||
return
|
||||
changed: Final = _changed_litellm_param_names(before, after) if before is not None and after is not None else ()
|
||||
await create_object_audit_log(
|
||||
object_id=guardrail_id,
|
||||
action=action,
|
||||
litellm_changed_by=None,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
litellm_proxy_admin_name=litellm_proxy_admin_name,
|
||||
table_name=LitellmTableNames.GUARDRAILS_TABLE_NAME,
|
||||
before_value=None if before is None else safe_dumps(_guardrail_audit_snapshot(before)),
|
||||
after_value=None
|
||||
if after is None
|
||||
else safe_dumps(
|
||||
{**_guardrail_audit_snapshot(after), **({"changed_litellm_params": list(changed)} if changed else {})}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _guardrails_table(prisma_client: "PrismaClient") -> "TableActions[LiteLLM_GuardrailsTable]":
|
||||
return GuardrailsRepository(prisma_client).table
|
||||
|
||||
|
|
@ -479,6 +559,12 @@ async def create_guardrail(
|
|||
init_error,
|
||||
)
|
||||
|
||||
await _audit_guardrail_change(
|
||||
guardrail_id=str(guardrail_id),
|
||||
action="created",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
after={**result, "status": _NEW_GUARDRAIL_STATUS},
|
||||
)
|
||||
return result
|
||||
except HTTPException:
|
||||
raise
|
||||
|
|
@ -579,6 +665,7 @@ async def update_guardrail(
|
|||
)
|
||||
|
||||
guardrail_name: Final = result.get("guardrail_name", "Unknown")
|
||||
saved_guardrail: Final = copy.deepcopy(dict(result))
|
||||
|
||||
try:
|
||||
IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(
|
||||
|
|
@ -612,6 +699,13 @@ async def update_guardrail(
|
|||
update_error,
|
||||
)
|
||||
|
||||
await _audit_guardrail_change(
|
||||
guardrail_id=guardrail_id,
|
||||
action="updated",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
before=existing_guardrail,
|
||||
after=saved_guardrail,
|
||||
)
|
||||
return result
|
||||
except HTTPException as e:
|
||||
raise e
|
||||
|
|
@ -687,6 +781,12 @@ async def delete_guardrail(
|
|||
delete_error,
|
||||
)
|
||||
|
||||
await _audit_guardrail_change(
|
||||
guardrail_id=guardrail_id,
|
||||
action="deleted",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
before=existing_guardrail,
|
||||
)
|
||||
return result
|
||||
except HTTPException as e:
|
||||
raise e
|
||||
|
|
@ -853,6 +953,12 @@ async def register_guardrail(
|
|||
"updated_at": now,
|
||||
},
|
||||
)
|
||||
await _audit_guardrail_change(
|
||||
guardrail_id=created.guardrail_id,
|
||||
action="created",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
after=created,
|
||||
)
|
||||
return RegisterGuardrailResponse(
|
||||
guardrail_id=created.guardrail_id,
|
||||
guardrail_name=created.guardrail_name,
|
||||
|
|
@ -1095,10 +1201,17 @@ async def approve_guardrail_submission(
|
|||
)
|
||||
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
await _guardrails_table(prisma_client).update(
|
||||
approved: Final = await _guardrails_table(prisma_client).update(
|
||||
where={"guardrail_id": guardrail_id},
|
||||
data={"status": "active", "reviewed_at": now, "updated_at": now},
|
||||
)
|
||||
await _audit_guardrail_change(
|
||||
guardrail_id=guardrail_id,
|
||||
action="updated",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
before=row,
|
||||
after=approved,
|
||||
)
|
||||
|
||||
guardrail_info: Final = _parse_json_field(row.guardrail_info)
|
||||
if not litellm_params:
|
||||
|
|
@ -1174,10 +1287,17 @@ async def reject_guardrail_submission(
|
|||
)
|
||||
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
await _guardrails_table(prisma_client).update(
|
||||
rejected: Final = await _guardrails_table(prisma_client).update(
|
||||
where={"guardrail_id": guardrail_id},
|
||||
data={"status": "rejected", "reviewed_at": now, "updated_at": now},
|
||||
)
|
||||
await _audit_guardrail_change(
|
||||
guardrail_id=guardrail_id,
|
||||
action="updated",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
before=row,
|
||||
after=rejected,
|
||||
)
|
||||
return {
|
||||
"guardrail_id": guardrail_id,
|
||||
"status": "rejected",
|
||||
|
|
@ -1378,6 +1498,13 @@ async def patch_guardrail(
|
|||
update_error,
|
||||
)
|
||||
|
||||
await _audit_guardrail_change(
|
||||
guardrail_id=guardrail_id,
|
||||
action="updated",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
before=existing_guardrail,
|
||||
after=result,
|
||||
)
|
||||
return result
|
||||
except HTTPException as e:
|
||||
raise e
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import json
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import Dict, Final, List, Literal, Optional
|
||||
from unittest.mock import AsyncMock
|
||||
from typing import Final, List, Literal, Optional
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -48,9 +48,9 @@ from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_fil
|
|||
|
||||
MOCK_ADMIN_USER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
from litellm.proxy.guardrails.guardrail_registry import (
|
||||
IN_MEMORY_GUARDRAIL_HANDLER,
|
||||
InMemoryGuardrailHandler,
|
||||
encrypt_guardrail_litellm_params,
|
||||
guardrail_from_db_row,
|
||||
)
|
||||
from litellm.types.guardrails import (
|
||||
ApplyGuardrailRequest,
|
||||
|
|
@ -1020,7 +1020,7 @@ def test_optional_params_not_returned_when_not_overridden():
|
|||
"""Test that optional_params is not returned when the config model doesn't override it"""
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import Field
|
||||
|
||||
from litellm.proxy.guardrails.guardrail_endpoints import _get_fields_from_model
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
|
|
@ -1080,7 +1080,7 @@ def test_optional_params_returned_when_properly_overridden():
|
|||
@pytest.mark.asyncio
|
||||
async def test_bedrock_guardrail_prepare_request_with_api_key():
|
||||
"""Test _prepare_request method uses Bearer token when api_key is provided in data"""
|
||||
from unittest.mock import Mock, patch
|
||||
from unittest.mock import Mock
|
||||
|
||||
from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
|
||||
BedrockGuardrail,
|
||||
|
|
@ -2029,6 +2029,286 @@ async def test_delete_guardrail_endpoint(
|
|||
assert "Failed to remove guardrail" in str(mock_logger.warning.call_args)
|
||||
|
||||
|
||||
PLANTED_API_KEY: Final = "sk-planted-guardrail-api-key"
|
||||
PLANTED_AUTH_HEADER: Final = "Bearer planted-guardrail-auth-header"
|
||||
PLANTED_APIKEY_HEADER: Final = "planted-guardrail-apikey-header"
|
||||
ROTATED_API_KEY: Final = "sk-planted-rotated-api-key"
|
||||
PLANTED_SECRETS: Final = (PLANTED_API_KEY, ROTATED_API_KEY, PLANTED_AUTH_HEADER, PLANTED_APIKEY_HEADER, "litellm_enc::")
|
||||
AUDIT_ADMIN: Final = UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN, user_id="security-admin", api_key="hashed-admin-key"
|
||||
)
|
||||
STORED_PARAMS: Final = MappingProxyType(
|
||||
{
|
||||
"guardrail": "generic_guardrail_api",
|
||||
"mode": "pre_call",
|
||||
"default_on": False,
|
||||
"api_base": "https://guardrails.example.com/validate",
|
||||
"api_key": PLANTED_API_KEY,
|
||||
"headers": {"Authorization": PLANTED_AUTH_HEADER, "apikey": PLANTED_APIKEY_HEADER, "X-Tenant": "acme"},
|
||||
}
|
||||
)
|
||||
STORED_GUARDRAIL_SNAPSHOT: Final = MappingProxyType(
|
||||
{
|
||||
"guardrail_id": "gr-audit",
|
||||
"guardrail_name": "audited-guard",
|
||||
"guardrail_info": {"description": "audited"},
|
||||
"status": "active",
|
||||
"litellm_params": {"guardrail": "generic_guardrail_api", "mode": "pre_call", "default_on": False},
|
||||
}
|
||||
)
|
||||
SAVED_AT: Final = datetime(2026, 11, 11, 12, 0, 0)
|
||||
|
||||
|
||||
def _requested_guardrail(**params: object) -> Guardrail:
|
||||
return Guardrail(
|
||||
guardrail_name="audited-guard",
|
||||
litellm_params=LitellmParams(**{**STORED_PARAMS, **params}),
|
||||
guardrail_info={"description": "audited"},
|
||||
)
|
||||
|
||||
|
||||
def _row_as_written(where: Mapping[str, object], data: Mapping[str, object]):
|
||||
"""The guardrails table row holding exactly what the registry wrote, as Prisma returns it."""
|
||||
from prisma.models import LiteLLM_GuardrailsTable
|
||||
|
||||
return LiteLLM_GuardrailsTable(
|
||||
**{"guardrail_id": where["guardrail_id"], "status": "active", "created_at": SAVED_AT, **data}
|
||||
)
|
||||
|
||||
|
||||
def _stored_row():
|
||||
return _row_as_written(
|
||||
{"guardrail_id": "gr-audit"},
|
||||
{
|
||||
"guardrail_name": "audited-guard",
|
||||
"litellm_params": json.dumps(
|
||||
encrypt_guardrail_litellm_params(LitellmParams(**STORED_PARAMS).model_dump()), default=str
|
||||
),
|
||||
"guardrail_info": json.dumps({"description": "audited"}),
|
||||
"updated_at": SAVED_AT,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def audit_prisma(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch) -> Mock:
|
||||
"""Audit logging on, with a prisma client whose LiteLLM_AuditLog inserts are recorded and whose guardrails
|
||||
table holds one stored guardrail and returns whatever the registry writes."""
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "store_audit_logs", True)
|
||||
monkeypatch.setattr(litellm, "audit_log_callbacks", [])
|
||||
monkeypatch.delenv("GENERIC_GUARDRAIL_API_BASE", raising=False)
|
||||
mocker.patch("litellm.proxy.proxy_server.premium_user", True)
|
||||
prisma: Final = Mock()
|
||||
prisma.db.litellm_auditlog.create = AsyncMock()
|
||||
prisma.db.litellm_guardrailstable.find_unique = AsyncMock(return_value=_stored_row())
|
||||
prisma.db.litellm_guardrailstable.update = AsyncMock(side_effect=_row_as_written)
|
||||
prisma.db.litellm_guardrailstable.create = AsyncMock(
|
||||
side_effect=lambda data: _row_as_written({"guardrail_id": "gr-audit"}, data)
|
||||
)
|
||||
prisma.db.litellm_guardrailstable.delete = AsyncMock()
|
||||
mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma)
|
||||
return prisma
|
||||
|
||||
|
||||
def _live_guardrails(mocker: MockerFixture, loaded: Sequence[object] = ()) -> InMemoryGuardrailHandler:
|
||||
handler: Final = InMemoryGuardrailHandler()
|
||||
for row in loaded:
|
||||
handler.initialize_guardrail(guardrail=guardrail_from_db_row(row), source="db")
|
||||
mocker.patch("litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", handler)
|
||||
return handler
|
||||
|
||||
|
||||
async def _written_audit_rows(prisma: Mock) -> tuple[Mapping[str, object], ...]:
|
||||
from litellm.proxy.management_helpers.audit_logs import drain_audit_tasks
|
||||
|
||||
await drain_audit_tasks()
|
||||
insert: Final[AsyncMock] = prisma.db.litellm_auditlog.create
|
||||
return tuple(MappingProxyType(call.kwargs["data"]) for call in insert.call_args_list)
|
||||
|
||||
|
||||
def _audit_snapshot(row: Mapping[str, object], field: str) -> Mapping[str, object]:
|
||||
value: Final = row[field]
|
||||
assert isinstance(value, str), f"{field} should be a JSON string, got {value!r}"
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
def _assert_no_planted_secret(row: Mapping[str, object]) -> None:
|
||||
serialized: Final = json.dumps(dict(row), default=str)
|
||||
for secret in PLANTED_SECRETS:
|
||||
assert secret not in serialized, f"{secret!r} leaked into the audit row: {serialized}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_guardrail_audit_row_records_the_caller_and_no_provider_secret(
|
||||
mocker: MockerFixture, audit_prisma: Mock
|
||||
):
|
||||
_live_guardrails(mocker)
|
||||
|
||||
await create_guardrail(CreateGuardrailRequest(guardrail=_requested_guardrail()), user_api_key_dict=AUDIT_ADMIN)
|
||||
|
||||
rows: Final = await _written_audit_rows(audit_prisma)
|
||||
assert [
|
||||
{
|
||||
k: row.get(k)
|
||||
for k in ("action", "table_name", "object_id", "changed_by", "changed_by_api_key", "before_value")
|
||||
}
|
||||
for row in rows
|
||||
] == [
|
||||
{
|
||||
"action": "created",
|
||||
"table_name": "LiteLLM_GuardrailsTable",
|
||||
"object_id": "gr-audit",
|
||||
"changed_by": "security-admin",
|
||||
"changed_by_api_key": "hashed-admin-key",
|
||||
"before_value": None,
|
||||
}
|
||||
], rows
|
||||
assert _audit_snapshot(rows[0], "updated_values") == STORED_GUARDRAIL_SNAPSHOT
|
||||
_assert_no_planted_secret(rows[0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("method", ["put", "patch"])
|
||||
async def test_update_guardrail_audit_row_shows_the_mode_change_and_names_only_the_rotated_key(
|
||||
method: str, mocker: MockerFixture, audit_prisma: Mock
|
||||
):
|
||||
_live_guardrails(mocker, loaded=(_stored_row(),))
|
||||
|
||||
if method == "put":
|
||||
await update_guardrail(
|
||||
"gr-audit",
|
||||
UpdateGuardrailRequest(guardrail=_requested_guardrail(mode="post_call", api_key=ROTATED_API_KEY)),
|
||||
user_api_key_dict=AUDIT_ADMIN,
|
||||
)
|
||||
else:
|
||||
await patch_guardrail(
|
||||
"gr-audit",
|
||||
PatchGuardrailRequest(litellm_params={"api_key": ROTATED_API_KEY, "mode": "post_call"}),
|
||||
user_api_key_dict=AUDIT_ADMIN,
|
||||
)
|
||||
|
||||
rows: Final = await _written_audit_rows(audit_prisma)
|
||||
assert [(r["action"], r["object_id"], r["changed_by"]) for r in rows] == [
|
||||
("updated", "gr-audit", "security-admin")
|
||||
], rows
|
||||
assert _audit_snapshot(rows[0], "before_value") == STORED_GUARDRAIL_SNAPSHOT
|
||||
assert _audit_snapshot(rows[0], "updated_values") == {
|
||||
**STORED_GUARDRAIL_SNAPSHOT,
|
||||
"litellm_params": {"guardrail": "generic_guardrail_api", "mode": "post_call", "default_on": False},
|
||||
"changed_litellm_params": ["api_key"],
|
||||
}, "only api_key changed, so headers the live guardrail adds for itself must not show up"
|
||||
_assert_no_planted_secret(rows[0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_guardrail_audit_row_keeps_the_deleted_guardrail_without_secrets(
|
||||
mocker: MockerFixture, audit_prisma: Mock
|
||||
):
|
||||
_live_guardrails(mocker, loaded=(_stored_row(),))
|
||||
|
||||
await delete_guardrail(guardrail_id="gr-audit", user_api_key_dict=AUDIT_ADMIN)
|
||||
|
||||
rows: Final = await _written_audit_rows(audit_prisma)
|
||||
assert [(r["action"], r["object_id"], r.get("updated_values")) for r in rows] == [("deleted", "gr-audit", None)], (
|
||||
rows
|
||||
)
|
||||
assert _audit_snapshot(rows[0], "before_value") == STORED_GUARDRAIL_SNAPSHOT
|
||||
_assert_no_planted_secret(rows[0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejected_guardrail_update_writes_no_audit_row(mocker: MockerFixture, audit_prisma: Mock):
|
||||
_live_guardrails(mocker, loaded=(_stored_row(),))
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await update_guardrail(
|
||||
"gr-audit",
|
||||
UpdateGuardrailRequest(guardrail=_requested_guardrail(mode="post_call", api_base=None)),
|
||||
user_api_key_dict=AUDIT_ADMIN,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 422, exc_info.value.detail
|
||||
assert await _written_audit_rows(audit_prisma) == (), "a rolled-back update must not be audited"
|
||||
|
||||
|
||||
def _team_guardrail_row(status: str, params_stored_as_json_string: bool = False):
|
||||
encrypted: Final = encrypt_guardrail_litellm_params(dict(STORED_PARAMS))
|
||||
return _row_as_written(
|
||||
{"guardrail_id": "team-gr"},
|
||||
{
|
||||
"guardrail_name": "team-guard",
|
||||
"litellm_params": json.dumps(json.dumps(encrypted) if params_stored_as_json_string else encrypted),
|
||||
"guardrail_info": json.dumps({"description": "team"}),
|
||||
"status": status,
|
||||
"team_id": "team-1",
|
||||
"updated_at": SAVED_AT,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _team_guardrail_snapshot(status: str) -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
"guardrail_id": "team-gr",
|
||||
"guardrail_name": "team-guard",
|
||||
"guardrail_info": {"description": "team"},
|
||||
"status": status,
|
||||
"team_id": "team-1",
|
||||
"litellm_params": {"guardrail": "generic_guardrail_api", "mode": "pre_call", "default_on": False},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_guardrail_audit_row_records_the_pending_submission_without_secrets(
|
||||
audit_prisma: Mock,
|
||||
) -> None:
|
||||
audit_prisma.db.litellm_guardrailstable.find_unique = AsyncMock(return_value=None)
|
||||
audit_prisma.db.litellm_guardrailstable.create = AsyncMock(return_value=_team_guardrail_row("pending_review"))
|
||||
request: Final = RegisterGuardrailRequest(
|
||||
guardrail_name="team-guard", litellm_params=dict(STORED_PARAMS), team_id="team-1"
|
||||
)
|
||||
|
||||
await register_guardrail(request, AUDIT_ADMIN)
|
||||
|
||||
rows: Final = await _written_audit_rows(audit_prisma)
|
||||
assert [(r["action"], r["object_id"]) for r in rows] == [("created", "team-gr")], rows
|
||||
assert _audit_snapshot(rows[0], "updated_values") == _team_guardrail_snapshot("pending_review")
|
||||
_assert_no_planted_secret(rows[0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"review, new_status", [(approve_guardrail_submission, "active"), (reject_guardrail_submission, "rejected")]
|
||||
)
|
||||
@pytest.mark.parametrize("params_stored_as_json_string", [False, True], ids=["json_object", "json_string"])
|
||||
async def test_guardrail_submission_review_audit_row_shows_the_status_change(
|
||||
review: Callable[[str, UserAPIKeyAuth], Awaitable[Mapping[str, object]]],
|
||||
new_status: str,
|
||||
params_stored_as_json_string: bool,
|
||||
mocker: MockerFixture,
|
||||
audit_prisma: Mock,
|
||||
):
|
||||
audit_prisma.db.litellm_guardrailstable.find_unique = AsyncMock(
|
||||
return_value=_team_guardrail_row("pending_review", params_stored_as_json_string)
|
||||
)
|
||||
audit_prisma.db.litellm_guardrailstable.update = AsyncMock(
|
||||
return_value=_team_guardrail_row(new_status, params_stored_as_json_string)
|
||||
)
|
||||
_live_guardrails(mocker)
|
||||
|
||||
response: Final = await review("team-gr", AUDIT_ADMIN)
|
||||
|
||||
assert response["status"] == new_status, response
|
||||
rows: Final = await _written_audit_rows(audit_prisma)
|
||||
assert [(r["action"], r["object_id"]) for r in rows] == [("updated", "team-gr")], rows
|
||||
assert _audit_snapshot(rows[0], "before_value") == _team_guardrail_snapshot("pending_review")
|
||||
assert _audit_snapshot(rows[0], "updated_values") == _team_guardrail_snapshot(new_status)
|
||||
_assert_no_planted_secret(rows[0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_not_found(mocker):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ const TABLE_OPTIONS: readonly FilterOption[] = [
|
|||
{ labelKey: "logs.audit.tableNames.organizations", value: "LiteLLM_OrganizationTable" },
|
||||
{ labelKey: "logs.audit.tableNames.models", value: "LiteLLM_ProxyModelTable" },
|
||||
{ labelKey: "logs.audit.tableNames.agents", value: "LiteLLM_AgentsTable" },
|
||||
{ labelKey: "logs.audit.tableNames.guardrails", value: "LiteLLM_GuardrailsTable" },
|
||||
];
|
||||
|
||||
const ACTION_FILTER_ITEMS: readonly FilterOption[] = [
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ export const AUDIT_TABLE_NAME_DISPLAY: Record<string, ParseKeys> = {
|
|||
LiteLLM_OrganizationTable: "logs.audit.tableNames.organizations",
|
||||
LiteLLM_ProxyModelTable: "logs.audit.tableNames.models",
|
||||
LiteLLM_AgentsTable: "logs.audit.tableNames.agents",
|
||||
LiteLLM_GuardrailsTable: "logs.audit.tableNames.guardrails",
|
||||
};
|
||||
|
||||
const ACTION_TONE: Record<string, StatusTone> = {
|
||||
|
|
|
|||
|
|
@ -14935,7 +14935,8 @@
|
|||
"users": "Users",
|
||||
"organizations": "Organizations",
|
||||
"models": "Models",
|
||||
"agents": "Agents"
|
||||
"agents": "Agents",
|
||||
"guardrails": "Guardrails"
|
||||
},
|
||||
"drawer": {
|
||||
"notAvailable": "N/A",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue