From 38406d7efe927da23a338ab7cda5c305a6601b0a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 19 Feb 2026 19:20:27 -0800 Subject: [PATCH] feat(ui/): new policy editor view --- .gitignore | 4 +- .../proxy/policy_engine/policy_endpoints.py | 281 ++++++++++- .../proxy/policy_engine/policy_registry.py | 420 +++++++++++++++++ litellm/proxy/schema.prisma | 67 ++- .../proxy/policy_engine/pipeline_types.py | 6 +- .../types/proxy/policy_engine/policy_types.py | 98 ++++ .../proxy/policy_engine/resolver_types.py | 29 ++ .../src/components/networking.tsx | 120 +++++ .../components/policies/add_policy_form.tsx | 163 ++++++- .../src/components/policies/index.tsx | 16 +- .../policies/pipeline_flow_builder.tsx | 222 ++++++++- .../components/policies/policy_templates.tsx | 6 +- .../components/policies/policy_test_panel.tsx | 18 +- .../src/components/policies/types.ts | 49 ++ .../policies/version_comparison.tsx | 322 +++++++++++++ .../components/policies/version_sidebar.tsx | 442 ++++++++++++++++++ .../policies/version_status_badge.tsx | 49 ++ 17 files changed, 2264 insertions(+), 48 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/policies/version_comparison.tsx create mode 100644 ui/litellm-dashboard/src/components/policies/version_sidebar.tsx create mode 100644 ui/litellm-dashboard/src/components/policies/version_status_badge.tsx diff --git a/.gitignore b/.gitignore index c43df98a9e5..fc7a8147156 100644 --- a/.gitignore +++ b/.gitignore @@ -100,4 +100,6 @@ STABILIZATION_TODO.md **/test-results **/playwright-report **/*.storageState.json -**/coverage \ No newline at end of file +**/coverage + +ui/litellm-dashboard/out/__next.* diff --git a/litellm/proxy/policy_engine/policy_endpoints.py b/litellm/proxy/policy_engine/policy_endpoints.py index af12a8598f6..29dc2bdee78 100644 --- a/litellm/proxy/policy_engine/policy_endpoints.py +++ b/litellm/proxy/policy_engine/policy_endpoints.py @@ -352,6 +352,257 @@ async def get_resolved_guardrails(policy_id: str): raise HTTPException(status_code=500, detail=str(e)) +# ───────────────────────────────────────────────────────────────────────────── +# Policy Version Management Endpoints +# ───────────────────────────────────────────────────────────────────────────── + + +@router.post( + "/policies/{policy_id}/versions", + tags=["Policies"], + dependencies=[Depends(user_api_key_auth)], + response_model=PolicyDBResponse, +) +async def create_policy_version( + policy_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create a new version from an existing policy. + + The new version will be created as a draft with version_number incremented. + All policy configuration (guardrails, pipeline, condition) will be copied from the source version. + + Example Request: + ```bash + curl -X POST "http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000/versions" \\ + -H "Authorization: Bearer " + ``` + + Example Response: + ```json + { + "policy_id": "new-uuid-here", + "policy_name": "global-baseline", + "version_number": 2, + "version_status": "draft", + "parent_version_id": "123e4567-e89b-12d3-a456-426614174000", + "description": "Base guardrails for all requests", + "guardrails_add": ["pii_masking"], + "guardrails_remove": [] + } + ``` + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + + try: + created_by = user_api_key_dict.user_id + result = await get_policy_registry().create_policy_version( + policy_id=policy_id, + prisma_client=prisma_client, + created_by=created_by, + ) + return result + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"Error creating policy version: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get( + "/policies/name/{policy_name}/versions", + tags=["Policies"], + dependencies=[Depends(user_api_key_auth)], + response_model=PolicyListDBResponse, +) +async def list_policy_versions(policy_name: str): + """ + List all versions of a policy by policy name. + + Returns all versions ordered by version_number descending (latest first). + + Example Request: + ```bash + curl -X GET "http://localhost:4000/policies/name/global-baseline/versions" \\ + -H "Authorization: Bearer " + ``` + + Example Response: + ```json + { + "policies": [ + { + "policy_id": "uuid-v2", + "policy_name": "global-baseline", + "version_number": 2, + "version_status": "draft", + "is_latest": true + }, + { + "policy_id": "uuid-v1", + "policy_name": "global-baseline", + "version_number": 1, + "version_status": "production", + "is_latest": false + } + ], + "total_count": 2 + } + ``` + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + + try: + versions = await get_policy_registry().get_policy_versions( + policy_name=policy_name, + prisma_client=prisma_client, + ) + return PolicyListDBResponse(policies=versions, total_count=len(versions)) + except Exception as e: + verbose_proxy_logger.exception(f"Error listing policy versions: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.put( + "/policies/{policy_id}/status", + tags=["Policies"], + dependencies=[Depends(user_api_key_auth)], + response_model=PolicyDBResponse, +) +async def update_policy_version_status( + policy_id: str, + status: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Change the status of a policy version. + + Valid status transitions: + - draft → published + - published → production + - production → published (demote) + + When promoting to production, the previous production version (if any) will be demoted to published. + + Example Request: + ```bash + curl -X PUT "http://localhost:4000/policies/123e4567-e89b-12d3-a456-426614174000/status?status=published" \\ + -H "Authorization: Bearer " + ``` + + Example Response: + ```json + { + "policy_id": "123e4567-e89b-12d3-a456-426614174000", + "policy_name": "global-baseline", + "version_number": 2, + "version_status": "published", + "published_at": "2024-01-15T10:30:00Z" + } + ``` + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + + # Validate status + valid_statuses = {"draft", "published", "production"} + if status not in valid_statuses: + raise HTTPException( + status_code=400, + detail=f"Invalid status '{status}'. Must be one of: {valid_statuses}", + ) + + try: + updated_by = user_api_key_dict.user_id + result = await get_policy_registry().update_policy_status( + policy_id=policy_id, + new_status=status, + prisma_client=prisma_client, + updated_by=updated_by, + ) + return result + except HTTPException: + raise + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + verbose_proxy_logger.exception(f"Error updating policy status: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get( + "/policies/{policy_id}/compare/{compare_with_id}", + tags=["Policies"], + dependencies=[Depends(user_api_key_auth)], +) +async def compare_policy_versions(policy_id: str, compare_with_id: str): + """ + Compare two policy versions and return their differences. + + Returns a structured diff showing what changed between the two versions. + + Example Request: + ```bash + curl -X GET "http://localhost:4000/policies/uuid-v2/compare/uuid-v1" \\ + -H "Authorization: Bearer " + ``` + + Example Response: + ```json + { + "policy_1": { + "policy_id": "uuid-v2", + "version_number": 2, + "version_status": "draft" + }, + "policy_2": { + "policy_id": "uuid-v1", + "version_number": 1, + "version_status": "production" + }, + "differences": { + "guardrails_add": { + "added": ["toxicity_filter"], + "removed": [], + "unchanged": ["pii_masking"] + }, + "description": { + "changed": true, + "old": "Base guardrails", + "new": "Enhanced base guardrails" + } + } + } + ``` + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + + try: + comparison = await get_policy_registry().compare_policy_versions( + policy_id_1=policy_id, + policy_id_2=compare_with_id, + prisma_client=prisma_client, + ) + return comparison + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"Error comparing policy versions: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + # ───────────────────────────────────────────────────────────────────────────── # Pipeline Test Endpoint # ───────────────────────────────────────────────────────────────────────────── @@ -373,6 +624,8 @@ async def test_pipeline( step-by-step results showing which guardrails passed/failed, actions taken, and timing information. + Optionally filter to test only specific guardrails using the guardrail_filter parameter. + Example Request: ```bash curl -X POST "http://localhost:4000/policies/test-pipeline" \\ @@ -388,12 +641,38 @@ async def test_pipeline( "test_messages": [{"role": "user", "content": "My SSN is 123-45-6789"}] }' ``` + + Example with guardrail filtering: + ```bash + curl -X POST "http://localhost:4000/policies/test-pipeline" \\ + -H "Authorization: Bearer " \\ + -H "Content-Type: application/json" \\ + -d '{ + "pipeline": {...}, + "test_messages": [...], + "guardrail_filter": ["pii-guard"] + }' + ``` """ try: validated_pipeline = GuardrailPipeline(**request.pipeline) except Exception as e: raise HTTPException(status_code=400, detail=f"Invalid pipeline: {e}") + # Filter steps if guardrail_filter is provided + steps_to_execute = validated_pipeline.steps + if request.guardrail_filter: + steps_to_execute = [ + step + for step in validated_pipeline.steps + if step.guardrail in request.guardrail_filter + ] + if not steps_to_execute: + raise HTTPException( + status_code=400, + detail=f"No steps match the guardrail filter: {request.guardrail_filter}", + ) + data = { "messages": request.test_messages, "model": "test", @@ -402,7 +681,7 @@ async def test_pipeline( try: result = await PipelineExecutor.execute_steps( - steps=validated_pipeline.steps, + steps=steps_to_execute, mode=validated_pipeline.mode, data=data, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/policy_engine/policy_registry.py b/litellm/proxy/policy_engine/policy_registry.py index 1ca6062e5ac..d5ca0ac75bc 100644 --- a/litellm/proxy/policy_engine/policy_registry.py +++ b/litellm/proxy/policy_engine/policy_registry.py @@ -285,6 +285,12 @@ class PolicyRegistry: guardrails_remove=created_policy.guardrails_remove or [], condition=created_policy.condition, pipeline=created_policy.pipeline, + version_number=getattr(created_policy, "version_number", 1), + version_status=getattr(created_policy, "version_status", "production"), + parent_version_id=getattr(created_policy, "parent_version_id", None), + is_latest=getattr(created_policy, "is_latest", True), + published_at=getattr(created_policy, "published_at", None), + production_at=getattr(created_policy, "production_at", None), created_at=created_policy.created_at, updated_at=created_policy.updated_at, created_by=created_policy.created_by, @@ -366,6 +372,12 @@ class PolicyRegistry: guardrails_remove=updated_policy.guardrails_remove or [], condition=updated_policy.condition, pipeline=updated_policy.pipeline, + version_number=getattr(updated_policy, "version_number", 1), + version_status=getattr(updated_policy, "version_status", "production"), + parent_version_id=getattr(updated_policy, "parent_version_id", None), + is_latest=getattr(updated_policy, "is_latest", True), + published_at=getattr(updated_policy, "published_at", None), + production_at=getattr(updated_policy, "production_at", None), created_at=updated_policy.created_at, updated_at=updated_policy.updated_at, created_by=updated_policy.created_by, @@ -444,6 +456,12 @@ class PolicyRegistry: guardrails_remove=policy.guardrails_remove or [], condition=policy.condition, pipeline=policy.pipeline, + version_number=getattr(policy, "version_number", 1), + version_status=getattr(policy, "version_status", "production"), + parent_version_id=getattr(policy, "parent_version_id", None), + is_latest=getattr(policy, "is_latest", True), + published_at=getattr(policy, "published_at", None), + production_at=getattr(policy, "production_at", None), created_at=policy.created_at, updated_at=policy.updated_at, created_by=policy.created_by, @@ -481,6 +499,12 @@ class PolicyRegistry: guardrails_remove=p.guardrails_remove or [], condition=p.condition, pipeline=p.pipeline, + version_number=getattr(p, "version_number", 1), + version_status=getattr(p, "version_status", "production"), + parent_version_id=getattr(p, "parent_version_id", None), + is_latest=getattr(p, "is_latest", True), + published_at=getattr(p, "published_at", None), + production_at=getattr(p, "production_at", None), created_at=p.created_at, updated_at=p.updated_at, created_by=p.created_by, @@ -582,6 +606,402 @@ class PolicyRegistry: verbose_proxy_logger.exception(f"Error resolving guardrails from DB: {e}") raise Exception(f"Error resolving guardrails from DB: {str(e)}") + # ───────────────────────────────────────────────────────────────────────── + # Policy Versioning Methods + # ───────────────────────────────────────────────────────────────────────── + + async def create_policy_version( + self, + policy_id: str, + prisma_client: "PrismaClient", + created_by: Optional[str] = None, + ) -> PolicyDBResponse: + """ + Create a new version from an existing policy. + + The new version will: + - Be created as a draft + - Have version_number incremented from the latest version + - Copy all configuration from the source policy + - Set parent_version_id to the source policy_id + + Args: + policy_id: ID of the policy to create a version from + prisma_client: The Prisma client instance + created_by: User who created the version + + Returns: + PolicyDBResponse with the new version + """ + try: + # Get the source policy + source_policy = await prisma_client.db.litellm_policytable.find_unique( + where={"policy_id": policy_id} + ) + + if source_policy is None: + from fastapi import HTTPException + + raise HTTPException( + status_code=404, detail=f"Policy with ID {policy_id} not found" + ) + + # Get the latest version number for this policy name + existing_versions = await prisma_client.db.litellm_policytable.find_many( + where={"policy_name": source_policy.policy_name}, + order_by={"version_number": "desc"}, + ) + + new_version_number = ( + existing_versions[0].version_number + 1 if existing_versions else 1 + ) + + # Mark previous versions as not latest + for version in existing_versions: + if version.is_latest: + await prisma_client.db.litellm_policytable.update( + where={"policy_id": version.policy_id}, + data={"is_latest": False}, + ) + + # Create new version with copied configuration + new_version_data = { + "policy_name": source_policy.policy_name, + "inherit": source_policy.inherit, + "description": source_policy.description, + "guardrails_add": source_policy.guardrails_add or [], + "guardrails_remove": source_policy.guardrails_remove or [], + "condition": source_policy.condition, + "pipeline": source_policy.pipeline, + "version_number": new_version_number, + "version_status": "draft", + "parent_version_id": policy_id, + "is_latest": True, + "created_at": datetime.now(timezone.utc), + "updated_at": datetime.now(timezone.utc), + } + + if created_by is not None: + new_version_data["created_by"] = created_by + new_version_data["updated_by"] = created_by + + created_version = await prisma_client.db.litellm_policytable.create( + data=new_version_data + ) + + return PolicyDBResponse( + policy_id=created_version.policy_id, + policy_name=created_version.policy_name, + inherit=created_version.inherit, + description=created_version.description, + guardrails_add=created_version.guardrails_add or [], + guardrails_remove=created_version.guardrails_remove or [], + condition=created_version.condition, + pipeline=created_version.pipeline, + version_number=created_version.version_number, + version_status=created_version.version_status, + parent_version_id=created_version.parent_version_id, + is_latest=created_version.is_latest, + published_at=created_version.published_at, + production_at=created_version.production_at, + created_at=created_version.created_at, + updated_at=created_version.updated_at, + created_by=created_version.created_by, + updated_by=created_version.updated_by, + ) + except Exception as e: + verbose_proxy_logger.exception(f"Error creating policy version: {e}") + raise + + async def get_policy_versions( + self, + policy_name: str, + prisma_client: "PrismaClient", + ) -> List[PolicyDBResponse]: + """ + Get all versions of a policy by policy name. + + Args: + policy_name: Name of the policy + prisma_client: The Prisma client instance + + Returns: + List of PolicyDBResponse objects ordered by version_number desc + """ + try: + versions = await prisma_client.db.litellm_policytable.find_many( + where={"policy_name": policy_name}, + order_by={"version_number": "desc"}, + ) + + return [ + PolicyDBResponse( + policy_id=v.policy_id, + policy_name=v.policy_name, + inherit=v.inherit, + description=v.description, + guardrails_add=v.guardrails_add or [], + guardrails_remove=v.guardrails_remove or [], + condition=v.condition, + pipeline=v.pipeline, + version_number=v.version_number, + version_status=v.version_status, + parent_version_id=v.parent_version_id, + is_latest=v.is_latest, + published_at=v.published_at, + production_at=v.production_at, + created_at=v.created_at, + updated_at=v.updated_at, + created_by=v.created_by, + updated_by=v.updated_by, + ) + for v in versions + ] + except Exception as e: + verbose_proxy_logger.exception(f"Error getting policy versions: {e}") + raise Exception(f"Error getting policy versions: {str(e)}") + + async def update_policy_status( + self, + policy_id: str, + new_status: str, + prisma_client: "PrismaClient", + updated_by: Optional[str] = None, + ) -> PolicyDBResponse: + """ + Update the status of a policy version. + + Valid transitions: + - draft → published + - published → production + - production → published (demote) + + When promoting to production, previous production versions are demoted to published. + + Args: + policy_id: ID of the policy to update + new_status: New status (draft, published, or production) + prisma_client: The Prisma client instance + updated_by: User who updated the status + + Returns: + PolicyDBResponse with the updated policy + + Raises: + ValueError: If the status transition is invalid + """ + try: + # Get the policy + policy = await prisma_client.db.litellm_policytable.find_unique( + where={"policy_id": policy_id} + ) + + if policy is None: + from fastapi import HTTPException + + raise HTTPException( + status_code=404, detail=f"Policy with ID {policy_id} not found" + ) + + current_status = policy.version_status + + # Validate status transition + valid_transitions = { + "draft": {"published"}, + "published": {"production", "draft"}, + "production": {"published"}, + } + + if ( + new_status != current_status + and new_status not in valid_transitions.get(current_status, set()) + ): + raise ValueError( + f"Invalid status transition from '{current_status}' to '{new_status}'. " + f"Valid transitions: {valid_transitions.get(current_status, set())}" + ) + + # Build update data + update_data: Dict[str, Any] = { + "version_status": new_status, + "updated_at": datetime.now(timezone.utc), + } + + if updated_by is not None: + update_data["updated_by"] = updated_by + + # Set timestamp fields based on new status + if new_status == "published" and current_status == "draft": + update_data["published_at"] = datetime.now(timezone.utc) + elif new_status == "production": + update_data["production_at"] = datetime.now(timezone.utc) + + # Demote other production versions of this policy to published + await prisma_client.db.litellm_policytable.update_many( + where={ + "policy_name": policy.policy_name, + "version_status": "production", + }, + data={"version_status": "published"}, + ) + + # Update the policy + updated_policy = await prisma_client.db.litellm_policytable.update( + where={"policy_id": policy_id}, + data=update_data, + ) + + # If promoted to production, sync to in-memory registry + if new_status == "production": + policy_obj = self._parse_policy( + updated_policy.policy_name, + { + "inherit": updated_policy.inherit, + "description": updated_policy.description, + "guardrails": { + "add": updated_policy.guardrails_add, + "remove": updated_policy.guardrails_remove, + }, + "condition": updated_policy.condition, + "pipeline": updated_policy.pipeline, + }, + ) + self.add_policy(updated_policy.policy_name, policy_obj) + + return PolicyDBResponse( + policy_id=updated_policy.policy_id, + policy_name=updated_policy.policy_name, + inherit=updated_policy.inherit, + description=updated_policy.description, + guardrails_add=updated_policy.guardrails_add or [], + guardrails_remove=updated_policy.guardrails_remove or [], + condition=updated_policy.condition, + pipeline=updated_policy.pipeline, + version_number=updated_policy.version_number, + version_status=updated_policy.version_status, + parent_version_id=updated_policy.parent_version_id, + is_latest=updated_policy.is_latest, + published_at=updated_policy.published_at, + production_at=updated_policy.production_at, + created_at=updated_policy.created_at, + updated_at=updated_policy.updated_at, + created_by=updated_policy.created_by, + updated_by=updated_policy.updated_by, + ) + except Exception as e: + verbose_proxy_logger.exception(f"Error updating policy status: {e}") + raise + + async def compare_policy_versions( + self, + policy_id_1: str, + policy_id_2: str, + prisma_client: "PrismaClient", + ) -> Dict[str, Any]: + """ + Compare two policy versions and return their differences. + + Args: + policy_id_1: ID of the first policy + policy_id_2: ID of the second policy + prisma_client: The Prisma client instance + + Returns: + Dict with comparison results showing differences + """ + try: + # Get both policies + policy1 = await self.get_policy_by_id_from_db(policy_id_1, prisma_client) + policy2 = await self.get_policy_by_id_from_db(policy_id_2, prisma_client) + + if policy1 is None: + from fastapi import HTTPException + + raise HTTPException( + status_code=404, detail=f"Policy with ID {policy_id_1} not found" + ) + if policy2 is None: + from fastapi import HTTPException + + raise HTTPException( + status_code=404, detail=f"Policy with ID {policy_id_2} not found" + ) + + # Helper to compare lists + def compare_lists(list1: List[str], list2: List[str]) -> Dict[str, Any]: + set1, set2 = set(list1), set(list2) + return { + "added": sorted(set1 - set2), + "removed": sorted(set2 - set1), + "unchanged": sorted(set1 & set2), + } + + # Build differences + differences: Dict[str, Any] = {} + + # Compare guardrails_add + if policy1.guardrails_add != policy2.guardrails_add: + differences["guardrails_add"] = compare_lists( + policy1.guardrails_add, policy2.guardrails_add + ) + + # Compare guardrails_remove + if policy1.guardrails_remove != policy2.guardrails_remove: + differences["guardrails_remove"] = compare_lists( + policy1.guardrails_remove, policy2.guardrails_remove + ) + + # Compare description + if policy1.description != policy2.description: + differences["description"] = { + "changed": True, + "new": policy1.description, + "old": policy2.description, + } + + # Compare inherit + if policy1.inherit != policy2.inherit: + differences["inherit"] = { + "changed": True, + "new": policy1.inherit, + "old": policy2.inherit, + } + + # Compare condition + if policy1.condition != policy2.condition: + differences["condition"] = { + "changed": True, + "new": policy1.condition, + "old": policy2.condition, + } + + # Compare pipeline + if policy1.pipeline != policy2.pipeline: + differences["pipeline"] = { + "changed": True, + "new": policy1.pipeline, + "old": policy2.pipeline, + } + + return { + "policy_1": { + "policy_id": policy1.policy_id, + "policy_name": policy1.policy_name, + "version_number": policy1.version_number, + "version_status": policy1.version_status, + }, + "policy_2": { + "policy_id": policy2.policy_id, + "policy_name": policy2.policy_name, + "version_number": policy2.version_number, + "version_status": policy2.version_status, + }, + "differences": differences, + } + except Exception as e: + verbose_proxy_logger.exception(f"Error comparing policy versions: {e}") + raise + # Global singleton instance _policy_registry: Optional[PolicyRegistry] = None diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 6eaeabe8916..57f02afe4ae 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -963,20 +963,36 @@ model LiteLLM_SkillsTable { updated_by String? } -// Policy table for storing guardrail policies +// Policy table for storing guardrail policies with versioning support model LiteLLM_PolicyTable { policy_id String @id @default(uuid()) - policy_name String @unique + policy_name String // Policy name (multiple versions can share same name) inherit String? // Name of parent policy to inherit from description String? guardrails_add String[] @default([]) guardrails_remove String[] @default([]) condition Json? @default("{}") // Policy conditions (e.g., model matching) pipeline Json? // Optional guardrail pipeline (mode + steps[]) + + // Versioning fields + version_number Int @default(1) + version_status String @default("production") // "draft", "published", or "production" + parent_version_id String? // Reference to the policy version this was created from + is_latest Boolean @default(true) + published_at DateTime? // When this version was published + production_at DateTime? // When this version was promoted to production + created_at DateTime @default(now()) created_by String? updated_at DateTime @default(now()) @updatedAt updated_by String? + + // Relations + mirroring_configs LiteLLM_PolicyMirroringTable[] + + @@unique([policy_name, version_number]) + @@index([policy_name, version_status]) + @@index([version_status]) } // Policy attachment table for defining where policies apply @@ -994,6 +1010,53 @@ model LiteLLM_PolicyAttachmentTable { updated_by String? } +// Policy mirroring configuration for silent/shadow testing +model LiteLLM_PolicyMirroringTable { + mirroring_id String @id @default(uuid()) + policy_id String // References LiteLLM_PolicyTable.policy_id + traffic_percentage Float @default(0.0) // 0.0-100.0 + max_requests Int? // Optional limit on total mirrored requests + current_requests Int @default(0) + enabled Boolean @default(true) + fail_silently Boolean @default(true) // Don't block requests if mirroring fails + log_results Boolean @default(true) // Store results in mirror logs + started_at DateTime @default(now()) + expires_at DateTime? // Optional expiration time for mirroring + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? + + // Relations + policy LiteLLM_PolicyTable @relation(fields: [policy_id], references: [policy_id]) + mirror_logs LiteLLM_PolicyMirrorLogs[] + + @@index([policy_id, enabled]) + @@index([expires_at]) +} + +// Policy mirroring execution logs +model LiteLLM_PolicyMirrorLogs { + log_id String @id @default(uuid()) + mirroring_id String // References LiteLLM_PolicyMirroringTable.mirroring_id + request_id String? // Original request ID if available + executed_at DateTime @default(now()) + execution_time_ms Float? // How long the mirrored policy execution took + result String // "pass", "fail", "error" + terminal_action String? // Final action taken by policy (allow/block/modify) + step_results Json? // Detailed step-by-step results + error_message String? // Error details if result = "error" + metadata Json? @default("{}") + created_at DateTime @default(now()) + + // Relations + mirroring_config LiteLLM_PolicyMirroringTable @relation(fields: [mirroring_id], references: [mirroring_id]) + + @@index([mirroring_id, executed_at]) + @@index([result]) + @@index([executed_at]) +} + //Unified Access Groups table for storing unified access groups model LiteLLM_AccessGroupTable { access_group_id String @id @default(uuid()) diff --git a/litellm/types/proxy/policy_engine/pipeline_types.py b/litellm/types/proxy/policy_engine/pipeline_types.py index 29d2e576000..979f7c67d87 100644 --- a/litellm/types/proxy/policy_engine/pipeline_types.py +++ b/litellm/types/proxy/policy_engine/pipeline_types.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List, Literal, Optional from pydantic import BaseModel, ConfigDict, Field, field_validator -VALID_PIPELINE_ACTIONS = {"allow", "block", "next", "modify_response"} +VALID_PIPELINE_ACTIONS = {"allow", "block", "next", "skip_to_end", "modify_response"} VALID_PIPELINE_MODES = {"pre_call", "post_call"} @@ -24,11 +24,11 @@ class PipelineStep(BaseModel): guardrail: str = Field(description="Name of the guardrail to run.") on_fail: str = Field( default="block", - description="Action when guardrail rejects: next | block | allow | modify_response", + description="Action when guardrail rejects: next | block | allow | skip_to_end | modify_response", ) on_pass: str = Field( default="allow", - description="Action when guardrail passes: next | block | allow | modify_response", + description="Action when guardrail passes: next | block | allow | skip_to_end | modify_response", ) pass_data: bool = Field( default=False, diff --git a/litellm/types/proxy/policy_engine/policy_types.py b/litellm/types/proxy/policy_engine/policy_types.py index 53a74ca6fd8..542fc1dad38 100644 --- a/litellm/types/proxy/policy_engine/policy_types.py +++ b/litellm/types/proxy/policy_engine/policy_types.py @@ -27,14 +27,76 @@ Key concepts: - `policies`: Define WHAT guardrails to apply (with inheritance via `inherit` and `guardrails.add`/`remove`) - `policy_attachments`: Define WHERE policies apply (teams, keys, models) - `condition`: Optional model condition for when guardrails apply +- `versioning`: Policies support draft/published/production workflow for safe iteration """ +from datetime import datetime +from enum import Enum from typing import Dict, List, Optional, Union from pydantic import BaseModel, ConfigDict, Field from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline +# ───────────────────────────────────────────────────────────────────────────── +# Policy Versioning +# ───────────────────────────────────────────────────────────────────────────── + + +class PolicyVersionStatus(str, Enum): + """ + Status of a policy version in the version workflow. + + - DRAFT: Policy is being edited, not yet published + - PUBLISHED: Policy is ready for testing/mirroring + - PRODUCTION: Policy is active and applied to matching requests + """ + + DRAFT = "draft" + PUBLISHED = "published" + PRODUCTION = "production" + + +class SilentMirroringConfig(BaseModel): + """ + Configuration for silent/shadow testing of policy versions. + + Silent mirroring executes a policy version on a percentage of traffic + without blocking requests, allowing safe testing before promotion. + + Example usage: + - Test new policy on 10% of traffic + - Limit to 1000 test executions + - Log all results for analysis + - Automatically expire after testing period + """ + + traffic_percentage: float = Field( + default=0.0, + ge=0.0, + le=100.0, + description="Percentage of traffic to mirror (0.0-100.0)", + ) + max_requests: Optional[int] = Field( + default=None, + description="Maximum number of requests to mirror (optional limit)", + ) + fail_silently: bool = Field( + default=True, + description="Continue request processing if mirroring fails", + ) + log_results: bool = Field( + default=True, + description="Store execution results in mirror logs", + ) + expires_at: Optional[datetime] = Field( + default=None, + description="Optional expiration time for mirroring", + ) + + model_config = ConfigDict(extra="forbid") + + # ───────────────────────────────────────────────────────────────────────────── # Policy Condition # ───────────────────────────────────────────────────────────────────────────── @@ -182,6 +244,12 @@ class Policy(BaseModel): Policies can have a `condition` for model-based guardrail application. + Versioning Support: + - Policies support version workflow: draft → published → production + - Multiple versions of the same policy can coexist + - Only production versions are applied to requests by default + - Published versions can be tested via silent mirroring + Example configuration: ```yaml policies: @@ -238,6 +306,36 @@ class Policy(BaseModel): description="Optional pipeline for ordered, conditional guardrail execution.", ) + # Versioning fields (populated from database) + version_number: int = Field( + default=1, + description="Version number of this policy (1-indexed)", + ) + version_status: PolicyVersionStatus = Field( + default=PolicyVersionStatus.PRODUCTION, + description="Status of this policy version (draft/published/production)", + ) + parent_version_id: Optional[str] = Field( + default=None, + description="Policy ID of the version this was created from", + ) + is_latest: bool = Field( + default=True, + description="Whether this is the latest version of the policy", + ) + published_at: Optional[datetime] = Field( + default=None, + description="When this version was published", + ) + production_at: Optional[datetime] = Field( + default=None, + description="When this version was promoted to production", + ) + silent_mirroring: Optional[SilentMirroringConfig] = Field( + default=None, + description="Silent mirroring configuration (for testing)", + ) + model_config = ConfigDict(extra="forbid") diff --git a/litellm/types/proxy/policy_engine/resolver_types.py b/litellm/types/proxy/policy_engine/resolver_types.py index a5a2334ae4b..fade8205acb 100644 --- a/litellm/types/proxy/policy_engine/resolver_types.py +++ b/litellm/types/proxy/policy_engine/resolver_types.py @@ -10,6 +10,8 @@ from typing import Any, Dict, List, Optional from pydantic import BaseModel, ConfigDict, Field +from litellm.types.proxy.policy_engine.policy_types import PolicyVersionStatus + class PolicyMatchContext(BaseModel): """ @@ -158,6 +160,10 @@ class PolicyCreateRequest(BaseModel): default=None, description="Optional guardrail pipeline for ordered execution. Contains 'mode' and 'steps'.", ) + version_status: PolicyVersionStatus = Field( + default=PolicyVersionStatus.DRAFT, + description="Initial status for the policy version (default: draft).", + ) class PolicyUpdateRequest(BaseModel): @@ -212,6 +218,25 @@ class PolicyDBResponse(BaseModel): pipeline: Optional[Dict[str, Any]] = Field( default=None, description="Optional guardrail pipeline." ) + + # Versioning fields + version_number: int = Field(default=1, description="Version number (1-indexed).") + version_status: str = Field( + default="production", description="Status: draft, published, or production." + ) + parent_version_id: Optional[str] = Field( + default=None, description="Policy ID this version was created from." + ) + is_latest: bool = Field( + default=True, description="Whether this is the latest version." + ) + published_at: Optional[datetime] = Field( + default=None, description="When this version was published." + ) + production_at: Optional[datetime] = Field( + default=None, description="When this version was promoted to production." + ) + created_at: Optional[datetime] = Field( default=None, description="When the policy was created." ) @@ -311,6 +336,10 @@ class PipelineTestRequest(BaseModel): test_messages: List[Dict[str, str]] = Field( description="Test messages to run through the pipeline, e.g. [{'role': 'user', 'content': '...'}].", ) + guardrail_filter: Optional[List[str]] = Field( + default=None, + description="Optional list of guardrail names to test. If provided, only these guardrails will be executed.", + ) class PolicyResolveRequest(BaseModel): diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 3517822985a..a72e5aee07d 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -5805,6 +5805,126 @@ export const getPolicyInfo = async (accessToken: string, policyId: string) => { } }; +// Policy Version Management API Calls + +export const createPolicyVersion = async (accessToken: string, policyId: string) => { + try { + const url = proxyBaseUrl ? `${proxyBaseUrl}/policies/${policyId}/versions` : `/policies/${policyId}/versions`; + const response = await fetch(url, { + method: "POST", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + + const data = await response.json(); + return data; + } catch (error) { + console.error("Failed to create policy version:", error); + throw error; + } +}; + +export const listPolicyVersions = async (accessToken: string, policyName: string) => { + try { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/policies/name/${encodeURIComponent(policyName)}/versions` + : `/policies/name/${encodeURIComponent(policyName)}/versions`; + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + + const data = await response.json(); + return data; + } catch (error) { + console.error("Failed to list policy versions:", error); + throw error; + } +}; + +export const updatePolicyVersionStatus = async ( + accessToken: string, + policyId: string, + status: "draft" | "published" | "production" +) => { + try { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/policies/${policyId}/status?status=${status}` + : `/policies/${policyId}/status?status=${status}`; + const response = await fetch(url, { + method: "PUT", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + + const data = await response.json(); + return data; + } catch (error) { + console.error("Failed to update policy version status:", error); + throw error; + } +}; + +export const comparePolicyVersions = async ( + accessToken: string, + policyId: string, + compareWithId: string +) => { + try { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/policies/${policyId}/compare/${compareWithId}` + : `/policies/${policyId}/compare/${compareWithId}`; + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + + const data = await response.json(); + return data; + } catch (error) { + console.error("Failed to compare policy versions:", error); + throw error; + } +}; + // Policy Attachments API Calls export const getPolicyAttachmentsList = async (accessToken: string) => { diff --git a/ui/litellm-dashboard/src/components/policies/add_policy_form.tsx b/ui/litellm-dashboard/src/components/policies/add_policy_form.tsx index 4ccc151bd13..c892a6554f0 100644 --- a/ui/litellm-dashboard/src/components/policies/add_policy_form.tsx +++ b/ui/litellm-dashboard/src/components/policies/add_policy_form.tsx @@ -3,9 +3,11 @@ import { Form, Select, Modal, Divider, Typography, Tag, Alert, Radio } from "ant import { Button, TextInput, Textarea } from "@tremor/react"; import { Policy, PolicyCreateRequest, PolicyUpdateRequest } from "./types"; import { Guardrail } from "../guardrails/types"; -import { getResolvedGuardrails, modelAvailableCall } from "../networking"; +import { getResolvedGuardrails, modelAvailableCall, createPolicyVersion } from "../networking"; import NotificationsManager from "../molecules/notifications_manager"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import VersionStatusBadge from "./version_status_badge"; +import VersionSidebar from "./version_sidebar"; const { Text } = Typography; const { Option } = Select; @@ -21,6 +23,9 @@ interface AddPolicyFormProps { availableGuardrails: Guardrail[]; createPolicy: (accessToken: string, policyData: any) => Promise; updatePolicy: (accessToken: string, policyId: string, policyData: any) => Promise; + onVersionSelect?: (version: Policy) => void; + onVersionCreated?: () => void; + onOpenSimulator?: (policy: Policy) => void; } // ───────────────────────────────────────────────────────────────────────────── @@ -140,6 +145,9 @@ const AddPolicyForm: React.FC = ({ availableGuardrails, createPolicy, updatePolicy, + onVersionSelect, + onVersionCreated, + onOpenSimulator, }) => { const [form] = Form.useForm(); const [isSubmitting, setIsSubmitting] = useState(false); @@ -173,14 +181,13 @@ const AddPolicyForm: React.FC = ({ loadResolvedGuardrails(editingPolicy.policy_id); } - // If editing a pipeline policy, go directly to flow builder - if (editingPolicy.pipeline) { - onClose(); + // When editing any policy, open in flow builder instead of the form. + // Only call onOpenFlowBuilder (not onClose) so the parent keeps editingPolicy + // and the flow builder can pre-populate from it. + if (editingPolicy.policy_id) { onOpenFlowBuilder(); return; } - // If editing a simple policy, skip mode picker - setStep("simple_form"); } else if (visible) { form.resetFields(); setResolvedGuardrails([]); @@ -288,6 +295,34 @@ const AddPolicyForm: React.FC = ({ } }; + const handleCreateNewVersion = async () => { + if (!accessToken || !editingPolicy?.policy_id) return; + + Modal.confirm({ + title: "Create New Version", + content: "This will create a new draft version based on the current policy. You can then edit the new version.", + okText: "Create", + cancelText: "Cancel", + onOk: async () => { + setIsSubmitting(true); + try { + await createPolicyVersion(accessToken, editingPolicy.policy_id); + NotificationsManager.success("New version created successfully"); + resetForm(); + onSuccess(); + onClose(); + } catch (error) { + console.error("Failed to create version:", error); + NotificationsManager.fromBackend( + "Failed to create version: " + (error instanceof Error ? error.message : String(error)) + ); + } finally { + setIsSubmitting(false); + } + }, + }); + }; + const handleSubmit = async () => { try { setIsSubmitting(true); @@ -352,6 +387,7 @@ const AddPolicyForm: React.FC = ({ footer={null} width={620} > +
{selectedMode === "flow_builder" && ( @@ -386,14 +422,61 @@ const AddPolicyForm: React.FC = ({ } // ── Simple Form Step ────────────────────────────────────────────────────── - return ( - + const versionStatus = editingPolicy?.version_status || "draft"; + const versionNumber = editingPolicy?.version_number || 1; + const isReadOnlyVersion = isEditing && versionStatus !== "draft"; + const showVersionSidebar = isEditing && editingPolicy && onVersionSelect && onVersionCreated; + + const formContent = ( + <> + {/* Version Indicator (compact when sidebar is shown) */} + {isEditing && editingPolicy && ( +
+
+
+
+ + Version {versionNumber} + + + {editingPolicy.policy_name} + +
+ +
+
+ {onOpenSimulator && ( + + )} + {(versionStatus === "published" || versionStatus === "production") && ( + + )} +
+
+ {isReadOnlyVersion && ( + + )} +
+ )} + = ({