diff --git a/apps/arc-web/app/data/verifications.ts b/apps/arc-web/app/data/verifications.ts new file mode 100644 index 000000000..d1a6ad6cd --- /dev/null +++ b/apps/arc-web/app/data/verifications.ts @@ -0,0 +1,216 @@ +export type VerificationStatus = "pass" | "fail" | "na"; + +export type VerificationType = "ai" | "automated" | "analysis" | "ai-analysis"; + +export interface Criterion { + name: string; + description: string; + type: VerificationType | null; + status: VerificationStatus; +} + +export interface VerificationCategory { + name: string; + question: string; + status: VerificationStatus; + criteria: Criterion[]; +} + +export const statusConfig = { + pass: { + label: "Pass", + color: "text-mint", + bg: "bg-mint/15", + dot: "bg-mint", + border: "border-l-mint/50", + }, + fail: { + label: "Fail", + color: "text-coral", + bg: "bg-coral/15", + dot: "bg-coral", + border: "border-l-coral/50", + }, + na: { + label: "N/A", + color: "text-navy-600", + bg: "bg-white/[0.04]", + dot: "bg-navy-600", + border: "border-l-navy-600/50", + }, +} as const satisfies Record< + VerificationStatus, + { label: string; color: string; bg: string; dot: string; border: string } +>; + +export const typeConfig = { + ai: { label: "AI", color: "text-teal-300", bg: "bg-teal-500/10" }, + automated: { label: "Automated", color: "text-mint", bg: "bg-mint/10" }, + analysis: { label: "Analysis", color: "text-amber", bg: "bg-amber/10" }, + "ai-analysis": { label: "AI + Analysis", color: "text-teal-300", bg: "bg-teal-500/10" }, +} as const satisfies Record< + VerificationType, + { label: string; color: string; bg: string } +>; + +export const verificationCategories: VerificationCategory[] = [ + { + name: "Traceability", + question: "Do we understand what this change is and why we're making it?", + status: "pass", + criteria: [ + { name: "Motivation", description: "Origin of proposal identified", type: "ai", status: "pass" }, + { name: "Specifications", description: "Requirements written down", type: "ai", status: "pass" }, + { name: "Documentation", description: "Developer and user docs added", type: "ai", status: "pass" }, + { name: "Minimization", description: "No extraneous changes", type: "ai", status: "pass" }, + ], + }, + { + name: "Readability", + question: "Can a human or agent quickly read this and understand what it does?", + status: "pass", + criteria: [ + { name: "Formatting", description: "Code layout matches standard", type: "automated", status: "pass" }, + { name: "Linting", description: "Linter issues resolved", type: "automated", status: "pass" }, + { name: "Style", description: "House style applied", type: "ai", status: "pass" }, + ], + }, + { + name: "Reliability", + question: "Will this behave correctly and safely under real-world conditions and failures?", + status: "pass", + criteria: [ + { name: "Completeness", description: "Implementation covers requirements", type: "ai", status: "pass" }, + { name: "Defects", description: "Potential or likely bugs remediated", type: "ai-analysis", status: "pass" }, + { name: "Performance", description: "Hot path impact identified", type: "ai", status: "pass" }, + ], + }, + { + name: "Code Coverage", + question: "Do we have trustworthy, automated evidence that it works and won't regress?", + status: "fail", + criteria: [ + { name: "Test Coverage", description: "Production code exercised by unit tests", type: "analysis", status: "pass" }, + { name: "Test Quality", description: "Tests are robust and clear", type: "ai", status: "fail" }, + { name: "E2E Coverage", description: "Browser automation exercises UX", type: "analysis", status: "na" }, + ], + }, + { + name: "Maintainability", + question: "Will this be easy to modify or extend later without creating new risk?", + status: "pass", + criteria: [ + { name: "Architecture", description: "Layering and dependency graph meets design", type: "analysis", status: "pass" }, + { name: "Interfaces", description: "", type: null, status: "pass" }, + { name: "Duplication", description: "Similar and identical code blocks identified", type: "analysis", status: "pass" }, + { name: "Simplicity", description: "Extra review for reducing complexity", type: "ai", status: "pass" }, + { name: "Dead Code", description: "Unexecuted code and dependencies removed", type: "analysis", status: "pass" }, + ], + }, + { + name: "Security", + question: "Does this preserve or improve our security posture and avoid vulnerabilities?", + status: "pass", + criteria: [ + { name: "Vulnerabilities", description: "Security issues are remediated", type: "ai-analysis", status: "pass" }, + { name: "IaC Scanning", description: "", type: null, status: "pass" }, + { name: "Dependency Alerts", description: "Known CVEs are patched", type: "analysis", status: "pass" }, + { name: "Security Controls", description: "Organization standards applied", type: "ai", status: "pass" }, + ], + }, + { + name: "Deployability", + question: "Is this changeset safe to ship to production immediately?", + status: "fail", + criteria: [ + { name: "Compatibility", description: "Breaking changes are avoided", type: "analysis", status: "pass" }, + { name: "Rollout / Rollback", description: "Known rollback plan if deploy fails", type: "ai", status: "fail" }, + { name: "Observability", description: "Logging, metrics, tracing instrumented", type: "ai", status: "fail" }, + { name: "Cost", description: "Tech ops costs estimated", type: "analysis", status: "pass" }, + ], + }, + { + name: "Compliance", + question: "Does this meet our regulatory, contractual, and policy obligations?", + status: "pass", + criteria: [ + { name: "Change Control", description: "Separation of Duties policy met", type: "analysis", status: "pass" }, + { name: "AI Governance", description: "AI involvement was acceptable", type: "analysis", status: "pass" }, + { name: "Privacy", description: "PII is identified and handled to standards", type: "ai", status: "pass" }, + { name: "Accessibility", description: "Software meets accessibility requirements", type: "analysis", status: "pass" }, + { name: "Licensing", description: "Supply chain meets IP policy", type: "analysis", status: "pass" }, + ], + }, +]; + +export type EvaluationResult = "pass" | "fail" | "skip"; + +export type VerificationMode = "active" | "evaluate" | "disabled"; + +export interface CriterionPerformance { + f1: number | null; + passAt1: number | null; + mode: VerificationMode; + evaluations: EvaluationResult[]; +} + +export const modeConfig = { + active: { label: "Active", color: "text-mint", bg: "bg-mint/10" }, + evaluate: { label: "Evaluate", color: "text-amber", bg: "bg-amber/10" }, + disabled: { label: "Disabled", color: "text-navy-600", bg: "bg-white/[0.04]" }, +} as const satisfies Record< + VerificationMode, + { label: string; color: string; bg: string } +>; + +export const criterionPerformance: Record = { + "Motivation": { f1: 0.87, passAt1: 0.82, mode: "active", evaluations: ["pass","pass","fail","pass","pass","pass","pass","fail","pass","pass"] }, + "Specifications": { f1: 0.83, passAt1: 0.78, mode: "active", evaluations: ["pass","fail","pass","pass","pass","fail","pass","pass","pass","pass"] }, + "Documentation": { f1: 0.79, passAt1: 0.74, mode: "active", evaluations: ["pass","pass","pass","fail","pass","pass","fail","pass","pass","fail"] }, + "Minimization": { f1: 0.72, passAt1: 0.68, mode: "evaluate", evaluations: ["pass","fail","pass","fail","pass","pass","fail","pass","pass","pass"] }, + "Formatting": { f1: 0.99, passAt1: 0.98, mode: "active", evaluations: ["pass","pass","pass","pass","pass","pass","pass","pass","pass","pass"] }, + "Linting": { f1: 0.98, passAt1: 0.97, mode: "active", evaluations: ["pass","pass","pass","pass","pass","pass","pass","pass","fail","pass"] }, + "Style": { f1: 0.81, passAt1: 0.76, mode: "active", evaluations: ["pass","fail","pass","pass","pass","pass","fail","pass","pass","pass"] }, + "Completeness": { f1: 0.76, passAt1: 0.71, mode: "active", evaluations: ["pass","pass","fail","pass","fail","pass","pass","pass","fail","pass"] }, + "Defects": { f1: 0.84, passAt1: 0.79, mode: "active", evaluations: ["pass","pass","pass","fail","pass","pass","pass","pass","pass","fail"] }, + "Performance": { f1: 0.69, passAt1: 0.63, mode: "evaluate", evaluations: ["fail","pass","pass","fail","pass","fail","pass","pass","fail","pass"] }, + "Test Coverage": { f1: 0.95, passAt1: 0.93, mode: "active", evaluations: ["pass","pass","pass","pass","pass","pass","fail","pass","pass","pass"] }, + "Test Quality": { f1: 0.71, passAt1: 0.65, mode: "evaluate", evaluations: ["pass","fail","fail","pass","pass","fail","pass","fail","pass","pass"] }, + "E2E Coverage": { f1: 0.91, passAt1: 0.88, mode: "active", evaluations: ["pass","pass","pass","fail","pass","pass","pass","pass","pass","pass"] }, + "Architecture": { f1: 0.88, passAt1: 0.84, mode: "active", evaluations: ["pass","pass","pass","pass","fail","pass","pass","pass","pass","pass"] }, + "Interfaces": { f1: null, passAt1: null, mode: "disabled", evaluations: [] }, + "Duplication": { f1: 0.96, passAt1: 0.94, mode: "active", evaluations: ["pass","pass","pass","pass","pass","pass","pass","fail","pass","pass"] }, + "Simplicity": { f1: 0.74, passAt1: 0.69, mode: "active", evaluations: ["pass","fail","pass","pass","fail","pass","pass","fail","pass","pass"] }, + "Dead Code": { f1: 0.93, passAt1: 0.90, mode: "active", evaluations: ["pass","pass","pass","pass","pass","fail","pass","pass","pass","pass"] }, + "Vulnerabilities": { f1: 0.86, passAt1: 0.81, mode: "active", evaluations: ["pass","pass","fail","pass","pass","pass","pass","pass","fail","pass"] }, + "IaC Scanning": { f1: null, passAt1: null, mode: "disabled", evaluations: [] }, + "Dependency Alerts": { f1: 0.97, passAt1: 0.95, mode: "active", evaluations: ["pass","pass","pass","pass","pass","pass","pass","pass","pass","fail"] }, + "Security Controls": { f1: 0.80, passAt1: 0.75, mode: "active", evaluations: ["pass","pass","fail","pass","pass","fail","pass","pass","pass","pass"] }, + "Compatibility": { f1: 0.89, passAt1: 0.85, mode: "active", evaluations: ["pass","pass","pass","pass","fail","pass","pass","pass","pass","pass"] }, + "Rollout / Rollback": { f1: 0.66, passAt1: 0.60, mode: "evaluate", evaluations: ["fail","pass","fail","pass","fail","pass","pass","fail","pass","fail"] }, + "Observability": { f1: 0.73, passAt1: 0.67, mode: "evaluate", evaluations: ["pass","fail","pass","fail","pass","pass","fail","pass","fail","pass"] }, + "Cost": { f1: 0.78, passAt1: 0.72, mode: "evaluate", evaluations: ["pass","pass","fail","pass","fail","pass","pass","fail","pass","pass"] }, + "Change Control": { f1: 0.94, passAt1: 0.91, mode: "active", evaluations: ["pass","pass","pass","pass","pass","pass","pass","pass","fail","pass"] }, + "AI Governance": { f1: 0.85, passAt1: 0.80, mode: "active", evaluations: ["pass","pass","pass","fail","pass","pass","pass","pass","pass","pass"] }, + "Privacy": { f1: 0.77, passAt1: 0.72, mode: "active", evaluations: ["pass","fail","pass","pass","pass","fail","pass","pass","pass","fail"] }, + "Accessibility": { f1: 0.90, passAt1: 0.87, mode: "active", evaluations: ["pass","pass","pass","pass","pass","fail","pass","pass","pass","pass"] }, + "Licensing": { f1: 0.96, passAt1: 0.93, mode: "active", evaluations: ["pass","pass","pass","pass","pass","pass","pass","pass","pass","pass"] }, +}; + +export function getCategorySummary(categories: readonly VerificationCategory[]) { + const passing = categories.filter((c) => c.status === "pass").length; + return { passing, total: categories.length }; +} + +export function getCriteriaSummary(criteria: readonly Criterion[]) { + return { + passing: criteria.filter((c) => c.status === "pass").length, + failing: criteria.filter((c) => c.status === "fail").length, + na: criteria.filter((c) => c.status === "na").length, + total: criteria.length, + }; +} + +export function getAllCriteria(categories: readonly VerificationCategory[]) { + return categories.flatMap((c) => c.criteria); +} diff --git a/apps/arc-web/app/routes.ts b/apps/arc-web/app/routes.ts index 45756ddd6..6de1dcd90 100644 --- a/apps/arc-web/app/routes.ts +++ b/apps/arc-web/app/routes.ts @@ -23,6 +23,7 @@ export default [ route("configuration", "routes/run-configuration.tsx"), route("graph", "routes/run-graph.tsx"), route("files", "routes/run-files-changed.tsx"), + route("verifications", "routes/run-verifications.tsx"), route("usage", "routes/run-usage.tsx"), route("retro", "routes/run-retro.tsx"), ]), diff --git a/apps/arc-web/app/routes/run-detail.tsx b/apps/arc-web/app/routes/run-detail.tsx index e0a892c78..091ae5ca6 100644 --- a/apps/arc-web/app/routes/run-detail.tsx +++ b/apps/arc-web/app/routes/run-detail.tsx @@ -8,6 +8,7 @@ const tabs = [ { name: "Overview", path: "", count: null }, { name: "Stages", path: "/stages/detect-drift", count: 4 }, { name: "Files Changed", path: "/files", count: 3 }, + { name: "Verifications", path: "/verifications", count: null }, { name: "Retro", path: "/retro", count: null }, { name: "Usage", path: "/usage", count: null }, ]; diff --git a/apps/arc-web/app/routes/run-verifications.tsx b/apps/arc-web/app/routes/run-verifications.tsx new file mode 100644 index 000000000..fdab7f510 --- /dev/null +++ b/apps/arc-web/app/routes/run-verifications.tsx @@ -0,0 +1,128 @@ +import { + Disclosure, + DisclosureButton, + DisclosurePanel, +} from "@headlessui/react"; +import { + CheckCircleIcon, + XCircleIcon, + MinusCircleIcon, + ChevronRightIcon, +} from "@heroicons/react/20/solid"; +import { + verificationCategories, + statusConfig, + typeConfig, + getCriteriaSummary, +} from "../data/verifications"; +import type { + VerificationStatus, + VerificationType, + VerificationCategory, +} from "../data/verifications"; + +function StatusIcon({ + status, + className = "size-5", +}: { + status: VerificationStatus; + className?: string; +}) { + switch (status) { + case "pass": + return ; + case "fail": + return ; + case "na": + return ; + } +} + +function TypeBadge({ type }: { type: VerificationType | null }) { + if (type === null) return null; + const config = typeConfig[type]; + return ( + + {config.label} + + ); +} + +function CategoryCard({ category }: { category: VerificationCategory }) { + const criteriaStats = getCriteriaSummary(category.criteria); + const applicable = criteriaStats.total - criteriaStats.na; + const config = statusConfig[category.status]; + + return ( + + + +
+
+ + {category.name} + + + {category.question} + +
+
+ + {criteriaStats.passing}/{applicable} + + +
+ + +
+ + + {category.criteria.map((criterion) => ( + + + + + + + ))} + +
+ + + {criterion.name} + + {criterion.description || ( + Not configured + )} + + +
+
+
+
+ ); +} + +export default function RunVerifications() { + return ( +
+ {verificationCategories.map((category) => ( + + ))} +
+ ); +} diff --git a/apps/arc-web/app/routes/verifications.tsx b/apps/arc-web/app/routes/verifications.tsx new file mode 100644 index 000000000..3142a0bcd --- /dev/null +++ b/apps/arc-web/app/routes/verifications.tsx @@ -0,0 +1,411 @@ +import { useState } from "react"; +import { + Disclosure, + DisclosureButton, + DisclosurePanel, +} from "@headlessui/react"; +import { + ChevronRightIcon, + LightBulbIcon, + ClipboardDocumentListIcon, + BookOpenIcon, + FunnelIcon, + Bars3BottomLeftIcon, + WrenchIcon, + PaintBrushIcon, + CheckBadgeIcon, + BugAntIcon, + BoltIcon, + BeakerIcon, + StarIcon, + ComputerDesktopIcon, + CubeTransparentIcon, + ArrowsRightLeftIcon, + DocumentDuplicateIcon, + SparklesIcon, + ArchiveBoxXMarkIcon, + ShieldExclamationIcon, + ServerStackIcon, + ExclamationTriangleIcon, + LockClosedIcon, + PuzzlePieceIcon, + ArrowUturnLeftIcon, + EyeIcon, + CurrencyDollarIcon, + ClipboardDocumentCheckIcon, + CpuChipIcon, + FingerPrintIcon, + HandRaisedIcon, + ScaleIcon, + MapPinIcon, + DocumentTextIcon, + ShieldCheckIcon, + WrenchScrewdriverIcon, + KeyIcon, + RocketLaunchIcon, + BuildingLibraryIcon, +} from "@heroicons/react/20/solid"; +import { + MagnifyingGlassIcon, + ChevronDownIcon, +} from "@heroicons/react/24/outline"; +import { + verificationCategories, + typeConfig, + modeConfig, + criterionPerformance, +} from "../data/verifications"; +import type { + VerificationType, + VerificationMode, + EvaluationResult, + VerificationCategory, +} from "../data/verifications"; +import type { Route } from "./+types/verifications"; + +export const handle = { wide: true }; + +export function meta({}: Route.MetaArgs) { + return [{ title: "Verifications — Arc" }]; +} + +type IconComponent = React.ComponentType<{ className?: string }>; + +function TrafficLightIcon({ className }: { className?: string }) { + return ( + + + + ); +} + +const criterionIcons: Record = { + "Motivation": LightBulbIcon, + "Specifications": ClipboardDocumentListIcon, + "Documentation": BookOpenIcon, + "Minimization": FunnelIcon, + "Formatting": Bars3BottomLeftIcon, + "Linting": WrenchIcon, + "Style": PaintBrushIcon, + "Completeness": CheckBadgeIcon, + "Defects": BugAntIcon, + "Performance": BoltIcon, + "Test Coverage": BeakerIcon, + "Test Quality": StarIcon, + "E2E Coverage": ComputerDesktopIcon, + "Architecture": CubeTransparentIcon, + "Interfaces": ArrowsRightLeftIcon, + "Duplication": DocumentDuplicateIcon, + "Simplicity": SparklesIcon, + "Dead Code": ArchiveBoxXMarkIcon, + "Vulnerabilities": ShieldExclamationIcon, + "IaC Scanning": ServerStackIcon, + "Dependency Alerts": ExclamationTriangleIcon, + "Security Controls": LockClosedIcon, + "Compatibility": PuzzlePieceIcon, + "Rollout / Rollback": ArrowUturnLeftIcon, + "Observability": EyeIcon, + "Cost": CurrencyDollarIcon, + "Change Control": ClipboardDocumentCheckIcon, + "AI Governance": CpuChipIcon, + "Privacy": FingerPrintIcon, + "Accessibility": HandRaisedIcon, + "Licensing": ScaleIcon, +}; + +const categoryIcons: Record = { + "Traceability": MapPinIcon, + "Readability": DocumentTextIcon, + "Reliability": ShieldCheckIcon, + "Code Coverage": TrafficLightIcon, + "Maintainability": WrenchScrewdriverIcon, + "Security": KeyIcon, + "Deployability": RocketLaunchIcon, + "Compliance": BuildingLibraryIcon, +}; + +function TypeBadge({ type }: { type: VerificationType | null }) { + if (type === null) return null; + const config = typeConfig[type]; + return ( + + {config.label} + + ); +} + +type ViewMode = "grouped" | "ungrouped"; + +function CategoryCard({ category }: { category: VerificationCategory }) { + return ( + + + {(() => { + const CatIcon = categoryIcons[category.name]; + return CatIcon ? : null; + })()} +
+
+ + {category.name} + + + {category.question} + +
+
+ + {category.criteria.length} controls + + +
+ + +
+ + + {category.criteria.map((criterion) => { + const Icon = criterionIcons[criterion.name]; + const perf = criterionPerformance[criterion.name]; + return ( + + + + + + + + + ); + })} + +
+ {Icon && } + + {criterion.name} + + {criterion.description || ( + Not configured + )} + + + + {perf && } + + {perf && } +
+
+
+
+ ); +} + +function GroupedView({ categories }: { categories: readonly VerificationCategory[] }) { + return ( +
+ {categories.map((category) => ( + + ))} +
+ ); +} + +function ModeBadge({ mode }: { mode: VerificationMode }) { + const config = modeConfig[mode]; + return ( + + {config.label} + + ); +} + +function EvaluationDots({ evaluations }: { evaluations: readonly EvaluationResult[] }) { + if (evaluations.length === 0) { + return ; + } + return ( +
+ {evaluations.map((result, i) => ( + + ))} +
+ ); +} + +function UngroupedView({ categories }: { categories: readonly VerificationCategory[] }) { + return ( +
+ + + + + + + + + + + + + + + {categories.flatMap((category) => + category.criteria.map((criterion) => { + const Icon = criterionIcons[criterion.name]; + const perf = criterionPerformance[criterion.name]; + return ( + + + + + + + + + + + + ); + }), + )} + +
+ VerificationDescriptionCategoryTypeAccuracy (F1)pass@1ModeEvaluations
+ {Icon && } + + {criterion.name} + + {criterion.description || ( + Not configured + )} + + {category.name} + + + + {perf?.f1 != null ? perf.f1.toFixed(2) : } + + {perf?.passAt1 != null ? perf.passAt1.toFixed(2) : } + + {perf && } + + {perf && } +
+
+ ); +} + +function filterCategories( + categories: readonly VerificationCategory[], + query: string, + modeFilter: VerificationMode | "all", +): VerificationCategory[] { + const lowerQuery = query.toLowerCase(); + return categories + .map((category) => { + const filtered = category.criteria.filter((c) => { + const perf = criterionPerformance[c.name]; + const matchesMode = modeFilter === "all" || perf?.mode === modeFilter; + const matchesQuery = + lowerQuery === "" || + c.name.toLowerCase().includes(lowerQuery) || + c.description.toLowerCase().includes(lowerQuery) || + category.name.toLowerCase().includes(lowerQuery); + return matchesMode && matchesQuery; + }); + return { ...category, criteria: filtered }; + }) + .filter((category) => category.criteria.length > 0); +} + +export default function Verifications() { + const [view, setView] = useState("grouped"); + const [query, setQuery] = useState(""); + const [modeFilter, setModeFilter] = useState("all"); + + const filtered = filterCategories(verificationCategories, query, modeFilter); + + return ( +
+ {/* Toolbar */} +
+
+ + setQuery(e.target.value)} + className="w-full rounded-md border border-white/[0.06] bg-navy-800/80 py-2 pl-9 pr-3 text-sm text-ice-100 placeholder-navy-600 outline-none transition-colors focus:border-teal-500/40 focus:ring-0" + /> +
+
+ + +
+
+ + +
+
+ + {view === "grouped" ? : } +
+ ); +}