diff --git a/docs/my-website/docs/proxy/high_availability_control_plane.md b/docs/my-website/docs/proxy/high_availability_control_plane.md new file mode 100644 index 00000000000..324fba3a180 --- /dev/null +++ b/docs/my-website/docs/proxy/high_availability_control_plane.md @@ -0,0 +1,190 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import { ControlPlaneArchitecture } from '@site/src/components/ControlPlaneArchitecture'; + +# [BETA] High Availability Control Plane + +Deploy a single LiteLLM UI that manages multiple independent LiteLLM proxy instances, each with its own database, Redis, and master key. + +:::info + +This is an Enterprise feature. + +[Enterprise Pricing](https://www.litellm.ai/#pricing) + +[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial) + +::: + +## Why This Architecture? + +In the [standard multi-region setup](./control_plane_and_data_plane.md), all instances share a single database and master key. This works, but introduces a shared dependency. If the database goes down, every instance is affected. + +The **High Availability Control Plane** takes a different approach: + +| | Shared Database (Standard) | High Availability Control Plane | +|---|---|---| +| **Database** | Single shared DB for all instances | Each instance has its own DB | +| **Redis** | Shared Redis | Each instance has its own Redis | +| **Master Key** | Same key across all instances | Each instance has its own key | +| **Failure isolation** | DB outage affects all instances | Failure is isolated to one instance | +| **User management** | Centralized, one user table | Independent, each worker manages its own users | +| **UI** | One UI per admin instance | Single control plane UI manages all workers | + +### Benefits + +- **True high availability**: no shared infrastructure means no single point of failure +- **Blast radius containment**: a misconfiguration or outage on one worker doesn't affect others +- **Regional isolation**: workers can run in different regions with data residency requirements +- **Simpler operations**: each worker is a self-contained LiteLLM deployment + +## Architecture + + + +The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It does not proxy LLM requests, it is purely for administration. + +Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own users, keys, teams, and budgets. + +## Setup + +### 1. Control Plane Configuration + +The control plane needs a `worker_registry` that lists all worker instances. + +```yaml title="cp_config.yaml" +model_list: [] + +general_settings: + master_key: sk-1234 + database_url: os.environ/DATABASE_URL + +worker_registry: + - worker_id: "worker-a" + name: "Worker A" + url: "http://localhost:4001" + - worker_id: "worker-b" + name: "Worker B" + url: "http://localhost:4002" +``` + +Start the control plane: + +```bash +litellm --config cp_config.yaml --port 4000 +``` + +### 2. Worker Configuration + +Each worker needs `control_plane_url` in its `general_settings` to enable cross-origin authentication from the control plane UI. + +`PROXY_BASE_URL` must also be set for each worker so that SSO callback redirects resolve correctly. + + + + +```yaml title="worker_a_config.yaml" +model_list: [] + +general_settings: + master_key: sk-worker-a-1234 + database_url: os.environ/WORKER_A_DATABASE_URL + control_plane_url: "http://localhost:4000" +``` + +```bash +PROXY_BASE_URL=http://localhost:4001 litellm --config worker_a_config.yaml --port 4001 +``` + + + + +```yaml title="worker_b_config.yaml" +model_list: [] + +general_settings: + master_key: sk-worker-b-1234 + database_url: os.environ/WORKER_B_DATABASE_URL + control_plane_url: "http://localhost:4000" +``` + +```bash +PROXY_BASE_URL=http://localhost:4002 litellm --config worker_b_config.yaml --port 4002 +``` + + + + +:::important +Each worker must have its own `master_key` and `database_url`. The whole point of this architecture is that workers are independent. +::: + +### 3. SSO Configuration (Optional) + +SSO is configured on the **control plane** instance the same way as a standard LiteLLM proxy. See the [SSO setup guide](./admin_ui_sso.md) for full instructions. + +If using SSO, make sure to register each worker URL and the control plane URL as allowed callback URLs in your SSO provider's dashboard. + +## How It Works + +### Login Flow + +1. User visits the control plane UI (`http://localhost:4000/ui`) +2. The login page shows a **worker selector** dropdown listing all registered workers +3. User selects a worker (e.g. "Worker A") and logs in with username/password or SSO +4. The UI authenticates against the **selected worker** using the `/v3/login` endpoint +5. On success, the UI stores the worker's JWT and points all subsequent API calls at the worker +6. The user can now manage keys, teams, models, and budgets on that worker, all from the control plane UI + +### Switching Workers + +Once logged in, users can switch workers from the **navbar dropdown** without leaving the UI. Switching redirects back to the login page to authenticate against the new worker. + +### Discovery + +The control plane exposes a `/.well-known/litellm-ui-config` endpoint that the UI reads on load. This endpoint returns: +- `is_control_plane: true` +- The list of workers with their IDs, names, and URLs + +This is how the login page knows to show the worker selector. + +## Local Testing + +To try this out locally, start each instance in a separate terminal: + +```bash +# Terminal 1: Control Plane +litellm --config cp_config.yaml --port 4000 + +# Terminal 2: Worker A +PROXY_BASE_URL=http://localhost:4001 litellm --config worker_a_config.yaml --port 4001 + +# Terminal 3: Worker B +PROXY_BASE_URL=http://localhost:4002 litellm --config worker_b_config.yaml --port 4002 +``` + +Then open `http://localhost:4000/ui`. You should see the worker selector on the login page. + +## Configuration Reference + +### Control Plane Settings + +| Field | Location | Description | +|---|---|---| +| `worker_registry` | Top-level config | List of worker instances | +| `worker_registry[].worker_id` | Required | Unique identifier for the worker | +| `worker_registry[].name` | Required | Display name shown in the UI | +| `worker_registry[].url` | Required | Full URL of the worker instance | + +### Worker Settings + +| Field | Location | Description | +|---|---|---| +| `general_settings.control_plane_url` | Required | URL of the control plane instance. Enables `/v3/login` and `/v3/login/exchange` endpoints on this worker. | +| `PROXY_BASE_URL` | Environment variable | The worker's own external URL. Required for SSO callback redirects. | + +## Related Documentation + +- [Standard Multi-Region Setup](./control_plane_and_data_plane.md) - shared-database architecture for admin/worker split +- [SSO Setup](./admin_ui_sso.md) - configuring SSO for the admin UI +- [Production Deployment](./prod.md) - production best practices diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 4c0471fb8f4..888ec3f5f69 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -430,6 +430,7 @@ const sidebars = { "proxy/architecture", "proxy/multi_tenant_architecture", "proxy/control_plane_and_data_plane", + "proxy/high_availability_control_plane", "proxy/db_deadlocks", "proxy/db_info", "proxy/image_handling", diff --git a/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx b/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx new file mode 100644 index 00000000000..5cc41979b02 --- /dev/null +++ b/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx @@ -0,0 +1,95 @@ +import React from 'react'; +import styles from './styles.module.css'; + +/* ────────────────────── Shared small pieces ────────────────────── */ + +function InfraChip({ color, label }: { color: string; label: string }) { + const dotClass = + color === 'green' + ? styles.infraDotGreen + : color === 'blue' + ? styles.infraDotBlue + : styles.infraDotOrange; + + return ( + + + {label} + + ); +} + +/* ────────────────────── Architecture tab ────────────────────── */ + +function ArchitectureView() { + return ( +
+ {/* User */} +
+
👤
+ Admin +
+ +
+ + {/* Control Plane */} +
+
+ Control Plane + UI +
+
cp.example.com
+
+ + + +
+
+ + {/* Branch connector */} +
+
+
+
+ + {/* Workers */} +
+
+
+ Worker A + US East +
+
worker-a.example.com
+
+ + + +
+
+ +
+
+ Worker B + EU West +
+
worker-b.example.com
+
+ + + +
+
+
+
+ ); +} + +/* ────────────────────── Main component ────────────────────── */ + +export default function ControlPlaneArchitecture() { + return ( +
+ +
+ ); +} diff --git a/docs/my-website/src/components/ControlPlaneArchitecture/index.tsx b/docs/my-website/src/components/ControlPlaneArchitecture/index.tsx new file mode 100644 index 00000000000..826b4d68818 --- /dev/null +++ b/docs/my-website/src/components/ControlPlaneArchitecture/index.tsx @@ -0,0 +1 @@ +export { default as ControlPlaneArchitecture } from './ControlPlaneArchitecture'; diff --git a/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css b/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css new file mode 100644 index 00000000000..8400a8920e0 --- /dev/null +++ b/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css @@ -0,0 +1,517 @@ +/* ── Custom properties ── */ +:root { + --cp-bg: #ffffff; + --cp-border: #e5e7eb; + --cp-text: #1a1a2e; + --cp-text-secondary: #6b7280; + --cp-text-muted: #9ca3af; + --cp-accent: #3b82f6; + --cp-accent-light: #dbeafe; + --cp-accent-glow: rgba(59, 130, 246, 0.15); + --cp-green: #10b981; + --cp-green-light: #d1fae5; + --cp-green-glow: rgba(16, 185, 129, 0.15); + --cp-orange: #f59e0b; + --cp-orange-light: #fef3c7; + --cp-purple: #8b5cf6; + --cp-purple-light: #ede9fe; + --cp-red: #ef4444; + --cp-red-light: #fee2e2; + --cp-card-bg: #f9fafb; + --cp-infra-bg: #f1f5f9; + --cp-infra-border: #cbd5e1; + --cp-connector: #d1d5db; + --cp-dot-size: 8px; +} + +[data-theme='dark'] { + --cp-bg: #111827; + --cp-border: #374151; + --cp-text: #e5e7eb; + --cp-text-secondary: #9ca3af; + --cp-text-muted: #6b7280; + --cp-accent: #60a5fa; + --cp-accent-light: #1e3a5f; + --cp-accent-glow: rgba(96, 165, 250, 0.2); + --cp-green: #34d399; + --cp-green-light: #064e3b; + --cp-green-glow: rgba(52, 211, 153, 0.2); + --cp-orange: #fbbf24; + --cp-orange-light: #78350f; + --cp-purple: #a78bfa; + --cp-purple-light: #3b0764; + --cp-red: #f87171; + --cp-red-light: #451a1a; + --cp-card-bg: #1f2937; + --cp-infra-bg: #1e293b; + --cp-infra-border: #475569; + --cp-connector: #4b5563; +} + +/* ── Wrapper ── */ +.wrapper { + margin: 1.5rem 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; +} + +/* ── Tab bar ── */ +.tabs { + display: flex; + gap: 0; + margin-bottom: 1.5rem; + border-bottom: 2px solid var(--cp-border); +} + +.tab { + padding: 0.6rem 1.25rem; + font-size: 0.85rem; + font-weight: 600; + color: var(--cp-text-secondary); + background: none; + border: none; + border-bottom: 2px solid transparent; + margin-bottom: -2px; + cursor: pointer; + transition: color 0.2s, border-color 0.2s; +} + +.tab:hover { + color: var(--cp-text); +} + +.tabActive { + color: var(--cp-accent); + border-bottom-color: var(--cp-accent); +} + +/* ── Architecture diagram ── */ +.diagram { + display: flex; + flex-direction: column; + align-items: center; + gap: 0; +} + +/* ── User icon ── */ +.userRow { + display: flex; + flex-direction: column; + align-items: center; + margin-bottom: 0.5rem; +} + +.userIcon { + width: 40px; + height: 40px; + border-radius: 50%; + background: var(--cp-accent-light); + border: 2px solid var(--cp-accent); + display: flex; + align-items: center; + justify-content: center; + font-size: 1.1rem; +} + +.userLabel { + font-size: 0.75rem; + color: var(--cp-text-secondary); + margin-top: 0.3rem; + font-weight: 500; +} + +/* ── Connectors ── */ +.connectorDown { + width: 2px; + height: 28px; + background: var(--cp-connector); + position: relative; +} + +.connectorDown::after { + content: ''; + position: absolute; + bottom: -4px; + left: 50%; + transform: translateX(-50%); + width: 0; + height: 0; + border-left: 5px solid transparent; + border-right: 5px solid transparent; + border-top: 5px solid var(--cp-connector); +} + +.connectorBranch { + display: flex; + align-items: flex-start; + justify-content: center; + position: relative; + width: 100%; + max-width: 700px; + height: 36px; +} + +.connectorBranch::before { + content: ''; + position: absolute; + top: 0; + left: 50%; + width: 2px; + height: 12px; + background: var(--cp-connector); + transform: translateX(-50%); +} + +.connectorBranch::after { + content: ''; + position: absolute; + top: 12px; + left: calc(25% + 12px); + right: calc(25% + 12px); + height: 2px; + background: var(--cp-connector); +} + +.branchLeg { + position: absolute; + top: 12px; + width: 2px; + height: 24px; + background: var(--cp-connector); +} + +.branchLeg::after { + content: ''; + position: absolute; + bottom: -4px; + left: 50%; + transform: translateX(-50%); + width: 0; + height: 0; + border-left: 5px solid transparent; + border-right: 5px solid transparent; + border-top: 5px solid var(--cp-connector); +} + +.branchLegLeft { + left: calc(25% + 12px); +} + +.branchLegRight { + right: calc(25% + 12px); +} + +/* ── Node cards ── */ +.node { + border: 2px solid var(--cp-border); + border-radius: 12px; + background: var(--cp-card-bg); + padding: 1rem 1.25rem; + text-align: center; + transition: border-color 0.3s, box-shadow 0.3s; + position: relative; +} + +.nodeControlPlane { + border-color: var(--cp-accent); + box-shadow: 0 0 0 3px var(--cp-accent-glow); + min-width: 280px; +} + +.nodeWorker { + min-width: 220px; +} + +.nodeWorkerA { + border-color: var(--cp-green); + box-shadow: 0 0 0 3px var(--cp-green-glow); +} + +.nodeWorkerB { + border-color: var(--cp-purple); + box-shadow: 0 0 0 3px rgba(139, 92, 246, 0.15); +} + +[data-theme='dark'] .nodeWorkerB { + box-shadow: 0 0 0 3px rgba(167, 139, 250, 0.2); +} + +.nodeHeader { + display: flex; + align-items: center; + justify-content: center; + gap: 0.5rem; + margin-bottom: 0.5rem; +} + +.nodeIcon { + font-size: 1.1rem; +} + +.nodeTitle { + font-size: 0.95rem; + font-weight: 700; + color: var(--cp-text); +} + +.nodeSubtitle { + font-size: 0.75rem; + color: var(--cp-text-secondary); + margin-bottom: 0.75rem; +} + +.badge { + display: inline-block; + font-size: 0.65rem; + font-weight: 600; + padding: 0.15rem 0.5rem; + border-radius: 9999px; + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.badgeBlue { + background: var(--cp-accent-light); + color: var(--cp-accent); +} + +.badgeGreen { + background: var(--cp-green-light); + color: var(--cp-green); +} + +.badgePurple { + background: var(--cp-purple-light); + color: var(--cp-purple); +} + +/* ── Infrastructure chips ── */ +.infraRow { + display: flex; + gap: 0.4rem; + justify-content: center; + flex-wrap: wrap; + margin-top: 0.5rem; +} + +.infraChip { + display: flex; + align-items: center; + gap: 0.3rem; + font-size: 0.7rem; + font-weight: 500; + color: var(--cp-text-secondary); + background: var(--cp-infra-bg); + border: 1px solid var(--cp-infra-border); + border-radius: 6px; + padding: 0.2rem 0.5rem; +} + +.infraDot { + width: 6px; + height: 6px; + border-radius: 50%; + flex-shrink: 0; +} + +.infraDotGreen { + background: var(--cp-green); +} + +.infraDotBlue { + background: var(--cp-accent); +} + +.infraDotOrange { + background: var(--cp-orange); +} + +/* ── Workers row ── */ +.workersRow { + display: flex; + gap: 2rem; + justify-content: center; + flex-wrap: wrap; +} + +/* ── Animated flow ── */ +.flowLabel { + font-size: 0.7rem; + color: var(--cp-accent); + font-weight: 600; + position: absolute; + white-space: nowrap; +} + +/* ── Comparison view ── */ +.comparisonGrid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1.5rem; + margin-top: 0.5rem; +} + +.comparisonColumn { + border: 2px solid var(--cp-border); + border-radius: 12px; + padding: 1.25rem; + background: var(--cp-card-bg); +} + +.comparisonColumnOld { + border-color: var(--cp-red); +} + +.comparisonColumnNew { + border-color: var(--cp-green); +} + +.comparisonTitle { + font-size: 0.9rem; + font-weight: 700; + color: var(--cp-text); + text-align: center; + margin-bottom: 1rem; + display: flex; + align-items: center; + justify-content: center; + gap: 0.4rem; +} + +.comparisonTitleOld { + color: var(--cp-red); +} + +.comparisonTitleNew { + color: var(--cp-green); +} + +/* ── Mini diagram inside comparison ── */ +.miniDiagram { + display: flex; + flex-direction: column; + align-items: center; + gap: 0.5rem; +} + +.miniNode { + border: 1.5px solid var(--cp-border); + border-radius: 8px; + background: var(--cp-bg); + padding: 0.5rem 0.75rem; + text-align: center; + font-size: 0.75rem; + font-weight: 600; + color: var(--cp-text); + width: 100%; + max-width: 180px; +} + +.miniNodeHighlight { + border-color: var(--cp-accent); + background: var(--cp-accent-light); +} + +.miniNodeDanger { + border-color: var(--cp-red); + background: var(--cp-red-light); +} + +.miniNodeSuccess { + border-color: var(--cp-green); + background: var(--cp-green-light); +} + +.miniConnector { + width: 1.5px; + height: 16px; + background: var(--cp-connector); +} + +.miniWorkersRow { + display: flex; + gap: 0.5rem; + justify-content: center; + width: 100%; +} + +.miniWorkerStack { + display: flex; + flex-direction: column; + align-items: center; + gap: 0.3rem; + flex: 1; + max-width: 140px; +} + +.miniInfra { + font-size: 0.65rem; + color: var(--cp-text-muted); + font-weight: 500; +} + +.miniInfraShared { + color: var(--cp-red); + font-weight: 600; +} + +.miniInfraOwn { + color: var(--cp-green); + font-weight: 600; +} + +/* ── Callout box ── */ +.callout { + display: flex; + align-items: flex-start; + gap: 0.6rem; + padding: 0.75rem 1rem; + border-radius: 8px; + margin-top: 1rem; + font-size: 0.8rem; + color: var(--cp-text); + line-height: 1.5; +} + +.calloutDanger { + background: var(--cp-red-light); + border: 1px solid var(--cp-red); +} + +.calloutSuccess { + background: var(--cp-green-light); + border: 1px solid var(--cp-green); +} + +.calloutIcon { + font-size: 1rem; + flex-shrink: 0; + margin-top: 0.1rem; +} + +/* ── Responsive ── */ +@media (max-width: 768px) { + .comparisonGrid { + grid-template-columns: 1fr; + } + + .workersRow { + flex-direction: column; + align-items: center; + } + + .nodeControlPlane { + min-width: auto; + width: 100%; + max-width: 300px; + } + + .nodeWorker { + min-width: auto; + width: 100%; + max-width: 260px; + } + + .connectorBranch { + display: none; + } +}