feat(control-plane): runtime dataplane and role writes via redis stream and postgres (#51762)

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Yassin Kortam 2026-11-16 09:36:25 -08:00 • committed by GitHub
parent 74666b1621
commit 35db165829
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
47 changed files with 3033 additions and 55 deletions

View file

@ -67,6 +67,36 @@ backend:
gatewayUrl: https://gateway.example.com
```
## Control plane migrations
Set `migrationJob.mode` to `control-plane` to migrate the control plane's PostgreSQL database. The default `data-plane` mode runs Prisma migrations for the gateway and backend. The control plane mode runs `litellm --control-plane --skip_server_startup` through the shared image's entrypoint.
```yaml
migrationJob:
mode: control-plane
controlPlane:
databaseSecret:
name: control-plane-migrations
key: database-url
```
Create this Secret before installation. Its key must contain a direct PostgreSQL URL for the control plane database. The Job needs at least two database connections. Control plane migrations use this Secret and `migrationJob.extraEnv`, without the data plane's database or Redis settings.
Helm runs the Job before install and upgrade by default. For Argo CD, select its PreSync hook:
```yaml
migrationJob:
hooks:
helm:
enabled: false
argocd:
enabled: true
```
Argo CD deletes successful Jobs and recreates the named hook on the next sync. Failed Jobs remain until the next sync. The chart omits `ttlSecondsAfterFinished` when the Argo CD hook is enabled so the TTL controller cannot delete the Job before Argo CD records its result. Helm hooks retain the configured TTL. Run a full application sync because selective syncs skip hooks.
`migrationJob.command` and `migrationJob.args` override either mode's command. Setting only `command` clears the default arguments.
## Testing
```bash

View file

@ -1,14 +1,9 @@
{{- if .Values.migrationJob.enabled -}}
# Pre-install / pre-upgrade hook that runs `prisma migrate deploy` against
# the writer database before the application Deployments are rolled out.
# Required because the proxy, gateway and backend all spin up Prisma at
# startup and assume the LiteLLM schema (LiteLLM_Config,
# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...) already exists.
#
# Running this pre-upgrade closes the window where new application pods would
# otherwise serve traffic against the previous release's unmigrated schema.
# Argo CD users can swap the Helm hook for a PreSync hook through
# `migrationJob.hooks`, which re-runs the Job on every sync.
{{- $mode := .Values.migrationJob.mode -}}
{{- if not (has $mode (list "data-plane" "control-plane")) -}}
{{- fail (printf "migrationJob.mode must be data-plane or control-plane (got %q)" $mode) -}}
{{- end -}}
{{- $controlPlane := eq $mode "control-plane" -}}
apiVersion: batch/v1
kind: Job
metadata:
@ -29,7 +24,7 @@ metadata:
{{- end }}
{{- if .Values.migrationJob.hooks.argocd.enabled }}
argocd.argoproj.io/hook: PreSync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
{{- end }}
{{- with .Values.migrationJob.jobAnnotations }}
{{- toYaml . | nindent 4 }}
@ -37,7 +32,9 @@ metadata:
{{- end }}
spec:
backoffLimit: {{ .Values.migrationJob.backoffLimit }}
{{- if not .Values.migrationJob.hooks.argocd.enabled }}
ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
{{- end }}
{{- with .Values.migrationJob.activeDeadlineSeconds }}
activeDeadlineSeconds: {{ . }}
{{- end }}
@ -71,7 +68,7 @@ spec:
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
containers:
- name: prisma-migrations
- name: {{ ternary "control-plane-migrations" "prisma-migrations" $controlPlane }}
image: {{ include "litellm.image" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- with .Values.migrationJob.securityContext }}
@ -85,12 +82,28 @@ spec:
args: {{ toYaml .Values.migrationJob.args | nindent 12 }}
{{- else if .Values.migrationJob.command }}
args: []
{{- else if $controlPlane }}
args:
- proxy
- --control-plane
- --skip_server_startup
{{- else }}
args:
- migrations
{{- end }}
env:
{{- if $controlPlane }}
- name: CONTROL_PLANE_DATABASE_URL
valueFrom:
secretKeyRef:
name: {{ required "migrationJob.controlPlane.databaseSecret.name is required in control-plane mode" .Values.migrationJob.controlPlane.databaseSecret.name | quote }}
key: {{ required "migrationJob.controlPlane.databaseSecret.key is required in control-plane mode" .Values.migrationJob.controlPlane.databaseSecret.key | quote }}
{{- with .Values.migrationJob.extraEnv }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- else }}
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.migrationJob) | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.volumeMounts }}
volumeMounts:
{{- toYaml . | nindent 12 }}

View file

@ -27,7 +27,9 @@ tests:
value: PreSync
- equal:
path: metadata.annotations["argocd.argoproj.io/hook-delete-policy"]
value: BeforeHookCreation
value: BeforeHookCreation,HookSucceeded
- notExists:
path: spec.ttlSecondsAfterFinished
- it: drops the Helm hook so Argo CD owns the Job
set:
@ -61,3 +63,23 @@ tests:
- equal:
path: metadata.annotations["helm.sh/hook-weight"]
value: "-5"
- it: retains the configured cleanup TTL for Helm hooks
set:
migrationJob.ttlSecondsAfterFinished: 600
asserts:
- equal:
path: spec.ttlSecondsAfterFinished
value: 600
- it: gives Argo CD cleanup ownership even when a Job TTL is configured
set:
migrationJob.hooks.helm.enabled: false
migrationJob.hooks.argocd.enabled: true
migrationJob.ttlSecondsAfterFinished: 1
asserts:
- notExists:
path: spec.ttlSecondsAfterFinished
- equal:
path: metadata.annotations["argocd.argoproj.io/hook-delete-policy"]
value: BeforeHookCreation,HookSucceeded

View file

@ -327,3 +327,112 @@ tests:
asserts:
- notExists:
path: spec.template.metadata.annotations
- it: migrates the control plane without data plane database settings or credentials
set:
migrationJob.mode: control-plane
migrationJob.controlPlane.databaseSecret.name: cp-migrations
database.writer.host: ""
database.writer.dbname: ""
redis.host: data-plane-redis
migrationJob.extraEnv:
- name: SSL_CERT_FILE
value: /certs/postgres-ca.pem
asserts:
- notExists:
path: spec.template.spec.containers[0].command
- equal:
path: spec.template.spec.containers[0].args
value: [proxy, --control-plane, --skip_server_startup]
- equal:
path: spec.template.spec.containers[0].env
value:
- name: CONTROL_PLANE_DATABASE_URL
valueFrom:
secretKeyRef:
name: cp-migrations
key: database-url
- name: SSL_CERT_FILE
value: /certs/postgres-ca.pem
- equal:
path: metadata.annotations["helm.sh/hook"]
value: pre-install,pre-upgrade
- it: runs control plane migrations as an Argo CD hook with a custom Secret key
set:
migrationJob.mode: control-plane
migrationJob.controlPlane.databaseSecret.name: cp-migrations
migrationJob.controlPlane.databaseSecret.key: direct-url
migrationJob.hooks.helm.enabled: false
migrationJob.hooks.argocd.enabled: true
asserts:
- equal:
path: metadata.annotations["argocd.argoproj.io/hook"]
value: PreSync
- equal:
path: metadata.annotations["argocd.argoproj.io/hook-delete-policy"]
value: BeforeHookCreation,HookSucceeded
- notExists:
path: metadata.annotations["helm.sh/hook"]
- notExists:
path: spec.ttlSecondsAfterFinished
- equal:
path: spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef
value:
name: cp-migrations
key: direct-url
- it: preserves an explicit command and args in control plane mode
set:
migrationJob.mode: control-plane
migrationJob.controlPlane.databaseSecret.name: cp-migrations
migrationJob.command: [sh]
migrationJob.args: [-c, echo migrating]
asserts:
- equal:
path: spec.template.spec.containers[0].command
value: [sh]
- equal:
path: spec.template.spec.containers[0].args
value: [-c, echo migrating]
- it: clears the control plane arguments when only a command override is set
set:
migrationJob.mode: control-plane
migrationJob.controlPlane.databaseSecret.name: cp-migrations
migrationJob.command: [sh]
asserts:
- equal:
path: spec.template.spec.containers[0].args
value: []
- it: rejects an unsupported migration mode
set:
migrationJob.mode: invalid
asserts:
- failedTemplate:
errorMessage: 'migrationJob.mode must be data-plane or control-plane (got "invalid")'
- it: requires the control plane database Secret name
set:
migrationJob.mode: control-plane
asserts:
- failedTemplate:
errorMessage: migrationJob.controlPlane.databaseSecret.name is required in control-plane mode
- it: rejects an empty control plane database Secret key
set:
migrationJob.mode: control-plane
migrationJob.controlPlane.databaseSecret.name: cp-migrations
migrationJob.controlPlane.databaseSecret.key: ""
asserts:
- failedTemplate:
errorMessage: migrationJob.controlPlane.databaseSecret.key is required in control-plane mode
- it: requires no control plane Secret when the migration Job is disabled
set:
migrationJob.enabled: false
migrationJob.mode: control-plane
asserts:
- hasDocuments:
count: 0

View file

@ -134,16 +134,13 @@ serviceAccounts:
annotations: {}
name: ""
# Pre-install / pre-upgrade Helm hook that runs `prisma migrate deploy`
# against the writer database, creating the LiteLLM schema (tables that
# gateway + backend assume exist at startup: LiteLLM_Config,
# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...). Disable if your
# pipeline runs migrations out-of-band.
#
# Runs the shared image with `args: [migrations]`, which the entrypoint
# dispatches to the prisma migration runner without importing the proxy.
migrationJob:
enabled: true
mode: data-plane
controlPlane:
databaseSecret:
name: ""
key: database-url
# Which controller is responsible for running the Job.
#
# `helm.enabled` renders the Helm pre-install / pre-upgrade hook, so the Job

View file

@ -29,6 +29,8 @@ from litellm.proxy.control_plane.dataplanes.router import router as dataplanes_r
from litellm.proxy.control_plane.fleet.router import router as fleet_router
from litellm.proxy.control_plane.health.router import router as health_router
from litellm.proxy.control_plane.proxy.router import router as proxy_router
from litellm.proxy.control_plane.roles.router import router as roles_router
from litellm.proxy.control_plane.store.dependencies import get_runtime_config
from litellm.proxy.control_plane.usage.router import router as usage_router
from litellm.proxy.middleware.request_size_limit_middleware import RequestSizeLimitMiddleware
@ -89,15 +91,19 @@ class _Lifespan:
return self
async def __aenter__(self) -> None:
await self.container.runtime_config.start()
self.container.audit_log.pipeline.start()
async def __aexit__(
self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None
) -> None:
try:
await self.container.audit_log.pipeline.close()
await self.container.runtime_config.close()
finally:
self.container.telemetry.shutdown()
try:
await self.container.audit_log.pipeline.close()
finally:
self.container.telemetry.shutdown()
def create_app(container: Container, license_check: License) -> FastAPI:
@ -121,6 +127,7 @@ def create_app(container: Container, license_check: License) -> FastAPI:
app.dependency_overrides[get_dataplane_client] = lambda: container.dataplane_client
app.dependency_overrides[get_fanout] = lambda: container.fanout
app.dependency_overrides[get_audit_log] = lambda: container.audit_log
app.dependency_overrides[get_runtime_config] = lambda: container.runtime_config
install_exception_handlers(app)
app.include_router(health_router)
app.include_router(auth_router, prefix=container.settings.api_prefix)
@ -129,4 +136,5 @@ def create_app(container: Container, license_check: License) -> FastAPI:
app.include_router(fleet_router, prefix=container.settings.api_prefix)
app.include_router(proxy_router, prefix=container.settings.api_prefix)
app.include_router(audit_router, prefix=container.settings.api_prefix)
app.include_router(roles_router, prefix=container.settings.api_prefix)
return app

View file

@ -63,7 +63,7 @@ class AuditLog:
trace: Callable[[], Trace] = current_trace,
unauthenticated_records: deque[AuditRecord] | None = None, # mutable-ok: bounded failure ring
logger: logging.Logger = AUDIT_LOGGER,
known_dataplane_ids: frozenset[str] = frozenset(),
known_dataplane_ids: frozenset[str] | Callable[[], frozenset[str]] = frozenset(),
) -> None:
self._records = records
self._unauthenticated_records: deque[AuditRecord] = ( # mutable-ok: bounded failure ring
@ -109,9 +109,12 @@ class AuditLog:
safe_path: Final = _bounded_redacted(redact_path(_safe_text(path)[:MAX_AUDIT_FIELD_CHARS]))
safe_detail: Final = None if detail is None else _bounded_redacted(_safe_text(detail))
safe_dataplane_id: Final = _safe_text(dataplane_id)
known_ids: Final = (
self._known_dataplane_ids() if callable(self._known_dataplane_ids) else self._known_dataplane_ids
)
recorded_dataplane_id: Final = (
safe_dataplane_id[:MAX_AUDIT_FIELD_CHARS]
if dataplane_id in self._known_dataplane_ids
if dataplane_id in known_ids
else _bounded_redacted(safe_dataplane_id)
)
event: Final = AuditEvent(
@ -179,7 +182,7 @@ def build_audit_log(
product: Product = UNKNOWN_PRODUCT,
trace: Callable[[], Trace] = current_trace,
stream: TextIO | None = None,
known_dataplane_ids: frozenset[str] = frozenset(),
known_dataplane_ids: frozenset[str] | Callable[[], frozenset[str]] = frozenset(),
) -> AuditLog:
logger: Final = AUDIT_LOGGER if stream is None else build_audit_logger(stream)
return AuditLog(

View file

@ -18,8 +18,8 @@ from litellm.proxy.control_plane.auth.rbac import (
ADMIN_ROLE,
ALL_SCOPES,
DataplaneBinding,
Policy,
Principal,
RolePolicy,
build_principal,
parse_scopes,
)
@ -160,7 +160,7 @@ class SessionTokenCodec:
token minted under the same master key by another component never authorizes a control plane route."""
def __init__(
self, master_key: str, issuer: str, ttl: timedelta, policy: RolePolicy, audience: str = SESSION_AUDIENCE
self, master_key: str, issuer: str, ttl: timedelta, policy: Policy, audience: str = SESSION_AUDIENCE
) -> None:
self._master_key = master_key
self._issuer = issuer
@ -192,7 +192,7 @@ class SessionTokenCodec:
class OidcTokenVerifier:
def __init__(self, settings: OidcSettings, keys: JwksKeyLookup, policy: RolePolicy) -> None:
def __init__(self, settings: OidcSettings, keys: JwksKeyLookup, policy: Policy) -> None:
self._settings = settings
self._keys = keys
self._policy = policy
@ -222,7 +222,7 @@ class OidcTokenVerifier:
def _principal_from_claims(
policy: RolePolicy,
policy: Policy,
claims: Mapping[str, object],
roles_claim: str,
scope_claim: str,
@ -283,7 +283,7 @@ class TokenVerifier:
"""Resolves a bearer credential of any supported kind to a Principal."""
def __init__(
self, master_key: str, sessions: SessionTokenCodec, oidc: OidcTokenVerifier | None, policy: RolePolicy
self, master_key: str, sessions: SessionTokenCodec, oidc: OidcTokenVerifier | None, policy: Policy
) -> None:
self._master_key = master_key
self._sessions = sessions
@ -313,7 +313,7 @@ def _issuer_of(credential: str) -> str | None:
def build_token_verifier(
settings: ControlPlaneSettings, policy: RolePolicy, keys: JwksKeyLookup | None = None
settings: ControlPlaneSettings, policy: Policy, keys: JwksKeyLookup | None = None
) -> TokenVerifier:
master_key: Final = settings.master_key.get_secret_value()
sessions: Final = SessionTokenCodec(

View file

@ -8,7 +8,7 @@ from types import MappingProxyType
from typing import Final, TypeAlias
from litellm.proxy.control_plane.auth.jwt import SessionTokenCodec
from litellm.proxy.control_plane.auth.rbac import ADMIN_ROLE, RolePolicy, build_principal
from litellm.proxy.control_plane.auth.rbac import ADMIN_ROLE, Policy, build_principal
from litellm.types.proxy.control_plane.auth import SessionTokenResponse
LOCKOUT_FAILURES: Final = 5
@ -85,7 +85,7 @@ class LoginService:
def __init__(
self,
sessions: SessionTokenCodec,
policy: RolePolicy,
policy: Policy,
ttl_seconds: int,
admin_username: str | None,
admin_password: str | None,

View file

@ -1,11 +1,12 @@
"""Role and scope policy for the control plane. Roles grant scopes, scopes gate routes, the dataplane binding
on the token decides where the scopes apply (``*`` or no binding is fleet-wide)."""
from collections import deque
from collections.abc import Mapping
from dataclasses import dataclass
from enum import Enum
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing import Final, Literal, Protocol, TypeAlias
class Scope(str, Enum):
@ -62,6 +63,17 @@ class UnknownParent:
RolePolicyError: TypeAlias = UnknownScope | UnknownParent
class Policy(Protocol):
@property
def names(self) -> frozenset[str]: ...
def scopes_for(self, role: str) -> frozenset[Scope]: ...
def known(self, raw: tuple[str, ...]) -> frozenset[str]: ...
def grant(self, roles: frozenset[str]) -> frozenset[Scope]: ...
class RolePolicy:
"""Resolved role -> scopes table. Built once from defaults plus operator-defined roles."""
@ -86,10 +98,23 @@ def default_policy() -> RolePolicy:
return RolePolicy(DEFAULT_ROLES)
def _resolve(name: str, definitions: Mapping[str, RoleDefinition], seen: frozenset[str]) -> frozenset[Scope]:
definition: Final = definitions[name]
parents: Final = definition.inherits - seen
return definition.scopes.union(*(_resolve(parent, definitions, seen | {name}) for parent in parents))
def _resolve(definitions: Mapping[str, RoleDefinition]) -> Mapping[str, frozenset[Scope]]:
children: Final[dict[str, deque[str]]] = { # mutable-ok: build reverse edges in linear time
name: deque() for name in definitions
}
for name, definition in definitions.items():
for ancestor in definition.inherits:
children[ancestor].append(name)
grants: Final[dict[str, frozenset[Scope]]] = { # mutable-ok: scope unions converge without rebuilding the graph
name: definition.scopes for name, definition in definitions.items()
}
pending: Final[deque[str]] = deque(definitions) # mutable-ok: queue roles only when their inherited grants grow
while (parent := pending.popleft() if pending else None) is not None:
for child in children[parent]:
if (scopes := grants[child] | grants[parent]) != grants[child]:
grants[child] = scopes
pending.append(child)
return grants
def build_policy(custom: Mapping[str, RoleDefinition], include_defaults: bool = True) -> RolePolicy | RolePolicyError:
@ -107,7 +132,7 @@ def build_policy(custom: Mapping[str, RoleDefinition], include_defaults: bool =
) # comprehension-ok: flattening (role, parent) pairs
if unknown_parents:
return unknown_parents[0]
return RolePolicy({name: _resolve(name, definitions, frozenset()) for name in definitions})
return RolePolicy(_resolve(definitions))
def parse_role_definition(
@ -173,7 +198,7 @@ def parse_scopes(raw: tuple[str, ...] | None) -> frozenset[Scope] | None:
def build_principal(
policy: RolePolicy,
policy: Policy,
subject: str,
email: str | None,
roles: frozenset[str],

View file

@ -37,6 +37,7 @@ from litellm.proxy.control_plane.audit.transport import HttpxPoster
from litellm.proxy.control_plane.auth.jwt import JwksKeyLookup, SessionTokenCodec, TokenVerifier, build_token_verifier
from litellm.proxy.control_plane.auth.login import LoginService, LoginThrottle
from litellm.proxy.control_plane.auth.rbac import (
Policy,
RoleDefinition,
RolePolicy,
RolePolicyError,
@ -49,8 +50,13 @@ from litellm.proxy.control_plane.common.fanout import Fanout
from litellm.proxy.control_plane.common.tracing import Telemetry, build_log_exporter, build_resource, build_telemetry
from litellm.proxy.control_plane.dataplanes.client import DataplaneClient, HttpTransport, RetryPolicy
from litellm.proxy.control_plane.dataplanes.credentials import TokenExchanger, http_token_exchanger
from litellm.proxy.control_plane.dataplanes.registry import DataplaneRegistry, build_registry
from litellm.proxy.control_plane.dataplanes.registry import DataplaneRegistry
from litellm.proxy.control_plane.settings import ControlPlaneSettings
from litellm.proxy.control_plane.store.crypto import SecretCodec
from litellm.proxy.control_plane.store.postgres import ConfigStore, PostgresStore
from litellm.proxy.control_plane.store.runtime import RuntimeConfig
from litellm.proxy.control_plane.store.state import LivePolicy, LiveRegistry, LiveState
from litellm.proxy.control_plane.store.stream import ConfigStream, RedisStream
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
@ -60,7 +66,7 @@ if TYPE_CHECKING:
@dataclass(frozen=True, slots=True)
class Container:
settings: ControlPlaneSettings
policy: RolePolicy
policy: Policy
token_verifier: TokenVerifier
login_service: LoginService
registry: DataplaneRegistry
@ -68,6 +74,7 @@ class Container:
fanout: Fanout
audit_log: AuditLog
telemetry: Telemetry
runtime_config: RuntimeConfig
class InvalidRolePolicy(ValueError):
@ -222,15 +229,34 @@ def build_container(
exchanger: TokenExchanger | None = None,
tracer_provider: TracerProvider | None = None,
audit_sinks: tuple[AuditSink, ...] | None = None,
config_store: ConfigStore | None = None,
config_stream: ConfigStream | None = None,
) -> Container:
master_key: Final = settings.master_key.get_secret_value()
policy: Final = build_role_policy(settings)
initial_policy: Final = build_role_policy(settings)
http: Final = transport if transport is not None else _shared_http_client(settings)
exchange: Final = (
exchanger
if exchanger is not None
else http_token_exchanger(_shared_http_handler(settings), settings.dataplane_timeout_seconds)
)
codec: Final = SecretCodec(settings.salt_key) if settings.salt_key is not None else None
state: Final = LiveState(settings, exchange, initial_policy, codec)
policy: Final = LivePolicy(state)
registry: Final = LiveRegistry(state)
runtime: Final = RuntimeConfig(
state,
config_store
if config_store is not None
else PostgresStore.build(settings.database_url)
if settings.database_url is not None
else None,
config_stream
if config_stream is not None
else RedisStream.build(settings.redis_url, settings.config_stream, settings.config_stream_maxlen)
if settings.redis_url is not None
else None,
)
sessions: Final = SessionTokenCodec(
master_key=master_key,
issuer=settings.session_issuer,
@ -240,7 +266,6 @@ def build_container(
)
password: Final = settings.ui_password
telemetry: Final = build_telemetry(settings, tracer_provider)
registry: Final = build_registry(settings.dataplanes, exchange)
return Container(
settings=settings,
policy=policy,
@ -265,7 +290,8 @@ def build_container(
settings.audit_capacity,
pipeline=build_audit_pipeline(settings, audit_sinks),
product=litellm_product(),
known_dataplane_ids=frozenset(dataplane.id for dataplane in registry.all()),
known_dataplane_ids=lambda: frozenset(dataplane.id for dataplane in registry.all()),
),
telemetry=telemetry,
runtime_config=runtime,
)

View file

@ -1,7 +1,7 @@
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Protocol
from typing import Final, Literal, Protocol
from litellm.litellm_core_utils.oauth_token_exchange import OAuthTokenExchangeConfig, OAuthTokenExchangeError
from litellm.proxy.control_plane.auth.rbac import Principal
@ -23,6 +23,7 @@ class Dataplane:
url: str
credential: DataplaneCredential
credential_kinds: tuple[CredentialKind, ...]
source: Literal["config", "runtime"] = "config"
class DataplaneRegistry(Protocol):

View file

@ -1,11 +1,16 @@
from http import HTTPStatus
from typing import Annotated, Final
from fastapi import APIRouter, Depends, Security
from fastapi import APIRouter, Depends, Response, Security
from litellm.proxy.control_plane.auth.dependencies import require_scopes
from litellm.proxy.control_plane.auth.rbac import Principal, Scope
from litellm.proxy.control_plane.common.exceptions import ControlPlaneException, bad_gateway
from litellm.proxy.control_plane.common.exceptions import (
ControlPlaneException,
ErrorBody,
bad_gateway,
service_unavailable,
)
from litellm.proxy.control_plane.dataplanes.client import (
DataplaneClient,
DataplaneReply,
@ -21,6 +26,10 @@ from litellm.proxy.control_plane.dataplanes.dependencies import (
)
from litellm.proxy.control_plane.dataplanes.identity import USER_INFO_PATH, UserInfo, parse_user_info
from litellm.proxy.control_plane.dataplanes.registry import Dataplane, DataplaneRegistry, visible_to
from litellm.proxy.control_plane.settings import DataplaneSpec
from litellm.proxy.control_plane.store.dependencies import audited_write, get_runtime_config
from litellm.proxy.control_plane.store.models import Change
from litellm.proxy.control_plane.store.runtime import NOT_CONFIGURED, RuntimeConfig
from litellm.types.proxy.control_plane.dataplanes import (
DataplaneHealthResponse,
DataplaneIdentityResponse,
@ -33,9 +42,49 @@ router: Final = APIRouter(prefix="/dataplanes", tags=["dataplanes"])
LIVENESS_PATH: Final = "/health/liveliness"
@router.put("/{dataplane_id}", response_model=DataplaneResponse)
async def put_dataplane(
dataplane_id: str,
spec: DataplaneSpec,
principal: Annotated[Principal, Security(audited_write, scopes=[Scope.DATAPLANES_WRITE.value])],
runtime: Annotated[RuntimeConfig, Depends(get_runtime_config)],
) -> DataplaneResponse:
codec: Final = runtime.state.codec
if codec is None:
raise ControlPlaneException(service_unavailable(NOT_CONFIGURED))
change: Final = Change(
op="upsert", kind="dataplane", name=dataplane_id, entity=codec.encode(spec), actor=principal.subject
)
result: Final = await runtime.write(change, principal)
if isinstance(result, ErrorBody):
raise ControlPlaneException(result)
dataplane: Final = result.snapshot.registry.get(dataplane_id)
if dataplane is None:
raise ControlPlaneException(service_unavailable("Runtime dataplane is unavailable."))
return _to_response(dataplane)
@router.delete("/{dataplane_id}", status_code=204)
async def delete_dataplane(
dataplane_id: str,
principal: Annotated[Principal, Security(audited_write, scopes=[Scope.DATAPLANES_WRITE.value])],
runtime: Annotated[RuntimeConfig, Depends(get_runtime_config)],
) -> Response:
result: Final = await runtime.write(
Change(op="delete", kind="dataplane", name=dataplane_id, actor=principal.subject), principal
)
if isinstance(result, ErrorBody):
raise ControlPlaneException(result)
return Response(status_code=204)
def _to_response(dataplane: Dataplane) -> DataplaneResponse:
return DataplaneResponse(
id=dataplane.id, name=dataplane.name, url=dataplane.url, credentials=dataplane.credential_kinds
id=dataplane.id,
name=dataplane.name,
url=dataplane.url,
credentials=dataplane.credential_kinds,
source=dataplane.source,
)

View file

@ -0,0 +1,70 @@
from typing import Annotated, Final
from fastapi import APIRouter, Depends, Response, Security
from litellm.proxy.control_plane.auth.dependencies import require_scopes
from litellm.proxy.control_plane.auth.rbac import Principal, Scope
from litellm.proxy.control_plane.common.exceptions import ControlPlaneException, ErrorBody
from litellm.proxy.control_plane.settings import RoleSpec
from litellm.proxy.control_plane.store.dependencies import audited_write, get_runtime_config
from litellm.proxy.control_plane.store.models import ENTITY, Change
from litellm.proxy.control_plane.store.runtime import RuntimeConfig
from litellm.types.proxy.control_plane.roles import RoleListResponse, RoleResponse
router: Final = APIRouter(prefix="/roles", tags=["roles"])
@router.get("", response_model=RoleListResponse)
async def list_roles(
_principal: Annotated[Principal, Security(require_scopes, scopes=[Scope.ROLES_WRITE.value])],
runtime: Annotated[RuntimeConfig, Depends(get_runtime_config)],
) -> RoleListResponse:
view: Final = runtime.state.view
return RoleListResponse(
items=tuple(
RoleResponse(
name=name,
scopes=tuple(sorted(scope.value for scope in view.policy.scopes_for(name))),
inherits=spec.inherits,
source="config" if runtime.state.protected("role", name) else "runtime",
)
for name, spec in sorted(view.roles.items())
)
)
@router.put("/{name}", response_model=RoleResponse)
async def put_role(
name: str,
spec: RoleSpec,
principal: Annotated[Principal, Security(audited_write, scopes=[Scope.ROLES_WRITE.value])],
runtime: Annotated[RuntimeConfig, Depends(get_runtime_config)],
) -> RoleResponse:
change: Final = Change(
op="upsert",
kind="role",
name=name,
entity=ENTITY.validate_json(spec.model_dump_json()),
actor=principal.subject,
)
result: Final = await runtime.write(change, principal)
if isinstance(result, ErrorBody):
raise ControlPlaneException(result)
return RoleResponse(
name=name,
scopes=tuple(sorted(scope.value for scope in result.snapshot.policy.scopes_for(name))),
inherits=spec.inherits,
source="runtime",
)
@router.delete("/{name}", status_code=204)
async def delete_role(
name: str,
principal: Annotated[Principal, Security(audited_write, scopes=[Scope.ROLES_WRITE.value])],
runtime: Annotated[RuntimeConfig, Depends(get_runtime_config)],
) -> Response:
result: Final = await runtime.write(Change(op="delete", kind="role", name=name, actor=principal.subject), principal)
if isinstance(result, ErrorBody):
raise ControlPlaneException(result)
return Response(status_code=204)

View file

@ -7,7 +7,16 @@ from typing import Final, Literal, TypeAlias
from urllib.parse import urlsplit
import yaml
from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, field_validator, model_validator
from pydantic import (
AliasChoices,
BaseModel,
ConfigDict,
Field,
SecretStr,
TypeAdapter,
field_validator,
model_validator,
)
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm.litellm_core_utils.oauth_token_exchange import TokenExchangeProfile
@ -161,7 +170,9 @@ _YAML_DOCUMENT: Final = TypeAdapter(dict[str, object])
class ControlPlaneSettings(BaseSettings):
model_config = SettingsConfigDict(frozen=True, env_prefix="CONTROL_PLANE_", extra="ignore")
model_config = SettingsConfigDict(
frozen=True, env_prefix="CONTROL_PLANE_", extra="ignore", hide_input_in_errors=True
)
master_key: SecretStr = Field(validation_alias="LITELLM_MASTER_KEY")
ui_username: str | None = Field(default=None, max_length=USERNAME_MAX_LENGTH, validation_alias="UI_USERNAME")
@ -171,6 +182,13 @@ class ControlPlaneSettings(BaseSettings):
session_audience: str = "litellm-control-plane-api"
roles: Mapping[str, RoleSpec] = Field(default_factory=dict)
include_default_roles: bool = True
database_url: SecretStr | None = None
redis_url: SecretStr | None = None
config_stream: str = Field(default="litellm:control_plane:config", min_length=1)
config_stream_maxlen: int = Field(default=10_000, ge=1)
salt_key: SecretStr | None = Field(
default=None, validation_alias=AliasChoices("CONTROL_PLANE_SALT_KEY", "LITELLM_SALT_KEY")
)
oidc_issuer: str | None = None
oidc_audience: str | None = None
oidc_jwks_url: str | None = None
@ -219,6 +237,14 @@ class ControlPlaneSettings(BaseSettings):
audit_sentinel_token_scope: str | None = None
audit_sentinel_timeout_seconds: float = 30.0
@model_validator(mode="after")
def _runtime_store_settings(self) -> "ControlPlaneSettings":
if (self.database_url is None) != (self.redis_url is None):
raise ValueError("Set both CONTROL_PLANE_DATABASE_URL and CONTROL_PLANE_REDIS_URL, or neither.")
if self.database_url is not None and (self.salt_key is None or not self.salt_key.get_secret_value().strip()):
raise ValueError("Runtime config writes require CONTROL_PLANE_SALT_KEY or LITELLM_SALT_KEY.")
return self
@field_validator("audit_s3_bucket")
@classmethod
def _s3_bucket_is_not_blank(cls, value: str | None) -> str | None:

View file

@ -0,0 +1,62 @@
import base64
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Final
from pydantic import SecretStr, TypeAdapter
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value, encrypt_value
from litellm.proxy.control_plane.settings import DataplaneSpec
from litellm.proxy.control_plane.store.models import ENTITY, Entity
_TEXT: Final = TypeAdapter(str)
_BYTES: Final = TypeAdapter(bytes)
_SECRET_FIELDS: Final = (("oidc_exchange", "client_secret"), ("service_token", "token"))
def _transform_field(name: str, value: object, transform: Callable[[str], str]) -> object:
section, secret = _secret_field(name)
if name != section or value is None:
return value
fields: Final = ENTITY.validate_python(value)
return {**fields, secret: transform(_TEXT.validate_python(fields[secret]))}
def _secret_field(name: str) -> tuple[str, str]:
return next((pair for pair in _SECRET_FIELDS if pair[0] == name), ("", ""))
@dataclass(frozen=True, slots=True)
class SecretCodec:
key: SecretStr = field(repr=False)
def encrypt(self, value: str) -> str:
encrypted: Final = _BYTES.validate_python(encrypt_value(value, self.key.get_secret_value()))
return base64.urlsafe_b64encode(encrypted).decode("ascii")
def decrypt(self, value: str) -> str:
return decrypt_value(base64.urlsafe_b64decode(value), self.key.get_secret_value())
def encode(self, spec: DataplaneSpec) -> Entity:
encoded: Final = ENTITY.validate_json(spec.model_dump_json())
oidc: Final = spec.oidc_exchange
service: Final = spec.service_token
return {
**encoded,
**(
{}
if oidc is None
else {
"oidc_exchange": {
**ENTITY.validate_python(encoded["oidc_exchange"]),
"client_secret": self.encrypt(oidc.client_secret.get_secret_value()),
}
}
),
**({} if service is None else {"service_token": {"token": self.encrypt(service.token.get_secret_value())}}),
}
def decode(self, entity: Entity) -> DataplaneSpec:
return DataplaneSpec.model_validate(
{name: _transform_field(name, value, self.decrypt) for name, value in entity.items()}
)

View file

@ -0,0 +1,112 @@
import asyncio
from collections.abc import AsyncIterator
from typing import Annotated, Final
from fastapi import Depends, Request
from fastapi.exceptions import RequestValidationError
from fastapi.security import SecurityScopes
from litellm.proxy.control_plane.audit.dependencies import get_audit_log
from litellm.proxy.control_plane.audit.event import CONTROL_PLANE_ID
from litellm.proxy.control_plane.audit.log import AuditLog
from litellm.proxy.control_plane.auth.dependencies import get_principal
from litellm.proxy.control_plane.auth.rbac import Principal, Scope
from litellm.proxy.control_plane.common.exceptions import (
ControlPlaneException,
ErrorBody,
forbidden,
service_unavailable,
)
from litellm.proxy.control_plane.store.runtime import RuntimeConfig
def get_runtime_config() -> RuntimeConfig:
raise RuntimeError("get_runtime_config is not bound, build the app with create_app")
def _write_denial(principal: Principal, required: frozenset[Scope]) -> ErrorBody | None:
if principal.missing_scopes(required):
return forbidden("Missing runtime write scope.")
if not principal.dataplanes.all:
return forbidden("Runtime writes require a fleet-wide dataplane binding.")
return None
async def audited_write(
request: Request,
security_scopes: SecurityScopes,
principal: Annotated[Principal, Depends(get_principal)],
audit: Annotated[AuditLog, Depends(get_audit_log)],
) -> AsyncIterator[Principal]:
denial: Final = _write_denial(principal, frozenset(Scope(scope) for scope in security_scopes.scopes))
if denial is not None:
audit.record(
principal.subject,
principal.email,
CONTROL_PLANE_ID,
request.method,
request.url.path,
"denied",
denial.status,
detail=denial.detail,
kind="authentication",
)
raise ControlPlaneException(denial)
reservation: Final = audit.reserve()
try:
if reservation is None:
raise ControlPlaneException(service_unavailable("Audit sink backpressure, try again shortly."))
yield principal
except ControlPlaneException as error:
audit.record(
principal.subject,
principal.email,
CONTROL_PLANE_ID,
request.method,
request.url.path,
"denied",
error.body.status,
detail=error.body.detail,
reservation=reservation,
)
raise
except RequestValidationError:
audit.record(
principal.subject,
principal.email,
CONTROL_PLANE_ID,
request.method,
request.url.path,
"denied",
422,
detail="Invalid runtime write body.",
reservation=reservation,
)
raise
except (Exception, asyncio.CancelledError) as error:
audit.record(
principal.subject,
principal.email,
CONTROL_PLANE_ID,
request.method,
request.url.path,
"unreachable",
503 if isinstance(error, asyncio.CancelledError) else 500,
detail="Runtime write failed.",
reservation=reservation,
)
raise
else:
audit.record(
principal.subject,
principal.email,
CONTROL_PLANE_ID,
request.method,
request.url.path,
"allowed",
204 if request.method == "DELETE" else 200,
reservation=reservation,
)
finally:
if reservation is not None:
reservation.release()

View file

@ -0,0 +1,65 @@
from collections.abc import Callable
from importlib.resources import files
from typing import Final
import click
from alembic import command
from alembic.config import Config
from alembic.script import ScriptDirectory
from pydantic import SecretStr
from pydantic_settings import BaseSettings, SettingsConfigDict
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from sqlalchemy.pool import NullPool
VERSION_TABLE: Final = "control_plane_alembic_version"
ADVISORY_LOCK: Final = 714883513942
MIGRATE_COMMAND: Final = "litellm --control-plane --skip_server_startup"
def database_url(value: SecretStr) -> str:
raw: Final = value.get_secret_value()
return raw.replace("postgresql://", "postgresql+psycopg://", 1) if raw.startswith("postgresql://") else raw
def engine_for(url: SecretStr) -> AsyncEngine:
return create_async_engine(
database_url(url),
poolclass=NullPool,
hide_parameters=True,
connect_args={"prepare_threshold": None, "connect_timeout": 5},
)
def migration_config() -> Config:
config: Final = Config()
config.set_main_option("script_location", str(files("litellm.proxy.control_plane.store").joinpath("migrations")))
return config
def head_revision() -> str | None:
return ScriptDirectory.from_config(migration_config()).get_current_head()
def migrate(url: SecretStr, upgrade: Callable[[Config, str], None] = command.upgrade) -> None:
config: Final = migration_config()
config.attributes["database_url"] = url
upgrade(config, "head")
class MigrationSettings(BaseSettings):
model_config = SettingsConfigDict(frozen=True, env_prefix="CONTROL_PLANE_", extra="ignore")
database_url: SecretStr | None = None
def migrate_from_environment(migrate_database: Callable[[SecretStr], None] = migrate) -> None:
url: Final = MigrationSettings().database_url
if url is None or not url.get_secret_value().strip():
raise click.ClickException("CONTROL_PLANE_DATABASE_URL is required for control-plane migrations.")
try:
migrate_database(url)
except Exception as error: # noqa: BLE001 # migration jobs must fail closed without exposing DSN credentials
raise click.ClickException(
f"Control-plane migration failed ({type(error).__name__}). Check the dedicated CONTROL_PLANE_DATABASE_URL and migration job logs."
) from None
click.echo("Control-plane database migrations are current.")

View file

@ -0,0 +1,36 @@
import asyncio
from typing import Final
from alembic import context
from pydantic import SecretStr, TypeAdapter
from sqlalchemy import Connection, text
from litellm.proxy.control_plane.store.migrate import ADVISORY_LOCK, VERSION_TABLE, engine_for
from litellm.proxy.control_plane.store.schema import metadata
def configure(connection: Connection) -> None:
context.configure(
connection=connection,
target_metadata=metadata,
version_table=VERSION_TABLE,
transaction_per_migration=True,
)
with context.begin_transaction():
context.run_migrations()
async def run() -> None:
url: Final = TypeAdapter(SecretStr).validate_python(context.config.attributes["database_url"])
engine: Final = engine_for(url)
try:
# Keep the lock across per-revision commits, including through transaction pooling.
async with engine.begin() as lock_connection:
await lock_connection.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": ADVISORY_LOCK})
async with engine.connect() as connection:
await connection.run_sync(configure)
finally:
await engine.dispose()
asyncio.run(run())

View file

@ -0,0 +1,35 @@
from typing import Final
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: Final = "0001"
down_revision: Final = None
branch_labels: Final = None
depends_on: Final = None
def upgrade() -> None:
op.create_table(
"control_plane_dataplanes",
sa.Column("id", sa.Text(), primary_key=True),
sa.Column("spec", postgresql.JSONB(), nullable=False),
sa.Column("revision", sa.Text(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("updated_by", sa.Text(), nullable=False),
)
op.create_table(
"control_plane_roles",
sa.Column("name", sa.Text(), primary_key=True),
sa.Column("scopes", postgresql.JSONB(), nullable=False),
sa.Column("inherits", postgresql.JSONB(), nullable=False),
sa.Column("revision", sa.Text(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("updated_by", sa.Text(), nullable=False),
)
def downgrade() -> None:
op.drop_table("control_plane_roles")
op.drop_table("control_plane_dataplanes")

View file

@ -0,0 +1,34 @@
from collections.abc import Mapping
from typing import Final, Literal, TypeAlias
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter
Kind: TypeAlias = Literal["dataplane", "role"]
Entity: TypeAlias = Mapping[str, JsonValue]
ENTITY: Final = TypeAdapter(Entity)
class Change(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
op: Literal["upsert", "delete", "resync"]
kind: Kind = "dataplane"
name: str = Field(default="", max_length=2048)
entity: Entity = Field(default_factory=dict)
actor: str
class Entry(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
revision: str = Field(pattern=r"^\d+-\d+$")
change: Change
def revision_key(revision: str) -> tuple[int, int]:
milliseconds, sequence = revision.split("-")
return int(milliseconds), int(sequence)
def maximum_revision(entries: tuple[Entry, ...]) -> str:
return max((entry.revision for entry in entries), key=revision_key, default="0-0")

View file

@ -0,0 +1,112 @@
from collections.abc import AsyncGenerator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from dataclasses import dataclass
from typing import Final, Protocol
from pydantic import SecretStr
from sqlalchemy import inspect, select, text
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
from litellm.proxy.control_plane.store.migrate import ADVISORY_LOCK, VERSION_TABLE, engine_for, head_revision
from litellm.proxy.control_plane.store.models import Entry, Kind
from litellm.proxy.control_plane.store.schema import dataplanes, roles
class Transaction(Protocol):
async def load(self) -> tuple[Entry, ...]: ...
async def write(self, entry: Entry) -> None: ...
class ConfigStore(Protocol):
async def current(self) -> bool: ...
def transaction(self) -> AbstractAsyncContextManager[Transaction]: ...
async def close(self) -> None: ...
@dataclass(frozen=True, slots=True)
class PostgresTransaction:
connection: AsyncConnection
async def _load_kind(self, kind: Kind) -> tuple[Entry, ...]:
table: Final = dataplanes if kind == "dataplane" else roles
result: Final = await self.connection.execute(select(table))
return tuple(
Entry.model_validate(
{
"revision": row["revision"],
"change": {
"op": "upsert",
"kind": kind,
"name": row["id" if kind == "dataplane" else "name"],
"entity": row["spec"]
if kind == "dataplane"
else {"scopes": row["scopes"], "inherits": row["inherits"]},
"actor": row["updated_by"],
},
}
)
for row in result.mappings()
)
async def load(self) -> tuple[Entry, ...]:
dp: Final = await self._load_kind("dataplane")
role: Final = await self._load_kind("role")
return (*dp, *role)
async def write(self, entry: Entry) -> None:
change: Final = entry.change
table: Final = dataplanes if change.kind == "dataplane" else roles
name_column: Final = table.c.id if change.kind == "dataplane" else table.c.name
if change.op == "delete":
await self.connection.execute(table.delete().where(name_column == change.name))
return
values: Final = {
name_column.name: change.name,
**(
{"spec": dict(change.entity)}
if change.kind == "dataplane"
else {
"scopes": change.entity.get("scopes", []),
"inherits": change.entity.get("inherits", []),
}
),
"revision": entry.revision,
"updated_by": change.actor,
}
statement: Final = insert(table).values(**values)
await self.connection.execute(
statement.on_conflict_do_update(
index_elements=[name_column],
set_={
**{name: value for name, value in values.items() if name != name_column.name},
"updated_at": text("CURRENT_TIMESTAMP"),
},
)
)
@dataclass(frozen=True, slots=True)
class PostgresStore:
engine: AsyncEngine
@classmethod
def build(cls, url: SecretStr) -> "PostgresStore":
return cls(engine_for(url))
async def current(self) -> bool:
async with self.engine.connect() as connection:
exists: Final = await connection.run_sync(lambda sync: inspect(sync).has_table(VERSION_TABLE))
if not exists:
return False
result: Final = await connection.execute(text("SELECT version_num FROM control_plane_alembic_version"))
return tuple(result.scalars()) == (head_revision(),)
@asynccontextmanager
async def transaction(self) -> AsyncGenerator[Transaction]:
async with self.engine.begin() as connection:
await connection.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": ADVISORY_LOCK})
yield PostgresTransaction(connection)
async def close(self) -> None:
await self.engine.dispose()

View file

@ -0,0 +1,213 @@
import asyncio
import logging
from collections.abc import Awaitable, Callable
from contextlib import suppress
from dataclasses import dataclass, replace
from typing import Final
from nacl.exceptions import CryptoError
from redis.exceptions import RedisError
from sqlalchemy.exc import SQLAlchemyError
from litellm.proxy.control_plane.auth.rbac import Principal, Scope
from litellm.proxy.control_plane.common.exceptions import (
ErrorBody,
bad_request,
error_body,
forbidden,
not_found,
service_unavailable,
)
from litellm.proxy.control_plane.store.migrate import MIGRATE_COMMAND
from litellm.proxy.control_plane.store.models import Change, Entry, maximum_revision, revision_key
from litellm.proxy.control_plane.store.postgres import ConfigStore
from litellm.proxy.control_plane.store.state import InvalidSnapshot, LiveState, Snapshot
from litellm.proxy.control_plane.store.stream import ConfigStream
LOGGER: Final = logging.getLogger("litellm.control_plane.config")
NOT_CONFIGURED: Final = (
"Runtime config store is not configured or not migrated. Set CONTROL_PLANE_DATABASE_URL and CONTROL_PLANE_REDIS_URL, then run "
+ MIGRATE_COMMAND
+ "."
)
async def retry_pause() -> None:
await asyncio.sleep(0.2)
@dataclass(frozen=True, slots=True)
class Written:
entry: Entry
snapshot: Snapshot
class RuntimeConfig:
def __init__(
self,
state: LiveState,
store: ConfigStore | None,
stream: ConfigStream | None,
pause: Callable[[], Awaitable[None]] = retry_pause,
) -> None:
self.state = state
self.store = store
self.stream = stream
self.pause = pause
self.ready = False
self.cursor = "0-0"
self.task: asyncio.Task[None] | None = None
async def reload(self) -> bool:
if self.store is None:
return False
async with self.store.transaction() as transaction:
entries: Final = await transaction.load()
if not self.state.reload(entries):
LOGGER.error("Invalid runtime config snapshot. Reads retain the last valid configuration.")
return False
self.cursor = max((self.cursor, maximum_revision(entries)), key=revision_key)
return True
async def start(self) -> None:
if self.store is None or self.stream is None:
return
try:
self.ready = await self.recover()
except (SQLAlchemyError, OSError, ValueError, CryptoError):
self.ready = False
LOGGER.error("Runtime config unavailable. Run %s before serving runtime writes.", MIGRATE_COMMAND)
self.task = asyncio.create_task(self.consume(), name="control-plane-config")
async def recover(self) -> bool:
store: Final = self.store
if store is None:
return False
if not await store.current():
LOGGER.error(NOT_CONFIGURED)
return False
return await self.reload()
async def close(self) -> None:
self.ready = False
if self.task is not None:
self.task.cancel()
with suppress(asyncio.CancelledError):
await self.task
try:
if self.stream is not None:
await self.stream.close()
finally:
if self.store is not None:
await self.store.close()
async def process(self, entry: Entry) -> bool:
if entry.change.op == "resync":
return await self.reload()
if self.store is None:
return False
async with self.store.transaction() as transaction:
entries: Final = await transaction.load()
committed: Final = next(
(row for row in entries if (row.change.kind, row.change.name) == (entry.change.kind, entry.change.name)),
None,
)
if (
entry.change.op == "upsert"
and committed is not None
and revision_key(committed.revision) >= revision_key(entry.revision)
):
return self.state.apply(committed) or await self.reload()
elif entry.change.op == "delete" and committed is None:
return self.state.apply(entry) or await self.reload()
return True
async def _consume_batch(self, stream: ConfigStream) -> None:
batch: Final[tuple[Entry, ...]] = await stream.read(self.cursor)
for entry in batch:
if not await self.process(entry):
self.ready = False
return
self.cursor = max((self.cursor, entry.revision), key=revision_key)
if self.ready:
self.ready = await self.reload()
async def consume(self) -> None:
stream: Final = self.stream
if stream is None:
return
while True:
if not self.ready:
await self.pause()
try:
if not self.ready:
self.ready = await self.recover()
continue
await self._consume_batch(stream)
except (RedisError, SQLAlchemyError, OSError, ValueError, CryptoError):
self.ready = False
LOGGER.warning("Runtime config consumer reconnecting.")
def validate(self, change: Change, principal: Principal) -> ErrorBody | Snapshot:
if self.state.protected(change.kind, change.name):
return error_body("conflict", "Conflict", 409, "Config-defined entries and default roles are read-only.")
key: Final = (change.kind, change.name)
if change.op == "delete" and key not in self.state.view.entries and key not in self.state.view.quarantined:
return not_found("Runtime entry not found.")
if (
change.kind == "role"
and change.op == "delete"
and any(change.name in spec.inherits for name, spec in self.state.view.roles.items() if name != change.name)
):
return error_body("conflict", "Conflict", 409, "Another role inherits this role.")
desired: Final = tuple(row for row_key, row in self.state.view.entries.items() if row_key != key)
candidate: Final = Entry(revision="0-0", change=change)
view: Final = self.state.build((*desired, candidate) if change.op == "upsert" else desired, {})
if isinstance(view, InvalidSnapshot):
return bad_request(view.detail)
granted: Final = (
view.policy.scopes_for(change.name)
if change.op == "upsert"
else self.state.view.policy.scopes_for(change.name)
)
if change.kind == "role" and change.op == "upsert" and not granted.issubset(principal.scopes):
return forbidden("A role cannot grant scopes beyond the caller's own scopes.")
return view
async def write(self, desired: Change, principal: Principal) -> Written | ErrorBody:
change: Final = desired.model_copy(update={"actor": principal.subject})
store: Final = self.store
stream: Final = self.stream
if not self.ready or store is None or stream is None:
return service_unavailable(NOT_CONFIGURED)
published: str | None = None # rebind-ok: compensate only after XADD succeeded
try:
async with store.transaction() as transaction:
current: Final = await transaction.load()
if not self.state.reload(current):
return service_unavailable("Runtime config snapshot is invalid.")
writer: Final = (
principal
if principal.token_kind == "master_key"
else replace(principal, scopes=principal.scopes & self.state.view.policy.grant(principal.roles))
)
required: Final = Scope.DATAPLANES_WRITE if change.kind == "dataplane" else Scope.ROLES_WRITE
if not writer.has_scope(required):
return forbidden("Missing runtime write scope in the committed policy.")
invalid: Final = self.validate(change, writer)
if isinstance(invalid, ErrorBody):
return invalid
published = await stream.append(change) # rebind-ok: compensate only after XADD succeeded
entry: Final = Entry(revision=published, change=change)
await transaction.write(entry)
self.state.apply(entry)
return Written(entry, invalid)
except (SQLAlchemyError, RedisError, OSError, asyncio.CancelledError) as error:
if published is not None:
with suppress(RedisError, OSError):
await asyncio.shield(stream.append(Change(op="resync", actor=principal.subject)))
if isinstance(error, asyncio.CancelledError):
raise
return service_unavailable(
"Runtime config persistence failed. Reload from the authoritative database is required."
)

View file

@ -0,0 +1,25 @@
from typing import Final
from sqlalchemy import Column, DateTime, MetaData, Table, Text, func
from sqlalchemy.dialects.postgresql import JSONB
metadata: Final = MetaData()
dataplanes: Final = Table(
"control_plane_dataplanes",
metadata,
Column("id", Text, primary_key=True),
Column("spec", JSONB, nullable=False),
Column("revision", Text, nullable=False),
Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
Column("updated_by", Text, nullable=False),
)
roles: Final = Table(
"control_plane_roles",
metadata,
Column("name", Text, primary_key=True),
Column("scopes", JSONB, nullable=False),
Column("inherits", JSONB, nullable=False),
Column("revision", Text, nullable=False),
Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
Column("updated_by", Text, nullable=False),
)

View file

@ -0,0 +1,220 @@
import logging
from collections.abc import Mapping
from dataclasses import dataclass, replace
from types import MappingProxyType
from typing import Final, TypeAlias
from nacl.exceptions import CryptoError
from litellm.proxy.control_plane.audit.event import CONTROL_PLANE_ID
from litellm.proxy.control_plane.auth.rbac import (
DEFAULT_ROLES,
RoleDefinition,
RolePolicy,
Scope,
build_policy,
parse_role_definition,
)
from litellm.proxy.control_plane.dataplanes.credentials import TokenExchanger
from litellm.proxy.control_plane.dataplanes.registry import (
Dataplane,
StaticRegistry,
build_dataplane,
build_registry,
)
from litellm.proxy.control_plane.settings import ControlPlaneSettings, RoleSpec
from litellm.proxy.control_plane.store.crypto import SecretCodec
from litellm.proxy.control_plane.store.models import Entry, Kind, revision_key
Key: TypeAlias = tuple[Kind, str]
LOGGER: Final = logging.getLogger("litellm.control_plane.config")
@dataclass(frozen=True, slots=True)
class Snapshot:
entries: Mapping[Key, Entry]
revisions: Mapping[Key, str]
registry: StaticRegistry
policy: RolePolicy
roles: Mapping[str, RoleSpec]
quarantined: frozenset[Key] = frozenset()
@dataclass(frozen=True, slots=True)
class InvalidSnapshot:
detail: str
class LiveState:
def __init__(
self,
settings: ControlPlaneSettings,
exchanger: TokenExchanger,
policy: RolePolicy,
codec: SecretCodec | None,
) -> None:
self.settings = settings
self.exchanger = exchanger
self.codec = codec
self.config_roles: Final = MappingProxyType(
{
**(
{
name: RoleSpec(scopes=tuple(scope.value for scope in scopes))
for name, scopes in DEFAULT_ROLES.items()
}
if settings.include_default_roles
else {}
),
**settings.roles,
}
)
self.view = Snapshot(
MappingProxyType({}),
MappingProxyType({}),
build_registry(settings.dataplanes, exchanger),
policy,
self.config_roles,
)
def protected(self, kind: Kind, name: str) -> bool:
return name in (self.settings.dataplanes if kind == "dataplane" else self.config_roles)
def build(self, entries: tuple[Entry, ...], revisions: Mapping[Key, str]) -> Snapshot | InvalidSnapshot:
try:
return self._build(entries, revisions)
except (ValueError, CryptoError):
return InvalidSnapshot("Invalid runtime entity. Check credential strategy, scopes and role parents.")
def _build(self, entries: tuple[Entry, ...], revisions: Mapping[Key, str]) -> Snapshot | InvalidSnapshot:
active: Final[Mapping[Key, Entry]] = {
(entry.change.kind, entry.change.name): entry
for entry in entries
if entry.change.op == "upsert" and not self.protected(entry.change.kind, entry.change.name)
}
role_specs: Final = {
**{
name: RoleSpec.model_validate(entry.change.entity)
for (kind, name), entry in active.items()
if kind == "role"
},
**self.config_roles,
}
parsed: Final = {
name: parse_role_definition(name, spec.scopes, spec.inherits) for name, spec in role_specs.items()
}
if any(not isinstance(definition, RoleDefinition) for definition in parsed.values()):
return InvalidSnapshot("Unknown role scope.")
policy: Final = build_policy(
{name: definition for name, definition in parsed.items() if isinstance(definition, RoleDefinition)},
include_defaults=False,
)
if not isinstance(policy, RolePolicy):
return InvalidSnapshot("Unknown inherited role.")
codec: Final = self.codec
if active and codec is None:
return InvalidSnapshot("Runtime secret codec is not configured.")
runtime_dataplanes: Final = {
name: replace(build_dataplane(name, codec.decode(entry.change.entity), self.exchanger), source="runtime")
for (kind, name), entry in active.items()
if kind == "dataplane" and codec is not None
}
if CONTROL_PLANE_ID in runtime_dataplanes:
return InvalidSnapshot("The control-plane dataplane id is reserved.")
config_registry: Final = build_registry(self.settings.dataplanes, self.exchanger)
return Snapshot(
MappingProxyType(active),
MappingProxyType(dict(revisions)),
StaticRegistry(MappingProxyType({**runtime_dataplanes, **config_registry.entries})),
policy,
MappingProxyType(role_specs),
)
def _readable(self, entry: Entry) -> bool:
try:
if entry.change.kind == "role":
spec: Final = RoleSpec.model_validate(entry.change.entity)
return isinstance(parse_role_definition(entry.change.name, spec.scopes, spec.inherits), RoleDefinition)
if self.codec is None or entry.change.name == CONTROL_PLANE_ID:
return False
build_dataplane(entry.change.name, self.codec.decode(entry.change.entity), self.exchanger)
except (ValueError, CryptoError, KeyError, TypeError):
return False
return True
def _retain_parents(self, entries: tuple[Entry, ...]) -> tuple[Entry, ...]:
names: Final = frozenset(row.change.name for row in entries if row.change.kind == "role")
known: Final = names | frozenset(self.config_roles)
return tuple(
row
for row in entries
if row.change.kind != "role" or frozenset(RoleSpec.model_validate(row.change.entity).inherits) <= known
)
def _without_orphans(self, entries: tuple[Entry, ...]) -> tuple[Entry, ...]:
remaining: tuple[Entry, ...] = entries # rebind-ok: iterate to a fixed point without recursing on deep graphs
while len(retained := self._retain_parents(remaining)) < len(remaining):
remaining = retained
return retained
def reload(self, entries: tuple[Entry, ...]) -> bool:
incoming: Final[Mapping[Key, str]] = {(row.change.kind, row.change.name): row.revision for row in entries}
versions: Final[Mapping[Key, str]] = {
key: max((self.view.revisions.get(key, "0-0"), incoming.get(key, "0-0")), key=revision_key)
for key in self.view.revisions.keys() | incoming.keys()
}
candidates: Final = tuple(
row for row in entries if row.change.op == "upsert" and not self.protected(row.change.kind, row.change.name)
)
readable: Final = self._without_orphans(tuple(row for row in candidates if self._readable(row)))
snapshot: Final = self.build(readable, versions)
if isinstance(snapshot, InvalidSnapshot):
return False
quarantined: Final = frozenset((row.change.kind, row.change.name) for row in candidates).difference(
snapshot.entries
)
self.view = replace(snapshot, quarantined=quarantined)
if quarantined:
LOGGER.warning("Quarantined %d invalid runtime entities and dependent roles.", len(quarantined))
return True
def apply(self, entry: Entry) -> bool:
key: Final = (entry.change.kind, entry.change.name)
if revision_key(entry.revision) <= revision_key(self.view.revisions.get(key, "0-0")):
return False
entries: Final = tuple(row for row_key, row in self.view.entries.items() if row_key != key)
desired: Final = (*entries, entry) if entry.change.op == "upsert" else entries
snapshot: Final = self.build(desired, {**self.view.revisions, key: entry.revision})
if isinstance(snapshot, InvalidSnapshot):
return False
self.view = replace(snapshot, quarantined=self.view.quarantined - {key})
return True
@dataclass(frozen=True, slots=True)
class LiveRegistry:
state: LiveState
def get(self, dataplane_id: str) -> Dataplane | None:
return self.state.view.registry.get(dataplane_id)
def all(self) -> tuple[Dataplane, ...]:
return self.state.view.registry.all()
@dataclass(frozen=True, slots=True)
class LivePolicy:
state: LiveState
@property
def names(self) -> frozenset[str]:
return self.state.view.policy.names
def scopes_for(self, role: str) -> frozenset[Scope]:
return self.state.view.policy.scopes_for(role)
def known(self, raw: tuple[str, ...]) -> frozenset[str]:
return self.state.view.policy.known(raw)
def grant(self, roles: frozenset[str]) -> frozenset[Scope]:
return self.state.view.policy.grant(roles)

View file

@ -0,0 +1,73 @@
import logging
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, Protocol, runtime_checkable
from pydantic import SecretStr, TypeAdapter
from redis.asyncio import Redis
from litellm.proxy.control_plane.audit.event import CONTROL_PLANE_ID
from litellm.proxy.control_plane.store.models import Change, Entry
_READ: Final = TypeAdapter(tuple[tuple[str, tuple[tuple[str, dict[str, str]], ...]], ...])
_ID: Final = TypeAdapter(str)
LOGGER: Final = logging.getLogger("litellm.control_plane.config")
def _entry(revision: str, fields: Mapping[str, str]) -> Entry:
try:
return Entry(revision=revision, change=Change.model_validate_json(fields.get("event", "")))
except ValueError:
LOGGER.warning("Malformed runtime config stream event, reloading authoritative state.")
return Entry(revision=revision, change=Change(op="resync", actor=CONTROL_PLANE_ID))
class ConfigStream(Protocol):
async def append(self, change: Change) -> str: ...
async def read(self, after: str) -> tuple[Entry, ...]: ...
async def close(self) -> None: ...
@runtime_checkable
class RedisCommands(Protocol):
async def xadd(self, name: str, fields: Mapping[str, str], *, maxlen: int, approximate: bool) -> object: ...
async def xread(self, streams: Mapping[str, str], *, count: int, block: int) -> object: ...
async def aclose(self) -> None: ...
@dataclass(frozen=True, slots=True)
class RedisStream:
client: RedisCommands
name: str
maxlen: int
@classmethod
def build(cls, url: SecretStr, name: str, maxlen: int) -> "RedisStream":
client: Final = Redis.from_url(
url.get_secret_value(),
decode_responses=True,
socket_connect_timeout=5,
socket_timeout=5,
)
if not isinstance(client, RedisCommands):
raise TypeError("The Redis async client must support stream commands and aclose.")
return cls(client, name, maxlen)
async def append(self, change: Change) -> str:
return _ID.validate_python(
await self.client.xadd(
self.name,
{"event": change.model_dump_json()},
maxlen=self.maxlen,
approximate=True,
)
)
async def read(self, after: str) -> tuple[Entry, ...]:
result: Final = _READ.validate_python(await self.client.xread({self.name: after}, count=100, block=1000))
return tuple(
_entry(revision, fields) for _, entries in result for revision, fields in entries
) # comprehension-ok: flatten Redis stream batches
async def close(self) -> None:
await self.client.aclose()

View file

@ -961,7 +961,7 @@ class ProxyInitializationHelpers:
def control_plane_requested(control_plane: bool, skip_server_startup: bool, utility_command: bool = False) -> bool:
return control_plane and not skip_server_startup and not utility_command
return control_plane and not utility_command
class _ControlPlaneLicense(Protocol):
@ -980,10 +980,17 @@ def _control_plane_license_check() -> _ControlPlaneLicense:
return LicenseCheck()
def _migrate_control_plane() -> None:
from litellm.proxy.control_plane.store.migrate import migrate_from_environment
migrate_from_environment()
@dataclass(frozen=True, slots=True)
class CliDependencies:
decrypt_env: Callable[[], Mapping[str, str]] = _decrypt_env_var
license_check: Callable[[], _ControlPlaneLicense] = _control_plane_license_check
migrate_control_plane: Callable[[], None] = _migrate_control_plane
def _decrypt_kms_env_vars(decrypt_env: Callable[[], Mapping[str, str]]) -> None:
@ -1432,6 +1439,10 @@ def run_server(
version or validate_config or health or test or check_migrations_against_existing_data
)
if control_plane_requested(control_plane, skip_server_startup, utility_command):
if skip_server_startup:
_decrypt_kms_env_vars(dependencies.decrypt_env)
dependencies.migrate_control_plane()
return
if num_workers > 1:
raise click.UsageError("scale the control plane with replicas.")
from litellm.proxy.control_plane.main import serve

View file

@ -15,6 +15,7 @@ class DataplaneResponse(LiteLLMBaseModel):
name: str
url: str
credentials: tuple[CredentialKindName, ...]
source: Literal["config", "runtime"] = "config"
class DataplaneListResponse(LiteLLMBaseModel):

View file

@ -0,0 +1,18 @@
from typing import Literal
from pydantic import BaseModel, ConfigDict
class RoleResponse(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
name: str
scopes: tuple[str, ...]
inherits: tuple[str, ...]
source: Literal["config", "runtime"]
class RoleListResponse(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
items: tuple[RoleResponse, ...]

View file

@ -56,6 +56,10 @@ Documentation = "https://docs.litellm.ai"
[project.optional-dependencies]
switchyard = ["nemo-switchyard==0.4.0a1"]
proxy = [
"sqlalchemy>=2.0.49,<2.1",
"alembic>=1.18.4,<2.0",
"psycopg>=3.2,<4.0",
"psycopg-binary>=3.2,<4.0",
"casbin==1.43.0",
"gunicorn>=23.0.0,<24.0",
"uvicorn>=0.33.0,<1.0",
@ -264,6 +268,9 @@ admin-mcp = [
"litellm-admin-mcp @ https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz#sha256=5803a20a00a18177f4591706956f31be311879543049b8b38143ad90bb1cb0a6 ; python_version >= '3.12'",
]
proxy-dev = [
"sqlalchemy==2.0.49",
"pynacl==1.6.2",
"alembic==1.18.4",
{ include-group = "admin-mcp" },
"prisma==0.11.0",
"hypercorn==0.17.3",

View file

@ -0,0 +1,330 @@
import asyncio
import json
import os
import shutil
from collections.abc import AsyncIterator, Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Final
from uuid import uuid4
import httpx
import pytest
import pytest_asyncio
from alembic import command
from alembic.autogenerate import compare_metadata
from alembic.migration import MigrationContext
from pydantic import SecretStr
from redis.asyncio import Redis
from redis.exceptions import RedisError
from sqlalchemy import inspect, select, text
from sqlalchemy.engine import make_url
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.schema import CreateSchema, DropSchema
from litellm.proxy.control_plane.app import create_app
from litellm.proxy.control_plane.container import build_container
from litellm.proxy.control_plane.settings import ControlPlaneSettings
from litellm.proxy.control_plane.store.migrate import (
ADVISORY_LOCK,
VERSION_TABLE,
database_url,
engine_for,
migrate,
migration_config,
)
from litellm.proxy.control_plane.store.models import Change, Entry
from litellm.proxy.control_plane.store.postgres import PostgresStore
from litellm.proxy.control_plane.store.schema import dataplanes, metadata
from litellm.proxy.control_plane.store.stream import ConfigStream, RedisStream
MASTER_KEY: Final = "01234567890123456789012345678901"
class Licensed:
def is_premium(self) -> bool:
return True
@dataclass(frozen=True, slots=True)
class Services:
url: SecretStr
redis_url: SecretStr
stream_name: str
@pytest_asyncio.fixture
async def services() -> AsyncIterator[Services]:
url: Final = os.environ.get("CONTROL_PLANE_TEST_DATABASE_URL")
if url is None:
pytest.skip("Set CONTROL_PLANE_TEST_DATABASE_URL to a dedicated disposable Postgres database.")
redis_url: Final = os.environ.get("CONTROL_PLANE_TEST_REDIS_URL", "redis://127.0.0.1:6379/0")
engine: Final = engine_for(SecretStr(url))
redis: Final = Redis.from_url(redis_url, decode_responses=True, socket_connect_timeout=2)
schema: Final = "cp_test_" + uuid4().hex
stream_name: Final = "litellm:control_plane:test:" + uuid4().hex
try:
async with engine.connect() as probe:
await probe.execute(text("SELECT 1"))
await redis.ping()
except (SQLAlchemyError, RedisError, OSError) as error:
await redis.aclose()
await engine.dispose()
pytest.skip(f"Local control-plane services unavailable ({type(error).__name__}).")
try:
async with engine.begin() as connection:
await connection.execute(CreateSchema(schema))
isolated: Final = make_url(database_url(SecretStr(url))).update_query_dict(
{"options": "-csearch_path=" + schema}
)
yield Services(SecretStr(isolated.render_as_string(hide_password=False)), SecretStr(redis_url), stream_name)
finally:
await redis.delete(stream_name)
await redis.aclose()
async with engine.begin() as cleanup:
await cleanup.execute(DropSchema(schema, cascade=True, if_exists=True))
await engine.dispose()
@pytest.mark.asyncio
async def test_migrations_are_idempotent_schema_version_matches_and_metadata_has_no_autogenerate_diff(
services: Services,
) -> None:
store: Final = PostgresStore.build(services.url)
try:
assert not await store.current()
async with store.engine.connect() as empty:
assert await empty.run_sync(lambda sync: inspect(sync).get_table_names()) == []
await asyncio.to_thread(migrate, services.url)
assert await store.current()
await asyncio.to_thread(migrate, services.url)
assert await store.current()
async with store.engine.connect() as connection:
differences: Final = await connection.run_sync(
lambda sync: compare_metadata(
MigrationContext.configure(sync, opts={"version_table": VERSION_TABLE}), metadata
)
)
for _ in range(8):
assert await connection.scalar(text("SELECT CAST(:number AS integer)"), {"number": 1}) == 1
assert await connection.scalar(text("SELECT COUNT(*) FROM pg_prepared_statements")) == 0
assert differences == []
finally:
await store.close()
@pytest.mark.asyncio
async def test_pending_revisions_run_in_order_and_each_commits_independently(
services: Services, tmp_path: Path
) -> None:
config: Final = migration_config()
source: Final = config.get_main_option("script_location")
assert source is not None
scripts: Final = tmp_path / "migrations"
shutil.copytree(source, scripts)
(scripts / "versions" / "0002_probe.py").write_text(
'from alembic import op\nrevision = "0002"\ndown_revision = "0001"\n'
'def upgrade():\n op.execute("CREATE TABLE ordering_probe AS SELECT COUNT(*) AS count FROM control_plane_roles")\n'
)
(scripts / "versions" / "0003_failure.py").write_text(
'from alembic import op\nrevision = "0003"\ndown_revision = "0002"\n'
'def upgrade():\n op.execute("CREATE TABLE rolled_back (value TEXT)")\n raise RuntimeError("planned migration failure")\n'
)
config.set_main_option("script_location", str(scripts))
config.attributes["database_url"] = services.url
with pytest.raises(RuntimeError, match="planned migration failure"):
await asyncio.to_thread(command.upgrade, config, "head")
store: Final = PostgresStore.build(services.url)
try:
async with store.engine.connect() as connection:
assert await connection.scalar(text("SELECT count FROM ordering_probe")) == 0
assert await connection.scalar(text("SELECT version_num FROM control_plane_alembic_version")) == "0002"
assert not await connection.run_sync(lambda sync: inspect(sync).has_table("rolled_back"))
assert not await store.current()
finally:
await store.close()
async def eventually(predicate: Callable[[], bool]) -> None:
async with asyncio.timeout(30):
while not predicate():
await asyncio.sleep(0)
@pytest.mark.asyncio
async def test_migration_job_waits_for_the_postgres_advisory_lock(services: Services) -> None:
engine: Final = engine_for(services.url)
try:
async with engine.begin() as holder:
await holder.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": ADVISORY_LOCK})
migrating: Final = asyncio.create_task(asyncio.to_thread(migrate, services.url))
async with asyncio.timeout(30):
async with engine.connect() as observer:
while True:
waiting: Final = await observer.scalar(
text(
"SELECT EXISTS (SELECT 1 FROM pg_locks WHERE locktype = 'advisory' "
"AND classid = :high AND objid = :low AND NOT granted)"
),
{"high": ADVISORY_LOCK >> 32, "low": ADVISORY_LOCK & 0xFFFFFFFF},
)
if waiting:
break
assert not migrating.done()
await migrating
assert await PostgresStore(engine).current()
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_real_postgres_redis_workers_propagate_encrypted_dataplanes_roles_and_deletes(services: Services) -> None:
await asyncio.to_thread(migrate, services.url)
settings: Final = ControlPlaneSettings.model_validate(
{
"LITELLM_MASTER_KEY": MASTER_KEY,
"database_url": services.url,
"redis_url": services.redis_url,
"CONTROL_PLANE_SALT_KEY": "integration-encryption-key",
"config_stream": services.stream_name,
"audit_sinks": (),
}
)
first: Final = build_container(settings)
second: Final = build_container(settings)
redis: Final = Redis.from_url(services.redis_url.get_secret_value(), decode_responses=True)
store: Final = PostgresStore.build(services.url)
app_first: Final = create_app(first, Licensed())
app_second: Final = create_app(second, Licensed())
try:
async with app_first.router.lifespan_context(app_first), app_second.router.lifespan_context(app_second):
async with (
httpx.AsyncClient(transport=httpx.ASGITransport(app=app_first), base_url="http://writer") as writer,
httpx.AsyncClient(transport=httpx.ASGITransport(app=app_second), base_url="http://reader") as reader,
):
headers: Final = {"Authorization": f"Bearer {MASTER_KEY}"}
changed: Final = await writer.put(
"/api/v1/dataplanes/live",
headers=headers,
json={
"url": "https://runtime.test",
"service_token": {"token": "must-not-persist-in-plaintext"},
},
)
assert changed.status_code == 200, changed.text
await eventually(
lambda: first.registry.get("live") is not None and second.registry.get("live") is not None
)
seen: Final = await reader.get("/api/v1/dataplanes/live", headers=headers)
assert seen.json() == changed.json()
async with store.engine.connect() as connection:
persisted: Final = await connection.scalar(
select(dataplanes.c.spec).where(dataplanes.c.id == "live")
)
published: Final = await redis.xrange(services.stream_name)
assert "must-not-persist-in-plaintext" not in json.dumps(persisted) + str(published)
role: Final = await writer.put(
"/api/v1/roles/live-reader", headers=headers, json={"scopes": ["usage:read"]}
)
assert role.status_code == 200, role.text
await eventually(lambda: "live-reader" in second.policy.names)
assert tuple(scope.value for scope in second.policy.scopes_for("live-reader")) == ("usage:read",)
deleted: Final = await writer.delete("/api/v1/dataplanes/live", headers=headers)
assert deleted.status_code == 204, deleted.text
await eventually(lambda: first.registry.get("live") is None and second.registry.get("live") is None)
assert (await reader.get("/api/v1/dataplanes/live", headers=headers)).status_code == 404
async with store.transaction() as transaction:
rows: Final = await transaction.load()
assert tuple(row.change.name for row in rows) == ("live-reader",)
finally:
await redis.aclose()
await store.close()
@dataclass(frozen=True, slots=True)
class PausedDeleteStream:
delegate: ConfigStream
deleting: asyncio.Event
release: asyncio.Event
async def append(self, change: Change) -> str:
if change.op == "delete":
self.deleting.set()
await self.release.wait()
return await self.delegate.append(change)
async def read(self, after: str) -> tuple[Entry, ...]:
return await self.delegate.read(after)
async def close(self) -> None:
await self.delegate.close()
@pytest.mark.asyncio
async def test_concurrent_role_delete_and_inherit_cannot_commit_an_orphan(services: Services) -> None:
await asyncio.to_thread(migrate, services.url)
settings: Final = ControlPlaneSettings.model_validate(
{
"LITELLM_MASTER_KEY": MASTER_KEY,
"database_url": services.url,
"redis_url": services.redis_url,
"CONTROL_PLANE_SALT_KEY": "integration-key",
"audit_sinks": (),
}
)
stream: Final = PausedDeleteStream(
RedisStream.build(services.redis_url, services.stream_name, 100),
asyncio.Event(),
asyncio.Event(),
)
first: Final = build_container(settings, config_stream=stream)
second: Final = build_container(settings, config_stream=stream)
first.runtime_config.ready = second.runtime_config.ready = True
store: Final = PostgresStore.build(services.url)
headers: Final = {"Authorization": "Bearer " + MASTER_KEY}
try:
async with (
httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(first, Licensed())), base_url="http://delete"
) as deleting,
httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(second, Licensed())), base_url="http://inherit"
) as inheriting,
):
seeded: Final = await deleting.put("/api/v1/roles/parent", headers=headers, json={"scopes": []})
assert seeded.status_code == 200, seeded.text
delete: Final = asyncio.create_task(deleting.delete("/api/v1/roles/parent", headers=headers))
async with asyncio.timeout(30):
await stream.deleting.wait()
child: Final = asyncio.create_task(
inheriting.put(
"/api/v1/roles/child",
headers=headers,
json={"inherits": ["parent"]},
)
)
try:
async with asyncio.timeout(30):
async with store.engine.connect() as observer:
while not child.done():
waiting: Final = await observer.scalar(
text(
"SELECT EXISTS (SELECT 1 FROM pg_locks WHERE locktype = 'advisory' "
"AND classid = :high AND objid = :low AND NOT granted)"
),
{"high": ADVISORY_LOCK >> 32, "low": ADVISORY_LOCK & 0xFFFFFFFF},
)
if waiting:
break
finally:
stream.release.set()
removed, refused = await asyncio.gather(delete, child)
assert removed.status_code == 204, removed.text
assert refused.status_code == 400, refused.text
async with store.transaction() as transaction:
assert await transaction.load() == ()
finally:
stream.release.set()
await first.runtime_config.close()
await second.runtime_config.close()
await store.close()

View file

@ -97,6 +97,59 @@ def test_inheritance_cycles_terminate() -> None:
assert policy.scopes_for("a") == policy.scopes_for("b") == frozenset({Scope.USAGE_READ, Scope.FLEET_READ})
@pytest.mark.parametrize("cycle", [False, True])
def test_deep_inheritance_resolves_without_recursion_and_preserves_cycle_grants(cycle: bool) -> None:
count: Final = 1500
definitions: Final = {
str(index): RoleDefinition(
frozenset({Scope.USAGE_READ}) if index == 0 else frozenset({Scope.FLEET_READ}),
frozenset({str(index - 1)}) if index else frozenset({str(count - 1)}) if cycle else frozenset(),
)
for index in range(count)
}
policy: Final = build_policy(definitions, include_defaults=False)
assert isinstance(policy, RolePolicy)
assert policy.scopes_for(str(count - 1)) == frozenset({Scope.USAGE_READ, Scope.FLEET_READ})
assert policy.scopes_for("0") == (
frozenset({Scope.USAGE_READ, Scope.FLEET_READ}) if cycle else frozenset({Scope.USAGE_READ})
)
def test_dense_inheritance_resolves_every_ancestor_grant() -> None:
scopes: Final = tuple(Scope)
definitions: Final = {
str(index): RoleDefinition(
frozenset({scopes[index % len(scopes)]}), frozenset(str(parent) for parent in range(index))
)
for index in range(14)
}
policy: Final = build_policy(definitions, include_defaults=False)
assert isinstance(policy, RolePolicy)
assert tuple(policy.scopes_for(str(index)) for index in range(14)) == tuple(
frozenset(scopes[: min(index + 1, len(scopes))]) for index in range(14)
)
def test_shared_cycle_revisits_descendants_without_granting_scopes_to_ancestors() -> None:
policy: Final = build_policy(
{
"leaf": RoleDefinition(frozenset({Scope.AUDIT_READ}), frozenset({"left", "right"})),
"left": RoleDefinition(frozenset({Scope.PROXY_READ}), frozenset({"source", "right"})),
"source": RoleDefinition(frozenset({Scope.USAGE_READ}), frozenset()),
"right": RoleDefinition(frozenset({Scope.FLEET_READ}), frozenset({"left"})),
},
include_defaults=False,
)
assert isinstance(policy, RolePolicy)
inherited: Final = frozenset({Scope.PROXY_READ, Scope.USAGE_READ, Scope.FLEET_READ})
assert tuple(policy.scopes_for(name) for name in ("leaf", "left", "source", "right")) == (
inherited | {Scope.AUDIT_READ},
inherited,
frozenset({Scope.USAGE_READ}),
inherited,
)
def test_role_definition_rejects_unknown_scopes() -> None:
assert parse_role_definition("r", ("usage:read", "nope"), ()) == UnknownScope(role="r", scope="nope")
parsed: Final = parse_role_definition("r", ("usage:read",), ("viewer",))

View file

@ -15,6 +15,9 @@ from litellm.proxy.control_plane.auth.jwt import SessionTokenCodec
from litellm.proxy.control_plane.auth.rbac import USAGE_VIEWER_ROLE, VIEWER_ROLE, RolePolicy, build_principal
from litellm.proxy.control_plane.container import build_container
from litellm.proxy.control_plane.settings import ControlPlaneSettings, read_dataplanes_file
from tests.unit.proxy.control_plane.store.support import MemoryStore, MemoryStream
from tests.unit.proxy.control_plane.store.support import container as runtime_container
from tests.unit.proxy.control_plane.support import RoutedTransport, response
MASTER_KEY: Final = "01234567890123456789012345678901"
DATAPLANES: Final = {
@ -90,6 +93,7 @@ def test_list_shows_only_the_dataplanes_the_caller_is_bound_to() -> None:
"name": "US East",
"url": "https://us.test",
"credentials": ["service_token"],
"source": "config",
}
bound: Final = client.get("/api/v1/dataplanes", headers=_viewer(codec, "us")).json()
assert [item["id"] for item in bound["items"]] == ["us"]
@ -199,3 +203,137 @@ def test_malformed_user_info_body_is_a_502_error_not_a_crash() -> None:
for _ in range(2):
malformed: Final = client.get("/api/v1/dataplanes/eu/whoami", headers=_viewer(codec))
assert malformed.status_code == 502 and "malformed" in malformed.json()["detail"]
@pytest.mark.asyncio
async def test_runtime_dataplane_is_secret_free_and_live_for_list_get_fanout_proxy_and_audit() -> None:
dataplane_id: Final = "sk-" + "r" * 40
store: Final = MemoryStore()
stream: Final = MemoryStream()
transport: Final = RoutedTransport(
{"https://runtime.test/model/info": iter((response(200, {"data": []}), response(200, {"data": []})))}
)
rig: Final = runtime_container(store, stream, transport=transport)
await rig.runtime_config.start()
headers: Final = {"Authorization": f"Bearer {MASTER_KEY}"}
try:
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
put: Final = await client.put(
f"/api/v1/dataplanes/{dataplane_id}",
headers=headers,
json={"url": "https://runtime.test/", "service_token": {"token": "private-service-credential"}},
)
assert put.status_code == 200, put.text
assert put.json() == {
"id": dataplane_id,
"name": dataplane_id,
"url": "https://runtime.test",
"credentials": ["service_token"],
"source": "runtime",
}
await stream.consumed(stream.entries[-1].revision)
listed: Final = await client.get("/api/v1/dataplanes", headers=headers)
fetched: Final = await client.get(f"/api/v1/dataplanes/{dataplane_id}", headers=headers)
fleet: Final = await client.get("/api/v1/fleet/models", headers=headers)
proxied: Final = await client.get(f"/api/v1/dataplanes/{dataplane_id}/proxy/model/info", headers=headers)
assert listed.json() == {"items": [put.json()]}
assert fetched.json() == put.json()
assert fleet.status_code == proxied.status_code == 200
assert len(transport.calls) == 2
assert rig.audit_log.recent(1)[0].dataplane_id == dataplane_id
persisted: Final = " ".join(row.model_dump_json() for row in store.rows)
published: Final = " ".join(entry.model_dump_json() for entry in stream.entries)
assert "private-service-credential" not in persisted + published + put.text + listed.text + fetched.text
deleted: Final = await client.delete(f"/api/v1/dataplanes/{dataplane_id}", headers=headers)
assert deleted.status_code == 204, deleted.text
await stream.consumed(stream.entries[-1].revision)
assert rig.registry.get(dataplane_id) is None
assert (await client.get(f"/api/v1/dataplanes/{dataplane_id}", headers=headers)).status_code == 404
assert store.rows == ()
finally:
await rig.runtime_config.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["PUT", "DELETE"])
@pytest.mark.parametrize("gate", ["scope", "binding"])
async def test_runtime_dataplane_write_scope_and_fleet_binding_are_required_and_audited(method: str, gate: str) -> None:
stream: Final = MemoryStream()
rig: Final = runtime_container(MemoryStore(), stream, roles={"writer": {"scopes": ["dataplanes:write"]}})
rig.runtime_config.ready = True
codec: Final = SessionTokenCodec(MASTER_KEY, rig.settings.session_issuer, timedelta(minutes=5), rig.policy)
actor: Final = build_principal(
rig.policy,
"operator",
None,
frozenset({"viewer" if gate == "scope" else "writer"}),
None,
frozenset() if gate == "scope" else frozenset({"eu"}),
"session",
)
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
result: Final = await client.request(
method,
"/api/v1/dataplanes/new",
headers={"Authorization": f"Bearer {codec.mint(actor)}"},
**(
{"json": {"url": "https://runtime.test", "service_token": {"token": "private"}}}
if method == "PUT"
else {}
),
)
assert result.status_code == 403, result.text
assert stream.entries == ()
event: Final = rig.audit_log.recent(1)[0]
assert (event.actor, event.status, event.decision) == ("operator", 403, "denied")
@pytest.mark.asyncio
async def test_config_dataplanes_are_read_only_and_unconfigured_writes_are_unavailable() -> None:
configured: Final = runtime_container(MemoryStore(), MemoryStream(), dataplanes=DATAPLANES)
configured.runtime_config.ready = True
unconfigured: Final = runtime_container()
for rig, name, expected in ((configured, "eu", 409), (unconfigured, "new", 503)):
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
put: Final = await client.put(
f"/api/v1/dataplanes/{name}",
headers={"Authorization": f"Bearer {MASTER_KEY}"},
json={"url": "https://runtime.test", "service_token": {"token": "private"}},
)
delete: Final = await client.delete(
f"/api/v1/dataplanes/{name}", headers={"Authorization": f"Bearer {MASTER_KEY}"}
)
assert put.status_code == delete.status_code == expected
assert len(rig.audit_log.recent(10)) == 2
@pytest.mark.asyncio
@pytest.mark.parametrize(
("spec", "status"),
[
({"url": "https://user:password@runtime.test", "service_token": {"token": "private"}}, 422),
({"url": "http://runtime.test", "service_token": {"token": "private"}}, 422),
({"url": "https://runtime.test"}, 400),
],
)
async def test_invalid_runtime_dataplanes_are_rejected_and_audited_before_publication(
spec: dict[str, object], status: int
) -> None:
stream: Final = MemoryStream()
rig: Final = runtime_container(MemoryStore(), stream)
rig.runtime_config.ready = True
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
result: Final = await client.put(
"/api/v1/dataplanes/new", headers={"Authorization": f"Bearer {MASTER_KEY}"}, json=spec
)
assert result.status_code == status, result.text
assert stream.entries == ()
assert rig.audit_log.recent(1)[0].status == status

View file

@ -0,0 +1,292 @@
from datetime import timedelta
from typing import Final
import httpx
import pytest
from litellm.proxy.control_plane.app import create_app
from litellm.proxy.control_plane.auth.jwt import SessionTokenCodec
from litellm.proxy.control_plane.auth.rbac import Principal, Scope, build_principal
from tests.unit.proxy.control_plane.store.support import MemoryStore, MemoryStream, container
from tests.unit.proxy.control_plane.support import MASTER_KEY, Licensed
HEADERS: Final = {"Authorization": f"Bearer {MASTER_KEY}"}
@pytest.mark.asyncio
async def test_roles_are_created_listed_propagated_to_existing_session_codec_and_deleted() -> None:
store: Final = MemoryStore()
stream: Final = MemoryStream()
rig: Final = container(store, stream)
codec: Final = SessionTokenCodec(
MASTER_KEY,
rig.settings.session_issuer,
timedelta(minutes=5),
rig.policy,
audience=rig.settings.session_audience,
)
await rig.runtime_config.start()
try:
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
created: Final = await client.put("/api/v1/roles/analyst", headers=HEADERS, json={"scopes": ["usage:read"]})
assert created.status_code == 200, created.text
assert created.json() == {"name": "analyst", "scopes": ["usage:read"], "inherits": [], "source": "runtime"}
await stream.consumed(stream.entries[-1].revision)
principal: Final = build_principal(
rig.policy, "analyst-user", None, frozenset({"analyst"}), None, frozenset(), "session"
)
verified: Final = rig.token_verifier.verify(codec.mint(principal))
assert isinstance(verified, Principal)
assert verified.scopes == frozenset({Scope.USAGE_READ})
listed: Final = await client.get("/api/v1/roles", headers=HEADERS)
assert listed.status_code == 200, listed.text
assert created.json() in listed.json()["items"]
assert any(role["source"] == "config" for role in listed.json()["items"])
deleted: Final = await client.delete("/api/v1/roles/analyst", headers=HEADERS)
assert deleted.status_code == 204, deleted.text
await stream.consumed(stream.entries[-1].revision)
assert "analyst" not in rig.policy.names
assert store.rows == ()
assert tuple(event.decision for event in rig.audit_log.recent(2)) == ("allowed", "allowed")
finally:
await rig.runtime_config.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["GET", "PUT", "DELETE"])
async def test_role_routes_require_roles_write(method: str) -> None:
rig: Final = container(MemoryStore(), MemoryStream())
codec: Final = SessionTokenCodec(MASTER_KEY, rig.settings.session_issuer, timedelta(minutes=5), rig.policy)
viewer: Final = build_principal(rig.policy, "viewer", None, frozenset({"viewer"}), None, frozenset(), "session")
headers: Final = {"Authorization": f"Bearer {codec.mint(viewer)}"}
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
result: Final = await client.request(
method,
"/api/v1/roles" if method == "GET" else "/api/v1/roles/new",
headers=headers,
**({"json": {"scopes": []}} if method == "PUT" else {}),
)
assert result.status_code == 403, result.text
assert rig.audit_log.recent(1)[0].decision == "denied"
assert rig.audit_log.recent(1)[0].status == 403
@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["PUT", "DELETE"])
async def test_dataplane_bound_role_writer_can_list_but_cannot_mutate_fleet_roles(method: str) -> None:
store: Final = MemoryStore()
stream: Final = MemoryStream()
rig: Final = container(store, stream, roles={"delegator": {"scopes": ["roles:write", "usage:read"]}})
rig.runtime_config.ready = True
codec: Final = SessionTokenCodec(MASTER_KEY, rig.settings.session_issuer, timedelta(minutes=5), rig.policy)
delegator: Final = build_principal(
rig.policy, "delegator", None, frozenset({"delegator"}), None, frozenset({"us"}), "session"
)
headers: Final = {"Authorization": f"Bearer {codec.mint(delegator)}"}
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
created: Final = await client.put("/api/v1/roles/analyst", headers=HEADERS, json={"scopes": ["usage:read"]})
assert created.status_code == 200, created.text
rows: Final = store.rows
entries: Final = stream.entries
listed: Final = await client.get("/api/v1/roles", headers=headers)
result: Final = await client.request(
method,
"/api/v1/roles/analyst",
headers=headers,
**({"json": {"scopes": []}} if method == "PUT" else {}),
)
assert listed.status_code == 200, listed.text
assert result.status_code == 403, result.text
assert result.json()["detail"] == "Runtime writes require a fleet-wide dataplane binding."
assert store.rows == rows
assert stream.entries == entries
assert rig.audit_log.recent(1)[0].decision == "denied"
assert rig.audit_log.recent(1)[0].status == 403
@pytest.mark.asyncio
async def test_role_escalation_is_rejected_even_through_an_inherited_role() -> None:
stream: Final = MemoryStream()
store: Final = MemoryStore()
rig: Final = container(store, stream, roles={"delegator": {"scopes": ["roles:write"]}})
rig.runtime_config.ready = True
codec: Final = SessionTokenCodec(MASTER_KEY, rig.settings.session_issuer, timedelta(minutes=5), rig.policy)
delegator: Final = build_principal(
rig.policy, "delegator", None, frozenset({"delegator"}), None, frozenset(), "session"
)
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
result: Final = await client.put(
"/api/v1/roles/escalated",
headers={"Authorization": f"Bearer {codec.mint(delegator)}"},
json={"inherits": ["admin"]},
)
assert result.status_code == 403, result.text
assert stream.entries == ()
assert store.rows == ()
assert rig.audit_log.recent(1)[0].status == 403
@pytest.mark.asyncio
async def test_default_and_config_roles_are_read_only_and_inherited_roles_cannot_be_deleted() -> None:
store: Final = MemoryStore()
stream: Final = MemoryStream()
rig: Final = container(store, stream, roles={"configured": {"scopes": ["usage:read"]}})
rig.runtime_config.ready = True
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
for name in ("admin", "configured"):
put: Final = await client.put(f"/api/v1/roles/{name}", headers=HEADERS, json={"scopes": []})
delete: Final = await client.delete(f"/api/v1/roles/{name}", headers=HEADERS)
assert (put.status_code, delete.status_code) == (409, 409)
parent: Final = await client.put("/api/v1/roles/parent", headers=HEADERS, json={"scopes": ["usage:read"]})
child: Final = await client.put("/api/v1/roles/child", headers=HEADERS, json={"inherits": ["parent"]})
denied: Final = await client.delete("/api/v1/roles/parent", headers=HEADERS)
missing: Final = await client.delete("/api/v1/roles/absent", headers=HEADERS)
assert parent.status_code == child.status_code == 200
assert child.json()["scopes"] == ["usage:read"]
assert child.json()["inherits"] == ["parent"]
assert denied.status_code == 409, denied.text
assert missing.status_code == 404, missing.text
assert len(store.rows) == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("body", [{"scopes": ["unknown:scope"]}, {"inherits": ["unknown-parent"]}])
async def test_invalid_role_is_rejected_before_stream_publication(body: dict[str, list[str]]) -> None:
stream: Final = MemoryStream()
rig: Final = container(MemoryStore(), stream)
rig.runtime_config.ready = True
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
result: Final = await client.put("/api/v1/roles/invalid", headers=HEADERS, json=body)
assert result.status_code == 400, result.text
assert stream.entries == ()
assert rig.audit_log.recent(1)[0].status == 400
@pytest.mark.asyncio
async def test_unconfigured_role_writes_are_unavailable_but_config_roles_can_be_read() -> None:
rig: Final = container()
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
listed: Final = await client.get("/api/v1/roles", headers=HEADERS)
put: Final = await client.put("/api/v1/roles/new", headers=HEADERS, json={"scopes": []})
deleted: Final = await client.delete("/api/v1/roles/new", headers=HEADERS)
assert listed.status_code == 200 and listed.json()["items"]
assert put.status_code == deleted.status_code == 503
assert put.json()["detail"] == deleted.json()["detail"]
assert "not configured or not migrated" in put.json()["detail"]
@pytest.mark.asyncio
async def test_strict_audit_backpressure_prevents_any_runtime_write_before_publication() -> None:
store: Final = MemoryStore()
stream: Final = MemoryStream()
rig: Final = container(store, stream, audit_sinks=("log",), audit_strict=True, audit_queue_size=2)
rig.runtime_config.ready = True
rig.audit_log.record("operator", None, "control_plane", "PUT", "/occupied", "allowed", 200)
rig.audit_log.record("operator", None, "control_plane", "PUT", "/occupied", "allowed", 200)
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
role: Final = await client.put("/api/v1/roles/new", headers=HEADERS, json={"scopes": []})
dataplane: Final = await client.put(
"/api/v1/dataplanes/new",
headers=HEADERS,
json={"url": "https://runtime.test", "service_token": {"token": "private"}},
)
assert role.status_code == dataplane.status_code == 503
assert "backpressure" in role.json()["detail"]
assert store.loads == 0
assert stream.entries == store.rows == ()
assert tuple(event.status for event in rig.audit_log.recent(2)) == (503, 503)
@pytest.mark.asyncio
async def test_persistence_failure_is_audited_and_emits_resync() -> None:
store: Final = MemoryStore()
stream: Final = MemoryStream()
store.fail_write = True
rig: Final = container(store, stream)
rig.runtime_config.ready = True
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
result: Final = await client.put("/api/v1/roles/new", headers=HEADERS, json={"scopes": []})
assert result.status_code == 503, result.text
assert tuple(entry.change.op for entry in stream.entries) == ("upsert", "resync")
assert rig.audit_log.recent(1)[0].status == 503
@pytest.mark.asyncio
@pytest.mark.parametrize("resource", ["roles", "dataplanes"])
@pytest.mark.parametrize("method", ["PUT", "DELETE"])
@pytest.mark.parametrize("gate", ["scope", "binding"])
async def test_runtime_denial_flood_preserves_strict_write_capacity_and_history(
resource: str, method: str, gate: str
) -> None:
store: Final = MemoryStore()
stream: Final = MemoryStream()
rig: Final = container(
store,
stream,
roles={"writer": {"scopes": ["roles:write", "dataplanes:write"]}},
audit_sinks=("log",),
audit_strict=True,
audit_queue_size=2,
audit_capacity=2,
)
rig.runtime_config.ready = True
codec: Final = SessionTokenCodec(MASTER_KEY, rig.settings.session_issuer, timedelta(minutes=5), rig.policy)
blocked: Final = build_principal(
rig.policy,
"blocked",
None,
frozenset({"viewer" if gate == "scope" else "writer"}),
None,
frozenset() if gate == "scope" else frozenset({"eu"}),
"session",
)
headers: Final = {"Authorization": f"Bearer {codec.mint(blocked)}"}
path: Final = f"/api/v1/{resource}/new"
body: Final = (
{"scopes": []}
if resource == "roles"
else {"url": "https://runtime.test", "service_token": {"token": "private"}}
)
rig.audit_log.record("seed", None, "control_plane", "PUT", "/existing", "allowed", 200)
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=create_app(rig, Licensed())), base_url="http://cp"
) as client:
for _ in range(8):
denied: Final = await client.request(
method, path, headers=headers, **({"json": body} if method == "PUT" else {})
)
assert denied.status_code == 403, denied.text
assert store.rows == stream.entries == ()
assert tuple(
(event.path, event.status, event.kind) for event in rig.audit_log.recent(10) if event.actor == "blocked"
) == ((path, 403, "authentication"),) * 2
assert tuple(event.path for event in rig.audit_log.recent(10) if event.actor == "seed") == ("/existing",)
assert rig.audit_log.accepting()
allowed: Final = await client.put(path, headers=HEADERS, json=body)
assert allowed.status_code == 200, allowed.text
assert not rig.audit_log.accepting()
denied_when_full: Final = await client.request(
method, path, headers=headers, **({"json": body} if method == "PUT" else {})
)
assert denied_when_full.status_code == 403, denied_when_full.text
assert len(store.rows) == len(stream.entries) == 1
event: Final = rig.audit_log.recent(1)[0]
assert (event.actor, event.decision, event.status, event.kind) == ("blocked", "denied", 403, "authentication")

View file

@ -0,0 +1,125 @@
import asyncio
from collections.abc import AsyncIterator, Mapping
from contextlib import asynccontextmanager
from typing import Final
from redis.exceptions import RedisError
from sqlalchemy.exc import SQLAlchemyError
from litellm.proxy.control_plane.container import Container, build_container
from litellm.proxy.control_plane.dataplanes.client import HttpTransport
from litellm.proxy.control_plane.settings import ControlPlaneSettings
from litellm.proxy.control_plane.store.models import Change, Entry, revision_key
from litellm.proxy.control_plane.store.postgres import Transaction
from tests.unit.proxy.control_plane.support import MASTER_KEY, no_exchange
class MemoryStore:
def __init__(self, rows: tuple[Entry, ...] = (), migrated: bool = True) -> None:
self.rows = rows
self.migrated = migrated
self.lock = asyncio.Lock()
self.fail_write = False
self.fail_load = False
self.fail_current = False
self.closed = False
self.loads = 0
self.writes: tuple[Entry, ...] = ()
async def current(self) -> bool:
if self.fail_current:
raise SQLAlchemyError("schema unavailable")
return self.migrated
@asynccontextmanager
async def transaction(self) -> AsyncIterator[Transaction]:
async with self.lock:
original: Final = self.rows
try:
yield self
except BaseException:
self.rows = original
raise
async def load(self) -> tuple[Entry, ...]:
self.loads += 1
if self.fail_load:
raise SQLAlchemyError("snapshot unavailable")
return self.rows
async def write(self, entry: Entry) -> None:
if self.fail_write:
raise SQLAlchemyError("database write failed")
self.writes = (*self.writes, entry)
remaining: Final = tuple(
row for row in self.rows if (row.change.kind, row.change.name) != (entry.change.kind, entry.change.name)
)
self.rows = (*remaining, entry) if entry.change.op == "upsert" else remaining
async def close(self) -> None:
self.closed = True
class MemoryStream:
def __init__(self) -> None:
self.entries: tuple[Entry, ...] = ()
self.changed = asyncio.Event()
self.reads: asyncio.Queue[str] = asyncio.Queue()
self.failure: RedisError | None = None
self.closed = False
async def append(self, change: Change) -> str:
revision: Final = f"{100 + len(self.entries)}-0"
self.entries = (*self.entries, Entry(revision=revision, change=change))
self.changed.set()
return revision
async def read(self, after: str) -> tuple[Entry, ...]:
self.reads.put_nowait(after)
if self.failure is not None:
error: Final = self.failure
self.failure = None
raise error
pending: Final = tuple(entry for entry in self.entries if revision_key(entry.revision) > revision_key(after))
if pending:
return pending
self.changed.clear()
await self.changed.wait()
return tuple(entry for entry in self.entries if revision_key(entry.revision) > revision_key(after))
async def consumed(self, revision: str) -> None:
while True:
cursor: Final = await asyncio.wait_for(self.reads.get(), timeout=30)
if revision_key(cursor) >= revision_key(revision):
return
async def close(self) -> None:
self.closed = True
def container(
store: MemoryStore | None = None,
stream: MemoryStream | None = None,
transport: HttpTransport | None = None,
**overrides: object,
) -> Container:
settings: Final = ControlPlaneSettings.model_validate(
{
"LITELLM_MASTER_KEY": MASTER_KEY,
"CONTROL_PLANE_SALT_KEY": "runtime-test-salt",
"audit_sinks": (),
**overrides,
}
)
return build_container(
settings, exchanger=no_exchange, transport=transport, config_store=store, config_stream=stream
)
def role_entry(revision: str, name: str = "runtime", *scopes: str) -> Entry:
return Entry(
revision=revision,
change=Change(
op="upsert", kind="role", name=name, entity={"scopes": list(scopes), "inherits": []}, actor="operator"
),
)

View file

@ -0,0 +1,40 @@
from typing import Final
import pytest
from pydantic import SecretStr
from litellm.proxy.control_plane.settings import DataplaneSpec
from litellm.proxy.control_plane.store.crypto import SecretCodec
from litellm.proxy.control_plane.store.models import ENTITY
def test_secrets_roundtrip_with_the_configured_key_not_the_global_environment(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_SALT_KEY", "different-global-key")
codec: Final = SecretCodec(SecretStr("explicit-store-key"))
spec: Final = DataplaneSpec.model_validate(
{
"url": "https://runtime.test",
"service_token": {"token": "private-service-credential"},
"oidc_exchange": {
"token_endpoint": "https://idp.test/token",
"client_id": "cp",
"client_secret": "private-client-credential",
},
}
)
encrypted: Final = codec.encode(spec)
wire: Final = ENTITY.dump_json(encrypted).decode()
assert "private-service-credential" not in wire
assert "private-client-credential" not in wire
decoded: Final = codec.decode(encrypted)
assert decoded.service_token is not None
assert decoded.oidc_exchange is not None
assert decoded.service_token.token.get_secret_value() == spec.service_token.token.get_secret_value()
assert decoded.oidc_exchange.client_secret.get_secret_value() == spec.oidc_exchange.client_secret.get_secret_value()
assert decoded.url == spec.url
def test_specs_without_secrets_roundtrip_unchanged() -> None:
codec: Final = SecretCodec(SecretStr("explicit-store-key"))
spec: Final = DataplaneSpec(url="https://runtime.test")
assert codec.decode(codec.encode(spec)) == spec

View file

@ -0,0 +1,31 @@
from typing import Final
from alembic.config import Config
from alembic.script import ScriptDirectory
from pydantic import SecretStr
from litellm.proxy.control_plane.store.migrate import database_url, head_revision, migrate, migration_config
def test_migrator_dispatches_upgrade_head_using_packaged_revisions_and_the_dedicated_url() -> None:
calls: Final[list[tuple[Config, str]]] = [] # mutable-ok: records the Alembic command boundary
def upgrade(config: Config, revision: str) -> None:
calls.append((config, revision))
url: Final = SecretStr("postgresql://dedicated/control")
migrate(url, upgrade=upgrade)
[(config, revision)] = calls
assert revision == "head"
assert config.attributes["database_url"] == url
script: Final = ScriptDirectory.from_config(config)
assert script.get_current_head() == head_revision()
assert tuple(rev.revision for rev in script.walk_revisions()) == (head_revision(),)
def test_plain_postgres_urls_use_psycopg_without_modifying_escaped_credentials() -> None:
assert (
database_url(SecretStr("postgresql://user:p%25ss@local/control"))
== "postgresql+psycopg://user:p%25ss@local/control"
)
assert database_url(SecretStr("postgresql+psycopg://local/control")) == "postgresql+psycopg://local/control"

View file

@ -0,0 +1,309 @@
import asyncio
from typing import Final
import pytest
from pydantic import SecretStr
from redis.exceptions import ConnectionError as RedisConnectionError
from litellm.proxy.control_plane.auth.rbac import Principal, Scope, build_principal
from litellm.proxy.control_plane.settings import DataplaneSpec
from litellm.proxy.control_plane.store.crypto import SecretCodec
from litellm.proxy.control_plane.store.models import Change, Entry
from litellm.proxy.control_plane.store.runtime import Written
from tests.unit.proxy.control_plane.store.support import MemoryStore, MemoryStream, container, role_entry
from tests.unit.proxy.control_plane.support import MASTER_KEY
@pytest.mark.asyncio
@pytest.mark.parametrize("bad_kind", ["undecryptable", "missing-secret", "unknown-scope", "unknown-parent"])
async def test_quarantined_rows_revoke_stale_grants_without_blocking_other_writes_or_their_own_deletion(
bad_kind: str,
) -> None:
encrypted: Final = SecretCodec(SecretStr("wrong-key")).encode(
DataplaneSpec.model_validate({"url": "https://runtime.test", "service_token": {"token": "private"}})
)
bad: Final = Entry(
revision="2-0",
change=Change(
op="upsert",
kind="dataplane" if bad_kind in ("undecryptable", "missing-secret") else "role",
name="bad",
actor="operator",
entity={
"undecryptable": encrypted,
"missing-secret": {"url": "https://runtime.test", "service_token": {}},
"unknown-scope": {"scopes": ["invalid:scope"]},
"unknown-parent": {"scopes": ["roles:write"], "inherits": ["absent"]},
}[bad_kind],
),
)
dependent: Final = Entry(
revision="3-0",
change=Change(op="upsert", kind="role", name="dependent", entity={"inherits": ["bad"]}, actor="operator"),
)
store: Final = MemoryStore((bad, dependent, role_entry("4-0", "safe", "usage:read")))
stream: Final = MemoryStream()
rig: Final = container(store, stream)
runtime: Final = rig.runtime_config
assert runtime.state.apply(role_entry("1-0", "bad", "roles:write"))
assert await runtime.reload()
runtime.ready = True
assert "bad" not in rig.policy.names and "dependent" not in rig.policy.names
assert rig.registry.get("bad") is None
assert rig.policy.scopes_for("safe") == frozenset({Scope.USAGE_READ})
assert runtime.state.view.quarantined == {(bad.change.kind, "bad"), ("role", "dependent")}
actor: Final = rig.token_verifier.verify(MASTER_KEY)
assert isinstance(actor, Principal)
created: Final = await runtime.write(role_entry("0-0", "created", "usage:read").change, actor)
assert isinstance(created, Written)
deleted: Final = await runtime.write(Change(op="delete", kind=bad.change.kind, name="bad", actor="operator"), actor)
assert isinstance(deleted, Written)
assert tuple(row.change.name for row in store.rows) == ("dependent", "safe", "created")
assert tuple(entry.change.op for entry in stream.entries) == ("upsert", "delete")
@pytest.mark.asyncio
async def test_committed_writes_update_local_live_views_without_a_consumer() -> None:
store: Final = MemoryStore()
rig: Final = container(store, MemoryStream())
runtime: Final = rig.runtime_config
runtime.ready = True
actor: Final = rig.token_verifier.verify(MASTER_KEY)
assert isinstance(actor, Principal)
created: Final = await runtime.write(role_entry("0-0", "local", "proxy:read").change, actor)
assert isinstance(created, Written)
assert rig.policy.scopes_for("local") == frozenset({Scope.PROXY_READ})
revoked: Final = await runtime.write(Change(op="delete", kind="role", name="local", actor="operator"), actor)
assert isinstance(revoked, Written)
assert "local" not in rig.policy.names
spec: Final = DataplaneSpec.model_validate({"url": "https://runtime.test", "service_token": {"token": "private"}})
codec: Final = runtime.state.codec
assert codec is not None
written: Final = await runtime.write(
Change(op="upsert", kind="dataplane", name="local", entity=codec.encode(spec), actor="operator"), actor
)
assert isinstance(written, Written)
assert rig.registry.get("local") is not None
removed: Final = await runtime.write(Change(op="delete", kind="dataplane", name="local", actor="operator"), actor)
assert isinstance(removed, Written)
assert rig.registry.get("local") is None and store.rows == ()
assert runtime.task is None
@pytest.mark.asyncio
async def test_background_consumer_loads_snapshot_and_observes_writes_and_stops_when_blocked() -> None:
stored: Final = role_entry("9-0", "existing", "fleet:read")
store: Final = MemoryStore((stored,))
stream: Final = MemoryStream()
rig: Final = container(store, stream)
runtime: Final = rig.runtime_config
await runtime.start()
assert runtime.cursor == "9-0"
assert rig.policy.scopes_for("existing") == frozenset({Scope.FLEET_READ})
actor: Final = rig.token_verifier.verify(MASTER_KEY)
assert isinstance(actor, Principal)
result: Final = await runtime.write(role_entry("0-0", "created", "usage:read").change, actor)
assert isinstance(result, Written)
await stream.consumed(result.entry.revision)
assert rig.policy.scopes_for("created") == frozenset({Scope.USAGE_READ})
assert store.rows[-1].revision == stream.entries[-1].revision == result.entry.revision
assert store.rows[-1].change.actor == actor.subject
await runtime.close()
assert runtime.task is not None and runtime.task.done()
assert stream.closed and store.closed
@pytest.mark.asyncio
async def test_uncommitted_or_failed_events_do_not_grant_permissions_and_resync_restores_database() -> None:
store: Final = MemoryStore((role_entry("1-0", "saved", "fleet:read"),))
rig: Final = container(store, MemoryStream())
runtime: Final = rig.runtime_config
await runtime.reload()
await runtime.process(role_entry("2-0", "uncommitted", "roles:write"))
assert "uncommitted" not in rig.policy.names
runtime.state.apply(role_entry("3-0", "stale", "usage:read"))
await runtime.process(Entry(revision="4-0", change=Change(op="resync", actor="operator")))
assert rig.policy.scopes_for("saved") == frozenset({Scope.FLEET_READ})
assert "stale" not in rig.policy.names
@pytest.mark.asyncio
async def test_redis_error_backs_off_and_reloads_before_consuming_again() -> None:
entered: Final = asyncio.Event()
release: Final = asyncio.Event()
async def pause() -> None:
entered.set()
await release.wait()
store: Final = MemoryStore()
stream: Final = MemoryStream()
stream.failure = RedisConnectionError("disconnected")
rig: Final = container(store, stream)
runtime: Final = rig.runtime_config
runtime.pause = pause
await runtime.start()
await asyncio.wait_for(entered.wait(), 30)
assert not runtime.ready
store.rows = (role_entry("12-0", "recovered", "usage:read"),)
release.set()
await stream.consumed("12-0")
assert rig.policy.scopes_for("recovered") == frozenset({Scope.USAGE_READ})
assert runtime.ready
await runtime.close()
@pytest.mark.asyncio
async def test_failed_database_write_appends_resync_and_returns_unavailable() -> None:
store: Final = MemoryStore()
store.fail_write = True
stream: Final = MemoryStream()
rig: Final = container(store, stream)
runtime: Final = rig.runtime_config
runtime.ready = True
actor: Final = rig.token_verifier.verify(MASTER_KEY)
assert isinstance(actor, Principal)
result: Final = await runtime.write(role_entry("0-0", "failed", "roles:write").change, actor)
assert not isinstance(result, Written)
assert result.status == 503
assert tuple(entry.change.op for entry in stream.entries) == ("upsert", "resync")
assert store.rows == ()
assert "failed" not in rig.policy.names
@pytest.mark.asyncio
async def test_behind_schema_keeps_reads_config_only_without_loading_unmigrated_tables(
caplog: pytest.LogCaptureFixture,
) -> None:
store: Final = MemoryStore((role_entry("1-0", "unmigrated", "roles:write"),), migrated=False)
stream: Final = MemoryStream()
rig: Final = container(store, stream)
await rig.runtime_config.start()
assert not rig.runtime_config.ready
assert rig.runtime_config.task is not None
assert "unmigrated" not in rig.policy.names
assert store.loads == 0
assert "litellm --control-plane --skip_server_startup" in caplog.text
await rig.runtime_config.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("fail_current", [False, True])
async def test_startup_database_failure_recovers_without_restarting(fail_current: bool) -> None:
entered: Final = asyncio.Event()
release: Final = asyncio.Event()
async def pause() -> None:
entered.set()
await release.wait()
store: Final = MemoryStore((role_entry("12-0", "recovered", "usage:read"),))
store.fail_current = fail_current
store.fail_load = not fail_current
stream: Final = MemoryStream()
rig: Final = container(store, stream)
runtime: Final = rig.runtime_config
runtime.pause = pause
try:
await runtime.start()
await asyncio.wait_for(entered.wait(), 1)
assert not runtime.ready and "recovered" not in rig.policy.names
store.fail_current = store.fail_load = False
release.set()
await stream.consumed("12-0")
assert runtime.ready and rig.policy.scopes_for("recovered") == frozenset({Scope.USAGE_READ})
finally:
await runtime.close()
@pytest.mark.asyncio
async def test_empty_stream_poll_reconciles_a_committed_role_revocation_without_an_event() -> None:
store: Final = MemoryStore((role_entry("12-0", "revoked", "proxy:read"),))
stream: Final = MemoryStream()
rig: Final = container(store, stream)
runtime: Final = rig.runtime_config
try:
await runtime.start()
await stream.consumed("12-0")
assert rig.policy.scopes_for("revoked") == frozenset({Scope.PROXY_READ})
store.rows = ()
stream.changed.set()
await stream.consumed("12-0")
assert runtime.ready and "revoked" not in rig.policy.names
assert rig.policy.scopes_for("revoked") == frozenset()
finally:
await runtime.close()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("remaining", "change"),
[
((), role_entry("0-0", "delegated", "roles:write", "usage:read").change),
(("roles:write",), role_entry("0-0", "delegated", "roles:write", "usage:read").change),
((), Change(op="delete", kind="role", name="other", actor="operator")),
(
("roles:write",),
Change(
op="upsert",
kind="dataplane",
name="new",
entity={"url": "https://runtime.test", "service_token": {"token": "unused"}},
actor="operator",
),
),
],
)
async def test_committed_revocation_rejects_a_stale_replica_writer_before_publication(
remaining: tuple[str, ...], change: Change
) -> None:
store: Final = MemoryStore(
(role_entry("1-0", "delegated", "roles:write", "dataplanes:write", "usage:read"), role_entry("2-0", "other"))
)
stream: Final = MemoryStream()
first: Final = container(store, stream).runtime_config
stale: Final = container(store, stream).runtime_config
await first.reload()
await stale.reload()
first.ready = stale.ready = True
actor: Final = build_principal(
stale.state.view.policy, "operator", None, frozenset({"delegated"}), None, frozenset(), "oidc"
)
assert actor.has_scope(Scope.ROLES_WRITE) and actor.has_scope(Scope.DATAPLANES_WRITE)
master: Final = container().token_verifier.verify(MASTER_KEY)
assert isinstance(master, Principal)
revoked: Final = await first.write(role_entry("0-0", "delegated", *remaining).change, master)
assert isinstance(revoked, Written)
result: Final = await stale.write(change, actor)
assert not isinstance(result, Written)
assert result.status == 403
assert stream.entries == store.writes == (revoked.entry,)
assert store.rows == (role_entry("2-0", "other"), revoked.entry)
@pytest.mark.asyncio
async def test_locked_authorization_keeps_claim_narrowing_and_master_key_without_an_admin_role() -> None:
store: Final = MemoryStore((role_entry("1-0", "delegated", "roles:write", "usage:read"),))
stream: Final = MemoryStream()
rig: Final = container(store, stream, include_default_roles=False)
runtime: Final = rig.runtime_config
await runtime.reload()
runtime.ready = True
actor: Final = build_principal(
rig.policy,
"operator",
None,
frozenset({"delegated"}),
frozenset({Scope.ROLES_WRITE}),
frozenset(),
"oidc",
)
desired: Final = role_entry("0-0", "new", "usage:read").change
denied: Final = await runtime.write(desired, actor)
assert not isinstance(denied, Written) and denied.status == 403
assert stream.entries == store.writes == ()
master: Final = rig.token_verifier.verify(MASTER_KEY)
assert isinstance(master, Principal) and "admin" not in rig.policy.names
allowed: Final = await runtime.write(desired, master)
assert isinstance(allowed, Written)
assert stream.entries == store.writes == (allowed.entry,)

View file

@ -0,0 +1,49 @@
from dataclasses import replace
from typing import Final
import pytest
from litellm.proxy.control_plane.auth.rbac import Scope
from litellm.proxy.control_plane.store.models import Change, Entry
from tests.unit.proxy.control_plane.store.support import container, role_entry
def test_replayed_out_of_order_and_deleted_entries_cannot_restore_stale_permissions() -> None:
rig: Final = container()
state: Final = rig.runtime_config.state
state.apply(role_entry("10-0", "runtime", "fleet:read"))
state.apply(role_entry("9-9", "runtime", "roles:write"))
assert rig.policy.scopes_for("runtime") == frozenset({Scope.FLEET_READ})
assert not state.apply(role_entry("10-0", "runtime", "roles:write"))
assert rig.policy.scopes_for("runtime") == frozenset({Scope.FLEET_READ})
deleted: Final = Entry(revision="11-0", change=Change(op="delete", kind="role", name="runtime", actor="operator"))
assert state.apply(deleted)
assert "runtime" not in rig.policy.names
state.reload(())
assert not state.apply(role_entry("10-1", "runtime", "roles:write"))
assert not state.apply(deleted)
assert "runtime" not in rig.policy.names
assert state.apply(role_entry("12-0", "runtime", "usage:read"))
assert rig.policy.scopes_for("runtime") == frozenset({Scope.USAGE_READ})
def test_config_entries_and_default_roles_cannot_be_replaced_by_runtime_rows() -> None:
rig: Final = container()
before: Final = rig.policy.scopes_for("admin")
rig.runtime_config.state.apply(role_entry("1-0", "admin"))
assert rig.policy.scopes_for("admin") == before
def test_invalid_parent_does_not_replace_the_live_snapshot() -> None:
rig: Final = container()
state: Final = rig.runtime_config.state
original: Final = state.view
assert not state.apply(
Entry(
revision="1-0",
change=Change(
op="upsert", kind="role", name="bad", entity={"scopes": [], "inherits": ["absent"]}, actor="operator"
),
)
)
assert state.view is original

View file

@ -0,0 +1,80 @@
from collections.abc import Mapping
from typing import Final
import pytest
from pydantic import SecretStr
from litellm.proxy.control_plane.audit.event import CONTROL_PLANE_ID
from litellm.proxy.control_plane.settings import DataplaneSpec
from litellm.proxy.control_plane.store.crypto import SecretCodec
from litellm.proxy.control_plane.store.models import Change, Entry
from litellm.proxy.control_plane.store.stream import RedisStream
class RedisRecorder:
def __init__(self) -> None:
self.fields: Mapping[str, str] = {}
self.limit = 0
self.approximate = False
self.stream = ""
self.after = ""
self.block = 0
self.closed = False
self.revision = "123-4"
async def xadd(self, name: str, fields: Mapping[str, str], *, maxlen: int, approximate: bool) -> object:
self.fields = fields
self.limit = maxlen
self.approximate = approximate
self.stream = name
return self.revision
async def xread(self, streams: Mapping[str, str], *, count: int, block: int) -> object:
self.after = streams[self.stream]
self.block = block
return [[self.stream, [[self.revision, self.fields]]]]
async def aclose(self) -> None:
self.closed = True
@pytest.mark.asyncio
async def test_stream_preserves_encrypted_entities_and_actor_and_uses_bounded_blocking_reads_and_approximate_trim() -> (
None
):
client: Final = RedisRecorder()
stream: Final = RedisStream(client, "cp:config", 100)
spec: Final = DataplaneSpec.model_validate(
{"url": "https://runtime.test", "service_token": {"token": "plaintext-secret"}}
)
codec: Final = SecretCodec(SecretStr("configured-key"))
change: Final = Change(op="upsert", kind="dataplane", name="live", entity=codec.encode(spec), actor="operator")
revision: Final = await stream.append(change)
assert revision == "123-4"
assert (client.stream, client.limit, client.approximate) == ("cp:config", 100, True)
assert "plaintext-secret" not in str(client.fields)
records: Final = await stream.read("123-3")
assert records == (Entry(revision=revision, change=change),)
assert client.after == "123-3" and client.block > 0
assert codec.decode(records[0].change.entity) == spec
await stream.close()
assert client.closed
@pytest.mark.asyncio
@pytest.mark.parametrize("fields", [{}, {"event": "not-json"}, {"event": "{}"}])
async def test_malformed_stream_entries_become_resync_markers_without_losing_their_cursor(
fields: dict[str, str],
) -> None:
client: Final = RedisRecorder()
stream: Final = RedisStream(client, "cp:config", 100)
await stream.append(Change(op="resync", actor="operator"))
client.fields = fields
assert await stream.read("123-3") == (
Entry(revision="123-4", change=Change(op="resync", actor=CONTROL_PLANE_ID)),
)
client.revision = "123-5"
await stream.append(Change(op="resync", actor="next-writer"))
assert await stream.read("123-4") == (
Entry(revision="123-5", change=Change(op="resync", actor="next-writer")),
)

View file

@ -11,6 +11,67 @@ from litellm.proxy.control_plane.settings import (
)
from litellm.types.proxy.control_plane.auth import PASSWORD_MAX_LENGTH, USERNAME_MAX_LENGTH
@pytest.mark.parametrize("configured", ["database_url", "redis_url"])
def test_runtime_store_requires_both_connections(configured: str) -> None:
with pytest.raises(ValidationError, match="Set both"):
ControlPlaneSettings.model_validate(
{"LITELLM_MASTER_KEY": "01234567890123456789012345678901", configured: "unused"}
)
def test_runtime_store_requires_explicit_or_fallback_salt(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("CONTROL_PLANE_SALT_KEY", raising=False)
monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
with pytest.raises(ValidationError, match="SALT_KEY"):
ControlPlaneSettings.model_validate(
{
"LITELLM_MASTER_KEY": "01234567890123456789012345678901",
"database_url": "postgresql://local/control",
"redis_url": "redis://local",
}
)
@pytest.mark.parametrize("connections", ["database", "redis", "both"])
def test_runtime_settings_tracebacks_hide_connection_credentials(
monkeypatch: pytest.MonkeyPatch, connections: str
) -> None:
monkeypatch.delenv("CONTROL_PLANE_SALT_KEY", raising=False)
monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
database: Final = "postgresql://operator:db-private-password@localhost/control"
redis: Final = "redis://operator:redis-private-password@localhost/1"
with pytest.raises(ValidationError) as error:
ControlPlaneSettings.model_validate(
{
"LITELLM_MASTER_KEY": "01234567890123456789012345678901",
**({"database_url": database} if connections in ("database", "both") else {}),
**({"redis_url": redis} if connections in ("redis", "both") else {}),
}
)
message: Final = str(error.value)
assert "db-private-password" not in message and "redis-private-password" not in message
assert ("SALT_KEY" if connections == "both" else "Set both") in message
def test_runtime_store_settings_use_dedicated_environment_and_salt_precedence(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_MASTER_KEY", "01234567890123456789012345678901")
monkeypatch.setenv("DATABASE_URL", "must-not-be-used")
monkeypatch.setenv("CONTROL_PLANE_DATABASE_URL", "postgresql://local/control")
monkeypatch.setenv("CONTROL_PLANE_REDIS_URL", "redis://local/1")
monkeypatch.delenv("CONTROL_PLANE_SALT_KEY", raising=False)
monkeypatch.setenv("LITELLM_SALT_KEY", "fallback")
settings: Final = ControlPlaneSettings()
assert (
settings.database_url is not None and settings.database_url.get_secret_value() == "postgresql://local/control"
)
assert settings.redis_url is not None and settings.redis_url.get_secret_value() == "redis://local/1"
assert settings.salt_key is not None and settings.salt_key.get_secret_value() == "fallback"
monkeypatch.setenv("CONTROL_PLANE_SALT_KEY", "dedicated")
explicit: Final = ControlPlaneSettings()
assert explicit.salt_key is not None and explicit.salt_key.get_secret_value() == "dedicated"
MASTER_KEY: Final = "01234567890123456789012345678901"

View file

@ -24,7 +24,7 @@ import yaml
from gunicorn.config import Config as GunicornConfig
from click.testing import CliRunner
from fastapi.testclient import TestClient
from pydantic import TypeAdapter
from pydantic import SecretStr, TypeAdapter
from uvicorn.config import LOOP_FACTORIES
from uvicorn.importer import import_from_string
@ -4776,7 +4776,7 @@ def test_utility_commands_do_not_decrypt_kms_environment(
("control_plane", "skip_server_startup", "utility_command", "expected"),
[
(True, False, False, True),
(True, True, False, False),
(True, True, False, True),
(True, False, True, False),
(False, False, False, False),
(False, True, True, False),
@ -4808,3 +4808,56 @@ def test_other_servers_and_reload_are_a_usage_error_with_the_control_plane_flag(
def test_control_plane_rejects_a_lone_tls_file() -> None:
result: Final = CliRunner().invoke(run_server, ["--control-plane", "--ssl_keyfile_path", "/c/tls.key"])
assert result.exit_code == 2 and "both" in result.output
def test_control_plane_migration_job_needs_only_its_own_database_and_never_starts_prisma_or_server(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.control_plane.store.migrate import migrate_from_environment
monkeypatch.setenv("CONTROL_PLANE_DATABASE_URL", "postgresql://dedicated/control")
monkeypatch.setenv("DATABASE_URL", "invalid-proxy-database")
monkeypatch.delenv("CONTROL_PLANE_REDIS_URL", raising=False)
monkeypatch.delenv("LITELLM_MASTER_KEY", raising=False)
monkeypatch.delenv("USE_AWS_KMS", raising=False)
migrated: Final[list[str]] = [] # mutable-ok: records the one-shot invocation
def upgrade(url: SecretStr) -> None:
migrated.append(url.get_secret_value())
result: Final = CliRunner().invoke(
run_server,
["--control-plane", "--skip_server_startup"],
obj=CliDependencies(migrate_control_plane=lambda: migrate_from_environment(upgrade)),
)
assert result.exit_code == 0, result.output
assert migrated == ["postgresql://dedicated/control"]
assert "migrations are current" in result.output
def test_control_plane_migration_job_rejects_missing_dedicated_database_instead_of_using_database_url(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("CONTROL_PLANE_DATABASE_URL", raising=False)
monkeypatch.delenv("USE_AWS_KMS", raising=False)
monkeypatch.setenv("DATABASE_URL", "postgresql://proxy/should-not-be-used")
result: Final = CliRunner().invoke(run_server, ["--control-plane", "--skip_server_startup"])
assert result.exit_code != 0
assert "CONTROL_PLANE_DATABASE_URL is required" in result.output
def test_control_plane_migration_failure_is_nonzero_and_does_not_disclose_dsn(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.control_plane.store.migrate import migrate_from_environment
monkeypatch.setenv("CONTROL_PLANE_DATABASE_URL", "postgresql://private-password@dedicated/control")
monkeypatch.delenv("USE_AWS_KMS", raising=False)
def upgrade(_url: SecretStr) -> None:
raise RuntimeError("private-password")
result: Final = CliRunner().invoke(
run_server,
["--control-plane", "--skip_server_startup"],
obj=CliDependencies(migrate_control_plane=lambda: migrate_from_environment(upgrade)),
)
assert result.exit_code != 0
assert "Control-plane migration failed" in result.output
assert "private-password" not in result.output

14
uv.lock generated
View file

@ -5114,6 +5114,7 @@ mlflow = [
{ name = "mlflow" },
]
proxy = [
{ name = "alembic" },
{ name = "apscheduler" },
{ name = "azure-identity" },
{ name = "azure-storage-blob" },
@ -5140,6 +5141,8 @@ proxy = [
{ name = "opentelemetry-sdk" },
{ name = "orjson" },
{ name = "polars" },
{ name = "psycopg" },
{ name = "psycopg-binary" },
{ name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" },
{ name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" },
{ name = "pyjwt" },
@ -5152,6 +5155,7 @@ proxy = [
{ name = "rq" },
{ name = "semver" },
{ name = "soundfile" },
{ name = "sqlalchemy" },
{ name = "starlette" },
{ name = "tomlkit" },
{ name = "tzdata" },
@ -5306,6 +5310,7 @@ healthcheck = [
]
proxy-dev = [
{ name = "a2a-sdk" },
{ name = "alembic" },
{ name = "azure-identity" },
{ name = "backports-zstd", marker = "python_full_version < '3.14'" },
{ name = "httptools" },
@ -5318,14 +5323,17 @@ proxy-dev = [
{ name = "opentelemetry-sdk" },
{ name = "prisma" },
{ name = "prometheus-client" },
{ name = "pynacl" },
{ name = "semver" },
{ name = "sentry-sdk" },
{ name = "sqlalchemy" },
]
[package.metadata]
requires-dist = [
{ name = "a2a-sdk", marker = "extra == 'extra-proxy'", specifier = ">=1.1.0,<2.0" },
{ name = "aiohttp", specifier = ">=3.14.2,<4.0" },
{ name = "alembic", marker = "extra == 'proxy'", specifier = ">=1.18.4,<2.0" },
{ name = "anthropic", extras = ["vertex"], marker = "extra == 'proxy-runtime'", specifier = ">=0.84.0,<1.0" },
{ name = "anyio", specifier = ">=4.14.0,<5.0" },
{ name = "apscheduler", marker = "extra == 'proxy'", specifier = ">=3.11.2,<4.0" },
@ -5407,7 +5415,9 @@ requires-dist = [
{ name = "prometheus-client", marker = "extra == 'proxy-runtime'", specifier = ">=0.20.0,<1.0" },
{ name = "protobuf", marker = "extra == 'stt-nvidia-riva'", specifier = ">=5.26.1,<7.0" },
{ name = "psycopg", marker = "extra == 'extra-proxy'", specifier = ">=3.2,<4.0" },
{ name = "psycopg", marker = "extra == 'proxy'", specifier = ">=3.2,<4.0" },
{ name = "psycopg-binary", marker = "extra == 'extra-proxy'", specifier = ">=3.2,<4.0" },
{ name = "psycopg-binary", marker = "extra == 'proxy'", specifier = ">=3.2,<4.0" },
{ name = "pydantic", marker = "python_full_version < '3.14'", specifier = ">=2.11.0,<3.0.0" },
{ name = "pydantic", marker = "python_full_version == '3.14.*'", specifier = ">=2.12.0,<3.0.0" },
{ name = "pydantic", marker = "python_full_version >= '3.15'", specifier = ">=2.14.0,<3.0.0" },
@ -5436,6 +5446,7 @@ requires-dist = [
{ name = "sentry-sdk", marker = "extra == 'proxy-runtime'", specifier = ">=2.21.0,<3.0" },
{ name = "soundfile", marker = "extra == 'proxy'", specifier = ">=0.12.1,<1.0" },
{ name = "soundfile", marker = "extra == 'stt-nvidia-riva'", specifier = ">=0.12.1" },
{ name = "sqlalchemy", marker = "extra == 'proxy'", specifier = ">=2.0.49,<2.1" },
{ name = "starlette", marker = "extra == 'proxy'", specifier = ">=1.0.1,<2.0" },
{ name = "tiktoken", marker = "python_full_version < '3.14'", specifier = ">=0.8.0,<1.0" },
{ name = "tiktoken", marker = "python_full_version == '3.14.*'", specifier = ">=0.12.0,<1.0" },
@ -5549,6 +5560,7 @@ healthcheck = [
]
proxy-dev = [
{ name = "a2a-sdk", specifier = "==1.1.0" },
{ name = "alembic", specifier = "==1.18.4" },
{ name = "azure-identity", specifier = "==1.25.2" },
{ name = "backports-zstd", marker = "python_full_version < '3.14'", specifier = "==1.8.0" },
{ name = "httptools", specifier = ">=0.7.1,<0.8" },
@ -5561,8 +5573,10 @@ proxy-dev = [
{ name = "opentelemetry-sdk", specifier = "==1.45.2" },
{ name = "prisma", specifier = "==0.11.0" },
{ name = "prometheus-client", specifier = "==0.20.0" },
{ name = "pynacl", specifier = "==1.6.2" },
{ name = "semver", specifier = ">=3.0.4,<4" },
{ name = "sentry-sdk", specifier = "==2.21.0" },
{ name = "sqlalchemy", specifier = "==2.0.49" },
]
[[package]]