mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
014cb8fa9d
Split the monolithic LiteLLM proxy into independently scalable Kubernetes components to allow separate horizontal scaling of the LLM data plane and management API surfaces - Add DatabaseURLSettings pydantic-settings model that assembles DATABASE_URL (and optional DATABASE_URL_READ_REPLICA) from discrete DATABASE_* env vars before Prisma initializes, supporting both IAM token auth (minting short-lived RDS tokens) and password auth; replaces the CLI-only path that componentized entrypoints bypass - Add gateway component (port 4000) that trims the proxy route table to the LLM data-plane surface (chat, embeddings, completions, audio, realtime, provider passthroughs, health/metrics) via an allowlist applied inside the lifespan context so plugin-registered routes are captured - Add backend component (port 4001) that exposes the management/admin surface (keys, users, teams, orgs, spend analytics, model management, SSO, audit logs) with a complementary allowlist - Add ui component — Next.js static export served by nginx (port 3000) with RSC payload routing, asset prefix aliasing, and SPA fallback for dashboard routes - Add migrations component with dedicated Dockerfile that runs prisma migrate deploy via a Helm pre-install/pre-upgrade Job, eliminating per-pod schema contention on the Prisma advisory lock - Add Helm chart (helm/litellm) with separate Deployments, Services, HPAs, and ConfigMap for each component; shared _helpers.tpl emits DATABASE_*, IAM_TOKEN_DB_AUTH, REDIS_*, and DISABLE_SCHEMA_UPDATE env vars from chart values; ingress template routes traffic to the correct component by path prefix - Add comprehensive tests for DatabaseURLSettings covering IAM auth, password auth, read replica fallbacks, operator-pinned URL preservation, and percent-encoding; add coverage test asserting gateway + backend allowlist union equals the full proxy route set - Add pydantic-settings>=2.14.1 as a proxy extra dependency and update liccheck allowlist Co-authored-by: Yassin Kortam <yassinkortam@g.ucla.edu>
121 lines
2.7 KiB
Python
121 lines
2.7 KiB
Python
"""Path allowlist for the gateway component.
|
|
|
|
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
|
|
audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
|
|
responses, vector stores, passthrough providers, realtime websockets, MCP
|
|
tool-call endpoints, and operational endpoints (/health, /metrics).
|
|
|
|
Any path not listed here is dropped from the gateway process so management/UI
|
|
endpoints don't ride on the same pods.
|
|
|
|
Versioned data-plane paths are enumerated explicitly rather than allowing a
|
|
blanket `/v1/` or `/v2/` prefix — those broad prefixes would otherwise also
|
|
match management routes like `/v1/access_group`, `/v1/tool/{tool_name}/logs`,
|
|
`/v2/key/info`, etc.
|
|
"""
|
|
|
|
GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|
# OpenAI-compatible data-plane surface (versioned + unversioned)
|
|
"/v1/chat/",
|
|
"/chat/",
|
|
"/v1/completions",
|
|
"/completions",
|
|
"/v1/embeddings",
|
|
"/embeddings",
|
|
"/v1/moderations",
|
|
"/moderations",
|
|
"/v1/audio/",
|
|
"/audio/",
|
|
"/v1/images/",
|
|
"/images/",
|
|
"/v1/files",
|
|
"/files",
|
|
"/v1/batches",
|
|
"/batches",
|
|
"/v1/fine_tuning/",
|
|
"/fine_tuning/",
|
|
"/v1/fine-tuning/",
|
|
"/fine-tuning/",
|
|
"/v1/responses",
|
|
"/responses",
|
|
"/v1/threads",
|
|
"/threads",
|
|
"/v1/assistants",
|
|
"/assistants",
|
|
"/v1/vector_stores",
|
|
"/vector_stores",
|
|
"/v1/indexes",
|
|
"/v1/models",
|
|
"/models",
|
|
"/openai/",
|
|
"/engines/",
|
|
# Anthropic / agentic data-plane surface
|
|
"/v1/messages",
|
|
"/messages",
|
|
"/v1/skills",
|
|
"/v1/a2a/",
|
|
# LiteLLM-native LLM surface
|
|
"/v1/rerank",
|
|
"/v2/rerank",
|
|
"/rerank",
|
|
"/v1/ocr",
|
|
"/ocr",
|
|
"/v1/rag/",
|
|
"/rag/",
|
|
"/v1/video",
|
|
"/v1/videos",
|
|
"/video/",
|
|
"/videos",
|
|
"/v1/search",
|
|
"/search",
|
|
"/v1/containers",
|
|
"/containers",
|
|
"/v1/evals",
|
|
"/v1/memory",
|
|
"/queue/chat/",
|
|
# Google data plane (v1beta is the Google AI Studio version)
|
|
"/v1beta/",
|
|
"/interactions",
|
|
# Provider passthrough
|
|
"/anthropic/",
|
|
"/azure/",
|
|
"/azure_ai/",
|
|
"/aws/",
|
|
"/bedrock/",
|
|
"/cohere/",
|
|
"/gemini/",
|
|
"/google/",
|
|
"/vertex_ai/",
|
|
"/vertex-ai/",
|
|
"/assemblyai/",
|
|
"/eu.assemblyai/",
|
|
"/langfuse/",
|
|
"/vllm/",
|
|
"/mistral/",
|
|
"/groq/",
|
|
"/voyage/",
|
|
"/cursor/",
|
|
"/milvus/",
|
|
"/openai_passthrough/",
|
|
# Dynamic provider / toolset passthrough (path templates)
|
|
"/{provider}/",
|
|
"/toolset/",
|
|
# Realtime / streaming
|
|
"/v1/realtime",
|
|
"/realtime",
|
|
# Health & ops
|
|
"/health",
|
|
"/metrics",
|
|
)
|
|
|
|
GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
|
|
{
|
|
"/",
|
|
"/routes",
|
|
"/openapi.json",
|
|
"/docs",
|
|
"/docs/oauth2-redirect",
|
|
"/redoc",
|
|
"/test",
|
|
}
|
|
)
|