From b2a3f984d31fed748d1290aa8442ce53cdb5e757 Mon Sep 17 00:00:00 2001 From: sanjibani <18418553+sanjibani@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:00:21 +0530 Subject: [PATCH 1/2] feat(terraform): add Azure module for Azure Container Apps + App Gateway Closes #31843. Adds terraform/litellm/azure/, mirroring the AWS / GCP module shape: terraform/litellm/azure/ versions.tf azurerm + random provider pins variables.tf contract mirroring aws/variables.tf locals.tf naming, tags, gateway/ui path prefixes outputs.tf app gateway FQDN, key vault URI, run cmd network.tf VNet, subnets, NSGs postgres.tf Azure DB for PostgreSQL Flexible Server + AAD admin redis.tf Azure Cache for Redis (Basic/Standard/Premium) storage.tf Storage Account (TLS 1.2, versioned, no shared key) + private endpoint + proxy_config blob upload keyvault.tf Key Vault (private-endpoint + access policies) iam.tf user-assigned MI + role assignments compute.tf Container Apps Environment + 3 apps (gateway, backend, ui) migrations.tf Container Apps Job for prisma migrate deploy loadbalancer.tf Application Gateway v2 with path-based routing README.md module-level usage guide examples/default/ one-command deploy The Application Gateway routes the LLM data-plane prefixes (/v1/chat/*, /v1/embeddings, etc.) to gateway, the UI asset paths to ui, and falls through to backend (management API), matching the AWS ALB path map. runtime auth uses Entra-managed-identity tokens against Postgres Flexible Server (DATABASE_AZURE_AUTH=true) at the application layer; local admin kept in Key Vault for break-glass. Validated with azurerm ~> 3.117; terraform validate passes clean. --- README.md | 62 ++- terraform/litellm/azure/README.md | 219 ++++++++++ terraform/litellm/azure/compute.tf | 315 ++++++++++++++ .../examples/default/.terraform.lock.hcl | 43 ++ .../litellm/azure/examples/default/main.tf | 43 ++ .../azure/examples/default/providers.tf | 8 + .../examples/default/terraform.tfvars.example | 42 ++ .../azure/examples/default/variables.tf | 82 ++++ .../azure/examples/default/versions.tf | 14 + terraform/litellm/azure/iam.tf | 55 +++ terraform/litellm/azure/keyvault.tf | 169 ++++++++ terraform/litellm/azure/loadbalancer.tf | 337 +++++++++++++++ terraform/litellm/azure/locals.tf | 114 +++++ terraform/litellm/azure/migrations.tf | 79 ++++ terraform/litellm/azure/network.tf | 78 ++++ terraform/litellm/azure/outputs.tf | 89 ++++ terraform/litellm/azure/postgres.tf | 78 ++++ terraform/litellm/azure/redis.tf | 52 +++ terraform/litellm/azure/storage.tf | 124 ++++++ terraform/litellm/azure/variables.tf | 400 ++++++++++++++++++ terraform/litellm/azure/versions.tf | 14 + 21 files changed, 2414 insertions(+), 3 deletions(-) create mode 100644 terraform/litellm/azure/README.md create mode 100644 terraform/litellm/azure/compute.tf create mode 100644 terraform/litellm/azure/examples/default/.terraform.lock.hcl create mode 100644 terraform/litellm/azure/examples/default/main.tf create mode 100644 terraform/litellm/azure/examples/default/providers.tf create mode 100644 terraform/litellm/azure/examples/default/terraform.tfvars.example create mode 100644 terraform/litellm/azure/examples/default/variables.tf create mode 100644 terraform/litellm/azure/examples/default/versions.tf create mode 100644 terraform/litellm/azure/iam.tf create mode 100644 terraform/litellm/azure/keyvault.tf create mode 100644 terraform/litellm/azure/loadbalancer.tf create mode 100644 terraform/litellm/azure/locals.tf create mode 100644 terraform/litellm/azure/migrations.tf create mode 100644 terraform/litellm/azure/network.tf create mode 100644 terraform/litellm/azure/outputs.tf create mode 100644 terraform/litellm/azure/postgres.tf create mode 100644 terraform/litellm/azure/redis.tf create mode 100644 terraform/litellm/azure/storage.tf create mode 100644 terraform/litellm/azure/variables.tf create mode 100644 terraform/litellm/azure/versions.tf diff --git a/README.md b/README.md index 90d3e944fcc..befafabb8aa 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Deploy on Railway Deploy on AWS Deploy on GCP + Deploy on Azure

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

@@ -535,16 +536,71 @@ terraform apply Provider API keys live in Secret Manager; reference resource IDs (e.g. `projects/my-gcp-project/secrets/openai-api-key`) via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/google/latest?tab=inputs). +#### Azure: Container Apps + App Gateway + +Container Apps + Application Gateway with path-based routing, matching the AWS / GCP stacks. Pulls the four componentized images directly from GHCR (no mirror needed for Azure). + +```bash +git clone https://github.com/BerriAI/litellm.git +cd litellm/terraform/litellm/azure/examples/default +cp terraform.tfvars.example terraform.tfvars # edit location/tenant/env +terraform init && terraform apply +``` + +[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/azure/latest) + +To call the module from your own root config: + +```hcl +# main.tf +terraform { + required_version = ">= 1.6.0" + required_providers { + azurerm = { source = "hashicorp/azurerm", version = "~> 3.117" } + } +} + +provider "azurerm" { + features {} +} + +module "litellm" { + source = "BerriAI/litellm/azure" + version = "~> 1.89" + + location = "eastus" + tenant = "acme" + env = "prod" + azs = ["1", "2"] + + # Production: upload a TLS cert to Key Vault and reference it here. + # Without one, set allow_plaintext_app_gateway = true (dev/trial only). + # key_vault_certificate_id = "https://acmekv.vault.azure.net/certificates/litellm-cert/" + allow_plaintext_app_gateway = true +} + +output "litellm_url" { + value = module.litellm.app_gateway_url +} +``` + +```bash +terraform init +terraform apply +``` + +Provider API keys live in Key Vault; reference secret IDs via `gateway_extra_secrets`. Full input list on the [module README](./terraform/litellm/azure/README.md). + #### Both stacks include - The full componentized split (gateway / backend / UI as independent services) -- Managed Postgres (writer + reader) and Redis +- Managed Postgres and Redis - Versioned object store for proxy state + file uploads - An auto-generated `LITELLM_MASTER_KEY` in your cloud's secret manager - A one-off migration job that runs `prisma migrate deploy` before the proxy starts -- The same `proxy_config` surface as the [Helm chart](./helm/litellm/) — pass YAML as a typed map +- The same `proxy_config` surface as the [Helm chart](./helm/litellm/), pass YAML as a typed map -The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws/) and [`terraform/litellm/gcp/`](./terraform/litellm/gcp/) in this repo; the registry entries are read-only mirrors updated on each release. +The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws/), [`terraform/litellm/gcp/`](./terraform/litellm/gcp/), and [`terraform/litellm/azure/`](./terraform/litellm/azure/) in this repo; the registry entries are read-only mirrors updated on each release. ### Run in Developer Mode #### Services diff --git a/terraform/litellm/azure/README.md b/terraform/litellm/azure/README.md new file mode 100644 index 00000000000..1b26c717c8b --- /dev/null +++ b/terraform/litellm/azure/README.md @@ -0,0 +1,219 @@ +# LiteLLM on Azure (Container Apps + App Gateway) + +Deploys the componentized LiteLLM proxy on Azure: + +- **VNet** with three isolated subnets: containers (delegated to Azure + Container Apps), private endpoints (Postgres / Redis / Storage / Key + Vault), Application Gateway +- **Azure Database for PostgreSQL Flexible Server**, single instance, + with Entra (Azure AD) authentication enabled. Password auth is also + enabled for bootstrap +- **Azure Cache for Redis** (Basic by default; Standard / Premium + recommended for production) +- **Storage Account** (private, versioned, TLS 1.2 minimum) with a + `proxy` blob container for cache / proxy_config + a `files` blob + container for /v1/files passthrough storage +- **Key Vault** holding `LITELLM_MASTER_KEY`, the optional license, the + optional UI password, and the bootstrap Postgres admin password +- **Container Apps Environment** running three apps (`gateway`, port + 4000; `backend`, port 4001; `ui`, port 3000) plus a one-off + `migrations` Container Apps Job that runs `prisma migrate deploy` + from the dedicated `ghcr.io/berriai/litellm-migrations` image +- **User-assigned managed identity** shared across the three apps + the + migrations job, granted: Storage Blob Data Contributor on the storage + account, Key Vault Secrets User on the vault +- **Application Gateway v2** with path-based routing: + - LLM data-plane prefixes (`/v1/chat/*`, `/v1/embeddings`, ...) -> + gateway + - UI asset paths (`/_next/*`, `/assets/*`, ...) -> ui + - Everything else (management API: `/key/*`, `/user/*`, ...) -> + backend + +## Quick start + +```bash +cd terraform/litellm/azure/examples/default +cp terraform.tfvars.example terraform.tfvars # edit location/tenant/env +terraform init && terraform apply +``` + +After the apply: + +1. As the Entra admin (matches the principal running terraform), run + the `db_bootstrap_sql` output once against the Postgres Flexible + Server. This creates the `var.db_username` user the proxy will + authenticate as via Entra-managed-identity tokens at runtime. +2. Run the `migration_run_command` output to trigger the one-off + prisma migration Container App Job. The gateway / backend revisions + do not auto-redeploy after this; that's intentional so traffic is + not cut over mid-migration. +3. Point DNS at `app_gateway_fqdn` output. For TLS, set + `key_vault_certificate_id` after the initial apply and re-apply; the + HTTPS listener picks up the cert automatically. + +## Components + +### `proxy_config` (preferred) + +Mirrors the helm chart's `gateway.config.proxy_config`. The map is +YAML-encoded and uploaded to the storage account blob +`config/litellm-config.yaml`; the gateway and backend container +start scripts download it to `/tmp/litellm-config.yaml` and set +`CONFIG_FILE_PATH` to match. Editing the value produces a new blob +content and a rolling redeploy of both services. + +```hcl +proxy_config = { + model_list = [ + { + model_name = "gpt-4o" + litellm_params = { + model = "openai/gpt-4o" + api_key = "os.environ/OPENAI_API_KEY" + } + }, + ] + general_settings = { + master_key = "os.environ/LITELLM_MASTER_KEY" + database_url = "os.environ/DATABASE_URL" + } +} +``` + +LiteLLM resolves `os.environ/` references in the YAML against +the container's environment. That means provider API keys belong in +`*_extra_secrets` (next section), and your YAML just references them +by name. + +### Extra env vars + +Non-sensitive plaintext (feature flags, observability hosts, etc.) via +`gateway_extra_env` / `backend_extra_env`: + +```hcl +gateway_extra_env = { + LITELLM_LOG = "INFO" + LANGFUSE_PUBLIC_KEY = "pk-lf-..." +} + +backend_extra_env = { + UI_USERNAME = "admin" +} +``` + +### Extra secrets + +Sensitive env vars (provider API keys) via `gateway_extra_secrets` / +`backend_extra_secrets`: + +```hcl +gateway_extra_secrets = { + OPENAI_API_KEY = azurerm_key_vault_secret.openai.id +} +``` + +The Container Apps managed identity has `Get` / `List` on the Key Vault, +so it can read the referenced secret URIs. Create those `*_secret` +resources separately and grant the same access policy. + +## Database authentication + +The Container Apps managed identity uses Entra (Azure AD) tokens at +runtime against the Postgres Flexible Server. The proxy assembles +`DATABASE_URL` from `DATABASE_HOST/PORT/USER/NAME` + a short-lived +AAD token minted via `azure-identity`'s `DefaultAzureCredential`. The +gateway / backend / migration Container Apps have +`DATABASE_AZURE_AUTH=true` in their environment. + +**Break-glass.** The local `litellm_admin` user (password lives in +Key Vault as `db-admin-password`) is kept for break-glass repairs. To +rotate, use the Azure Portal -> "Reset password" or pass a new +`random_password.db_admin_password` resource via a maintenance PR. + +**Prerequisite.** A first-time `terraform apply` requires either +`var.litellm_master_key != ""` (the secret is set by the caller) OR +that the current terraform principal is granted `Set` / `Get` on the +Key Vault (the access policy is created automatically by +`keyvault.tf`). + +## Container Apps pull + +By default the stack pulls the four component images from +`ghcr.io/berriai/litellm-{gateway,backend,ui,migrations}:`. + +Azure Container Apps can pull from `ghcr.io` directly without mirror +configuration (the AWS stack pulls from ECR post-push, the GCP stack +needs an Artifact Registry remote repo, Azure is the closest to a +direct pull-from-public-registry experience). + +If you want to pin to a private Azure Container Registry (ACR) instead, +push the images there first and override the `image_*` variables: + +```hcl +image_registry = "myregistry.azurecr.io" +image_namespace = "litellm" +image_tag = "v1.86.0" +``` + +The same role assignment (`AcrPull` for the Container Apps managed +identity) is granted automatically at apply time when the registry is +an `*.azurecr.io` host. + +## Observability + +Set `otel_endpoint` to an OTLP collector URL to turn on OpenTelemetry +v2 instrumentation in both gateway and backend. Service names are +stamped per component: + +``` +${local.name}-gateway +${local.name}-backend +``` + +The current terraform principal can also grant `Monitoring Metrics +Publisher` on the same resource group if you want Container Apps +metrics to feed to Application Insights. + +## Component sizing defaults + +Defaults work for a small team / production-trail pattern: + +| Component | CPU | Memory | Min replicas | Max replicas | +|-----------|-----|--------|--------------|--------------| +| gateway | 1.0 | 2Gi | 2 | 10 | +| backend | 1.0 | 2Gi | 1 | 5 | +| ui | 0.5 | 1Gi | 1 | 3 | +| migrations| 1.0 | 2Gi | (job) | (job) | + +Bump these via `gateway_cpu`, `gateway_memory`, etc. The full input list +is on the (forthcoming) Terraform Registry page; until then, see +`variables.tf` in this directory. + +## State and CI + +Apply from CI? Then pin the constructor's +`role_definition_name` to `Owner` or `Contributor + User Access +Administrator` on the subscription so the role assignments and +Key Vault access policies land. The current terraform principal's +object ID is read via `data.azurerm_client_config`; CI runners need to +be authenticated as the AAD admin for the bootstrap pattern to work +without manual steps. + +## Known gaps vs AWS / GCP + +- Azure Container Apps does not natively integrate with managed + Postgres IAM tokens the way Aurora + RDS-iam-token does on AWS. + Entra-managed-identity tokens at the application layer are the + equivalent. The `proxy_config` blob is the only state shared + between the two write paths; everything else is per-component env. +- Application Gateway WAF v2 is not enabled in this baseline. To turn + it on, swap the SKU `Standard_v2` to `WAF_v2` and add WAF policy + resources. The path rule structure does not change. +- Redis SSL on Azure: the `rediss://` URL is built from the SSL port + (default 6380). The proxy uses `REDIS_SSL=true` to flip the scheme, + matching AWS module's `transit_encryption_enabled = true`. +- Migration ordering: the AWS stack uses `depends_on` chains via + ECS task ordering; Azure Container Apps Jobs are not part of ACA + service ordering. The migration is run as a manual one-shot AFTER + the gateway / backend services are deployed. The README's + Quick-start ordering covers this. diff --git a/terraform/litellm/azure/compute.tf b/terraform/litellm/azure/compute.tf new file mode 100644 index 00000000000..c93481a1c5c --- /dev/null +++ b/terraform/litellm/azure/compute.tf @@ -0,0 +1,315 @@ +# Container Apps Environment (one per stack) + the three componentized +# apps (gateway / backend / ui) plus the migrations job. +# +# The Azure equivalent of AWS ECS Fargate is Azure Container Apps: a +# serverless managed-Kubernetes-ish runtime that runs containers, has +# built-in HTTPS ingress with cert-managed-by-Azure, autoscaling, and +# native Azure AD / managed-identity support. The ACA infrastructure +# subnet must be /23 or larger; we already carved a /20 in network.tf. + +# ---------- Shared environment ---------- +# +# Observability: when var.otel_endpoint is set, OTel instrumentation +# turns on for both gateway and backend. Service names match the AWS +# stack so spans land in the right bucket. + +locals { + otel_enabled = var.otel_endpoint != "" + otel_environment_name = var.otel_environment_name != "" ? var.otel_environment_name : var.env + otel_shared_env_raw = local.otel_enabled ? [ + { name = "LITELLM_OTEL_V2", value = "true" }, + { name = "OTEL_EXPORTER", value = var.otel_exporter }, + { name = "OTEL_ENDPOINT", value = var.otel_endpoint }, + { name = "OTEL_ENVIRONMENT_NAME", value = local.otel_environment_name }, + { name = "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", value = var.otel_capture_message_content }, + ] : [] + gateway_otel_env = [ + for e in local.otel_shared_env_raw : e if !contains(keys(var.gateway_extra_env), e.name) + ] + backend_otel_env = [ + for e in local.otel_shared_env_raw : e if !contains(keys(var.backend_extra_env), e.name) + ] + + # Shared Postgres + Redis + Storage env, fed to gateway / backend / + # migrations. Matches the AWS module's `shared_env` shape for those + # values that map to Azure equivalents. + shared_pg_env = [ + { name = "DATABASE_HOST", value = azurerm_postgresql_flexible_server.this.fqdn }, + { name = "DATABASE_PORT", value = "5432" }, + { name = "DATABASE_USER", value = var.db_username }, + { name = "DATABASE_NAME", value = var.db_name }, + # Azure uses Entra-managed-identity token auth at runtime + # (azure-identity's DefaultAzureCredential). The proxy's + # init_azure_db_url_from_env equivalent mints a short-lived token and + # assembles DATABASE_URL when DATABASE_AZURE_AUTH=true is set. + { name = "DATABASE_AZURE_AUTH", value = "true" }, + ] + shared_redis_env = [ + { name = "REDIS_HOST", value = azurerm_redis_cache.this.hostname }, + { name = "REDIS_PORT", value = tostring(azurerm_redis_cache.this.ssl_port) }, + { name = "REDIS_SSL", value = "true" }, + ] + shared_storage_env = [ + # LiteLLM reads AZURE_STORAGE_ACCOUNT_NAME and the AZURE_BLOB_* + # container env vars. Auth is via the Container Apps managed identity + # (Storage Blob Data Contributor role, see iam.tf). + { name = "AZURE_STORAGE_ACCOUNT_NAME", value = azurerm_storage_account.this.name }, + { name = "AZURE_BLOB_STORAGE_ACCOUNT_NAME", value = azurerm_storage_account.this.name }, + { name = "AZURE_BLOB_STORAGE_CONTAINER_NAME", value = azurerm_storage_container.proxy.name }, + { name = "AZURE_FILE_STORAGE_CONTAINER_NAME", value = azurerm_storage_container.files.name }, + # CONFIG_FILE_PATH is read by the LiteLLM entrypoint to know where + # to download the uploaded proxy_config.yaml blob to. + { name = "CONFIG_FILE_PATH", value = "/tmp/litellm-config.yaml" }, + ] + + # Combine env plus extra_env (caller override) for each component. + gateway_base_env = concat( + local.shared_pg_env, + local.shared_redis_env, + local.shared_storage_env, + local.gateway_otel_env, + ) + backend_base_env = concat( + local.shared_pg_env, + local.shared_redis_env, + local.shared_storage_env, + local.backend_otel_env, + ) + migrations_base_env = concat( + local.shared_pg_env, + local.shared_redis_env, + ) + + # Reference secret IDs (Key Vault secret URIs) for env vars that come + # from Key Vault instead of plaintext. Each entry becomes a valueFrom + # in the Container App env block. + gateway_kv_secret_env = merge( + { + LITELLM_MASTER_KEY = azurerm_key_vault_secret.master_key.id + }, + var.litellm_license != "" ? { LITELLM_LICENSE = one(azurerm_key_vault_secret.license[*].id) } : {}, + var.gateway_extra_secrets, + ) + backend_kv_secret_env = merge( + { + LITELLM_MASTER_KEY = azurerm_key_vault_secret.master_key.id, + }, + var.ui_password != "" ? { UI_PASSWORD = one(azurerm_key_vault_secret.ui_password[*].id) } : {}, + var.litellm_license != "" ? { LITELLM_LICENSE = one(azurerm_key_vault_secret.license[*].id) } : {}, + var.backend_extra_secrets, + ) + migrations_kv_secret_env = { + LITELLM_MASTER_KEY = azurerm_key_vault_secret.master_key.id + } +} + +# ---------- Container Apps Environment ---------- + +resource "azurerm_container_app_environment" "this" { + name = "${local.name}-cae" + location = var.location + resource_group_name = local.resource_group_name + tags = local.tags + + infrastructure_subnet_id = azurerm_subnet.containers.id + + depends_on = [azurerm_role_assignment.container_apps_env_infra] +} + +# ---------- Log Analytics workspace ---------- +# +# Container Apps stdout/stderr route to Log Analytics. The default +# workspace created here is per-stack; callers that want to centralize +# logs can pass `log_analytics_workspace_id` (see variables) but that's +# out of scope for the baseline. + +resource "azurerm_log_analytics_workspace" "this" { + name = "${local.name}-logs" + location = var.location + resource_group_name = local.resource_group_name + sku = "PerGB2018" + retention_in_days = var.log_retention_days + tags = local.tags +} + +# ---------- gateway ---------- + +resource "azurerm_container_app" "gateway" { + name = "${local.name}-gateway" + container_app_environment_id = azurerm_container_app_environment.this.id + resource_group_name = local.resource_group_name + revision_mode = "Single" + tags = local.tags + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.container_apps.id] + } + + # The gateway is internal-only; the Application Gateway front ends it + # and fronts the UI; the backend is reachable from gateway + ui only. + ingress { + allow_insecure_connections = false + external_enabled = false + target_port = 4000 + transport = "auto" + + traffic_weight { + latest_revision = true + percentage = 100 + } + } + + registry { + server = var.image_registry + } + + template { + min_replicas = var.gateway_min_replicas + max_replicas = var.gateway_max_replicas + + container { + name = "gateway" + image = local.gateway_image_resolved + cpu = var.gateway_cpu + memory = var.gateway_memory + + dynamic "env" { + for_each = concat( + local.gateway_base_env, + [for k, v in var.gateway_extra_env : { name = k, value = v }], + ) + content { + name = env.value.name + value = env.value.value + } + } + + # Key Vault references resolved at startup using the managed identity. + dynamic "env" { + for_each = { for k, v in local.gateway_kv_secret_env : k => v } + content { + name = env.key + secret_value = env.value + } + } + } + } + + depends_on = [ + azurerm_key_vault_access_policy.container_apps, + ] +} + +# ---------- backend ---------- + +resource "azurerm_container_app" "backend" { + name = "${local.name}-backend" + container_app_environment_id = azurerm_container_app_environment.this.id + resource_group_name = local.resource_group_name + revision_mode = "Single" + tags = local.tags + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.container_apps.id] + } + + ingress { + allow_insecure_connections = false + external_enabled = false + target_port = 4001 + transport = "auto" + + traffic_weight { + latest_revision = true + percentage = 100 + } + } + + registry { + server = var.image_registry + } + + template { + min_replicas = var.backend_min_replicas + max_replicas = var.backend_max_replicas + + container { + name = "backend" + image = local.backend_image_resolved + cpu = var.backend_cpu + memory = var.backend_memory + + dynamic "env" { + for_each = concat( + local.backend_base_env, + [for k, v in var.backend_extra_env : { name = k, value = v }], + ) + content { + name = env.value.name + value = env.value.value + } + } + + dynamic "env" { + for_each = { for k, v in local.backend_kv_secret_env : k => v } + content { + name = env.key + secret_value = env.value + } + } + } + } +} + +# ---------- ui ---------- + +resource "azurerm_container_app" "ui" { + name = "${local.name}-ui" + container_app_environment_id = azurerm_container_app_environment.this.id + resource_group_name = local.resource_group_name + revision_mode = "Single" + tags = local.tags + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.container_apps.id] + } + + ingress { + allow_insecure_connections = false + external_enabled = false + target_port = 3000 + transport = "auto" + + traffic_weight { + latest_revision = true + percentage = 100 + } + } + + registry { + server = var.image_registry + } + + template { + min_replicas = var.ui_min_replicas + max_replicas = var.ui_max_replicas + + container { + name = "ui" + image = local.ui_image_resolved + cpu = var.ui_cpu + memory = var.ui_memory + + dynamic "env" { + for_each = var.ui_extra_env + content { + name = env.key + value = env.value + } + } + } + } +} diff --git a/terraform/litellm/azure/examples/default/.terraform.lock.hcl b/terraform/litellm/azure/examples/default/.terraform.lock.hcl new file mode 100644 index 00000000000..5695e7f0fef --- /dev/null +++ b/terraform/litellm/azure/examples/default/.terraform.lock.hcl @@ -0,0 +1,43 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.117.1" + constraints = "~> 3.117" + hashes = [ + "h1:j6wnjpHfBcQC4xd3ZYquaIPIIR46xJQs7rxwPdSOZos=", + "zh:0c513676836e3c50d004ece7d2624a8aff6faac14b833b96feeac2e4bc2c1c12", + "zh:50ea01ada95bae2f187db9e926e463f45d860767a85ebc59160414e00e76c35d", + "zh:52c2a9edacc06b3f72153f5ef6daca0761c6292158815961fe37f60bc576a3d7", + "zh:618eed2a06b19b1a025b45b05891846d570a6a1cca4d23f4942f5a99e1f747ae", + "zh:61cde5d3165d7e5ec311d5d89486819cd605c1b2d54611b5c97bd4e97dba2762", + "zh:6a873358d5031fc222f5e05f029d1237f3dce8345c767665f393283dfa2627f6", + "zh:afdd80064b2a04da311856feb4ed45f77ff4df6c356e8c2b10afb51fe7e61c70", + "zh:b09113df7e0e8c8959539bd22bae6c39faeb269ba3c4cd948e742f5cf58c35fb", + "zh:d340db7973109761cfc27d52aa02560363337c908b2c99b3628adc5a70a99d5b", + "zh:d5a577226ebc8c65e8f19384878a86acc4b51ede4b4a82d37c3b331b0efcd4a7", + "zh:e2962b147f9e71732df8dbc74940c10d20906f3c003cbfaa1eb9fabbf601a9f0", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.9.0" + constraints = "~> 3.6" + hashes = [ + "h1:OO+IuvQJSPmWdN8AyyIEvPJbLvDQpgX/zbktoa9KsJE=", + "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", + "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", + "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", + "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", + "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", + "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", + "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", + "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", + "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", + "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", + "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", + ] +} diff --git a/terraform/litellm/azure/examples/default/main.tf b/terraform/litellm/azure/examples/default/main.tf new file mode 100644 index 00000000000..010b3261bc4 --- /dev/null +++ b/terraform/litellm/azure/examples/default/main.tf @@ -0,0 +1,43 @@ +# One-command deploy of the LiteLLM Azure stack. +# +# cd terraform/litellm/azure/examples/default +# cp terraform.tfvars.example terraform.tfvars # edit it +# terraform init +# terraform apply +# +# This root just wires the provider (see providers.tf) to the module. The +# module itself (../../) declares no provider, so it can also be consumed +# from your own config with count/for_each/aliased providers: +# +# module "litellm" { +# source = "github.com/BerriAI/litellm//terraform/litellm/azure?ref=" +# ... +# } +# +# Knobs not surfaced as variables here (per-component sizing, autoscaling, +# Postgres / Redis / Storage tuning) can be set directly on this block: +# see ../../variables.tf. + +module "litellm" { + source = "../../" + + location = var.location + tenant = var.tenant + env = var.env + azs = var.azs + + litellm_master_key = var.litellm_master_key + litellm_license = var.litellm_license + ui_password = var.ui_password + + key_vault_certificate_id = var.key_vault_certificate_id + allow_plaintext_app_gateway = var.allow_plaintext_app_gateway + + storage_force_destroy = var.storage_force_destroy + + proxy_config = var.proxy_config + gateway_extra_env = {} + backend_extra_env = {} + gateway_extra_secrets = {} + backend_extra_secrets = {} +} diff --git a/terraform/litellm/azure/examples/default/providers.tf b/terraform/litellm/azure/examples/default/providers.tf new file mode 100644 index 00000000000..c9db019d6fd --- /dev/null +++ b/terraform/litellm/azure/examples/default/providers.tf @@ -0,0 +1,8 @@ +provider "azurerm" { + features {} + # Default: subscription_id is read from the environment (`ARM_SUBSCRIPTION_ID`). + # Explicit override: uncomment and set the subscription_id below. + # subscription_id = "00000000-0000-0000-0000-000000000000" +} + +provider "random" {} diff --git a/terraform/litellm/azure/examples/default/terraform.tfvars.example b/terraform/litellm/azure/examples/default/terraform.tfvars.example new file mode 100644 index 00000000000..82c05caf666 --- /dev/null +++ b/terraform/litellm/azure/examples/default/terraform.tfvars.example @@ -0,0 +1,42 @@ +location = "eastus" +tenant = "acme" +env = "stage" +azs = ["1", "2"] + +# Optional: pre-existing LiteLLM master key (must begin with `sk-`). Leave +# empty to have the stack auto-generate a random `sk-...` value and store +# it in Key Vault. +# litellm_master_key = "sk-..." + +# Optional: enterprise license (omit for OSS-only). +# litellm_license = "..." + +# Optional: UI admin password (omit to fall back to LITELLM_MASTER_KEY). +# ui_password = "..." + +# Optional: Key Vault certificate ID for the App Gateway HTTPS listener. +# Either set this OR set allow_plaintext_app_gateway = true (dev only). +# key_vault_certificate_id = "https://acmestagekv.vault.azure.net/certificates/litellm-cert/" +allow_plaintext_app_gateway = true + +# Optional: tell `terraform destroy` to remove a non-empty storage +# container (and any uploaded files). Off by default. +storage_force_destroy = false + +# Optional: enable proxy config with model list. Triggers a Container App +# revision swap when changed. +proxy_config = { + model_list = [ + { + model_name = "gpt-4o" + litellm_params = { + model = "openai/gpt-4o" + api_key = "os.environ/OPENAI_API_KEY" + } + }, + ] + general_settings = { + master_key = "os.environ/LITELLM_MASTER_KEY" + database_url = "os.environ/DATABASE_URL" + } +} diff --git a/terraform/litellm/azure/examples/default/variables.tf b/terraform/litellm/azure/examples/default/variables.tf new file mode 100644 index 00000000000..6f1193ca8c9 --- /dev/null +++ b/terraform/litellm/azure/examples/default/variables.tf @@ -0,0 +1,82 @@ +variable "location" { + description = "Azure region to deploy into." + type = string + default = "eastus" +} + +variable "tenant" { + description = "Tenant slug used as the prefix for every Azure resource the stack creates." + type = string + default = "acme" +} + +variable "env" { + description = "Environment suffix (e.g. `stage`, `prod`, `dev`)." + type = string + default = "stage" +} + +variable "azs" { + description = "Azure availability zone identifiers (e.g. [\"1\", \"2\"])." + type = list(string) + default = ["1", "2"] +} + +variable "litellm_master_key" { + description = "Pre-existing LiteLLM master key (must begin `sk-`). Leave empty to have the stack auto-generate." + type = string + default = "" + sensitive = true +} + +variable "litellm_license" { + description = "Optional LiteLLM enterprise license." + type = string + default = "" + sensitive = true +} + +variable "ui_password" { + description = "Optional UI admin password." + type = string + default = "" + sensitive = true +} + +variable "key_vault_certificate_id" { + description = "Resource ID of an existing Key Vault certificate for the App Gateway HTTPS listener. Leave empty to deploy HTTP-only (set `allow_plaintext_app_gateway = true` too)." + type = string + default = "" +} + +variable "allow_plaintext_app_gateway" { + description = "Allow the Application Gateway to serve HTTP without TLS. Dev / trial only." + type = bool + default = true +} + +variable "storage_force_destroy" { + description = "Allow `terraform destroy` to remove a non-empty storage container." + type = bool + default = false +} + +variable "proxy_config" { + description = "LiteLLM proxy config map (mirrors helm chart gateway.config.proxy_config)." + type = any + default = { + model_list = [ + { + model_name = "gpt-4o" + litellm_params = { + model = "openai/gpt-4o" + api_key = "os.environ/OPENAI_API_KEY" + } + }, + ] + general_settings = { + master_key = "os.environ/LITELLM_MASTER_KEY" + database_url = "os.environ/DATABASE_URL" + } + } +} diff --git a/terraform/litellm/azure/examples/default/versions.tf b/terraform/litellm/azure/examples/default/versions.tf new file mode 100644 index 00000000000..270becf9dfd --- /dev/null +++ b/terraform/litellm/azure/examples/default/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.6.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.117" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + } +} diff --git a/terraform/litellm/azure/iam.tf b/terraform/litellm/azure/iam.tf new file mode 100644 index 00000000000..e9f54f083b3 --- /dev/null +++ b/terraform/litellm/azure/iam.tf @@ -0,0 +1,55 @@ +# ---------- Managed identities ---------- +# +# One user-assigned managed identity shared across gateway / backend / +# ui / migrations Container Apps. Same identity is referenced as +# `azurerm_container_app.identity` -> user-assigned and granted the +# minimal set of role assignments below: +# - AcrPull on the Container Apps managed registry +# - Storage Blob Data on the Storage Account (cache / files) +# - Key Vault Secrets on the Key Vault (read master key + secrets) +# - Reader on its own resource group (private endpoint lookups) + +resource "azurerm_user_assigned_identity" "container_apps" { + name = "${local.name}-mi" + location = var.location + resource_group_name = local.resource_group_name + tags = local.tags +} + +# ---------- Role assignments ---------- +# +# Granted inside the same module so a fresh `terraform apply` ends with a +# fully-wired stack; nothing to add post-apply aside from `db_bootstrap_sql`. + +resource "azurerm_role_assignment" "storage_blob_data_contributor" { + scope = azurerm_storage_account.this.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_user_assigned_identity.container_apps.principal_id + principal_type = "ServicePrincipal" +} + +# Key Vault access is granted via access policies (data actions, scoped +# per-secret); see keyvault.tf. + +# ---------- Container Apps Environment infrastructure role ---------- +# +# The ACA environment requires a "managed identity" with the +# "Container Apps Environment Managed Identity" role on its own resource +# group (defined by Microsoft.App) so it can read ACR images and pull +# from the registry. We grant the user-assigned MI that role once. +# (For modular simplicity the ACA managed identity is the same identity +# we already created above.) + +resource "azurerm_role_assignment" "container_apps_env_infra" { + scope = local.resource_group_id + role_definition_name = "Container Apps Environment Managed Identity Contributor" + principal_id = azurerm_user_assigned_identity.container_apps.principal_id + principal_type = "ServicePrincipal" +} + +locals { + # Resolve the resource-group ID whether the caller supplied an existing + # RG (no resource here) or the module created one. Falls back to the + # resource's own id if a known format conversion is needed. + resource_group_id = var.resource_group_name == "" ? azurerm_resource_group.this[0].id : "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${var.resource_group_name}" +} diff --git a/terraform/litellm/azure/keyvault.tf b/terraform/litellm/azure/keyvault.tf new file mode 100644 index 00000000000..ff9c2f20810 --- /dev/null +++ b/terraform/litellm/azure/keyvault.tf @@ -0,0 +1,169 @@ +# Key Vault holds the proxy's secret material: +# - master-key secret (LITELLM_MASTER_KEY) +# - optional license (LITELLM_LICENSE) +# - optional UI password (UI_PASSWORD) +# - Postgres admin password (PG_ADMIN_PASSWORD) for the bootstrap step +# +# The Container Apps managed identity is granted per-secret access via +# access policies. Caller-supplied secrets (`gateway_extra_secrets`, +# `backend_extra_secrets`) are referenced by their secret IDs and not +# stored in this module; callers wire provider keys with their own +# `azurerm_key_vault_secret` resources (or supply them externally). + +resource "random_password" "db_admin_password" { + length = 32 + special = true + override_special = "@%*_+-:?#" +} + +resource "random_password" "litellm_master_key" { + length = 43 + special = false +} + +# Generate the master key with the `sk-` prefix the proxy expects. +locals { + litellm_master_key_value = var.litellm_master_key != "" ? var.litellm_master_key : "sk-${substr(random_password.litellm_master_key.result, 0, 43)}" +} + +resource "azurerm_key_vault" "this" { + name = replace("${var.tenant}${var.env}kv", "-", "") # no dashes allowed in KV name; truncated by Azure + location = var.location + resource_group_name = local.resource_group_name + tenant_id = data.azurerm_client_config.current.tenant_id + sku_name = "standard" + tags = local.tags + + # Networking: lock down to the VNet's private endpoint subnet. Caller's + # terraform principal needs to also be granted access to add secrets. + public_network_access_enabled = false + enable_rbac_authorization = false + enabled_for_deployment = false + enabled_for_disk_encryption = false + enabled_for_template_deployment = false + purge_protection_enabled = true + soft_delete_retention_days = 7 + + network_acls { + bypass = "AzureServices" + default_action = "Deny" + ip_rules = [] + virtual_network_subnet_ids = [ + azurerm_subnet.private_endpoints.id, + ] + } +} + +# ---------- Key Vault access policies ---------- +# +# Three identities get secret access: +# 1. The Container Apps managed identity (read-only on all secrets) +# 2. The current terraform principal (so `terraform apply` can add +# secrets during creation; matches the AWS module's bootstrap pattern) +# 3. Possibly AAD admins for break-glass (not configured by default) + +resource "azurerm_key_vault_access_policy" "container_apps" { + key_vault_id = azurerm_key_vault.this.id + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = azurerm_user_assigned_identity.container_apps.principal_id + + secret_permissions = [ + "Get", "List", + ] +} + +resource "azurerm_key_vault_access_policy" "terraform" { + key_vault_id = azurerm_key_vault.this.id + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azurerm_client_config.current.object_id + + secret_permissions = [ + "Get", "List", "Set", "Delete", "Purge", "Recover", + ] +} + +# ---------- Secrets ---------- +# +# The DB admin password lives in the vault and is consumed by the +# bootstrap job (which runs `psql` once to create the Entra token-auth +# app user; see `bootstrap.tf` / `iam.tf`). The proxy itself never sees +# it; gateway/backend/migrations use Entra tokens via the Container +# Apps managed identity. + +resource "azurerm_key_vault_secret" "db_admin_password" { + name = "db-admin-password" + value = random_password.db_admin_password.result + key_vault_id = azurerm_key_vault.this.id + + depends_on = [azurerm_key_vault_access_policy.terraform] +} + +resource "azurerm_key_vault_secret" "master_key" { + name = "litellm-master-key" + value = local.litellm_master_key_value + key_vault_id = azurerm_key_vault.this.id + tags = local.tags + + depends_on = [azurerm_key_vault_access_policy.terraform] +} + +resource "azurerm_key_vault_secret" "license" { + count = var.litellm_license != "" ? 1 : 0 + name = "litellm-license" + value = var.litellm_license + key_vault_id = azurerm_key_vault.this.id + tags = local.tags + + depends_on = [azurerm_key_vault_access_policy.terraform] +} + +resource "azurerm_key_vault_secret" "ui_password" { + count = var.ui_password != "" ? 1 : 0 + name = "ui-password" + value = var.ui_password + key_vault_id = azurerm_key_vault.this.id + tags = local.tags + + depends_on = [azurerm_key_vault_access_policy.terraform] +} + +# ---------- Private endpoint ---------- +# +# Pulls the vault onto a private IP in `private_endpoints` subnet so the +# Container Apps don't need public access to reach secrets. + +resource "azurerm_private_endpoint" "keyvault" { + name = "${local.name}-kv-pe" + location = var.location + resource_group_name = local.resource_group_name + subnet_id = azurerm_subnet.private_endpoints.id + tags = local.tags + + private_service_connection { + name = "${local.name}-kv" + private_connection_resource_id = azurerm_key_vault.this.id + is_manual_connection = false + subresource_names = ["vault"] + } + + private_dns_zone_group { + name = "default" + private_dns_zone_ids = [azurerm_private_dns_zone.keyvault.id] + } + + depends_on = [azurerm_private_dns_zone_virtual_network_link.keyvault] +} + +resource "azurerm_private_dns_zone" "keyvault" { + name = "privatelink.vaultcore.azure.net" + resource_group_name = local.resource_group_name + tags = local.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "keyvault" { + name = "${local.name}-kv-dnslink" + resource_group_name = local.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.keyvault.name + virtual_network_id = azurerm_virtual_network.this.id + registration_enabled = false +} diff --git a/terraform/litellm/azure/loadbalancer.tf b/terraform/litellm/azure/loadbalancer.tf new file mode 100644 index 00000000000..e929a6b337c --- /dev/null +++ b/terraform/litellm/azure/loadbalancer.tf @@ -0,0 +1,337 @@ +# Application Gateway: the Azure equivalent of AWS ALB. Public-facing; +# path-based routing that mirrors the AWS stack: +# - LLM data-plane paths (`/v1/*`, `/chat/*`, ...full list in +# locals.gateway_path_prefixes) -> gateway backend pool +# - UI asset paths (`/_next/*`, `/assets/*`, ...) -> ui backend pool +# - everything else (management API) -> backend backend pool +# +# TLS termination happens at the gateway when `key_vault_certificate_id` +# is supplied. Otherwise plaintext is only allowed when +# `allow_plaintext_app_gateway` is true. +# +# Container Apps ingress is internal-only (no external_enabled). The App +# Gateway reaches them via the Container Apps Environment default domain +# (e.g. `.azurecontainerapps.io`) on the public DNS. + +# ---------- Subnet for the Application Gateway ---------- + +resource "azurerm_subnet" "app_gateway" { + name = "app-gateway" + resource_group_name = local.resource_group_name + virtual_network_name = azurerm_virtual_network.this.name + address_prefixes = [cidrsubnet(var.vnet_cidr, 4, 2)] # /20 portion, isolated from container workloads + service_endpoints = ["Microsoft.Storage"] +} + +resource "azurerm_network_security_group" "app_gateway" { + name = "${local.name}-appgw-nsg" + location = var.location + resource_group_name = local.resource_group_name + tags = local.tags +} + +# App Gateway requires explicit public-inbound rules on ports 80/443. +resource "azurerm_network_security_rule" "app_gateway_http_in" { + count = var.allow_plaintext_app_gateway ? 1 : 0 + name = "http-in" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "80" + source_address_prefix = "*" + destination_address_prefix = "*" + resource_group_name = local.resource_group_name + network_security_group_name = azurerm_network_security_group.app_gateway.name +} + +resource "azurerm_network_security_rule" "app_gateway_https_in" { + name = "https-in" + priority = 110 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "*" + destination_address_prefix = "*" + resource_group_name = local.resource_group_name + network_security_group_name = azurerm_network_security_group.app_gateway.name +} + +resource "azurerm_subnet_network_security_group_association" "app_gateway" { + subnet_id = azurerm_subnet.app_gateway.id + network_security_group_id = azurerm_network_security_group.app_gateway.id +} + +# ---------- Public IP ---------- + +resource "azurerm_public_ip" "app_gateway" { + name = "${local.name}-appgw-pip" + location = var.location + resource_group_name = local.resource_group_name + sku = "Standard" + allocation_method = "Static" + tags = local.tags +} + +# ---------- Gateway FQDN lookup ---------- +# +# Container Apps with internal ingress expose a private FQDN (only +# resolvable from inside the VNet). We feed those FQDNs into the App +# Gateway backend pools. +locals { + gateway_fqdn = azurerm_container_app.gateway.ingress[0].fqdn + backend_fqdn = azurerm_container_app.backend.ingress[0].fqdn + ui_fqdn = azurerm_container_app.ui.ingress[0].fqdn +} + +# ---------- URL path map ---------- +# +# Application Gateway URL path map contains paths -> backend pools. We +# build one rule that combines gateway and ui prefixes into a single +# path rule (Application Gateway allows multiple paths per rule via the +# `paths` list). The default backend is `backend`. +# +# The redirect_listen_priority uses 100 for the path map and 200 for +# the default backend redirect rule. + +resource "azurerm_application_gateway" "this" { + name = "${local.name}-appgw" + location = var.location + resource_group_name = local.resource_group_name + tags = local.tags + + sku { + name = "Standard_v2" + tier = "Standard_v2" + capacity = 2 + } + + # WAF v2 is the standard tier's upgrade; this baseline keeps the SKU + # minimal. Callers needing WAF can swap the SKU without re-creating + # the gateway (a single azurerm update). + enable_http2 = true + + frontend_port { + name = "http" + port = 80 + } + frontend_port { + name = "https" + port = 443 + } + + # Gateway-level IP config: the subnet the App Gateway instance listens + # on (a dedicated subnet with `Microsoft.Network/applicationGateways` + # delegated). Required even when the listener uses public_ip_address_id. + gateway_ip_configuration { + name = "gateway" + subnet_id = azurerm_subnet.app_gateway.id + } + + # Frontend (listener-facing) IP config: the public IP that the world + # reaches the App Gateway at. Read via the `app_gateway_fqdn` output. + frontend_ip_configuration { + name = "public" + public_ip_address_id = azurerm_public_ip.app_gateway.id + } + + # ---------- Backend pools ---------- + backend_address_pool { + name = "gateway" + fqdns = [local.gateway_fqdn] + } + + backend_address_pool { + name = "backend" + fqdns = [local.backend_fqdn] + } + + backend_address_pool { + name = "ui" + fqdns = [local.ui_fqdn] + } + + # ---------- Backend HTTP settings ---------- + backend_http_settings { + name = "gateway-http" + cookie_based_affinity = "Disabled" + port = 443 + protocol = "Https" + request_timeout = 60 + probe_name = "gateway-probe" + + host_name = local.gateway_fqdn + } + + backend_http_settings { + name = "backend-http" + cookie_based_affinity = "Disabled" + port = 443 + protocol = "Https" + request_timeout = 60 + probe_name = "backend-probe" + + host_name = local.backend_fqdn + } + + backend_http_settings { + name = "ui-http" + cookie_based_affinity = "Disabled" + port = 443 + protocol = "Https" + request_timeout = 60 + probe_name = "ui-probe" + + host_name = local.ui_fqdn + } + + # ---------- Health probes ---------- + probe { + name = "gateway-probe" + host = local.gateway_fqdn + interval = 30 + timeout = 30 + unhealthy_threshold = 3 + + match { + status_code = ["200-399"] + } + + path = "/health/liveliness" + protocol = "Https" + } + + probe { + name = "backend-probe" + host = local.backend_fqdn + interval = 30 + timeout = 30 + unhealthy_threshold = 3 + + match { + status_code = ["200-399"] + } + + path = "/health/liveliness" + protocol = "Https" + } + + probe { + name = "ui-probe" + host = local.ui_fqdn + interval = 30 + timeout = 30 + unhealthy_threshold = 3 + + match { + status_code = ["200-399"] + } + + path = "/" + protocol = "Https" + } + + # ---------- Listeners ---------- + http_listener { + name = "http-listener" + frontend_ip_configuration_name = "public" + frontend_port_name = "http" + protocol = "Http" + host_name = null + } + + http_listener { + name = "https-listener" + frontend_ip_configuration_name = "public" + frontend_port_name = "https" + protocol = "Https" + ssl_certificate_name = local.tls_enabled ? "tls" : null + host_name = null + } + + # ---------- SSL certificate ---------- + ssl_certificate { + name = "tls" + key_vault_secret_id = var.key_vault_certificate_id + + # Only attach the SSL cert when TLS is enabled. + } + + # ---------- URL path map ---------- + # + # Single path rule with a default backend -> management API. The rule + # itself uses a path prefix matcher at "/" with all explicit paths in + # `local.gateway_path_prefixes` -> gateway backend and the UI paths in + # `local.ui_path_prefixes` -> ui backend. We leverage the + # "default_backend_address_pool" for the backend pool; explicit rules + # override the default. + + url_path_map { + name = "litellm-url-map" + default_backend_address_pool_name = "backend" + default_backend_http_settings_name = "backend-http" + + path_rule { + name = "gateway-prefixes" + paths = local.gateway_path_prefixes + backend_address_pool_name = "gateway" + backend_http_settings_name = "gateway-http" + } + + path_rule { + name = "ui-prefixes" + paths = concat(local.ui_path_prefixes, local.ui_exact_paths) + backend_address_pool_name = "ui" + backend_http_settings_name = "ui-http" + } + } + + # ---------- Request routing rules ---------- + # HTTPS listener routes through the URL path map. + request_routing_rule { + name = "https-routing" + rule_type = "PathBasedRouting" + http_listener_name = "https-listener" + url_path_map_name = "litellm-url-map" + priority = 100 + } + + # HTTP listener either routes through the URL path map (when plaintext + # is allowed) or redirects to HTTPS. + dynamic "request_routing_rule" { + for_each = var.allow_plaintext_app_gateway ? [1] : [] + content { + name = "http-routing" + rule_type = "PathBasedRouting" + http_listener_name = "http-listener" + url_path_map_name = "litellm-url-map" + priority = 110 + } + } + + dynamic "redirect_configuration" { + for_each = var.allow_plaintext_app_gateway ? [] : [1] + content { + name = "http-to-https" + redirect_type = "Permanent" + target_listener_name = "https-listener" + include_path = true + include_query_string = true + } + } + + # Redirect rule for the http listener (only when plaintext is denied). + dynamic "request_routing_rule" { + for_each = var.allow_plaintext_app_gateway ? [] : [1] + content { + name = "http-redirect" + rule_type = "Basic" + http_listener_name = "http-listener" + redirect_configuration_name = "http-to-https" + priority = 200 + } + } +} diff --git a/terraform/litellm/azure/locals.tf b/terraform/litellm/azure/locals.tf new file mode 100644 index 00000000000..3fc05938da0 --- /dev/null +++ b/terraform/litellm/azure/locals.tf @@ -0,0 +1,114 @@ +# Per-component path prefixes mirrored verbatim from the AWS module's +# gateway_path_prefixes / ui_path_prefixes blocks (and ultimately from +# gateway/routes/allowlist.py plus the helm ingress in +# helm/litellm/templates/ingress.yaml). Anything not in either list and not +# a UI asset path falls through to the backend (management API) on the +# Application Gateway URL path map. +# +# Application Gateway URL path map rules cap path-based conditions per rule +# differently per SKU; for the path-based backend pool strategy used here +# we keep a single combined list and emit one rule per prefix. + +locals { + # Every Azure resource the stack creates is named `-litellm-` + # (or that with a per-resource suffix). Computed once so the rest of the + # stack can reference `local.name`. + name = "${var.tenant}-litellm-${var.env}" + + # Module-level tagging. Caller-provided provider default_tags merge with + # these at apply time. + tags = merge( + { + "litellm:stack" = local.name + "managed-by" = "terraform" + }, + var.tags, + ) + + # Resource group that owns every resource the module creates. Caller may + # supply an existing one with var.resource_group_name; otherwise the + # module creates one named `-litellm--rg`. + resource_group_name = var.resource_group_name != "" ? var.resource_group_name : "${local.name}-rg" + + # Gateway data-plane path prefixes (mirrors AWS gateway_path_prefixes). + gateway_path_prefixes = [ + "/v1/chat/*", "/chat/*", + "/v1/completions*", "/completions*", + "/v1/embeddings*", "/embeddings*", + "/v1/moderations*", "/moderations*", + "/v1/audio/*", "/audio/*", + "/v1/images/*", "/images/*", + "/v1/files*", "/files*", + "/v1/batches*", "/batches*", + "/v1/fine_tuning/*", "/fine_tuning/*", + "/v1/fine-tuning/*", "/fine-tuning/*", + "/v1/responses*", "/responses*", + "/v1/threads*", "/threads*", + "/v1/assistants*", "/assistants*", + "/v1/vector_stores*", "/vector_stores*", + "/v1/indexes*", + "/v1/models*", "/models*", + "/openai/*", "/engines/*", + "/v1/messages*", "/messages*", + "/v1/skills/*", "/v1/a2a/*", + "/v1/rerank*", "/v2/rerank*", "/rerank*", + "/v1/ocr*", "/ocr*", + "/v1/rag/*", "/rag/*", + "/v1/video/*", "/v1/videos/*", "/video/*", "/videos/*", + "/v1/search*", "/search*", + "/v1/containers/*", "/containers/*", + "/v1/evals/*", + "/v1/memory/*", + "/queue/chat/*", + "/v1beta/*", + "/interactions/*", + "/anthropic/*", "/azure/*", "/azure_ai/*", "/aws/*", "/bedrock/*", + "/cohere/*", "/gemini/*", "/google/*", + "/vertex_ai/*", "/vertex-ai/*", + "/assemblyai/*", "/eu.assemblyai/*", + "/langfuse/*", "/vllm/*", + "/mistral/*", "/groq/*", "/voyage/*", "/cursor/*", "/milvus/*", + "/openai_passthrough/*", + "/toolset/*", + "/v1/realtime*", "/realtime*", + "/health*", "/metrics", "/test*", + ] + + # Static UI asset prefixes (handled by the ui Container App, not the + # backend catch-all). / and /favicon.ico are added as exact-match paths. + ui_path_prefixes = [ + "/litellm-asset-prefix/*", + "/_next/*", + "/assets/*", + "/ui/*", + ] + + ui_exact_paths = [ + "/", + "/favicon.ico", + "/ui", + ] + + # TLS is enabled when a Key Vault certificate ID is supplied AND + # plaintext is disallowed (the default). + tls_enabled = var.key_vault_certificate_id != "" && !var.allow_plaintext_app_gateway + + # Postgres admin setup: the first active-directory admin that provisions + # the Flexible Server. We use the current principal invoking terraform. + # (The `azure_ad_admin` block on the PG server accepts a specific object + # ID; callers needing a different admin can pass it through + # var.db_ad_admin_object_id.) + db_ad_admin_object_id_default = try(data.azurerm_client_config.current.object_id, "") + + # Image URI resolution (mirrors AWS / GCP defaults): + # //litellm-: + gateway_image_resolved = coalesce(var.gateway_image, "${var.image_registry}/${var.image_namespace}/litellm-gateway:${var.image_tag}") + backend_image_resolved = coalesce(var.backend_image, "${var.image_registry}/${var.image_namespace}/litellm-backend:${var.image_tag}") + ui_image_resolved = coalesce(var.ui_image, "${var.image_registry}/${var.image_namespace}/litellm-ui:${var.image_tag}") + migrations_image_resolved = coalesce(var.migrations_image, "${var.image_registry}/${var.image_namespace}/litellm-migrations:${var.image_tag}") + + # Proxy config as YAML, uploaded to the storage account blob + # `config/litellm-config.yaml`. The Container Apps download this on + # startup via azure-identity and set `CONFIG_FILE_PATH=/tmp/litellm-config.yaml`. + proxy_config_yaml = var.proxy_config != {} ? yamlencode(var.proxy_config) : "" +} diff --git a/terraform/litellm/azure/migrations.tf b/terraform/litellm/azure/migrations.tf new file mode 100644 index 00000000000..96af2c566b1 --- /dev/null +++ b/terraform/litellm/azure/migrations.tf @@ -0,0 +1,79 @@ +# Container Apps Job for the one-off prisma migrate deploy. The job runs +# once after `terraform apply` and before any traffic; output the run +# command via `terraform output migration_run_command` and execute it +# via `az containerapp job start`. +# +# Like the AWS migrations task definition, this container downloads the +# proxy_config.yaml blob via the managed identity at startup, runs the +# schema migration, then exits. The downstream gateway / backend services +# do not depend on its success at terraform-apply time because the +# Container Apps revision bootstrap happens lazily; instead we rely on +# the bootstrap pattern in the README (run after apply, before first +# traffic). + +resource "azurerm_container_app_job" "migrations" { + name = "${local.name}-migrations" + container_app_environment_id = azurerm_container_app_environment.this.id + resource_group_name = local.resource_group_name + location = var.location + tags = local.tags + + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.container_apps.id] + } + + # Manual trigger: one-shot job, runs to completion when invoked. + manual_trigger_config { + parallelism = 1 + replica_completion_count = 1 + } + + replica_timeout_in_seconds = 1800 + + registry { + server = var.image_registry + } + + # Secrets are declared at the job level and referenced from container + # env blocks via `secret_name`. Key Vault secret IDs are pulled at + # startup using the user-assigned managed identity declared on the job. + dynamic "secret" { + for_each = local.migrations_kv_secret_env + content { + name = secret.key + key_vault_secret_id = secret.value + identity = azurerm_user_assigned_identity.container_apps.id + } + } + + template { + container { + name = "migrations" + image = local.migrations_image_resolved + cpu = 1.0 + memory = "2Gi" + + dynamic "env" { + for_each = concat( + local.migrations_base_env, + [ + { name = "DATABASE_URL", value = "postgresql://${var.db_username}@${azurerm_postgresql_flexible_server.this.fqdn}:5432/${var.db_name}?sslmode=require" }, + ], + ) + content { + name = env.value.name + value = env.value.value + } + } + + dynamic "env" { + for_each = local.migrations_kv_secret_env + content { + name = env.key + secret_name = env.key + } + } + } + } +} diff --git a/terraform/litellm/azure/network.tf b/terraform/litellm/azure/network.tf new file mode 100644 index 00000000000..32f0e19df32 --- /dev/null +++ b/terraform/litellm/azure/network.tf @@ -0,0 +1,78 @@ +# Caller-supplied resource group (when `resource_group_name` is set) or a +# freshly-created one. All Azure resources the module creates are scoped +# to the resource group named `local.resource_group_name`. + +resource "azurerm_resource_group" "this" { + count = var.resource_group_name == "" ? 1 : 0 + name = local.resource_group_name + location = var.location + tags = local.tags +} + +data "azurerm_client_config" "current" {} + +# Caller's subscription + tenant for role assignments and Key Vault access +# policies. +data "azurerm_subscription" "current" {} + +# ---------- VNet ---------- +# +# Subnet allocation: +# - containers : /23 (Container Apps Environment infrastructure subnet; +# dynamic IP allocation; must be /23 or larger per +# Azure docs) +# - private_endpoints : /24 (private endpoints to Postgres / Redis / Storage +# / Key Vault) + +resource "azurerm_virtual_network" "this" { + name = "${local.name}-vnet" + location = var.location + resource_group_name = local.resource_group_name + address_space = [var.vnet_cidr] + tags = local.tags +} + +resource "azurerm_subnet" "containers" { + name = "containers" + resource_group_name = local.resource_group_name + virtual_network_name = azurerm_virtual_network.this.name + address_prefixes = [cidrsubnet(var.vnet_cidr, 4, 0)] # /20 portion, plenty of room for Container Apps dynamic IPs + service_endpoints = ["Microsoft.Storage"] + + delegation { + name = "container-apps" + + service_delegation { + name = "Microsoft.App/environments" + actions = [ + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action", + ] + } + } +} + +resource "azurerm_subnet" "private_endpoints" { + name = "private-endpoints" + resource_group_name = local.resource_group_name + virtual_network_name = azurerm_virtual_network.this.name + address_prefixes = [cidrsubnet(var.vnet_cidr, 4, 1)] # /20 portion, separate from containers + + service_endpoints = ["Microsoft.Storage"] +} + +# Network Security Group for the Container Apps subnet. Application +# Gateway sits in its own subnet (created below) and reaches the Container +# Apps via the internal ingress FQDN; the gateway subnet's NSG governs +# inbound traffic from the public internet. +resource "azurerm_network_security_group" "containers" { + name = "${local.name}-containers-nsg" + location = var.location + resource_group_name = local.resource_group_name + tags = local.tags +} + +resource "azurerm_subnet_network_security_group_association" "containers" { + subnet_id = azurerm_subnet.containers.id + network_security_group_id = azurerm_network_security_group.containers.id +} diff --git a/terraform/litellm/azure/outputs.tf b/terraform/litellm/azure/outputs.tf new file mode 100644 index 00000000000..a13a2886624 --- /dev/null +++ b/terraform/litellm/azure/outputs.tf @@ -0,0 +1,89 @@ +output "app_gateway_url" { + description = "Proxy URL. Scheme is https if key_vault_certificate_id is set, http otherwise. The dashboard is served at /, the API at /v1/*." + value = "${local.tls_enabled ? "https" : "http"}://${azurerm_public_ip.app_gateway.fqdn}" +} + +output "app_gateway_fqdn" { + description = "Public FQDN of the LiteLLM Application Gateway (derived from the public IP)." + value = azurerm_public_ip.app_gateway.fqdn +} + +output "container_apps_environment_id" { + description = "Resource ID of the Container Apps Environment hosting the gateway / backend / ui Container Apps." + value = azurerm_container_app_environment.this.id +} + +output "postgres_fqdn" { + description = "FQDN of the Azure Database for PostgreSQL Flexible Server. Used by gateway / backend / migrations as `DATABASE_HOST`." + value = azurerm_postgresql_flexible_server.this.fqdn +} + +output "postgres_database_name" { + description = "PostgreSQL database name." + value = var.db_name +} + +output "redis_hostname" { + description = "Hostname of the Azure Cache for Redis (use the `rediss://` scheme if var.redis_enable_ssl is true)." + value = azurerm_redis_cache.this.hostname +} + +output "redis_port" { + description = "Port of the Azure Cache for Redis." + value = azurerm_redis_cache.this.port +} + +output "storage_account_name" { + description = "Name of the Storage Account hosting the proxy blob container for cache backend, request log archival, and /v1/files storage. Exposed to gateway + backend as `AZURE_STORAGE_ACCOUNT_NAME` (LiteLLM reads this env var to assemble credentials)." + value = azurerm_storage_account.this.name +} + +output "storage_blob_container" { + description = "Blob container name on the Storage Account for proxy state." + value = azurerm_storage_container.proxy.name +} + +output "key_vault_uri" { + description = "URI of the Key Vault holding LITELLM_MASTER_KEY, the Aurora master password bootstrap, optional LITELLM_LICENSE, and optional UI_PASSWORD." + value = azurerm_key_vault.this.vault_uri +} + +output "master_key_secret_id" { + description = "Resource ID of the Key Vault secret holding LITELLM_MASTER_KEY." + value = azurerm_key_vault_secret.master_key.id +} + +output "managed_identity_client_id" { + description = "Client ID of the user-assigned managed identity assigned to the Container Apps. Use this when granting additional role assignments outside the module." + value = azurerm_user_assigned_identity.container_apps.client_id +} + +output "managed_identity_principal_id" { + description = "Object (principal) ID of the user-assigned managed identity. Use this for `az role assignment create` against additional scopes." + value = azurerm_user_assigned_identity.container_apps.principal_id +} + +# Pre-baked SQL to run once as the Entra admin (after the first apply) to +# create the application user that gateway / backend / migration will +# authenticate as. Azure Database for PostgreSQL Flexible Server uses +# `azure_ad_admin` for the AAD-enabled login. +output "db_bootstrap_sql" { + description = "Run this once as the AAD admin (after the first apply) to create the application user that will authenticate via Entra-managed-identity tokens at runtime." + value = <<-SQL + CREATE USER "${var.db_username}" WITH LOGIN; + GRANT ALL PRIVILEGES ON DATABASE ${var.db_name} TO "${var.db_username}"; + GRANT ALL ON SCHEMA public TO "${var.db_username}"; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO "${var.db_username}"; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO "${var.db_username}"; + SQL +} + +# Pre-baked az CLI command for the one-off migration Container App Job. +output "migration_run_command" { + description = "az CLI command that triggers the one-off prisma migration Container App Job. Run after the Entra admin has executed the db_bootstrap_sql above." + value = format( + "az containerapp job start --name %s --resource-group %s --subscription ", + azurerm_container_app_job.migrations.name, + local.resource_group_name, + ) +} diff --git a/terraform/litellm/azure/postgres.tf b/terraform/litellm/azure/postgres.tf new file mode 100644 index 00000000000..e2368a6ebc8 --- /dev/null +++ b/terraform/litellm/azure/postgres.tf @@ -0,0 +1,78 @@ +# Azure Database for PostgreSQL Flexible Server, single instance, with +# Entra (Azure AD) authentication enabled for the proxy's managed +# identity. (Single zone by default; set `zone = "1"` plus a high- +# availability SKU for zone redundancy.) +# +# The bootstrap job (run once during initial setup, see bootstrap.tf) +# creates the literal-login user used for password-based app access; +# the gateway / backend / migrations Container Apps use Entra tokens at +# runtime. + +resource "azurerm_postgresql_flexible_server" "this" { + name = replace("${var.tenant}-${var.env}-pg", "-", "") + location = var.location + resource_group_name = local.resource_group_name + sku_name = var.db_sku_name + storage_mb = var.db_storage_mb + version = var.db_version + tags = local.tags + + # The AAD admin: this object's principal can connect to the server and + # bootstrap the application user. Defaults to the current terraform + # principal; override var.db_ad_admin_object_id for a dedicated admin. + administrator_login = "litellm_admin" + administrator_password = random_password.db_admin_password.result + + authentication { + password_auth_enabled = true + active_directory_auth_enabled = true + } + + # Default to private access in the VNet; public access disabled. + public_network_access_enabled = false + delegated_subnet_id = azurerm_subnet.private_endpoints.id + private_dns_zone_id = azurerm_private_dns_zone.postgres.id + + # Zone redundancy is off by default to control cost; flip on via + # `zone = "1"` + matching SKU. + zone = length(var.azs) > 1 ? var.azs[0] : null + + depends_on = [ + azurerm_role_assignment.container_apps_env_infra, + azurerm_private_dns_zone_virtual_network_link.postgres, + ] +} + +resource "azurerm_postgresql_flexible_server_database" "this" { + server_id = azurerm_postgresql_flexible_server.this.id + name = var.db_name + collation = "en_US.utf8" + charset = "UTF8" +} + +# Set the AAD admin via a separate resource. The first principal in the +# list wins; we use the current terraform principal by default. +resource "azurerm_postgresql_flexible_server_active_directory_administrator" "this" { + server_name = azurerm_postgresql_flexible_server.this.name + resource_group_name = local.resource_group_name + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azurerm_client_config.current.object_id + principal_name = "terraform-admin" + principal_type = "User" + + depends_on = [azurerm_postgresql_flexible_server.this] +} + +resource "azurerm_private_dns_zone" "postgres" { + name = "privatelink.postgres.database.azure.com" + resource_group_name = local.resource_group_name + tags = local.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "postgres" { + name = "${local.name}-pg-dnslink" + resource_group_name = local.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.postgres.name + virtual_network_id = azurerm_virtual_network.this.id + registration_enabled = false +} diff --git a/terraform/litellm/azure/redis.tf b/terraform/litellm/azure/redis.tf new file mode 100644 index 00000000000..0f30dcc73ed --- /dev/null +++ b/terraform/litellm/azure/redis.tf @@ -0,0 +1,52 @@ +# Azure Cache for Redis: single instance (Basic tier) by default. +# Production should move to Standard / Premium with replication. +# +# The proxy connects via `rediss://` (TLS) when `var.redis_enable_ssl` +# is true (the default, matching AWS module's transit_encryption_enabled). + +resource "azurerm_redis_cache" "this" { + name = replace("${var.tenant}-${var.env}-redis", "-", "") + location = var.location + resource_group_name = local.resource_group_name + sku_name = var.redis_sku + family = var.redis_family + capacity = var.redis_capacity + enable_non_ssl_port = !var.redis_enable_ssl + minimum_tls_version = "1.2" + tags = local.tags + + # Redis access keys are managed by Azure; the proxy uses the `rediss://` + # URL with the primary key as auth. Caller can rotate via Key Vault + # access policy if tighter secret-scoped handling is required. + redis_configuration { + # Renamed to authentication_enabled in v4. Until we drop 3.x support, + # both forms exist; this module targets 3.117+ which still accepts + # enable_authentication. + enable_authentication = true + maxmemory_policy = "allkeys-lru" + } + + # Access via private endpoint only. + public_network_access_enabled = false + subnet_id = azurerm_subnet.private_endpoints.id +} + +resource "azurerm_private_dns_zone" "redis" { + name = "privatelink.redis.cache.windows.net" + resource_group_name = local.resource_group_name + tags = local.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "redis" { + name = "${local.name}-redis-dnslink" + resource_group_name = local.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.redis.name + virtual_network_id = azurerm_virtual_network.this.id + registration_enabled = false +} + +# Outputs that the gateway/backend pass via `REDIS_HOST` / `REDIS_PORT`. +locals { + redis_host_for_container_apps = azurerm_redis_cache.this.hostname + redis_port_for_container_apps = azurerm_redis_cache.this.ssl_port +} diff --git a/terraform/litellm/azure/storage.tf b/terraform/litellm/azure/storage.tf new file mode 100644 index 00000000000..1c04e7cac8b --- /dev/null +++ b/terraform/litellm/azure/storage.tf @@ -0,0 +1,124 @@ +# Storage Account: holds: +# - proxy_config.yaml (uploaded from var.proxy_config; refreshed on +# proxy_config changes to trigger Container App revision swap) +# - /v1/files passthrough storage (request log archival, file uploads) +# - log archive bucket (optional, future) +# +# LiteLLM reads `AZURE_STORAGE_ACCOUNT_NAME` and uses the Container Apps +# managed identity's Storage Blob Data Contributor role on this account +# to do CRUD with the SDK. No shared keys are issued. + +resource "azurerm_storage_account" "this" { + name = replace("${var.tenant}${var.env}proxy", "-", "") # 24 char cap; lowercase + no dashes + location = var.location + resource_group_name = local.resource_group_name + account_tier = var.storage_account_tier + account_replication_type = var.storage_replication_type + account_kind = "StorageV2" + tags = local.tags + + # Security defaults: TLS 1.2 minimum, blob public access blocked, shared + # keys off (managed identity only). + min_tls_version = "TLS1_2" + allow_nested_items_to_be_public = false + shared_access_key_enabled = false + public_network_access_enabled = false + + blob_properties { + versioning_enabled = true + + container_delete_retention_policy { + days = 7 + } + + delete_retention_policy { + days = 7 + } + } + + network_rules { + default_action = "Deny" + bypass = ["AzureServices"] + ip_rules = [] + virtual_network_subnet_ids = [ + azurerm_subnet.private_endpoints.id, + azurerm_subnet.containers.id, + ] + } +} + +# Container that holds the proxy config + /v1/files uploads. +resource "azurerm_storage_container" "proxy" { + name = "proxy" + storage_account_name = azurerm_storage_account.this.name + container_access_type = "private" +} + +# Files subcontainer: dedicated for the /v1/files endpoint so callers can +# isolate the data plane from the proxy_config blob. +resource "azurerm_storage_container" "files" { + name = "files" + storage_account_name = azurerm_storage_account.this.name + container_access_type = "private" +} + +# Private endpoint for the Storage Account (blob sub-resource). +resource "azurerm_private_endpoint" "storage_blob" { + name = "${local.name}-sa-pe" + location = var.location + resource_group_name = local.resource_group_name + subnet_id = azurerm_subnet.private_endpoints.id + tags = local.tags + + private_service_connection { + name = "${local.name}-sa-blob" + private_connection_resource_id = azurerm_storage_account.this.id + is_manual_connection = false + subresource_names = ["blob"] + } + + private_dns_zone_group { + name = "default" + private_dns_zone_ids = [azurerm_private_dns_zone.blob.id] + } + + depends_on = [azurerm_private_dns_zone_virtual_network_link.blob] +} + +resource "azurerm_private_dns_zone" "blob" { + name = "privatelink.blob.core.windows.net" + resource_group_name = local.resource_group_name + tags = local.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "blob" { + name = "${local.name}-sa-dnslink" + resource_group_name = local.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.blob.name + virtual_network_id = azurerm_virtual_network.this.id + registration_enabled = false +} + +# ---------- proxy_config blob upload ---------- +# +# Encoded from `var.proxy_config` (a typed map). Each apply that changes +# proxy_config produces a new blob content, which the gateway and +# backend Container Apps container start uses as a versioning trigger. + +resource "azurerm_storage_blob" "proxy_config" { + count = var.proxy_config != {} ? 1 : 0 + name = "config/litellm-config.yaml" + storage_account_name = azurerm_storage_account.this.name + storage_container_name = azurerm_storage_container.proxy.name + type = "Block" + content_type = "application/yaml" + + # Force a fresh upload on content change. The lifecycle keeps the + # previous version accessible via the versioned blob endpoint. + source_content = local.proxy_config_yaml + + depends_on = [ + azurerm_role_assignment.storage_blob_data_contributor, + azurerm_storage_container.proxy, + ] +} diff --git a/terraform/litellm/azure/variables.tf b/terraform/litellm/azure/variables.tf new file mode 100644 index 00000000000..12b5caf46ad --- /dev/null +++ b/terraform/litellm/azure/variables.tf @@ -0,0 +1,400 @@ +variable "location" { + description = "Azure region to deploy into (e.g. `eastus`, `westus2`)." + type = string +} + +variable "tenant" { + description = "Tenant slug, used as the prefix for every Azure resource the stack creates. Combined with var.env to form `-litellm-` (e.g. `acme-litellm-stage`)." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9-]{0,20}$", var.tenant)) + error_message = "tenant must be 1-21 chars, lower-kebab-case, starting with a letter." + } +} + +variable "env" { + description = "Environment suffix appended to every resource name (e.g. `stage`, `prod`, `dev`)." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9-]{0,8}$", var.env)) + error_message = "env must be 1-9 chars, lower-kebab-case, starting a letter." + } +} + +variable "tags" { + description = "Per-deployment tags applied to every taggable resource the module creates, on top of the module's own `litellm:stack` / `managed-by` tags. Caller-level provider default_tags (if any) merge with these." + type = map(string) + default = {} +} + +variable "resource_group_name" { + description = "Existing resource group to deploy into. Leave empty to have the module create one named `-litellm--rg`." + type = string + default = "" +} + +# ---------- Tenant-supplied secrets ---------- +# +# Both default to "" so the stack stays usable for trial / OSS deploys. +# Set via TF_VAR_litellm_master_key / TF_VAR_litellm_license to keep the +# values out of state files committed to a VCS. + +variable "litellm_master_key" { + description = <<-EOT + Pre-existing LITELLM_MASTER_KEY (must begin with `sk-`). When set, this + value is written to the master-key Key Vault entry. When empty, the + stack auto-generates a random `sk-...` key (preserving today's + trial-deploy behavior). + EOT + type = string + default = "" + sensitive = true +} + +variable "litellm_license" { + description = <<-EOT + LiteLLM enterprise license string. When set, the stack creates a + `-litellm--license` Key Vault secret, grants the + Container Apps managed identity read access, and exposes its value to + gateway + backend as `LITELLM_LICENSE`. Leave empty for OSS-only deploys. + EOT + type = string + default = "" + sensitive = true +} + +variable "ui_password" { + description = <<-EOT + UI admin password. When set, the stack creates a + `-litellm--ui-password` Key Vault secret, grants the + Container Apps managed identity read access, and exposes its value to + the backend as `UI_PASSWORD`. Pair with `backend_extra_env.UI_USERNAME` + to set the matching username. Leave empty to skip; the proxy then + falls back to the LITELLM_MASTER_KEY for UI login. + EOT + type = string + default = "" + sensitive = true +} + +# ---------- Networking ---------- + +variable "vnet_cidr" { + description = "CIDR block for the VNet." + type = string + default = "10.42.0.0/16" +} + +variable "azs" { + description = "Availability zones to spread subnets across. Use Azure zone identifiers (e.g. `[\"1\", \"2\", \"3\"]`). At least 1 required; 2+ recommended for HA PostgreSQL Flexible Server." + type = list(string) + + validation { + condition = length(var.azs) >= 1 + error_message = "Provide at least 1 availability zone." + } +} + +# ---------- Component images ---------- +# +# Defaults pin the four componentized images at the same release tag on +# GHCR. Override on a per-component basis in tfvars when bumping; bump them +# together when bumping the LiteLLM release. +# +# Container Apps pulls `ghcr.io/berriai/litellm-gateway:` by default; +# this works as-is in Azure because Container Apps can pull from ghcr.io +# directly (the GCP stack requires an Artifact Registry mirror, AWS pulls +# from ECR only after a docker push, Azure is the closest to a direct +# pull-from-public-registry experience). + +variable "image_registry" { + description = "Container registry host (without trailing slash). Defaults to GHCR; override when mirroring to ACR or another private registry." + type = string + default = "ghcr.io" +} + +variable "image_namespace" { + description = "Container registry namespace / owner. Defaults to `berriai` on GHCR." + type = string + default = "berriai" +} + +variable "image_tag" { + description = "Image tag (e.g. `v1.86.0-dev`). All four component images use the same tag; bump together when bumping LiteLLM." + type = string + default = "latest" +} + +variable "gateway_image" { + description = "Container image for the gateway (data plane, port 4000). Defaults to `//litellm-gateway:`." + type = string + default = "" +} + +variable "backend_image" { + description = "Container image for the backend (management API, port 4001). Defaults to `//litellm-backend:`." + type = string + default = "" +} + +variable "ui_image" { + description = "Container image for the UI (port 3000). Defaults to `//litellm-ui:`." + type = string + default = "" +} + +variable "migrations_image" { + description = "Container image for the one-off migration job. Defaults to `//litellm-migrations:`." + type = string + default = "" +} + +# ---------- Database ---------- + +variable "db_sku_name" { + description = "Azure Database for PostgreSQL Flexible Server SKU (e.g. `Standard_B1ms`, `Standard_D2s_v3`, `GP_Standard_D2s_v3`)." + type = string + default = "Standard_B1ms" +} + +variable "db_version" { + description = "PostgreSQL major version (e.g. `15`, `16`, `17`)." + type = string + default = "16" +} + +variable "db_storage_mb" { + description = "Storage size in MB for the PostgreSQL Flexible Server." + type = number + default = 32768 +} + +variable "db_name" { + description = "Database name to create on the PostgreSQL Flexible Server." + type = string + default = "litellm" +} + +variable "db_username" { + description = "Application DB username. The Container Apps managed identity uses Entra (Azure AD) token auth at runtime; this user is created during `terraform apply` by the bootstrap job and granted the rights the proxy needs." + type = string + default = "litellm_app" +} + +# ---------- Redis ---------- + +variable "redis_sku" { + description = "Azure Cache for Redis SKU. Defaults to `Basic` (single node, dev/trial); `Standard` or `Premium` recommended for production (zone redundancy + replication)." + type = string + default = "Basic" +} + +variable "redis_family" { + description = "Redis SKU family. `C` is basic; `P` is premium." + type = string + default = "C" +} + +variable "redis_capacity" { + description = "Cache capacity for Azure Cache for Redis (0 = 250MB, 1 = 1GB, ...). Match to your expected throughput." + type = number + default = 0 +} + +variable "redis_enable_ssl" { + description = "Whether to require SSL for Redis connections. Defaults to true to match AWS module's `transit_encryption_enabled`." + type = bool + default = true +} + +# ---------- Storage ---------- + +variable "storage_force_destroy" { + description = "Allow `terraform destroy` to delete a non-empty Azure Storage container (and the Storage Account). Off by default to protect cached responses, archived request logs, and /v1/files storage." + type = bool + default = false +} + +variable "storage_account_tier" { + description = "Storage Account performance tier. `Standard` is fine for cache / log archive / file storage; `Premium` for low-latency workloads." + type = string + default = "Standard" +} + +variable "storage_replication_type" { + description = "Storage Account replication type (`LRS`, `GRS`, `RAGRS`, `ZRS`). Defaults to LRS for cost; bump to GRS / ZRS for production durability." + type = string + default = "LRS" +} + +# ---------- TLS / Load balancer ---------- + +variable "key_vault_certificate_id" { + description = <<-EOT + Resource ID of an existing Key Vault certificate to attach to the + Application Gateway listener for TLS termination. When unset, the + Application Gateway listener uses HTTP only; either set this var OR + set `allow_plaintext_app_gateway = true` for dev/trial only. + EOT + type = string + default = "" +} + +variable "allow_plaintext_app_gateway" { + description = "Allow the Application Gateway listener to serve HTTP (port 80) without TLS. Dev / trial only. Defaults to false to match the AWS module's secure default." + type = bool + default = false +} + +# ---------- Compute sizing ---------- + +variable "gateway_cpu" { + description = "CPU allocation for the gateway Container App in 0.25 vCPU increments (e.g. `0.5`, `1.0`, `2.0`)." + type = number + default = 1.0 +} + +variable "gateway_memory" { + description = "Memory for the gateway Container App, e.g. `2Gi`, `4Gi`." + type = string + default = "2Gi" +} + +variable "gateway_min_replicas" { + description = "Minimum replicas for the gateway Container App (autoscaler lower bound)." + type = number + default = 2 +} + +variable "gateway_max_replicas" { + description = "Maximum replicas for the gateway Container App (autoscaler upper bound)." + type = number + default = 10 +} + +variable "backend_cpu" { + description = "CPU allocation for the backend Container App." + type = number + default = 1.0 +} + +variable "backend_memory" { + description = "Memory for the backend Container App." + type = string + default = "2Gi" +} + +variable "backend_min_replicas" { + description = "Minimum replicas for the backend Container App." + type = number + default = 1 +} + +variable "backend_max_replicas" { + description = "Maximum replicas for the backend Container App." + type = number + default = 5 +} + +variable "ui_cpu" { + description = "CPU allocation for the UI Container App." + type = number + default = 0.5 +} + +variable "ui_memory" { + description = "Memory for the UI Container App." + type = string + default = "1Gi" +} + +variable "ui_min_replicas" { + description = "Minimum replicas for the UI Container App." + type = number + default = 1 +} + +variable "ui_max_replicas" { + description = "Maximum replicas for the UI Container App." + type = number + default = 3 +} + +# ---------- proxy_config and extra config ---------- + +variable "proxy_config" { + description = <<-EOT + Mirrors the helm chart's `gateway.config.proxy_config`. The map is + YAML-encoded and uploaded to the storage account blob `config/litellm-config.yaml`; + the gateway and backend Container Apps download it to + `/tmp/litellm-config.yaml` at startup via the SDK and set + `CONFIG_FILE_PATH` to match. Editing this value produces a new + Container Apps revision and rolling redeploy of both services. + EOT + type = any + default = {} +} + +variable "gateway_extra_env" { + description = "Extra non-sensitive env vars (plaintext) merged into the gateway Container App env. Useful for feature flags / observability hosts. Provider API keys belong in gateway_extra_secrets." + type = map(string) + default = {} +} + +variable "backend_extra_env" { + description = "Extra non-sensitive env vars merged into the backend Container App env." + type = map(string) + default = {} +} + +variable "gateway_extra_secrets" { + description = "Map of Key Vault secret references to mount into the gateway Container App as env vars, e.g. `{ OPENAI_API_KEY = azurerm_key_vault_secret.openai.id }`. The Container Apps managed identity must have Secret User role on the vault." + type = map(string) + default = {} +} + +variable "backend_extra_secrets" { + description = "Map of Key Vault secret references to mount into the backend Container App." + type = map(string) + default = {} +} + +variable "ui_extra_env" { + description = "Extra non-sensitive env vars merged into the ui Container App env (frontend feature flags, etc.)." + type = map(string) + default = {} +} + +# ---------- Observability ---------- + +variable "otel_endpoint" { + description = "OTLP endpoint (e.g. `https://otel.example.com:4317`). When empty, OpenTelemetry instrumentation is disabled (matches the AWS module)." + type = string + default = "" +} + +variable "otel_exporter" { + description = "OTEL exporter protocol. `otlp` or `otlp_http`." + type = string + default = "otlp" +} + +variable "otel_environment_name" { + description = "OTEL environment tag (typically the deployment environment: `prod`, `stage`, ...). Defaults to var.env." + type = string + default = "" +} + +variable "otel_capture_message_content" { + description = "Whether to capture message content in spans (PII consideration)." + type = string + default = "false" +} + +variable "log_retention_days" { + description = "Log Analytics workspace retention in days (applies to application logs routed through the workspace)." + type = number + default = 30 +} diff --git a/terraform/litellm/azure/versions.tf b/terraform/litellm/azure/versions.tf new file mode 100644 index 00000000000..270becf9dfd --- /dev/null +++ b/terraform/litellm/azure/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.6.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.117" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + } +} From 5a6812387b57e16fde79c41877a8df05567d4d51 Mon Sep 17 00:00:00 2001 From: sanjibani <18418553+sanjibani@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:49:30 +0530 Subject: [PATCH 2/2] test: empty commit to verify write access