diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 7d4ef0a14fb..40a9da66c70 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -158,6 +158,38 @@ AgentOps) live under `proxy_config.litellm_settings.callbacks` and are orthogonal to the OTLP variables above; their credentials still go in `*_extra_secrets`. +### Enterprise billing metrics + +License-gated request metering is opt-in and gated entirely on +`billing_metrics_endpoint`. Empty (default) and no billing env is added to +the container, so existing deployments are unchanged. Set it and both +gateway and backend export billable-request counts over OTLP/HTTP, +authenticating to the collector with the mTLS client certificate issued for +your deployment. + +The proxy accepts the certificate, key, and CA bundle as either a file path +or literal PEM content. This stack takes the PEM, writes each one to its own +Secrets Manager entry, grants the task-execution role +`secretsmanager:GetSecretValue` on them, and injects them as +`LITELLM_BILLING_METRICS_CLIENT_CERT` / `_CLIENT_KEY` (and `_CA_CERT` when +set), so no volume mount is needed on Fargate. + +```hcl +billing_metrics_endpoint = "https://telemetry.litellm.ai/v1/metrics" +``` + +```bash +export TF_VAR_billing_metrics_client_cert_pem="$(cat client.crt)" +export TF_VAR_billing_metrics_client_key_pem="$(cat client.key)" +``` + +`billing_metrics_ca_cert_pem` is only for private or test collectors whose +CA is not in the system trust store; leave it empty against +`telemetry.litellm.ai`. Metering requires an enterprise license, so pair +this with `litellm_license`. To tune the export cadence, set +`LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` / +`backend_extra_env` + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 54ab80de9f4..bf0487c0a93 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -76,6 +76,33 @@ locals { { name = "OTEL_HEADERS", valueFrom = var.otel_headers_secret_arn }, ] : [] + # Enterprise request metering, gated on billing_metrics_endpoint. The + # endpoint rides in as a plain env var; the mTLS material is stored in + # Secrets Manager (secrets.tf) and injected as PEM-valued env vars, which + # the proxy accepts in place of file paths. Each PEM is wired only when the + # operator supplied it, so an empty ca_cert_pem falls back to the system + # trust store. + billing_metrics_enabled = var.billing_metrics_endpoint != "" + billing_metrics_client_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_client_cert_pem != "" + billing_metrics_client_key_enabled = local.billing_metrics_enabled && var.billing_metrics_client_key_pem != "" + billing_metrics_ca_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_ca_cert_pem != "" + + billing_metrics_env = local.billing_metrics_enabled ? [ + { name = "LITELLM_BILLING_METRICS_ENDPOINT", value = var.billing_metrics_endpoint }, + ] : [] + + billing_metrics_secrets = concat( + local.billing_metrics_client_cert_enabled ? [ + { name = "LITELLM_BILLING_METRICS_CLIENT_CERT", valueFrom = aws_secretsmanager_secret.billing_metrics_client_cert[0].arn }, + ] : [], + local.billing_metrics_client_key_enabled ? [ + { name = "LITELLM_BILLING_METRICS_CLIENT_KEY", valueFrom = aws_secretsmanager_secret.billing_metrics_client_key[0].arn }, + ] : [], + local.billing_metrics_ca_cert_enabled ? [ + { name = "LITELLM_BILLING_METRICS_CA_CERT", valueFrom = aws_secretsmanager_secret.billing_metrics_ca_cert[0].arn }, + ] : [], + ) + shared_env = [ { name = "IAM_TOKEN_DB_AUTH", value = "true" }, { name = "DATABASE_HOST", value = aws_rds_cluster.this.endpoint }, @@ -108,6 +135,7 @@ locals { { name = "LITELLM_LICENSE", valueFrom = aws_secretsmanager_secret.license[0].arn }, ], local.otel_secrets, + local.billing_metrics_secrets, ) # Backend-only managed secrets. UI_PASSWORD is consumed by the management @@ -198,6 +226,7 @@ resource "aws_ecs_task_definition" "gateway" { environment = concat( local.shared_env, local.gateway_otel_env, + local.billing_metrics_env, local.gateway_extra_env_list, local.proxy_config_env, ) @@ -284,6 +313,7 @@ resource "aws_ecs_task_definition" "backend" { local.shared_env, local.backend_default_env, local.backend_otel_env, + local.billing_metrics_env, local.backend_extra_env_list, local.proxy_config_env, ) diff --git a/terraform/litellm/aws/iam.tf b/terraform/litellm/aws/iam.tf index 64e1b1ad5f9..63c6c26f184 100644 --- a/terraform/litellm/aws/iam.tf +++ b/terraform/litellm/aws/iam.tf @@ -53,6 +53,9 @@ data "aws_iam_policy_document" "secrets_access" { [aws_secretsmanager_secret.master_key.arn], aws_secretsmanager_secret.license[*].arn, aws_secretsmanager_secret.ui_password[*].arn, + aws_secretsmanager_secret.billing_metrics_client_cert[*].arn, + aws_secretsmanager_secret.billing_metrics_client_key[*].arn, + aws_secretsmanager_secret.billing_metrics_ca_cert[*].arn, local.extra_secret_arns, var.otel_headers_secret_arn == "" ? [] : [var.otel_headers_secret_arn], ) diff --git a/terraform/litellm/aws/secrets.tf b/terraform/litellm/aws/secrets.tf index 300d38e4053..85d3eb4502c 100644 --- a/terraform/litellm/aws/secrets.tf +++ b/terraform/litellm/aws/secrets.tf @@ -74,6 +74,61 @@ resource "aws_secretsmanager_secret_version" "ui_password" { secret_string = var.ui_password } +# Billing-metrics mTLS material — only created when metering is enabled +# (billing_metrics_endpoint non-empty) and the operator supplied the PEM. +# The task-execution role gets GetSecretValue via iam.tf, and gateway + +# backend pick the env vars up through shared_secrets in ecs.tf. +resource "aws_secretsmanager_secret" "billing_metrics_client_cert" { + count = local.billing_metrics_client_cert_enabled ? 1 : 0 + + name = "${local.name}-billing-metrics-client-cert" + description = "LITELLM_BILLING_METRICS_CLIENT_CERT for gateway + backend." + recovery_window_in_days = 0 + + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "billing_metrics_client_cert" { + count = local.billing_metrics_client_cert_enabled ? 1 : 0 + + secret_id = aws_secretsmanager_secret.billing_metrics_client_cert[0].id + secret_string = var.billing_metrics_client_cert_pem +} + +resource "aws_secretsmanager_secret" "billing_metrics_client_key" { + count = local.billing_metrics_client_key_enabled ? 1 : 0 + + name = "${local.name}-billing-metrics-client-key" + description = "LITELLM_BILLING_METRICS_CLIENT_KEY for gateway + backend." + recovery_window_in_days = 0 + + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "billing_metrics_client_key" { + count = local.billing_metrics_client_key_enabled ? 1 : 0 + + secret_id = aws_secretsmanager_secret.billing_metrics_client_key[0].id + secret_string = var.billing_metrics_client_key_pem +} + +resource "aws_secretsmanager_secret" "billing_metrics_ca_cert" { + count = local.billing_metrics_ca_cert_enabled ? 1 : 0 + + name = "${local.name}-billing-metrics-ca-cert" + description = "LITELLM_BILLING_METRICS_CA_CERT for gateway + backend." + recovery_window_in_days = 0 + + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "billing_metrics_ca_cert" { + count = local.billing_metrics_ca_cert_enabled ? 1 : 0 + + secret_id = aws_secretsmanager_secret.billing_metrics_ca_cert[0].id + secret_string = var.billing_metrics_ca_cert_pem +} + resource "aws_secretsmanager_secret" "db_master_password" { name = "${local.name}-db-master-password" description = "Aurora master-user password - bootstrap only. Runtime auth is IAM-token." diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index 8db4935664b..c2ed1db14b1 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -533,3 +533,65 @@ variable "otel_headers_secret_arn" { type = string default = "" } + +# ---------- Enterprise billing metrics ---------- +# +# License-gated request metering. Opt-in and gated entirely on +# billing_metrics_endpoint: leave it empty (the default) and nothing +# metering-related lands in the container env. Set it and gateway + backend +# export billable-request counts over OTLP/HTTP, authenticating to the +# collector with an mTLS client cert. The proxy accepts the cert, key, and CA +# as either a file path or literal PEM content, so on Fargate they are +# injected straight from Secrets Manager as env vars and no volume is needed. + +variable "billing_metrics_endpoint" { + description = <<-EOT + OTLP/HTTP endpoint for enterprise billing metrics (sets + LITELLM_BILLING_METRICS_ENDPOINT). Non-empty enables request metering; + empty (default) disables it and adds no billing env to the container. + Requires an enterprise license. Example: + "https://telemetry.litellm.ai/v1/metrics" + EOT + type = string + default = "" +} + +variable "billing_metrics_client_cert_pem" { + description = <<-EOT + PEM content of the mTLS client certificate issued for this deployment. + When billing_metrics_endpoint is set, the stack stores this in a + `-litellm--billing-metrics-client-cert` Secrets Manager + entry, grants the task-execution role GetSecretValue on it, and exposes + it to gateway + backend as LITELLM_BILLING_METRICS_CLIENT_CERT. Required + whenever metering is enabled. + EOT + type = string + default = "" + sensitive = true +} + +variable "billing_metrics_client_key_pem" { + description = <<-EOT + PEM content of the private key matching + billing_metrics_client_cert_pem. Stored in a + `-litellm--billing-metrics-client-key` Secrets Manager + entry and exposed as LITELLM_BILLING_METRICS_CLIENT_KEY. Required + whenever metering is enabled. + EOT + type = string + default = "" + sensitive = true +} + +variable "billing_metrics_ca_cert_pem" { + description = <<-EOT + PEM content of the CA bundle used to verify the metering collector. + Only needed for private or test collectors whose CA is not in the + system trust store; telemetry.litellm.ai is publicly trusted, so leave + this empty for production. When set, it is exposed as + LITELLM_BILLING_METRICS_CA_CERT. + EOT + type = string + default = "" + sensitive = true +} diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 1e0bf4319df..88e9979148f 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -204,6 +204,40 @@ Behavior matches the AWS stack 1:1; the only naming differences are `otel_headers_secret` (a Secret Manager resource ID) vs AWS's `otel_headers_secret_arn` (a Secrets Manager ARN). +### Enterprise billing metrics + +License-gated request metering is opt-in and gated entirely on +`billing_metrics_endpoint`. Empty (default) and no billing env is added to +the container, so existing deployments are unchanged. Set it and both +gateway and backend export billable-request counts over OTLP/HTTP, +authenticating to the collector with the mTLS client certificate issued for +your deployment. + +The proxy accepts the certificate, key, and CA bundle as either a file path +or literal PEM content. This stack takes the PEM, writes each one to its own +Secret Manager entry, grants the runtime service account +`roles/secretmanager.secretAccessor` on them, and injects them as Cloud Run +secret env vars `LITELLM_BILLING_METRICS_CLIENT_CERT` / `_CLIENT_KEY` (and +`_CA_CERT` when set), so no volume mount is needed. + +```hcl +billing_metrics_endpoint = "https://telemetry.litellm.ai/v1/metrics" +``` + +```bash +export TF_VAR_billing_metrics_client_cert_pem="$(cat client.crt)" +export TF_VAR_billing_metrics_client_key_pem="$(cat client.key)" +``` + +`billing_metrics_ca_cert_pem` is only for private or test collectors whose +CA is not in the system trust store; leave it empty against +`telemetry.litellm.ai`. Metering requires an enterprise license, so pair +this with `litellm_license`. To tune the export cadence, set +`LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` / +`backend_extra_env` + +Behavior matches the AWS stack 1:1; the variable names are identical + ## Tenant deployment Every resource the stack creates is named `${tenant}-litellm-${env}` (or diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index 7b1bb901e20..913ec2bbb3d 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -59,6 +59,33 @@ locals { { name = "OTEL_HEADERS", secret = var.otel_headers_secret, version = "latest" }, ] : [] + # Enterprise request metering, gated on billing_metrics_endpoint. The + # endpoint rides in as a plain env var; the mTLS material lives in Secret + # Manager (secrets.tf) and is injected as PEM-valued env vars, which the + # proxy accepts in place of file paths. Each PEM is wired only when the + # operator supplied it, so an empty ca_cert_pem falls back to the system + # trust store. + billing_metrics_enabled = var.billing_metrics_endpoint != "" + billing_metrics_client_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_client_cert_pem != "" + billing_metrics_client_key_enabled = local.billing_metrics_enabled && var.billing_metrics_client_key_pem != "" + billing_metrics_ca_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_ca_cert_pem != "" + + billing_metrics_env_kv = local.billing_metrics_enabled ? [ + { name = "LITELLM_BILLING_METRICS_ENDPOINT", value = var.billing_metrics_endpoint }, + ] : [] + + billing_metrics_env_secrets = concat( + local.billing_metrics_client_cert_enabled ? [ + { name = "LITELLM_BILLING_METRICS_CLIENT_CERT", secret = google_secret_manager_secret.billing_metrics_client_cert[0].id, version = "latest" }, + ] : [], + local.billing_metrics_client_key_enabled ? [ + { name = "LITELLM_BILLING_METRICS_CLIENT_KEY", secret = google_secret_manager_secret.billing_metrics_client_key[0].id, version = "latest" }, + ] : [], + local.billing_metrics_ca_cert_enabled ? [ + { name = "LITELLM_BILLING_METRICS_CA_CERT", secret = google_secret_manager_secret.billing_metrics_ca_cert[0].id, version = "latest" }, + ] : [], + ) + # Cloud Run v2 secret env vars use value_source.secret_key_ref pointing at a # secret resource ID. Shared between gateway and backend (the migrations # job has its own narrower env list — see migrations_env_secrets below). @@ -175,7 +202,7 @@ resource "google_cloud_run_v2_service" "gateway" { } dynamic "env" { - for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.gateway_extra_env_kv, local.proxy_config_env) + for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env) content { name = env.value.name value = env.value.value @@ -183,7 +210,7 @@ resource "google_cloud_run_v2_service" "gateway" { } dynamic "env" { - for_each = concat(local.shared_env_secrets, local.otel_env_secrets, local.gateway_extra_secret_kv) + for_each = concat(local.shared_env_secrets, local.otel_env_secrets, local.billing_metrics_env_secrets, local.gateway_extra_secret_kv) content { name = env.value.name value_source { @@ -242,6 +269,9 @@ resource "google_cloud_run_v2_service" "gateway" { google_secret_manager_secret_iam_member.license, google_secret_manager_secret_iam_member.extras, google_secret_manager_secret_iam_member.otel_headers, + google_secret_manager_secret_iam_member.billing_metrics_client_cert, + google_secret_manager_secret_iam_member.billing_metrics_client_key, + google_secret_manager_secret_iam_member.billing_metrics_ca_cert, google_storage_bucket_iam_member.proxy_config_runtime, google_sql_user.app, # Don't go live until the schema is migrated; otherwise the proxy boots, @@ -289,7 +319,7 @@ resource "google_cloud_run_v2_service" "backend" { } dynamic "env" { - for_each = concat(local.shared_env_kv, local.backend_default_env_kv, local.backend_otel_env_kv, local.backend_extra_env_kv, local.proxy_config_env) + for_each = concat(local.shared_env_kv, local.backend_default_env_kv, local.backend_otel_env_kv, local.billing_metrics_env_kv, local.backend_extra_env_kv, local.proxy_config_env) content { name = env.value.name value = env.value.value @@ -297,7 +327,7 @@ resource "google_cloud_run_v2_service" "backend" { } dynamic "env" { - for_each = concat(local.shared_env_secrets, local.backend_managed_env_secrets, local.otel_env_secrets, local.backend_extra_secret_kv) + for_each = concat(local.shared_env_secrets, local.backend_managed_env_secrets, local.otel_env_secrets, local.billing_metrics_env_secrets, local.backend_extra_secret_kv) content { name = env.value.name value_source { @@ -357,6 +387,9 @@ resource "google_cloud_run_v2_service" "backend" { google_secret_manager_secret_iam_member.ui_password, google_secret_manager_secret_iam_member.extras, google_secret_manager_secret_iam_member.otel_headers, + google_secret_manager_secret_iam_member.billing_metrics_client_cert, + google_secret_manager_secret_iam_member.billing_metrics_client_key, + google_secret_manager_secret_iam_member.billing_metrics_ca_cert, google_storage_bucket_iam_member.proxy_config_runtime, google_sql_user.app, terraform_data.migration, diff --git a/terraform/litellm/gcp/iam.tf b/terraform/litellm/gcp/iam.tf index dc3ae5e0912..09df5e7dff0 100644 --- a/terraform/litellm/gcp/iam.tf +++ b/terraform/litellm/gcp/iam.tf @@ -79,3 +79,29 @@ resource "google_secret_manager_secret_iam_member" "otel_headers" { role = "roles/secretmanager.secretAccessor" member = "serviceAccount:${google_service_account.runtime.email}" } + +# Billing-metrics mTLS accessors — only created when request metering is +# enabled and the matching PEM was supplied. +resource "google_secret_manager_secret_iam_member" "billing_metrics_client_cert" { + count = local.billing_metrics_client_cert_enabled ? 1 : 0 + + secret_id = google_secret_manager_secret.billing_metrics_client_cert[0].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.runtime.email}" +} + +resource "google_secret_manager_secret_iam_member" "billing_metrics_client_key" { + count = local.billing_metrics_client_key_enabled ? 1 : 0 + + secret_id = google_secret_manager_secret.billing_metrics_client_key[0].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.runtime.email}" +} + +resource "google_secret_manager_secret_iam_member" "billing_metrics_ca_cert" { + count = local.billing_metrics_ca_cert_enabled ? 1 : 0 + + secret_id = google_secret_manager_secret.billing_metrics_ca_cert[0].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.runtime.email}" +} diff --git a/terraform/litellm/gcp/secrets.tf b/terraform/litellm/gcp/secrets.tf index f93514bb70b..6ec77139996 100644 --- a/terraform/litellm/gcp/secrets.tf +++ b/terraform/litellm/gcp/secrets.tf @@ -63,3 +63,58 @@ resource "google_secret_manager_secret_version" "ui_password" { secret = google_secret_manager_secret.ui_password[0].id secret_data = var.ui_password } + +# Billing-metrics mTLS material — only created when metering is enabled +# (billing_metrics_endpoint non-empty) and the operator supplied the PEM. +# The runtime SA gets accessor permission via iam.tf, and gateway + backend +# pick the env vars up through billing_metrics_env_secrets in cloudrun.tf. +resource "google_secret_manager_secret" "billing_metrics_client_cert" { + count = local.billing_metrics_client_cert_enabled ? 1 : 0 + + secret_id = "${local.name}-billing-metrics-client-cert" + labels = local.labels + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "billing_metrics_client_cert" { + count = local.billing_metrics_client_cert_enabled ? 1 : 0 + + secret = google_secret_manager_secret.billing_metrics_client_cert[0].id + secret_data = var.billing_metrics_client_cert_pem +} + +resource "google_secret_manager_secret" "billing_metrics_client_key" { + count = local.billing_metrics_client_key_enabled ? 1 : 0 + + secret_id = "${local.name}-billing-metrics-client-key" + labels = local.labels + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "billing_metrics_client_key" { + count = local.billing_metrics_client_key_enabled ? 1 : 0 + + secret = google_secret_manager_secret.billing_metrics_client_key[0].id + secret_data = var.billing_metrics_client_key_pem +} + +resource "google_secret_manager_secret" "billing_metrics_ca_cert" { + count = local.billing_metrics_ca_cert_enabled ? 1 : 0 + + secret_id = "${local.name}-billing-metrics-ca-cert" + labels = local.labels + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "billing_metrics_ca_cert" { + count = local.billing_metrics_ca_cert_enabled ? 1 : 0 + + secret = google_secret_manager_secret.billing_metrics_ca_cert[0].id + secret_data = var.billing_metrics_ca_cert_pem +} diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index 4355192e9f1..1162e100bb2 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -490,3 +490,66 @@ variable "otel_capture_message_content" { error_message = "otel_capture_message_content must be one of: no_content, prompt_and_completion." } } + +# ---------- Enterprise billing metrics ---------- +# +# License-gated request metering. Opt-in and gated entirely on +# billing_metrics_endpoint: leave it empty (the default) and nothing +# metering-related is added to the container env. Set it and gateway + +# backend export billable-request counts over OTLP/HTTP, authenticating to +# the collector with an mTLS client cert. The proxy accepts the cert, key, +# and CA as either a file path or literal PEM content, so on Cloud Run they +# are injected straight from Secret Manager as env vars and no volume is +# needed. + +variable "billing_metrics_endpoint" { + description = <<-EOT + OTLP/HTTP endpoint for enterprise billing metrics (sets + LITELLM_BILLING_METRICS_ENDPOINT). Non-empty enables request metering; + empty (default) disables it and adds no billing env to the container. + Requires an enterprise license. Example: + "https://telemetry.litellm.ai/v1/metrics" + EOT + type = string + default = "" +} + +variable "billing_metrics_client_cert_pem" { + description = <<-EOT + PEM content of the mTLS client certificate issued for this deployment. + When billing_metrics_endpoint is set, the stack stores this in a + `-litellm--billing-metrics-client-cert` Secret Manager + entry, grants the runtime SA accessor on it, and exposes it to gateway + + backend as LITELLM_BILLING_METRICS_CLIENT_CERT. Required whenever + metering is enabled. + EOT + type = string + default = "" + sensitive = true +} + +variable "billing_metrics_client_key_pem" { + description = <<-EOT + PEM content of the private key matching + billing_metrics_client_cert_pem. Stored in a + `-litellm--billing-metrics-client-key` Secret Manager entry + and exposed as LITELLM_BILLING_METRICS_CLIENT_KEY. Required whenever + metering is enabled. + EOT + type = string + default = "" + sensitive = true +} + +variable "billing_metrics_ca_cert_pem" { + description = <<-EOT + PEM content of the CA bundle used to verify the metering collector. + Only needed for private or test collectors whose CA is not in the + system trust store; telemetry.litellm.ai is publicly trusted, so leave + this empty for production. When set, it is exposed as + LITELLM_BILLING_METRICS_CA_CERT. + EOT + type = string + default = "" + sensitive = true +}