scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-15 04:19:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 121

feat(terraform): billing-metrics variables for the aws and gcp templates

Browse source
This commit is contained in:
Yassin Kortam 2026-07-09 19:54:16 +03:00
parent 91f5e26ed5
commit 102307291e
10 changed files with 397 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
terraform/litellm/aws/README.md
View file

@ -158,6 +158,38 @@ AgentOps) live under `proxy_config.litellm_settings.callbacks` and are
orthogonal to the OTLP variables above; their credentials still go in
`*_extra_secrets`.
### Enterprise billing metrics
License-gated request metering is opt-in and gated entirely on
`billing_metrics_endpoint`. Empty (default) and no billing env is added to
the container, so existing deployments are unchanged. Set it and both
gateway and backend export billable-request counts over OTLP/HTTP,
authenticating to the collector with the mTLS client certificate issued for
your deployment.
The proxy accepts the certificate, key, and CA bundle as either a file path
or literal PEM content. This stack takes the PEM, writes each one to its own
Secrets Manager entry, grants the task-execution role
`secretsmanager:GetSecretValue` on them, and injects them as
`LITELLM_BILLING_METRICS_CLIENT_CERT` / `_CLIENT_KEY` (and `_CA_CERT` when
set), so no volume mount is needed on Fargate.
```hcl
billing_metrics_endpoint = "https://telemetry.litellm.ai/v1/metrics"
```
```bash
export TF_VAR_billing_metrics_client_cert_pem="$(cat client.crt)"
export TF_VAR_billing_metrics_client_key_pem="$(cat client.key)"
```
`billing_metrics_ca_cert_pem` is only for private or test collectors whose
CA is not in the system trust store; leave it empty against
`telemetry.litellm.ai`. Metering requires an enterprise license, so pair
this with `litellm_license`. To tune the export cadence, set
`LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` /
`backend_extra_env`
## Tenant deployment
Every resource the stack creates is named `${tenant}-litellm-${env}` (or

30
terraform/litellm/aws/ecs.tf
View file

@ -76,6 +76,33 @@ locals {
{ name = "OTEL_HEADERS", valueFrom = var.otel_headers_secret_arn },
] : []
# Enterprise request metering, gated on billing_metrics_endpoint. The
# endpoint rides in as a plain env var; the mTLS material is stored in
# Secrets Manager (secrets.tf) and injected as PEM-valued env vars, which
# the proxy accepts in place of file paths. Each PEM is wired only when the
# operator supplied it, so an empty ca_cert_pem falls back to the system
# trust store.
billing_metrics_enabled = var.billing_metrics_endpoint != ""
billing_metrics_client_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_client_cert_pem != ""
billing_metrics_client_key_enabled = local.billing_metrics_enabled && var.billing_metrics_client_key_pem != ""
billing_metrics_ca_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_ca_cert_pem != ""
billing_metrics_env = local.billing_metrics_enabled ? [
{ name = "LITELLM_BILLING_METRICS_ENDPOINT", value = var.billing_metrics_endpoint },
] : []
billing_metrics_secrets = concat(
local.billing_metrics_client_cert_enabled ? [
{ name = "LITELLM_BILLING_METRICS_CLIENT_CERT", valueFrom = aws_secretsmanager_secret.billing_metrics_client_cert[0].arn },
] : [],
local.billing_metrics_client_key_enabled ? [
{ name = "LITELLM_BILLING_METRICS_CLIENT_KEY", valueFrom = aws_secretsmanager_secret.billing_metrics_client_key[0].arn },
] : [],
local.billing_metrics_ca_cert_enabled ? [
{ name = "LITELLM_BILLING_METRICS_CA_CERT", valueFrom = aws_secretsmanager_secret.billing_metrics_ca_cert[0].arn },
] : [],
)
shared_env = [
{ name = "IAM_TOKEN_DB_AUTH", value = "true" },
{ name = "DATABASE_HOST", value = aws_rds_cluster.this.endpoint },
@ -108,6 +135,7 @@ locals {
{ name = "LITELLM_LICENSE", valueFrom = aws_secretsmanager_secret.license[0].arn },
],
local.otel_secrets,
local.billing_metrics_secrets,
)
# Backend-only managed secrets. UI_PASSWORD is consumed by the management
@ -198,6 +226,7 @@ resource "aws_ecs_task_definition" "gateway" {
environment = concat(
local.shared_env,
local.gateway_otel_env,
local.billing_metrics_env,
local.gateway_extra_env_list,
local.proxy_config_env,
)
@ -284,6 +313,7 @@ resource "aws_ecs_task_definition" "backend" {
local.shared_env,
local.backend_default_env,
local.backend_otel_env,
local.billing_metrics_env,
local.backend_extra_env_list,
local.proxy_config_env,
)

3
terraform/litellm/aws/iam.tf
View file

@ -53,6 +53,9 @@ data "aws_iam_policy_document" "secrets_access" {
[aws_secretsmanager_secret.master_key.arn],
aws_secretsmanager_secret.license[*].arn,
aws_secretsmanager_secret.ui_password[*].arn,
aws_secretsmanager_secret.billing_metrics_client_cert[*].arn,
aws_secretsmanager_secret.billing_metrics_client_key[*].arn,
aws_secretsmanager_secret.billing_metrics_ca_cert[*].arn,
local.extra_secret_arns,
var.otel_headers_secret_arn == "" ? [] : [var.otel_headers_secret_arn],
)

55
terraform/litellm/aws/secrets.tf
View file

@ -74,6 +74,61 @@ resource "aws_secretsmanager_secret_version" "ui_password" {
secret_string = var.ui_password
}
# Billing-metrics mTLS material — only created when metering is enabled
# (billing_metrics_endpoint non-empty) and the operator supplied the PEM.
# The task-execution role gets GetSecretValue via iam.tf, and gateway +
# backend pick the env vars up through shared_secrets in ecs.tf.
resource "aws_secretsmanager_secret" "billing_metrics_client_cert" {
count = local.billing_metrics_client_cert_enabled ? 1 : 0
name = "${local.name}-billing-metrics-client-cert"
description = "LITELLM_BILLING_METRICS_CLIENT_CERT for gateway + backend."
recovery_window_in_days = 0
tags = local.tags
}
resource "aws_secretsmanager_secret_version" "billing_metrics_client_cert" {
count = local.billing_metrics_client_cert_enabled ? 1 : 0
secret_id = aws_secretsmanager_secret.billing_metrics_client_cert[0].id
secret_string = var.billing_metrics_client_cert_pem
}
resource "aws_secretsmanager_secret" "billing_metrics_client_key" {
count = local.billing_metrics_client_key_enabled ? 1 : 0
name = "${local.name}-billing-metrics-client-key"
description = "LITELLM_BILLING_METRICS_CLIENT_KEY for gateway + backend."
recovery_window_in_days = 0
tags = local.tags
}
resource "aws_secretsmanager_secret_version" "billing_metrics_client_key" {
count = local.billing_metrics_client_key_enabled ? 1 : 0
secret_id = aws_secretsmanager_secret.billing_metrics_client_key[0].id
secret_string = var.billing_metrics_client_key_pem
}
resource "aws_secretsmanager_secret" "billing_metrics_ca_cert" {
count = local.billing_metrics_ca_cert_enabled ? 1 : 0
name = "${local.name}-billing-metrics-ca-cert"
description = "LITELLM_BILLING_METRICS_CA_CERT for gateway + backend."
recovery_window_in_days = 0
tags = local.tags
}
resource "aws_secretsmanager_secret_version" "billing_metrics_ca_cert" {
count = local.billing_metrics_ca_cert_enabled ? 1 : 0
secret_id = aws_secretsmanager_secret.billing_metrics_ca_cert[0].id
secret_string = var.billing_metrics_ca_cert_pem
}
resource "aws_secretsmanager_secret" "db_master_password" {
name = "${local.name}-db-master-password"
description = "Aurora master-user password - bootstrap only. Runtime auth is IAM-token."

62
terraform/litellm/aws/variables.tf
View file

@ -533,3 +533,65 @@ variable "otel_headers_secret_arn" {
type = string
default = ""
}
# ---------- Enterprise billing metrics ----------
#
# License-gated request metering. Opt-in and gated entirely on
# billing_metrics_endpoint: leave it empty (the default) and nothing
# metering-related lands in the container env. Set it and gateway + backend
# export billable-request counts over OTLP/HTTP, authenticating to the
# collector with an mTLS client cert. The proxy accepts the cert, key, and CA
# as either a file path or literal PEM content, so on Fargate they are
# injected straight from Secrets Manager as env vars and no volume is needed.
variable "billing_metrics_endpoint" {
description = <<-EOT
OTLP/HTTP endpoint for enterprise billing metrics (sets
LITELLM_BILLING_METRICS_ENDPOINT). Non-empty enables request metering;
empty (default) disables it and adds no billing env to the container.
Requires an enterprise license. Example:
"https://telemetry.litellm.ai/v1/metrics"
EOT
type = string
default = ""
}
variable "billing_metrics_client_cert_pem" {
description = <<-EOT
PEM content of the mTLS client certificate issued for this deployment.
When billing_metrics_endpoint is set, the stack stores this in a
`<tenant>-litellm-<env>-billing-metrics-client-cert` Secrets Manager
entry, grants the task-execution role GetSecretValue on it, and exposes
it to gateway + backend as LITELLM_BILLING_METRICS_CLIENT_CERT. Required
whenever metering is enabled.
EOT
type = string
default = ""
sensitive = true
}
variable "billing_metrics_client_key_pem" {
description = <<-EOT
PEM content of the private key matching
billing_metrics_client_cert_pem. Stored in a
`<tenant>-litellm-<env>-billing-metrics-client-key` Secrets Manager
entry and exposed as LITELLM_BILLING_METRICS_CLIENT_KEY. Required
whenever metering is enabled.
EOT
type = string
default = ""
sensitive = true
}
variable "billing_metrics_ca_cert_pem" {
description = <<-EOT
PEM content of the CA bundle used to verify the metering collector.
Only needed for private or test collectors whose CA is not in the
system trust store; telemetry.litellm.ai is publicly trusted, so leave
this empty for production. When set, it is exposed as
LITELLM_BILLING_METRICS_CA_CERT.
EOT
type = string
default = ""
sensitive = true
}

34
terraform/litellm/gcp/README.md
View file

@ -204,6 +204,40 @@ Behavior matches the AWS stack 1:1; the only naming differences are
`otel_headers_secret` (a Secret Manager resource ID) vs AWS's
`otel_headers_secret_arn` (a Secrets Manager ARN).
### Enterprise billing metrics
License-gated request metering is opt-in and gated entirely on
`billing_metrics_endpoint`. Empty (default) and no billing env is added to
the container, so existing deployments are unchanged. Set it and both
gateway and backend export billable-request counts over OTLP/HTTP,
authenticating to the collector with the mTLS client certificate issued for
your deployment.
The proxy accepts the certificate, key, and CA bundle as either a file path
or literal PEM content. This stack takes the PEM, writes each one to its own
Secret Manager entry, grants the runtime service account
`roles/secretmanager.secretAccessor` on them, and injects them as Cloud Run
secret env vars `LITELLM_BILLING_METRICS_CLIENT_CERT` / `_CLIENT_KEY` (and
`_CA_CERT` when set), so no volume mount is needed.
```hcl
billing_metrics_endpoint = "https://telemetry.litellm.ai/v1/metrics"
```
```bash
export TF_VAR_billing_metrics_client_cert_pem="$(cat client.crt)"
export TF_VAR_billing_metrics_client_key_pem="$(cat client.key)"
```
`billing_metrics_ca_cert_pem` is only for private or test collectors whose
CA is not in the system trust store; leave it empty against
`telemetry.litellm.ai`. Metering requires an enterprise license, so pair
this with `litellm_license`. To tune the export cadence, set
`LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` /
`backend_extra_env`
Behavior matches the AWS stack 1:1; the variable names are identical
## Tenant deployment
Every resource the stack creates is named `${tenant}-litellm-${env}` (or

41
terraform/litellm/gcp/cloudrun.tf
View file

@ -59,6 +59,33 @@ locals {
{ name = "OTEL_HEADERS", secret = var.otel_headers_secret, version = "latest" },
] : []
# Enterprise request metering, gated on billing_metrics_endpoint. The
# endpoint rides in as a plain env var; the mTLS material lives in Secret
# Manager (secrets.tf) and is injected as PEM-valued env vars, which the
# proxy accepts in place of file paths. Each PEM is wired only when the
# operator supplied it, so an empty ca_cert_pem falls back to the system
# trust store.
billing_metrics_enabled = var.billing_metrics_endpoint != ""
billing_metrics_client_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_client_cert_pem != ""
billing_metrics_client_key_enabled = local.billing_metrics_enabled && var.billing_metrics_client_key_pem != ""
billing_metrics_ca_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_ca_cert_pem != ""
billing_metrics_env_kv = local.billing_metrics_enabled ? [
{ name = "LITELLM_BILLING_METRICS_ENDPOINT", value = var.billing_metrics_endpoint },
] : []
billing_metrics_env_secrets = concat(
local.billing_metrics_client_cert_enabled ? [
{ name = "LITELLM_BILLING_METRICS_CLIENT_CERT", secret = google_secret_manager_secret.billing_metrics_client_cert[0].id, version = "latest" },
] : [],
local.billing_metrics_client_key_enabled ? [
{ name = "LITELLM_BILLING_METRICS_CLIENT_KEY", secret = google_secret_manager_secret.billing_metrics_client_key[0].id, version = "latest" },
] : [],
local.billing_metrics_ca_cert_enabled ? [
{ name = "LITELLM_BILLING_METRICS_CA_CERT", secret = google_secret_manager_secret.billing_metrics_ca_cert[0].id, version = "latest" },
] : [],
)
# Cloud Run v2 secret env vars use value_source.secret_key_ref pointing at a
# secret resource ID. Shared between gateway and backend (the migrations
# job has its own narrower env list — see migrations_env_secrets below).
@ -175,7 +202,7 @@ resource "google_cloud_run_v2_service" "gateway" {
}
dynamic "env" {
for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.gateway_extra_env_kv, local.proxy_config_env)
for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env)
content {
name = env.value.name
value = env.value.value
@ -183,7 +210,7 @@ resource "google_cloud_run_v2_service" "gateway" {
}
dynamic "env" {
for_each = concat(local.shared_env_secrets, local.otel_env_secrets, local.gateway_extra_secret_kv)
for_each = concat(local.shared_env_secrets, local.otel_env_secrets, local.billing_metrics_env_secrets, local.gateway_extra_secret_kv)
content {
name = env.value.name
value_source {
@ -242,6 +269,9 @@ resource "google_cloud_run_v2_service" "gateway" {
google_secret_manager_secret_iam_member.license,
google_secret_manager_secret_iam_member.extras,
google_secret_manager_secret_iam_member.otel_headers,
google_secret_manager_secret_iam_member.billing_metrics_client_cert,
google_secret_manager_secret_iam_member.billing_metrics_client_key,
google_secret_manager_secret_iam_member.billing_metrics_ca_cert,
google_storage_bucket_iam_member.proxy_config_runtime,
google_sql_user.app,
# Don't go live until the schema is migrated; otherwise the proxy boots,
@ -289,7 +319,7 @@ resource "google_cloud_run_v2_service" "backend" {
}
dynamic "env" {
for_each = concat(local.shared_env_kv, local.backend_default_env_kv, local.backend_otel_env_kv, local.backend_extra_env_kv, local.proxy_config_env)
for_each = concat(local.shared_env_kv, local.backend_default_env_kv, local.backend_otel_env_kv, local.billing_metrics_env_kv, local.backend_extra_env_kv, local.proxy_config_env)
content {
name = env.value.name
value = env.value.value
@ -297,7 +327,7 @@ resource "google_cloud_run_v2_service" "backend" {
}
dynamic "env" {
for_each = concat(local.shared_env_secrets, local.backend_managed_env_secrets, local.otel_env_secrets, local.backend_extra_secret_kv)
for_each = concat(local.shared_env_secrets, local.backend_managed_env_secrets, local.otel_env_secrets, local.billing_metrics_env_secrets, local.backend_extra_secret_kv)
content {
name = env.value.name
value_source {
@ -357,6 +387,9 @@ resource "google_cloud_run_v2_service" "backend" {
google_secret_manager_secret_iam_member.ui_password,
google_secret_manager_secret_iam_member.extras,
google_secret_manager_secret_iam_member.otel_headers,
google_secret_manager_secret_iam_member.billing_metrics_client_cert,
google_secret_manager_secret_iam_member.billing_metrics_client_key,
google_secret_manager_secret_iam_member.billing_metrics_ca_cert,
google_storage_bucket_iam_member.proxy_config_runtime,
google_sql_user.app,
terraform_data.migration,

26
terraform/litellm/gcp/iam.tf
View file

@ -79,3 +79,29 @@ resource "google_secret_manager_secret_iam_member" "otel_headers" {
role = "roles/secretmanager.secretAccessor"
member = "serviceAccount:${google_service_account.runtime.email}"
}
# Billing-metrics mTLS accessors — only created when request metering is
# enabled and the matching PEM was supplied.
resource "google_secret_manager_secret_iam_member" "billing_metrics_client_cert" {
count = local.billing_metrics_client_cert_enabled ? 1 : 0
secret_id = google_secret_manager_secret.billing_metrics_client_cert[0].id
role = "roles/secretmanager.secretAccessor"
member = "serviceAccount:${google_service_account.runtime.email}"
}
resource "google_secret_manager_secret_iam_member" "billing_metrics_client_key" {
count = local.billing_metrics_client_key_enabled ? 1 : 0
secret_id = google_secret_manager_secret.billing_metrics_client_key[0].id
role = "roles/secretmanager.secretAccessor"
member = "serviceAccount:${google_service_account.runtime.email}"
}
resource "google_secret_manager_secret_iam_member" "billing_metrics_ca_cert" {
count = local.billing_metrics_ca_cert_enabled ? 1 : 0
secret_id = google_secret_manager_secret.billing_metrics_ca_cert[0].id
role = "roles/secretmanager.secretAccessor"
member = "serviceAccount:${google_service_account.runtime.email}"
}

55
terraform/litellm/gcp/secrets.tf
View file

@ -63,3 +63,58 @@ resource "google_secret_manager_secret_version" "ui_password" {
secret = google_secret_manager_secret.ui_password[0].id
secret_data = var.ui_password
}
# Billing-metrics mTLS material — only created when metering is enabled
# (billing_metrics_endpoint non-empty) and the operator supplied the PEM.
# The runtime SA gets accessor permission via iam.tf, and gateway + backend
# pick the env vars up through billing_metrics_env_secrets in cloudrun.tf.
resource "google_secret_manager_secret" "billing_metrics_client_cert" {
count = local.billing_metrics_client_cert_enabled ? 1 : 0
secret_id = "${local.name}-billing-metrics-client-cert"
labels = local.labels
replication {
auto {}
}
}
resource "google_secret_manager_secret_version" "billing_metrics_client_cert" {
count = local.billing_metrics_client_cert_enabled ? 1 : 0
secret = google_secret_manager_secret.billing_metrics_client_cert[0].id
secret_data = var.billing_metrics_client_cert_pem
}
resource "google_secret_manager_secret" "billing_metrics_client_key" {
count = local.billing_metrics_client_key_enabled ? 1 : 0
secret_id = "${local.name}-billing-metrics-client-key"
labels = local.labels
replication {
auto {}
}
}
resource "google_secret_manager_secret_version" "billing_metrics_client_key" {
count = local.billing_metrics_client_key_enabled ? 1 : 0
secret = google_secret_manager_secret.billing_metrics_client_key[0].id
secret_data = var.billing_metrics_client_key_pem
}
resource "google_secret_manager_secret" "billing_metrics_ca_cert" {
count = local.billing_metrics_ca_cert_enabled ? 1 : 0
secret_id = "${local.name}-billing-metrics-ca-cert"
labels = local.labels
replication {
auto {}
}
}
resource "google_secret_manager_secret_version" "billing_metrics_ca_cert" {
count = local.billing_metrics_ca_cert_enabled ? 1 : 0
secret = google_secret_manager_secret.billing_metrics_ca_cert[0].id
secret_data = var.billing_metrics_ca_cert_pem
}

63
terraform/litellm/gcp/variables.tf
View file

@ -490,3 +490,66 @@ variable "otel_capture_message_content" {
error_message = "otel_capture_message_content must be one of: no_content, prompt_and_completion."
}
}
# ---------- Enterprise billing metrics ----------
#
# License-gated request metering. Opt-in and gated entirely on
# billing_metrics_endpoint: leave it empty (the default) and nothing
# metering-related is added to the container env. Set it and gateway +
# backend export billable-request counts over OTLP/HTTP, authenticating to
# the collector with an mTLS client cert. The proxy accepts the cert, key,
# and CA as either a file path or literal PEM content, so on Cloud Run they
# are injected straight from Secret Manager as env vars and no volume is
# needed.
variable "billing_metrics_endpoint" {
description = <<-EOT
OTLP/HTTP endpoint for enterprise billing metrics (sets
LITELLM_BILLING_METRICS_ENDPOINT). Non-empty enables request metering;
empty (default) disables it and adds no billing env to the container.
Requires an enterprise license. Example:
"https://telemetry.litellm.ai/v1/metrics"
EOT
type = string
default = ""
}
variable "billing_metrics_client_cert_pem" {
description = <<-EOT
PEM content of the mTLS client certificate issued for this deployment.
When billing_metrics_endpoint is set, the stack stores this in a
`<tenant>-litellm-<env>-billing-metrics-client-cert` Secret Manager
entry, grants the runtime SA accessor on it, and exposes it to gateway +
backend as LITELLM_BILLING_METRICS_CLIENT_CERT. Required whenever
metering is enabled.
EOT
type = string
default = ""
sensitive = true
}
variable "billing_metrics_client_key_pem" {
description = <<-EOT
PEM content of the private key matching
billing_metrics_client_cert_pem. Stored in a
`<tenant>-litellm-<env>-billing-metrics-client-key` Secret Manager entry
and exposed as LITELLM_BILLING_METRICS_CLIENT_KEY. Required whenever
metering is enabled.
EOT
type = string
default = ""
sensitive = true
}
variable "billing_metrics_ca_cert_pem" {
description = <<-EOT
PEM content of the CA bundle used to verify the metering collector.
Only needed for private or test collectors whose CA is not in the
system trust store; telemetry.litellm.ai is publicly trusted, so leave
this empty for production. When set, it is exposed as
LITELLM_BILLING_METRICS_CA_CERT.
EOT
type = string
default = ""
sensitive = true
}