fix(db): carry DATABASE_SSLMODE/DATABASE_SSLROOTCERT into the assembled writer and reader URLs (#40815)

* fix(db): carry DATABASE_SSLMODE/DATABASE_SSLROOTCERT into the assembled writer and reader URLs

The componentized gateway supervisor starts the in-container PgBouncer from the
DATABASE_URL assembled out of the discrete DATABASE_* vars before config.yaml is
read, so an IAM URL had no way to request verified TLS: PgBouncer dialed the
server with server_tls_sslmode = prefer (no SNI, no verification) and public
RDS endpoints rejected the handshake. Two new env vars, exposed by the chart as
database.writer.sslMode / sslRootCert, are appended as libpq sslmode/sslrootcert
to every writer and reader URL the settings assemble (never to a pinned URL),
then translated for Prisma as before. Token refresh now also carries Prisma's
sslmode/sslcert/sslaccept over into the re-minted URL

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(db): keep TLS params on the CLI password URL and the initial IAM reader mint

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(db): treat DATABASE_SSLROOTCERT on its own as verify-full and cover collector and migrations TLS env

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(db): type the reader mint TLS test double and drop its mutable capture

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-12 02:14:39 +00:00 committed by GitHub
parent 1fde15c1ec
commit b5bf09d22d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 323 additions and 6 deletions

View file

@ -257,6 +257,14 @@ IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets.
- name: DATABASE_SCHEMA
value: {{ .schema | quote }}
{{- end }}
{{- if .sslMode }}
- name: DATABASE_SSLMODE
value: {{ .sslMode | quote }}
{{- end }}
{{- if .sslRootCert }}
- name: DATABASE_SSLROOTCERT
value: {{ .sslRootCert | quote }}
{{- end }}
{{- if and .useIAMAuth .useAzureEntraAuth }}
{{- fail "database.writer.useIAMAuth and database.writer.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" }}
{{- end }}

View file

@ -4,6 +4,7 @@ templates:
- gateway/configmap.yaml
- backend/deployment.yaml
- backend/configmap.yaml
- migrations-job.yaml
values:
- ./values/required.yaml
tests:
@ -67,6 +68,82 @@ tests:
value: "true"
any: true
- it: emits no TLS env by default
template: gateway/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_SSLMODE
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_SSLROOTCERT
any: true
- it: writer sslMode and sslRootCert reach gateway and backend as DATABASE_SSLMODE and DATABASE_SSLROOTCERT
templates:
- gateway/deployment.yaml
- backend/deployment.yaml
set:
database.writer.useIAMAuth: true
database.writer.sslMode: verify-full
database.writer.sslRootCert: /etc/ssl/certs/ca-certificates.crt
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_SSLMODE
value: verify-full
any: true
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_SSLROOTCERT
value: /etc/ssl/certs/ca-certificates.crt
any: true
- it: writer sslMode and sslRootCert reach the collector sidecar and the migrations job, which dial Postgres themselves
set:
gateway.collector.enabled: true
database.connectionPool.enabled: true
database.writer.sslMode: verify-full
database.writer.sslRootCert: /etc/ssl/certs/ca-certificates.crt
asserts:
- equal:
path: spec.template.spec.containers[1].name
value: collector
template: gateway/deployment.yaml
- contains:
path: spec.template.spec.containers[1].env
content:
name: DATABASE_SSLMODE
value: verify-full
any: true
template: gateway/deployment.yaml
- contains:
path: spec.template.spec.containers[1].env
content:
name: DATABASE_SSLROOTCERT
value: /etc/ssl/certs/ca-certificates.crt
any: true
template: gateway/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_SSLMODE
value: verify-full
any: true
template: migrations-job.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_SSLROOTCERT
value: /etc/ssl/certs/ca-certificates.crt
any: true
template: migrations-job.yaml
- it: writer rejects both token sources at once
template: gateway/deployment.yaml
set:

View file

@ -208,6 +208,11 @@ database:
name: litellm-writer-secret
usernameKey: username
passwordKey: password
# libpq sslmode / sslrootcert applied to the writer and reader URLs (Prisma and the
# in-container PgBouncer); e.g. verify-full with /etc/ssl/certs/ca-certificates.crt for AWS RDS.
# sslRootCert on its own implies sslMode verify-full
sslMode: ""
sslRootCert: ""
# Optional read-replica routing. When `reader.host` is set, the proxy routes
# reads (find_*, count, group_by, query_raw/_first) to this endpoint while

View file

@ -74,6 +74,8 @@ DisablePreparedStatementsFlag = Annotated[
bool, BeforeValidator(partial(token_auth_flag_enabled, env_var=DISABLE_PREPARED_STATEMENTS_ENV_VAR))
]
MAX_IDLE_CONNECTION_LIFETIME_ENV_VAR: Final = "DATABASE_MAX_IDLE_CONNECTION_LIFETIME"
DATABASE_SSLMODE_ENV_VAR: Final = "DATABASE_SSLMODE"
DATABASE_SSLROOTCERT_ENV_VAR: Final = "DATABASE_SSLROOTCERT"
# schema.prisma pins `provider = "postgresql"`, so these are the only schemes
# Prisma can actually connect with.
@ -135,6 +137,7 @@ def add_missing_query_params(url: str, params: Mapping[str, str | int | float])
LIBPQ_VERIFY_SSLMODES: Final[frozenset[str]] = frozenset({"verify-ca", "verify-full"})
PRISMA_TLS_PARAM_KEYS: Final[frozenset[str]] = frozenset({"sslmode", "sslcert", "sslaccept"})
PEM_CERT_HEADER: Final = b"-----BEGIN CERTIFICATE-----"
PG_SSL_REQUEST: Final = struct.pack("!ii", 8, 80877103)
TLS_PROBE_TIMEOUT_SECONDS: Final = 10.0
@ -263,6 +266,19 @@ def connection_params_from_url(url: str) -> Mapping[str, str | int | float]:
)
def token_refresh_params_from_url(url: str) -> Mapping[str, str | int | float]:
"""Return the params a re-minted token URL carries over from the URL it replaces.
The pool and timeout params plus Prisma's TLS params (already translated from
libpq spelling), so a refreshed URL keeps verifying the server the way the
first one did.
"""
kept: Final = CONNECTION_PARAM_KEYS | PRISMA_TLS_PARAM_KEYS
return MappingProxyType(
{key: value for key, value in urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query) if key in kept}
)
def unsupported_db_scheme(database_url: str) -> str | None:
"""Return the connection URL scheme when it is not PostgreSQL, else None.
@ -312,6 +328,9 @@ class DatabaseURLSettings(BaseSettings):
default=None, validation_alias=MAX_IDLE_CONNECTION_LIFETIME_ENV_VAR
)
database_sslmode: str | None = Field(default=None, validation_alias=DATABASE_SSLMODE_ENV_VAR)
database_sslrootcert: str | None = Field(default=None, validation_alias=DATABASE_SSLROOTCERT_ENV_VAR)
# Writer
database_url: str | None = Field(default=None, validation_alias="DATABASE_URL")
direct_url: str | None = Field(default=None, validation_alias="DIRECT_URL")
@ -353,6 +372,25 @@ class DatabaseURLSettings(BaseSettings):
azure_postgresql_auth=self.azure_postgresql_auth,
)
def tls_params(self) -> Mapping[str, str]:
"""``sslmode`` / ``sslrootcert`` query params for every URL assembled from the discrete vars.
A root cert on its own means ``verify-full``: under libpq's default
``prefer`` the CA would never be consulted, and PgBouncer would dial
Postgres unverified with the bundle loaded.
"""
sslmode: Final = self.database_sslmode or ("verify-full" if self.database_sslrootcert else None)
return MappingProxyType(
{
key: value
for key, value in (
("sslmode", sslmode),
("sslrootcert", self.database_sslrootcert),
)
if value
}
)
def build_writer_url(self) -> str | None:
"""Return the writer URL to set, or ``None`` to leave it as-is.
@ -362,6 +400,12 @@ class DatabaseURLSettings(BaseSettings):
A ``DATABASE_URL`` the supervisor pointed at the in-container PgBouncer
is kept even under token auth: the pooler renews the token upstream.
"""
assembled: Final = self._assemble_writer_url()
if assembled is None:
return None
return add_missing_query_params(assembled, self.tls_params())
def _assemble_writer_url(self) -> str | None:
auth: Final = self.token_auth()
if auth is not None and database_url_is_pooled():
return None
@ -411,6 +455,12 @@ class DatabaseURLSettings(BaseSettings):
pre-existing ``DATABASE_URL_READ_REPLICA``. Reader fields fall back
to the writer's values.
"""
assembled: Final = self._assemble_reader_url()
if assembled is None:
return None
return add_missing_query_params(assembled, self.tls_params())
def _assemble_reader_url(self) -> str | None:
if not self.database_host_read_replica:
return None # reader is opt-in
if self.database_url_read_replica:

View file

@ -16,7 +16,7 @@ from datetime import datetime, timedelta
from typing import Any, Final, Protocol
from litellm._logging import verbose_proxy_logger
from litellm.proxy.db.db_url_settings import add_missing_query_params, connection_params_from_url
from litellm.proxy.db.db_url_settings import add_missing_query_params, token_refresh_params_from_url
from litellm.proxy.db.token_auth import (
DEFAULT_POSTGRES_PORT,
DatabaseTokenAuth,
@ -441,7 +441,7 @@ class PrismaWrapper:
endpoint: Final = self._iam_endpoint if self._iam_endpoint is not None else self._endpoint_from_env()
db_url: Final = add_missing_query_params(
endpoint.build_url(mint_database_token(auth, endpoint)),
connection_params_from_url(os.environ.get(self._db_url_env_var, "")),
token_refresh_params_from_url(os.environ.get(self._db_url_env_var, "")),
)
os.environ[self._db_url_env_var] = db_url
return db_url

View file

@ -121,6 +121,11 @@ from litellm.proxy.db.create_views import (
should_create_missing_views,
)
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.db.db_url_settings import (
DatabaseURLSettings,
add_missing_query_params,
token_refresh_params_from_url,
)
from litellm.proxy.db.exception_handler import (
PrismaDBExceptionHandler,
call_with_db_reconnect_retry,
@ -4054,7 +4059,10 @@ class PrismaClient:
# loop and times out after 30s.
if token_auth is not None and reader_iam_endpoint is not None:
reader_token: Final = mint_database_token(token_auth, reader_iam_endpoint)
read_replica_url = reader_iam_endpoint.build_url(reader_token)
read_replica_url = add_missing_query_params(
reader_iam_endpoint.build_url(reader_token),
token_refresh_params_from_url(read_replica_url),
)
os.environ["DATABASE_URL_READ_REPLICA"] = read_replica_url
reader_kwargs: Final[dict[str, Any]] = {"datasource": {"url": read_replica_url}}
if http_client is not None:
@ -7807,7 +7815,7 @@ def construct_database_url_from_env_vars() -> str | None:
if database_schema:
database_url += f"?schema={database_schema}"
return database_url
return add_missing_query_params(database_url, DatabaseURLSettings.from_env().tls_params())
return None

View file

@ -34,10 +34,12 @@ from pydantic import ValidationError
from litellm.proxy.db.db_url_settings import (
PG_SSL_REQUEST,
DatabaseURLSettings,
token_refresh_params_from_url,
translate_libpq_ssl_params,
unsupported_db_scheme,
unsupported_db_scheme_message,
)
from litellm.proxy.db.pgbouncer import PgBouncerPlan, PgBouncerSettings, plan_pgbouncer
from litellm.proxy.db.token_auth import AzureEntraTokenAuth, RdsIamTokenAuth
@ -51,6 +53,8 @@ _MANAGED_DB_ENV_VARS = (
"AZURE_POSTGRESQL_AUTH",
"DATABASE_DISABLE_PREPARED_STATEMENTS",
"DATABASE_MAX_IDLE_CONNECTION_LIFETIME",
"DATABASE_SSLMODE",
"DATABASE_SSLROOTCERT",
"DATABASE_URL",
"DIRECT_URL",
"DATABASE_URL_READ_REPLICA",
@ -781,6 +785,109 @@ def test_libpq_verify_full_and_sslrootcert_become_prisma_strict_sslcert(monkeypa
}
def _tls_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("DATABASE_HOST", "writer.example.com")
monkeypatch.setenv("DATABASE_USER", "litellm")
monkeypatch.setenv("DATABASE_NAME", "litellm_db")
monkeypatch.setenv("DATABASE_SSLMODE", "verify-full")
monkeypatch.setenv("DATABASE_SSLROOTCERT", "/certs/rds-bundle.pem")
def test_tls_env_vars_make_the_minted_iam_writer_url_verify_the_server(monkeypatch: pytest.MonkeyPatch):
"""The supervisor starts PgBouncer from the URL assembled here, before any
config.yaml is read, so an IAM URL with no TLS params leaves PgBouncer on
``prefer`` (no SNI, no verification) and the RDS handshake fails."""
monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true")
_tls_env(monkeypatch)
with _stub_iam_token("WRITER_TOKEN"):
assert _apply() is True
url: Final = os.environ["DATABASE_URL"]
assert url.startswith("postgresql://litellm:WRITER_TOKEN@writer.example.com:5432/litellm_db?")
assert _query(url) == {
"sslmode": ["require"],
"sslcert": ["/certs/rds-bundle.pem"],
"sslaccept": ["strict"],
"max_idle_connection_lifetime": ["60"],
}
def test_tls_env_vars_apply_to_the_password_writer_and_the_assembled_reader(monkeypatch: pytest.MonkeyPatch):
_tls_env(monkeypatch)
monkeypatch.setenv("DATABASE_PASSWORD", "s3cr3t")
monkeypatch.setenv("DATABASE_SCHEMA", "public")
monkeypatch.setenv("DATABASE_HOST_READ_REPLICA", "reader.example.com")
assert _apply() is True
expected: Final = {
"schema": ["public"],
"sslmode": ["require"],
"sslcert": ["/certs/rds-bundle.pem"],
"sslaccept": ["strict"],
"max_idle_connection_lifetime": ["60"],
}
assert os.environ["DATABASE_URL"].startswith("postgresql://litellm:s3cr3t@writer.example.com:5432/litellm_db?")
assert _query(os.environ["DATABASE_URL"]) == expected
assert os.environ["DATABASE_URL_READ_REPLICA"].startswith(
"postgresql://litellm:s3cr3t@reader.example.com:5432/litellm_db?"
)
assert _query(os.environ["DATABASE_URL_READ_REPLICA"]) == expected
def test_sslrootcert_env_var_alone_means_verify_full_for_prisma_and_pgbouncer(monkeypatch: pytest.MonkeyPatch):
"""Under libpq's default ``prefer`` a root cert is never consulted, so a URL
carrying only ``sslrootcert`` would leave PgBouncer on ``prefer`` with the CA
loaded but unused. Supplying a CA and nothing else must verify."""
_tls_env(monkeypatch)
monkeypatch.delenv("DATABASE_SSLMODE")
monkeypatch.setenv("DATABASE_PASSWORD", "s3cr3t")
assert _apply() is True
url: Final = os.environ["DATABASE_URL"]
assert _query(url) == {
"sslmode": ["require"],
"sslcert": ["/certs/rds-bundle.pem"],
"sslaccept": ["strict"],
"max_idle_connection_lifetime": ["60"],
}
plan: Final = plan_pgbouncer(url, PgBouncerSettings(enabled=True), Path("/run/pgb"), None)
assert isinstance(plan, PgBouncerPlan), plan
assert "server_tls_sslmode = verify-full" in plan.ini
assert "server_tls_ca_file = /run/pgb/server-ca.pem" in plan.ini
def test_tls_env_vars_never_override_a_pinned_database_url(monkeypatch: pytest.MonkeyPatch):
writer: Final = (
"postgresql://pinned:url@db.example.com:5432/litellm_db?sslmode=disable&max_idle_connection_lifetime=60"
)
reader: Final = "postgresql://pinned:url@reader.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
monkeypatch.setenv("DATABASE_URL", writer)
monkeypatch.setenv("DATABASE_URL_READ_REPLICA", reader)
monkeypatch.setenv("DATABASE_HOST_READ_REPLICA", "reader.example.com")
_tls_env(monkeypatch)
assert _apply() is False
assert os.environ["DATABASE_URL"] == writer
assert os.environ["DATABASE_URL_READ_REPLICA"] == reader
def test_token_refresh_params_keep_the_prisma_tls_dialect_but_not_the_schema():
kept: Final = token_refresh_params_from_url(
"postgresql://u:TOKEN@db.example.com:5432/litellm_db"
"?schema=tenant&connection_limit=5&sslmode=require&sslcert=/certs/root.pem&sslaccept=strict"
)
assert dict(kept) == {
"connection_limit": "5",
"sslmode": "require",
"sslcert": "/certs/root.pem",
"sslaccept": "strict",
}
def _issue_cert(
subject: str, issuer: x509.Certificate | None, issuer_key: ec.EllipticCurvePrivateKey | None, ca: bool
) -> tuple[x509.Certificate, ec.EllipticCurvePrivateKey]:

View file

@ -299,6 +299,10 @@ def test_azure_entra_mint_writes_an_encoded_url_into_the_db_url_env_var(azure_en
"connection_limit=20&pgbouncer=true&max_idle_connection_lifetime=45",
{"connection_limit": ["20"], "pgbouncer": ["true"], "max_idle_connection_lifetime": ["45"]},
),
(
"sslmode=require&sslcert=/certs/root.pem&sslaccept=strict&schema=tenant",
{"sslmode": ["require"], "sslcert": ["/certs/root.pem"], "sslaccept": ["strict"]},
),
],
)
def test_token_refresh_keeps_the_connection_params_of_the_url_it_replaces(

View file

@ -2,8 +2,9 @@ import asyncio
import logging
import os
import sys
from typing import Any, Dict
from unittest.mock import AsyncMock, MagicMock, patch
from typing import Any, Dict, Final
from unittest.mock import AsyncMock, MagicMock, call, patch
from urllib.parse import parse_qs, urlsplit
import pytest
@ -927,6 +928,47 @@ def test_prisma_client_init_falls_back_to_writer_when_reader_iam_token_fails(
)
def test_prisma_client_init_keeps_reader_tls_params_on_the_minted_iam_url(
monkeypatch: pytest.MonkeyPatch,
):
"""The initial reader mint rebuilds the URL from host/port/user/db, so the
Prisma TLS dialect on DATABASE_URL_READ_REPLICA must be carried over or
a verify-only database rejects the reader and reads fall to the writer."""
from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true")
monkeypatch.setenv(
"DATABASE_URL_READ_REPLICA",
"postgresql://reader_user@reader.aurora.local:5432/litellm"
"?schema=tenant&sslmode=require&sslcert=/certs/root.pem&sslaccept=strict",
)
prisma_factory: Final = MagicMock(name="Prisma")
fake_prisma_module: Final = MagicMock(Prisma=prisma_factory)
monkeypatch.setitem(sys.modules, "prisma", fake_prisma_module)
fake_iam_module: Final = MagicMock(generate_iam_auth_token=MagicMock(return_value="READER-TOKEN"))
monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_iam_module)
from litellm.proxy.utils import PrismaClient
client: Final = PrismaClient(
database_url="postgresql://writer@writer.aurora.local:5432/litellm",
proxy_logging_obj=MagicMock(),
)
assert isinstance(client.db, RoutingPrismaWrapper)
reader_url: Final = os.environ["DATABASE_URL_READ_REPLICA"]
assert reader_url.startswith("postgresql://reader_user:READER-TOKEN@reader.aurora.local:5432/litellm?")
assert parse_qs(urlsplit(reader_url).query) == {
"schema": ["tenant"],
"sslmode": ["require"],
"sslcert": ["/certs/root.pem"],
"sslaccept": ["strict"],
}
assert prisma_factory.call_args_list == [call(), call(datasource={"url": reader_url})]
@pytest.mark.asyncio
async def test_connect_degrades_writer_when_reader_available():
"""A writer connect failure with a healthy reader must NOT abort proxy

View file

@ -187,6 +187,22 @@ def test_construct_database_url_from_env_vars_with_schema(monkeypatch):
}
def test_construct_database_url_from_env_vars_carries_tls_env(monkeypatch: pytest.MonkeyPatch):
"""The CLI password path builds its URL here, so DATABASE_SSLMODE and
DATABASE_SSLROOTCERT must reach PgBouncer through it too."""
monkeypatch.setenv("DATABASE_HOST", "db.example.com")
monkeypatch.setenv("DATABASE_USERNAME", "user")
monkeypatch.setenv("DATABASE_PASSWORD", "pass")
monkeypatch.setenv("DATABASE_NAME", "litellm")
monkeypatch.setenv("DATABASE_SCHEMA", "public")
monkeypatch.setenv("DATABASE_SSLMODE", "verify-full")
monkeypatch.setenv("DATABASE_SSLROOTCERT", "/etc/ssl/certs/ca-certificates.crt")
assert construct_database_url_from_env_vars() == (
"postgresql://user:pass@db.example.com/litellm"
"?schema=public&sslmode=verify-full&sslrootcert=%2Fetc%2Fssl%2Fcerts%2Fca-certificates.crt"
)
def test_construct_database_url_from_env_vars_error_path_missing_host(monkeypatch):
monkeypatch.delenv("DATABASE_HOST", raising=False)
monkeypatch.setenv("DATABASE_USERNAME", "user")