fix(otel): honor SSL_CERT_FILE and ssl_verify in OTLP HTTP exporters

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-20 07:25:29 +00:00
parent 776f8bc309
commit 29fe595e6a
6 changed files with 179 additions and 16 deletions

View file

@ -24,6 +24,7 @@ DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}"
JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
@ -122,7 +123,7 @@ SERVER_ENV=(
"CONFIG_FILE_PATH=${CONFIG_PATH}"
"STORE_MODEL_IN_DB=True"
"OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
"OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
"OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}"
"SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
"PYTHONPATH=${REPO_ROOT}"
"JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
@ -147,16 +148,12 @@ start_server() {
echo $! > "${PIDS_DIR}/${name}.pid"
}
start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
if [[ "$(uname)" == "Linux" ]]; then
NGINX_UPSTREAM_HOST=127.0.0.1
NGINX_DOCKER_ARGS=(--network host)
else
NGINX_UPSTREAM_HOST=host.docker.internal
NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}")
NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}")
fi
cat > "${STACK_DIR}/nginx.conf" <<EOF
@ -186,12 +183,29 @@ http {
proxy_send_timeout 600s;
}
}
server {
listen ${JAEGER_OTLP_TLS_PORT} ssl;
ssl_certificate /certs/server.crt;
ssl_certificate_key /certs/server.key;
client_max_body_size 100m;
location / {
proxy_pass http://${NGINX_UPSTREAM_HOST}:${JAEGER_OTLP_PORT};
}
}
}
EOF
docker rm -f e2e-nginx >/dev/null 2>&1 || true
docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
-v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null
-v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \
-v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null
wait_for "Jaeger OTLP TLS listener" \
"curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'"
start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
@ -206,6 +220,7 @@ LITELLM_MASTER_KEY=${MASTER_KEY}
REDIS_HOST=127.0.0.1
REDIS_PORT=${REDIS_PORT}
E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
E2E_OTEL_EXPORTER_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}
E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
E2E_KEYCLOAK_ADMIN_USER=admin
E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret

View file

@ -6,7 +6,10 @@ from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, cast
import requests
from requests.adapters import HTTPAdapter
import litellm
from litellm._logging import verbose_logger
@ -99,6 +102,42 @@ _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
@dataclass(frozen=True, slots=True)
class _OtlpHttpTls:
certificate_file: str | None
session: requests.Session | None
class _NoVerifyAdapter(HTTPAdapter):
def cert_verify(
self,
conn: object,
url: str,
verify: bool | str,
cert: str | tuple[str, str] | None,
) -> None:
super().cert_verify( # pyright: ignore[reportUnknownMemberType] # requests stubs omit HTTPAdapter.cert_verify
conn, url, False, cert
)
def _resolve_otlp_http_tls(signal: Literal["TRACES", "METRICS", "LOGS"]) -> _OtlpHttpTls:
if os.getenv(f"OTEL_EXPORTER_OTLP_{signal}_CERTIFICATE") or os.getenv("OTEL_EXPORTER_OTLP_CERTIFICATE"):
return _OtlpHttpTls(certificate_file=None, session=None)
from litellm.llms.custom_httpx.http_handler import get_ssl_verify
verify: Final = get_ssl_verify()
if verify is False:
session: Final = requests.Session()
session.mount("https://", _NoVerifyAdapter())
return _OtlpHttpTls(certificate_file=None, session=session)
if isinstance(verify, str):
return _OtlpHttpTls(certificate_file=verify, session=None)
return _OtlpHttpTls(certificate_file=None, session=None)
LITELLM_TRACER_NAME: Final = os.getenv("OTEL_TRACER_NAME", "litellm")
LITELLM_METER_NAME: Final = os.getenv("LITELLM_METER_NAME", "litellm")
LITELLM_LOGGER_NAME: Final = os.getenv("LITELLM_LOGGER_NAME", "litellm")
@ -3084,8 +3123,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
otel_exporter,
)
normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces")
tls: Final = _resolve_otlp_http_tls("TRACES")
return BatchSpanProcessor(
OTLPSpanExporterHTTP(endpoint=normalized_endpoint, headers=_split_otel_headers),
OTLPSpanExporterHTTP(
endpoint=normalized_endpoint,
headers=_split_otel_headers,
certificate_file=tls.certificate_file,
session=tls.session,
),
)
elif otel_exporter == "otlp_grpc" or otel_exporter == "grpc":
try:
@ -3166,7 +3211,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
self.OTEL_EXPORTER,
normalized_endpoint,
)
return OTLPLogExporter(endpoint=normalized_endpoint, headers=_split_otel_headers)
tls: Final = _resolve_otlp_http_tls("LOGS")
return OTLPLogExporter(
endpoint=normalized_endpoint,
headers=_split_otel_headers,
certificate_file=tls.certificate_file,
session=tls.session,
)
elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc":
try:
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import (
@ -3229,9 +3280,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
OTLPMetricExporter,
)
tls: Final = _resolve_otlp_http_tls("METRICS")
exporter = OTLPMetricExporter(
endpoint=normalized_endpoint,
headers=_split_otel_headers,
certificate_file=tls.certificate_file,
session=tls.session,
)
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)

View file

@ -90,7 +90,7 @@ The suites run against a live proxy, so bring one up first by running the litell
They also need a proxy whose bundled UI contains the change under test, so run the proxy from your branch (an editable install serves the UI your checkout builds)
Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a Jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); the CI stack exports over TLS with its stack CA in `SSL_CERT_FILE`. Run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
The Presidio spend-log audit test also requires prompt storage on the proxy, because its assertions inspect the detected entities and their scores. Add the following to the proxy config before starting it:

View file

@ -58,6 +58,7 @@ LINEAR_READONLY_TOOL: Final = "list_teams" # as listed by tools/list on mcp.lin
# service in docker-compose.yml maps it to host 16686). Trace-completeness tests
# read exported spans back through it.
OTEL_QUERY_URL = os.environ.get("E2E_OTEL_QUERY_URL", "http://localhost:16686").rstrip("/")
OTEL_EXPORTER_ENDPOINT = os.environ.get("E2E_OTEL_EXPORTER_ENDPOINT", "")
# Real-DataDog read-back (no local sink - destination fakes cannot be deployed
# on the cluster): the proxy delivers with DD_API_KEY as in production, and the

View file

@ -12,21 +12,24 @@ commit 1bd603d1ac).
Both halves of the contract are asserted: the recorded state (the proxy reports
the OTEL v2 logger active via /health/readiness/details) and the enforced
behavior (the complete span tree at the destination, read back through the
destination's own query API - never proxy-side "export succeeded" logs).
destination's own query API - never proxy-side "export succeeded" logs). The
TLS coverage requires the stack to export OTLP over HTTPS with a certificate
signed by the CA in SSL_CERT_FILE, and treats a missing or plaintext endpoint
as a stack misconfiguration rather than skipping the test.
"""
from __future__ import annotations
import time
from typing import Final
import pytest
from pydantic import BaseModel, ConfigDict, ValidationError
from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, OTEL_EXPORTER_ENDPOINT, unique_marker
from lifecycle import ResourceManager
from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient, first_ok, readiness_details_body
from models import LiteLLMParamsBody
from otel_client import JaegerSpan, JaegerTrace, OtelReader
from pydantic import BaseModel, ConfigDict, ValidationError
pytestmark = pytest.mark.e2e
@ -312,6 +315,34 @@ class TestOtelTraceCompleteness:
)
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
def test_otel_export_over_tls_with_internal_ca_reaches_destination(
self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager
) -> None:
_assert_otel_destination_configured(client)
assert OTEL_EXPORTER_ENDPOINT.startswith("https://"), (
"the stack must export OTLP over TLS signed by the CA in SSL_CERT_FILE "
"(E2E_OTEL_EXPORTER_ENDPOINT) for this test to prove anything; a "
"missing or plaintext value is a stack misconfiguration"
)
route: Final = "/chat/completions"
key: Final = client.key_with_alias(f"otel-trace-tls-{unique_marker()}", models=[MODEL])
resources.defer(lambda: client.delete_key(key))
marker: Final = unique_marker()
outcome: Final = first_ok(
client, lambda: client.chat_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16)
)
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
hits: Final = otel_reader.poll_traces_for_call(
call_id=outcome.call_id,
settled_names=_settled_names(route=route, genai_span=f"chat {MODEL}"),
settled_prefixes={DB_SPAN_PREFIX},
)
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
def test_messages_exports_complete_trace(
self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager

View file

@ -9,10 +9,13 @@ import time
import unittest
import weakref
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import MappingProxyType
from parameterized import parameterized
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
# Adds the grandparent directory to sys.path to allow importing project modules
from opentelemetry import trace
from opentelemetry.sdk._logs import LogData
@ -23,6 +26,7 @@ from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from parameterized import parameterized
import litellm
from litellm.integrations import opentelemetry as otel_module
@ -2081,6 +2085,64 @@ class TestOpenTelemetryEndpointNormalization(unittest.TestCase):
self.assertEqual(traces, "http://collector:4318/v1/traces")
def test_resolve_otlp_http_tls_uses_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
certificate_file: Final = tmp_path / "ca.pem"
certificate_file.write_text("certificate")
monkeypatch.setenv("SSL_CERT_FILE", str(certificate_file))
monkeypatch.delenv("SSL_VERIFY", raising=False)
monkeypatch.delenv("OTEL_EXPORTER_OTLP_CERTIFICATE", raising=False)
monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", raising=False)
tls: Final = otel_module._resolve_otlp_http_tls("TRACES")
assert tls.certificate_file == str(certificate_file)
assert tls.session is None
def test_resolve_otlp_http_tls_disables_verification(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("SSL_VERIFY", "false")
monkeypatch.delenv("SSL_CERT_FILE", raising=False)
monkeypatch.delenv("OTEL_EXPORTER_OTLP_CERTIFICATE", raising=False)
monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", raising=False)
tls: Final = otel_module._resolve_otlp_http_tls("TRACES")
assert tls.certificate_file is None
assert tls.session is not None
adapter: Final = tls.session.get_adapter("https://example.invalid")
connection: Final = adapter.poolmanager.connection_from_url("https://example.invalid")
adapter.cert_verify(connection, "https://example.invalid", True, None)
assert connection.cert_reqs == "CERT_NONE"
def test_resolve_otlp_http_tls_prefers_otel_certificate(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
certificate_file: Final = tmp_path / "ca.pem"
certificate_file.write_text("certificate")
monkeypatch.setenv("SSL_CERT_FILE", str(certificate_file))
monkeypatch.setenv("OTEL_EXPORTER_OTLP_CERTIFICATE", "/otel-ca.pem")
tls: Final = otel_module._resolve_otlp_http_tls("TRACES")
assert tls.certificate_file is None
assert tls.session is None
def test_http_span_exporter_uses_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
certificate_file: Final = tmp_path / "ca.pem"
certificate_file.write_text("certificate")
monkeypatch.setenv("SSL_CERT_FILE", str(certificate_file))
monkeypatch.delenv("SSL_VERIFY", raising=False)
monkeypatch.delenv("OTEL_EXPORTER_OTLP_CERTIFICATE", raising=False)
monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", raising=False)
config: Final = OpenTelemetryConfig(exporter="otlp_http", endpoint="https://collector.invalid:4318")
otel: Final = OpenTelemetry(config=config)
processor: Final = otel._get_span_processor()
assert processor.span_exporter._certificate_file == str(certificate_file)
class TestOpenTelemetryProtocolSelection(unittest.TestCase):
"""Test suite for verifying correct exporter selection based on protocol"""