fix(otel): honor SSL_CERT_FILE and ssl_verify in OTLP HTTP exporters (#42106)

* fix(otel): honor SSL_CERT_FILE and ssl_verify in OTLP HTTP exporters

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(otel): assert OTLP HTTP TLS behavior against a real TLS sink

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(otel): assert rejected exports by outcome, not by exception type

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(otel): honor SSL_CERT_FILE and ssl_verify in the v2 OTLP HTTP exporters

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(otel): hoist otlp_tls imports and type the TLS sink fixture

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(e2e): gate the OTLP TLS export test behind an otel_tls opt-in

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* docs(e2e): drop CONTRIBUTING.md edit

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: yucheng <yucheng@berri.ai>
This commit is contained in:
devin-ai-integration[bot] 2026-09-22 20:50:53 +00:00 • committed by GitHub
parent e54f228ed8
commit b5b116a519
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 496 additions and 21 deletions

View file

@ -24,6 +24,7 @@ DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}"
JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
@ -122,7 +123,7 @@ SERVER_ENV=(
"CONFIG_FILE_PATH=${CONFIG_PATH}"
"STORE_MODEL_IN_DB=True"
"OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
"OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
"OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}"
"SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
"PYTHONPATH=${REPO_ROOT}"
"JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
@ -147,16 +148,12 @@ start_server() {
echo $! > "${PIDS_DIR}/${name}.pid"
}
start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
if [[ "$(uname)" == "Linux" ]]; then
NGINX_UPSTREAM_HOST=127.0.0.1
NGINX_DOCKER_ARGS=(--network host)
else
NGINX_UPSTREAM_HOST=host.docker.internal
NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}")
NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}")
fi
cat > "${STACK_DIR}/nginx.conf" <<EOF
@ -186,12 +183,29 @@ http {
proxy_send_timeout 600s;
}
}
server {
listen ${JAEGER_OTLP_TLS_PORT} ssl;
ssl_certificate /certs/server.crt;
ssl_certificate_key /certs/server.key;
client_max_body_size 100m;
location / {
proxy_pass http://${NGINX_UPSTREAM_HOST}:${JAEGER_OTLP_PORT};
}
}
}
EOF
docker rm -f e2e-nginx >/dev/null 2>&1 || true
docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
-v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null
-v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \
-v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null
wait_for "Jaeger OTLP TLS listener" \
"curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'"
start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
@ -206,6 +220,7 @@ LITELLM_MASTER_KEY=${MASTER_KEY}
REDIS_HOST=127.0.0.1
REDIS_PORT=${REDIS_PORT}
E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
E2E_OTEL_EXPORTER_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}
E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
E2E_KEYCLOAK_ADMIN_USER=admin
E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret

View file

@ -26,6 +26,7 @@ from litellm.integrations.otel.model.baggage import promoted_metadata
from litellm.integrations.otel.model.db_endpoint import db_span_attributes
from litellm.integrations.otel.model.metadata import flatten_metadata
from litellm.integrations.otel.model.semconv import Metric
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
@ -99,6 +100,7 @@ _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
LITELLM_TRACER_NAME: Final = os.getenv("OTEL_TRACER_NAME", "litellm")
LITELLM_METER_NAME: Final = os.getenv("LITELLM_METER_NAME", "litellm")
LITELLM_LOGGER_NAME: Final = os.getenv("LITELLM_LOGGER_NAME", "litellm")
@ -3090,8 +3092,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
otel_exporter,
)
normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces")
tls: Final = resolve_otlp_http_tls("TRACES")
return BatchSpanProcessor(
OTLPSpanExporterHTTP(endpoint=normalized_endpoint, headers=_split_otel_headers),
OTLPSpanExporterHTTP(
endpoint=normalized_endpoint,
headers=_split_otel_headers,
certificate_file=tls.certificate_file,
session=tls.session,
),
)
elif otel_exporter == "otlp_grpc" or otel_exporter == "grpc":
try:
@ -3172,7 +3180,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
self.OTEL_EXPORTER,
normalized_endpoint,
)
return OTLPLogExporter(endpoint=normalized_endpoint, headers=_split_otel_headers)
tls: Final = resolve_otlp_http_tls("LOGS")
return OTLPLogExporter(
endpoint=normalized_endpoint,
headers=_split_otel_headers,
certificate_file=tls.certificate_file,
session=tls.session,
)
elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc":
try:
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import (
@ -3235,9 +3249,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
OTLPMetricExporter,
)
tls: Final = resolve_otlp_http_tls("METRICS")
exporter = OTLPMetricExporter(
endpoint=normalized_endpoint,
headers=_split_otel_headers,
certificate_file=tls.certificate_file,
session=tls.session,
)
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)

View file

@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeAlias
import requests
from google.protobuf.json_format import MessageToDict
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
@ -62,8 +63,14 @@ def encode_spans_json(spans: Sequence[ReadableSpan]) -> bytes:
class OTLPJsonSpanExporter(OTLPSpanExporter):
def __init__(self, endpoint: str | None, headers: dict[str, str]) -> None: # mutable-ok: SDK __init__ takes Dict
super().__init__(endpoint=endpoint, headers=headers)
def __init__(
self,
endpoint: str | None,
headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict
certificate_file: str | None = None,
session: "requests.Session | None" = None,
) -> None:
super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session)
self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:

View file

@ -0,0 +1,41 @@
import os
from dataclasses import dataclass
from typing import Final, Literal
import requests
from requests.adapters import HTTPAdapter
@dataclass(frozen=True, slots=True)
class OtlpHttpTls:
certificate_file: str | None
session: requests.Session | None
class _NoVerifyAdapter(HTTPAdapter):
def cert_verify(
self,
conn: object,
url: str,
verify: bool | str,
cert: str | tuple[str, str] | None,
) -> None:
super().cert_verify( # pyright: ignore[reportUnknownMemberType] # requests stubs omit HTTPAdapter.cert_verify
conn, url, False, cert
)
def resolve_otlp_http_tls(signal: Literal["TRACES", "METRICS", "LOGS"]) -> OtlpHttpTls:
if os.getenv(f"OTEL_EXPORTER_OTLP_{signal}_CERTIFICATE") or os.getenv("OTEL_EXPORTER_OTLP_CERTIFICATE"):
return OtlpHttpTls(certificate_file=None, session=None)
from litellm.llms.custom_httpx.http_handler import get_ssl_verify
verify: Final = get_ssl_verify()
if verify is False:
session: Final = requests.Session()
session.mount("https://", _NoVerifyAdapter())
return OtlpHttpTls(certificate_file=None, session=session)
if isinstance(verify, str):
return OtlpHttpTls(certificate_file=verify, session=None)
return OtlpHttpTls(certificate_file=None, session=None)

View file

@ -58,6 +58,7 @@ from litellm.integrations.otel.plumbing.context import (
request_destinations,
suppressed_backends,
)
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
if TYPE_CHECKING:
from opentelemetry.metrics import Meter
@ -193,18 +194,24 @@ def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter:
if kind in _OTLP_HTTP_JSON_KINDS:
from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter
tls: Final = resolve_otlp_http_tls("TRACES")
return OTLPJsonSpanExporter(
endpoint=spec.traces_endpoint or _otlp_traces_endpoint(spec.endpoint),
headers=parse_headers(spec.headers),
certificate_file=tls.certificate_file,
session=tls.session,
)
if kind in _OTLP_HTTP_KINDS:
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter as HTTPExporter,
)
http_tls: Final = resolve_otlp_http_tls("TRACES")
return HTTPExporter(
endpoint=spec.traces_endpoint or _otlp_traces_endpoint(spec.endpoint),
headers=parse_headers(spec.headers),
certificate_file=http_tls.certificate_file,
session=http_tls.session,
)
if kind in _OTLP_GRPC_KINDS:
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
@ -902,9 +909,12 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader":
OTLPMetricExporter as HTTPMetricExporter,
)
tls: Final = resolve_otlp_http_tls("METRICS")
exporter: Any = HTTPMetricExporter(
endpoint=_otlp_metrics_endpoint(config.endpoint),
headers=parse_headers(config.headers),
certificate_file=tls.certificate_file,
session=tls.session,
)
elif kind in ("otlp_grpc", "grpc"):
try:
@ -962,9 +972,12 @@ def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter:
OTLPLogExporter as HTTPLogExporter,
)
tls: Final = resolve_otlp_http_tls("LOGS")
return HTTPLogExporter(
endpoint=_otlp_logs_endpoint(config.endpoint),
headers=parse_headers(config.headers),
certificate_file=tls.certificate_file,
session=tls.session,
)
if kind in ("otlp_grpc", "grpc"):
try:

View file

@ -30,6 +30,7 @@ from e2e_config import (
FIXTURE_MODE_RAW,
MANAGED_FILES_OPT_IN_ENV,
MCP_OAUTH_LIVE_OPT_IN_ENV,
OTEL_TLS_OPT_IN_ENV,
OTEL_V2_OPT_IN_ENV,
PROMPT_CACHING_OPT_IN_ENV,
PROVIDER_EDGE_HOST_OPT_IN_ENV,
@ -63,6 +64,7 @@ OPT_IN_MARKERS: Final = MappingProxyType(
"mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV,
"provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV,
"otel_v2": OTEL_V2_OPT_IN_ENV,
"otel_tls": OTEL_TLS_OPT_IN_ENV,
}
)
@ -156,6 +158,10 @@ def pytest_configure(config: pytest.Config) -> None:
"markers",
"otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set",
)
config.addinivalue_line(
"markers",
"otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set",
)
def pytest_sessionstart(session: pytest.Session) -> None:

View file

@ -58,6 +58,7 @@ LINEAR_READONLY_TOOL: Final = "list_teams" # as listed by tools/list on mcp.lin
# service in docker-compose.yml maps it to host 16686). Trace-completeness tests
# read exported spans back through it.
OTEL_QUERY_URL = os.environ.get("E2E_OTEL_QUERY_URL", "http://localhost:16686").rstrip("/")
OTEL_EXPORTER_ENDPOINT = os.environ.get("E2E_OTEL_EXPORTER_ENDPOINT", "")
# Real-DataDog read-back (no local sink - destination fakes cannot be deployed
# on the cluster): the proxy delivers with DD_API_KEY as in production, and the
@ -148,6 +149,7 @@ CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM"
MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE"
PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE"
OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2"
OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT"
ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))

View file

@ -12,21 +12,24 @@ commit 1bd603d1ac).
Both halves of the contract are asserted: the recorded state (the proxy reports
the OTEL v2 logger active via /health/readiness/details) and the enforced
behavior (the complete span tree at the destination, read back through the
destination's own query API - never proxy-side "export succeeded" logs).
destination's own query API - never proxy-side "export succeeded" logs). The
TLS coverage requires the stack to export OTLP over HTTPS with a certificate
signed by the CA in SSL_CERT_FILE, and treats a missing or plaintext endpoint
as a stack misconfiguration rather than skipping the test.
"""
from __future__ import annotations
import time
from typing import Final
import pytest
from pydantic import BaseModel, ConfigDict, ValidationError
from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, OTEL_EXPORTER_ENDPOINT, unique_marker
from lifecycle import ResourceManager
from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient, first_ok, readiness_details_body
from models import LiteLLMParamsBody
from otel_client import JaegerSpan, JaegerTrace, OtelReader
from pydantic import BaseModel, ConfigDict, ValidationError
pytestmark = pytest.mark.e2e
@ -312,6 +315,35 @@ class TestOtelTraceCompleteness:
)
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
@pytest.mark.otel_tls
def test_otel_export_over_tls_with_internal_ca_reaches_destination(
self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager
) -> None:
_assert_otel_destination_configured(client)
assert OTEL_EXPORTER_ENDPOINT.startswith("https://"), (
"the stack must export OTLP over TLS signed by the CA in SSL_CERT_FILE "
"(E2E_OTEL_EXPORTER_ENDPOINT) for this test to prove anything; a "
"missing or plaintext value is a stack misconfiguration"
)
route: Final = "/chat/completions"
key: Final = client.key_with_alias(f"otel-trace-tls-{unique_marker()}", models=[MODEL])
resources.defer(lambda: client.delete_key(key))
marker: Final = unique_marker()
outcome: Final = first_ok(
client, lambda: client.chat_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16)
)
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
hits: Final = otel_reader.poll_traces_for_call(
call_id=outcome.call_id,
settled_names=_settled_names(route=route, genai_span=f"chat {MODEL}"),
settled_prefixes={DB_SPAN_PREFIX},
)
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
def test_messages_exports_complete_trace(
self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager

View file

@ -15,3 +15,4 @@ markers =
mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set
provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set
otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set
otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set

View file

@ -0,0 +1,96 @@
import functools
import http.server
import ipaddress
import queue
import ssl
import threading
from collections.abc import Iterator
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Final
import pytest
@dataclass(frozen=True, slots=True)
class TlsSink:
url: str
certificate_path: str
received: "queue.Queue[str]"
class _RecordingOtelHandler(http.server.BaseHTTPRequestHandler):
def __init__(self, *args: object, received: "queue.Queue[str]", **kwargs: object) -> None:
self._received: Final = received
super().__init__(*args, **kwargs)
def do_POST(self) -> None:
length: Final = int(self.headers.get("Content-Length") or 0)
if length:
self.rfile.read(length)
self._received.put(self.path)
self.send_response(200)
self.send_header("Content-Type", "application/x-protobuf")
self.send_header("Content-Length", "0")
self.end_headers()
def log_message(self, format: str, *args: object) -> None:
pass
def write_self_signed_cert(directory: Path, stem: str) -> tuple[Path, Path]:
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])
certificate: Final = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(datetime.now(timezone.utc) - timedelta(minutes=1))
.not_valid_after(datetime.now(timezone.utc) + timedelta(hours=1))
.add_extension(
x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]),
critical=False,
)
.sign(key, hashes.SHA256())
)
certificate_path: Final = directory / f"{stem}.crt"
certificate_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
key_path: Final = directory / f"{stem}.key"
key_path.write_bytes(
key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.TraditionalOpenSSL,
serialization.NoEncryption(),
)
)
return certificate_path, key_path
@pytest.fixture
def tls_sink(tmp_path: Path) -> Iterator[TlsSink]:
certificate_path, key_path = write_self_signed_cert(tmp_path, "sink")
received: queue.Queue[str] = queue.Queue()
context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(str(certificate_path), str(key_path))
server: Final = http.server.ThreadingHTTPServer(
("127.0.0.1", 0), functools.partial(_RecordingOtelHandler, received=received)
)
server.socket = context.wrap_socket(server.socket, server_side=True)
thread: Final = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
yield TlsSink(
url=f"https://127.0.0.1:{server.server_port}",
certificate_path=str(certificate_path),
received=received,
)
server.shutdown()
server.server_close()
thread.join(timeout=5)

View file

@ -2,14 +2,17 @@
baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name
builders, and the registry validator's failure paths. Needs the OTel SDK."""
import contextlib
import json
import threading
import time
from collections.abc import Iterator
from contextvars import Context as ContextVarContext
from dataclasses import replace
from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer
import pytest
import requests
pytest.importorskip("opentelemetry")
@ -18,6 +21,9 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( # noqa:
)
from opentelemetry import baggage # noqa: E402
from opentelemetry.context import attach, detach # noqa: E402
from opentelemetry._logs.severity import SeverityNumber # noqa: E402
from opentelemetry.sdk._logs import LogData, LogRecord # noqa: E402
from opentelemetry.sdk._logs.export import LogExportResult # noqa: E402
from opentelemetry.sdk.metrics import MeterProvider # noqa: E402
from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402
from opentelemetry.sdk.trace import TracerProvider # noqa: E402
@ -29,11 +35,14 @@ from opentelemetry.sdk.trace.export import ( # noqa: E402
from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402
InMemorySpanExporter,
)
from opentelemetry.trace import SpanKind, get_current_span # noqa: E402
from opentelemetry.sdk.util.instrumentation import InstrumentationScope # noqa: E402
from opentelemetry.trace import SpanKind, TraceFlags, get_current_span # noqa: E402
from opentelemetry.trace.propagation.tracecontext import ( # noqa: E402
TraceContextTextMapPropagator,
)
import litellm # noqa: E402
from conftest import TlsSink # noqa: E402
from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402
from litellm.integrations.otel.plumbing import providers # noqa: E402
from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402
@ -1414,3 +1423,97 @@ def test_genai_mapper_guardrail_cost_in_spend_attr():
billed = dict(entry)
del billed["guardrail_cost_in_spend"]
assert LiteLLM.GUARDRAIL_COST_IN_SPEND not in GenAIMapper().map(GuardrailSpanData.from_logging_entry(billed))
def _isolate_v2_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None:
for key in (
"SSL_VERIFY",
"SSL_CERT_FILE",
"OTEL_EXPORTER_OTLP_CERTIFICATE",
"OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE",
"OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE",
"OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE",
):
monkeypatch.delenv(key, raising=False)
monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "2")
monkeypatch.setattr(litellm, "ssl_verify", True)
def test_v2_otlp_http_span_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None:
_isolate_v2_otlp_tls_env(monkeypatch)
monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path)
cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url)
_export_one_span(cfg)
assert tls_sink.received.get(timeout=5) == "/v1/traces"
def test_v2_http_json_span_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None:
_isolate_v2_otlp_tls_env(monkeypatch)
monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path)
cfg = OpenTelemetryV2Config(exporter="http/json", endpoint=tls_sink.url)
_export_one_span(cfg)
assert tls_sink.received.get(timeout=5) == "/v1/traces"
def test_v2_otlp_http_metric_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None:
_isolate_v2_otlp_tls_env(monkeypatch)
monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path)
cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url)
reader = providers.build_metric_reader(cfg)
provider = MeterProvider(metric_readers=[reader])
try:
provider.get_meter("v2-tls-test").create_counter("tls_export_test").add(1)
assert provider.force_flush(), "metric flush failed"
assert tls_sink.received.get(timeout=5) == "/v1/metrics"
finally:
provider.shutdown()
def test_v2_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None:
_isolate_v2_otlp_tls_env(monkeypatch)
monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path)
cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url)
exporter = providers.build_log_exporter(cfg)
try:
record = LogRecord(
timestamp=int(time.time() * 1e9),
observed_timestamp=int(time.time() * 1e9),
trace_id=0,
span_id=0,
trace_flags=TraceFlags(0),
severity_number=SeverityNumber.INFO,
body="v2-tls-test",
)
log_data = LogData(log_record=record, instrumentation_scope=InstrumentationScope("v2-tls-test"))
result = exporter.export([log_data])
assert result is LogExportResult.SUCCESS, f"log export failed: {result}"
assert tls_sink.received.get(timeout=5) == "/v1/logs"
finally:
exporter.shutdown()
def test_v2_otlp_http_export_skips_verification_when_ssl_verify_false(
monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink
) -> None:
_isolate_v2_otlp_tls_env(monkeypatch)
monkeypatch.setenv("SSL_VERIFY", "false")
cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url)
_export_one_span(cfg)
assert tls_sink.received.get(timeout=5) == "/v1/traces"
def test_v2_otlp_http_export_rejects_untrusted_collector_by_default(
monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink
) -> None:
_isolate_v2_otlp_tls_env(monkeypatch)
cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url)
provider = providers.build_tracer_provider(cfg)
provider.get_tracer("probe").start_span("probe").end()
try:
with contextlib.suppress(requests.exceptions.SSLError):
provider.force_flush()
assert tls_sink.received.empty(), "sink received a request it should never have trusted"
finally:
provider.shutdown()

View file

@ -1,5 +1,6 @@
import asyncio
import concurrent.futures
import contextlib
import gc
import json
import os
@ -9,21 +10,30 @@ import time
import unittest
import weakref
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import MappingProxyType
from parameterized import parameterized
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
# Adds the grandparent directory to sys.path to allow importing project modules
from opentelemetry import trace
from opentelemetry.sdk._logs import LogData
from opentelemetry._logs.severity import SeverityNumber
from opentelemetry.sdk._logs import LogData, LogRecord
from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor
from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult
from opentelemetry.sdk.util.instrumentation import InstrumentationScope
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from parameterized import parameterized
import requests
from conftest import TlsSink, write_self_signed_cert
import litellm
from litellm.integrations import opentelemetry as otel_module
from litellm.integrations.opentelemetry import (
@ -2081,6 +2091,138 @@ class TestOpenTelemetryEndpointNormalization(unittest.TestCase):
self.assertEqual(traces, "http://collector:4318/v1/traces")
def _isolate_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None:
for key in (
"SSL_VERIFY",
"SSL_CERT_FILE",
"OTEL_EXPORTER_OTLP_CERTIFICATE",
"OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE",
"OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE",
"OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE",
):
monkeypatch.delenv(key, raising=False)
monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "2")
monkeypatch.setattr(litellm, "ssl_verify", True)
def _ended_span() -> tuple[TracerProvider, ReadableSpan]:
provider: Final = TracerProvider()
span = provider.get_tracer(__name__).start_span("tls-export-test")
span.end()
return provider, span
def _assert_export_rejected(processor, span: ReadableSpan, sink: TlsSink) -> None:
with contextlib.suppress(requests.exceptions.SSLError):
result: Final = processor.span_exporter.export([span])
assert result is SpanExportResult.FAILURE, f"rejected export must report failure, got {result}"
assert sink.received.empty(), "sink received a request it should never have trusted"
def _otlp_http_otel(endpoint: str) -> OpenTelemetry:
return OpenTelemetry(config=OpenTelemetryConfig(exporter="otlp_http", endpoint=endpoint))
def test_otlp_http_span_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None:
_isolate_otlp_tls_env(monkeypatch)
monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path)
otel: Final = _otlp_http_otel(tls_sink.url)
processor: Final = otel._get_span_processor()
provider, span = _ended_span()
try:
result: Final = processor.span_exporter.export([span])
assert result is SpanExportResult.SUCCESS, f"span export failed: {result}"
assert tls_sink.received.get(timeout=5) == "/v1/traces"
finally:
processor.shutdown()
provider.shutdown()
def test_otlp_http_metric_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None:
_isolate_otlp_tls_env(monkeypatch)
monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path)
otel: Final = _otlp_http_otel(tls_sink.url)
reader: Final = otel._get_metric_reader()
provider: Final = MeterProvider(metric_readers=[reader])
try:
provider.get_meter(__name__).create_counter("tls_export_test").add(1)
assert provider.force_flush(), "metric flush failed"
assert tls_sink.received.get(timeout=5) == "/v1/metrics"
finally:
provider.shutdown()
def test_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None:
_isolate_otlp_tls_env(monkeypatch)
monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path)
otel: Final = _otlp_http_otel(tls_sink.url)
exporter: Final = otel._get_log_exporter()
try:
record: Final = LogRecord(
timestamp=int(time.time() * 1e9),
observed_timestamp=int(time.time() * 1e9),
trace_id=0,
span_id=0,
trace_flags=trace.TraceFlags(0),
severity_number=SeverityNumber.INFO,
body="tls-export-test",
)
log_data: Final = LogData(log_record=record, instrumentation_scope=InstrumentationScope("tls-export-test"))
result: Final = exporter.export([log_data])
assert result is LogExportResult.SUCCESS, f"log export failed: {result}"
assert tls_sink.received.get(timeout=5) == "/v1/logs"
finally:
exporter.shutdown()
def test_otlp_http_export_skips_verification_when_ssl_verify_false(
monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink
) -> None:
_isolate_otlp_tls_env(monkeypatch)
monkeypatch.setenv("SSL_VERIFY", "false")
otel: Final = _otlp_http_otel(tls_sink.url)
processor: Final = otel._get_span_processor()
provider, span = _ended_span()
try:
result: Final = processor.span_exporter.export([span])
assert result is SpanExportResult.SUCCESS, f"span export failed: {result}"
assert tls_sink.received.get(timeout=5) == "/v1/traces"
finally:
processor.shutdown()
provider.shutdown()
def test_otlp_http_export_rejects_untrusted_collector_by_default(
monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink
) -> None:
_isolate_otlp_tls_env(monkeypatch)
otel: Final = _otlp_http_otel(tls_sink.url)
processor: Final = otel._get_span_processor()
provider, span = _ended_span()
try:
_assert_export_rejected(processor, span, tls_sink)
finally:
processor.shutdown()
provider.shutdown()
def test_otel_certificate_env_takes_precedence_over_ssl_cert_file(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, tls_sink: TlsSink
) -> None:
_isolate_otlp_tls_env(monkeypatch)
unrelated_certificate, _ = write_self_signed_cert(tmp_path, "unrelated")
monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path)
monkeypatch.setenv("OTEL_EXPORTER_OTLP_CERTIFICATE", str(unrelated_certificate))
otel: Final = _otlp_http_otel(tls_sink.url)
processor: Final = otel._get_span_processor()
provider, span = _ended_span()
try:
_assert_export_rejected(processor, span, tls_sink)
finally:
processor.shutdown()
provider.shutdown()
class TestOpenTelemetryProtocolSelection(unittest.TestCase):
"""Test suite for verifying correct exporter selection based on protocol"""