From 29fe595e6a71bc8cb0bbeaa249eb12be3e99ff1a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 07:25:29 +0000 Subject: [PATCH] fix(otel): honor SSL_CERT_FILE and ssl_verify in OTLP HTTP exporters Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/e2e-stack/up.sh | 29 +++++++-- litellm/integrations/opentelemetry.py | 60 ++++++++++++++++- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/e2e_config.py | 1 + tests/e2e/logging/test_otel_trace_e2e.py | 39 +++++++++-- .../integrations/test_opentelemetry.py | 64 ++++++++++++++++++- 6 files changed, 179 insertions(+), 16 deletions(-) diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index a789a570483..9f3052fa58d 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -24,6 +24,7 @@ DATABASE_USER="${E2E_DATABASE_USER:-litellm}" DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" +JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}" JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" @@ -122,7 +123,7 @@ SERVER_ENV=( "CONFIG_FILE_PATH=${CONFIG_PATH}" "STORE_MODEL_IN_DB=True" "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" - "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" + "OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}" "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" "PYTHONPATH=${REPO_ROOT}" "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" @@ -147,16 +148,12 @@ start_server() { echo $! > "${PIDS_DIR}/${name}.pid" } -start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" -start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" -start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" - if [[ "$(uname)" == "Linux" ]]; then NGINX_UPSTREAM_HOST=127.0.0.1 NGINX_DOCKER_ARGS=(--network host) else NGINX_UPSTREAM_HOST=host.docker.internal - NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}") + NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}") fi cat > "${STACK_DIR}/nginx.conf" </dev/null 2>&1 || true docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \ - -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null + -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \ + -v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null + +wait_for "Jaeger OTLP TLS listener" \ + "curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'" + +start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" +start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" +start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300 wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300 @@ -206,6 +220,7 @@ LITELLM_MASTER_KEY=${MASTER_KEY} REDIS_HOST=127.0.0.1 REDIS_PORT=${REDIS_PORT} E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT} +E2E_OTEL_EXPORTER_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT} E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT} E2E_KEYCLOAK_ADMIN_USER=admin E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 180929bcfd4..ccaa0fc66f0 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -6,7 +6,10 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import datetime from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, TypedDict, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, cast + +import requests +from requests.adapters import HTTPAdapter import litellm from litellm._logging import verbose_logger @@ -99,6 +102,42 @@ _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256 # Dedicated so a slow exporter shutdown cannot starve the shared logging executor. _PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown") + +@dataclass(frozen=True, slots=True) +class _OtlpHttpTls: + certificate_file: str | None + session: requests.Session | None + + +class _NoVerifyAdapter(HTTPAdapter): + def cert_verify( + self, + conn: object, + url: str, + verify: bool | str, + cert: str | tuple[str, str] | None, + ) -> None: + super().cert_verify( # pyright: ignore[reportUnknownMemberType] # requests stubs omit HTTPAdapter.cert_verify + conn, url, False, cert + ) + + +def _resolve_otlp_http_tls(signal: Literal["TRACES", "METRICS", "LOGS"]) -> _OtlpHttpTls: + if os.getenv(f"OTEL_EXPORTER_OTLP_{signal}_CERTIFICATE") or os.getenv("OTEL_EXPORTER_OTLP_CERTIFICATE"): + return _OtlpHttpTls(certificate_file=None, session=None) + + from litellm.llms.custom_httpx.http_handler import get_ssl_verify + + verify: Final = get_ssl_verify() + if verify is False: + session: Final = requests.Session() + session.mount("https://", _NoVerifyAdapter()) + return _OtlpHttpTls(certificate_file=None, session=session) + if isinstance(verify, str): + return _OtlpHttpTls(certificate_file=verify, session=None) + return _OtlpHttpTls(certificate_file=None, session=None) + + LITELLM_TRACER_NAME: Final = os.getenv("OTEL_TRACER_NAME", "litellm") LITELLM_METER_NAME: Final = os.getenv("LITELLM_METER_NAME", "litellm") LITELLM_LOGGER_NAME: Final = os.getenv("LITELLM_LOGGER_NAME", "litellm") @@ -3084,8 +3123,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): otel_exporter, ) normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces") + tls: Final = _resolve_otlp_http_tls("TRACES") return BatchSpanProcessor( - OTLPSpanExporterHTTP(endpoint=normalized_endpoint, headers=_split_otel_headers), + OTLPSpanExporterHTTP( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + certificate_file=tls.certificate_file, + session=tls.session, + ), ) elif otel_exporter == "otlp_grpc" or otel_exporter == "grpc": try: @@ -3166,7 +3211,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): self.OTEL_EXPORTER, normalized_endpoint, ) - return OTLPLogExporter(endpoint=normalized_endpoint, headers=_split_otel_headers) + tls: Final = _resolve_otlp_http_tls("LOGS") + return OTLPLogExporter( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + certificate_file=tls.certificate_file, + session=tls.session, + ) elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc": try: from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( @@ -3229,9 +3280,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): OTLPMetricExporter, ) + tls: Final = _resolve_otlp_http_tls("METRICS") exporter = OTLPMetricExporter( endpoint=normalized_endpoint, headers=_split_otel_headers, + certificate_file=tls.certificate_file, + session=tls.session, ) return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 6c3dc4d0bd1..d73d1ffa958 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -90,7 +90,7 @@ The suites run against a live proxy, so bring one up first by running the litell They also need a proxy whose bundled UI contains the change under test, so run the proxy from your branch (an editable install serves the UI your checkout builds) -Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE` +Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a Jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); the CI stack exports over TLS with its stack CA in `SSL_CERT_FILE`. Run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE` The Presidio spend-log audit test also requires prompt storage on the proxy, because its assertions inspect the detected entities and their scores. Add the following to the proxy config before starting it: diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index a79c158f9c4..290caa422d4 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -58,6 +58,7 @@ LINEAR_READONLY_TOOL: Final = "list_teams" # as listed by tools/list on mcp.lin # service in docker-compose.yml maps it to host 16686). Trace-completeness tests # read exported spans back through it. OTEL_QUERY_URL = os.environ.get("E2E_OTEL_QUERY_URL", "http://localhost:16686").rstrip("/") +OTEL_EXPORTER_ENDPOINT = os.environ.get("E2E_OTEL_EXPORTER_ENDPOINT", "") # Real-DataDog read-back (no local sink - destination fakes cannot be deployed # on the cluster): the proxy delivers with DD_API_KEY as in production, and the diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py index 9f08fa6c4e7..7ad93b78e2d 100644 --- a/tests/e2e/logging/test_otel_trace_e2e.py +++ b/tests/e2e/logging/test_otel_trace_e2e.py @@ -12,21 +12,24 @@ commit 1bd603d1ac). Both halves of the contract are asserted: the recorded state (the proxy reports the OTEL v2 logger active via /health/readiness/details) and the enforced behavior (the complete span tree at the destination, read back through the -destination's own query API - never proxy-side "export succeeded" logs). +destination's own query API - never proxy-side "export succeeded" logs). The +TLS coverage requires the stack to export OTLP over HTTPS with a certificate +signed by the CA in SSL_CERT_FILE, and treats a missing or plaintext endpoint +as a stack misconfiguration rather than skipping the test. """ from __future__ import annotations import time +from typing import Final import pytest -from pydantic import BaseModel, ConfigDict, ValidationError - -from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker +from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, OTEL_EXPORTER_ENDPOINT, unique_marker from lifecycle import ResourceManager from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient, first_ok, readiness_details_body from models import LiteLLMParamsBody from otel_client import JaegerSpan, JaegerTrace, OtelReader +from pydantic import BaseModel, ConfigDict, ValidationError pytestmark = pytest.mark.e2e @@ -312,6 +315,34 @@ class TestOtelTraceCompleteness: ) _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"]) + def test_otel_export_over_tls_with_internal_ca_reaches_destination( + self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager + ) -> None: + _assert_otel_destination_configured(client) + assert OTEL_EXPORTER_ENDPOINT.startswith("https://"), ( + "the stack must export OTLP over TLS signed by the CA in SSL_CERT_FILE " + "(E2E_OTEL_EXPORTER_ENDPOINT) for this test to prove anything; a " + "missing or plaintext value is a stack misconfiguration" + ) + + route: Final = "/chat/completions" + key: Final = client.key_with_alias(f"otel-trace-tls-{unique_marker()}", models=[MODEL]) + resources.defer(lambda: client.delete_key(key)) + + marker: Final = unique_marker() + outcome: Final = first_ok( + client, lambda: client.chat_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16) + ) + assert outcome.call_id is not None, "success response must carry x-litellm-call-id" + + hits: Final = otel_reader.poll_traces_for_call( + call_id=outcome.call_id, + settled_names=_settled_names(route=route, genai_span=f"chat {MODEL}"), + settled_prefixes={DB_SPAN_PREFIX}, + ) + _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"]) def test_messages_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index bea9a38e9dd..bd84a9ac597 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -9,10 +9,13 @@ import time import unittest import weakref from datetime import datetime, timedelta, timezone +from pathlib import Path from types import MappingProxyType -from parameterized import parameterized +from typing import Final from unittest.mock import MagicMock, patch +import pytest + # Adds the grandparent directory to sys.path to allow importing project modules from opentelemetry import trace from opentelemetry.sdk._logs import LogData @@ -23,6 +26,7 @@ from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from parameterized import parameterized import litellm from litellm.integrations import opentelemetry as otel_module @@ -2081,6 +2085,64 @@ class TestOpenTelemetryEndpointNormalization(unittest.TestCase): self.assertEqual(traces, "http://collector:4318/v1/traces") +def test_resolve_otlp_http_tls_uses_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + certificate_file: Final = tmp_path / "ca.pem" + certificate_file.write_text("certificate") + monkeypatch.setenv("SSL_CERT_FILE", str(certificate_file)) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.delenv("OTEL_EXPORTER_OTLP_CERTIFICATE", raising=False) + monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", raising=False) + + tls: Final = otel_module._resolve_otlp_http_tls("TRACES") + + assert tls.certificate_file == str(certificate_file) + assert tls.session is None + + +def test_resolve_otlp_http_tls_disables_verification(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SSL_VERIFY", "false") + monkeypatch.delenv("SSL_CERT_FILE", raising=False) + monkeypatch.delenv("OTEL_EXPORTER_OTLP_CERTIFICATE", raising=False) + monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", raising=False) + + tls: Final = otel_module._resolve_otlp_http_tls("TRACES") + assert tls.certificate_file is None + assert tls.session is not None + + adapter: Final = tls.session.get_adapter("https://example.invalid") + connection: Final = adapter.poolmanager.connection_from_url("https://example.invalid") + adapter.cert_verify(connection, "https://example.invalid", True, None) + + assert connection.cert_reqs == "CERT_NONE" + + +def test_resolve_otlp_http_tls_prefers_otel_certificate(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + certificate_file: Final = tmp_path / "ca.pem" + certificate_file.write_text("certificate") + monkeypatch.setenv("SSL_CERT_FILE", str(certificate_file)) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_CERTIFICATE", "/otel-ca.pem") + + tls: Final = otel_module._resolve_otlp_http_tls("TRACES") + + assert tls.certificate_file is None + assert tls.session is None + + +def test_http_span_exporter_uses_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + certificate_file: Final = tmp_path / "ca.pem" + certificate_file.write_text("certificate") + monkeypatch.setenv("SSL_CERT_FILE", str(certificate_file)) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.delenv("OTEL_EXPORTER_OTLP_CERTIFICATE", raising=False) + monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", raising=False) + + config: Final = OpenTelemetryConfig(exporter="otlp_http", endpoint="https://collector.invalid:4318") + otel: Final = OpenTelemetry(config=config) + processor: Final = otel._get_span_processor() + + assert processor.span_exporter._certificate_file == str(certificate_file) + + class TestOpenTelemetryProtocolSelection(unittest.TestCase): """Test suite for verifying correct exporter selection based on protocol"""