mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(otel): honor SSL_CERT_FILE and ssl_verify in OTLP HTTP exporters
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
776f8bc309
commit
29fe595e6a
6 changed files with 179 additions and 16 deletions
29
.github/e2e-stack/up.sh
vendored
29
.github/e2e-stack/up.sh
vendored
|
|
@ -24,6 +24,7 @@ DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
|
|||
DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
|
||||
DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
|
||||
JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
|
||||
JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}"
|
||||
JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
|
||||
KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
|
||||
|
||||
|
|
@ -122,7 +123,7 @@ SERVER_ENV=(
|
|||
"CONFIG_FILE_PATH=${CONFIG_PATH}"
|
||||
"STORE_MODEL_IN_DB=True"
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}"
|
||||
"SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
|
||||
"PYTHONPATH=${REPO_ROOT}"
|
||||
"JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
|
||||
|
|
@ -147,16 +148,12 @@ start_server() {
|
|||
echo $! > "${PIDS_DIR}/${name}.pid"
|
||||
}
|
||||
|
||||
start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
|
||||
start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
|
||||
start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
|
||||
|
||||
if [[ "$(uname)" == "Linux" ]]; then
|
||||
NGINX_UPSTREAM_HOST=127.0.0.1
|
||||
NGINX_DOCKER_ARGS=(--network host)
|
||||
else
|
||||
NGINX_UPSTREAM_HOST=host.docker.internal
|
||||
NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}")
|
||||
NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}")
|
||||
fi
|
||||
|
||||
cat > "${STACK_DIR}/nginx.conf" <<EOF
|
||||
|
|
@ -186,12 +183,29 @@ http {
|
|||
proxy_send_timeout 600s;
|
||||
}
|
||||
}
|
||||
server {
|
||||
listen ${JAEGER_OTLP_TLS_PORT} ssl;
|
||||
ssl_certificate /certs/server.crt;
|
||||
ssl_certificate_key /certs/server.key;
|
||||
client_max_body_size 100m;
|
||||
location / {
|
||||
proxy_pass http://${NGINX_UPSTREAM_HOST}:${JAEGER_OTLP_PORT};
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
docker rm -f e2e-nginx >/dev/null 2>&1 || true
|
||||
docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
|
||||
-v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null
|
||||
-v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \
|
||||
-v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null
|
||||
|
||||
wait_for "Jaeger OTLP TLS listener" \
|
||||
"curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'"
|
||||
|
||||
start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
|
||||
start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
|
||||
start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
|
||||
|
||||
wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
|
||||
wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
|
||||
|
|
@ -206,6 +220,7 @@ LITELLM_MASTER_KEY=${MASTER_KEY}
|
|||
REDIS_HOST=127.0.0.1
|
||||
REDIS_PORT=${REDIS_PORT}
|
||||
E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
|
||||
E2E_OTEL_EXPORTER_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}
|
||||
E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
|
||||
E2E_KEYCLOAK_ADMIN_USER=admin
|
||||
E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@ from concurrent.futures import ThreadPoolExecutor
|
|||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, cast
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -99,6 +102,42 @@ _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
|
|||
# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
|
||||
_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OtlpHttpTls:
|
||||
certificate_file: str | None
|
||||
session: requests.Session | None
|
||||
|
||||
|
||||
class _NoVerifyAdapter(HTTPAdapter):
|
||||
def cert_verify(
|
||||
self,
|
||||
conn: object,
|
||||
url: str,
|
||||
verify: bool | str,
|
||||
cert: str | tuple[str, str] | None,
|
||||
) -> None:
|
||||
super().cert_verify( # pyright: ignore[reportUnknownMemberType] # requests stubs omit HTTPAdapter.cert_verify
|
||||
conn, url, False, cert
|
||||
)
|
||||
|
||||
|
||||
def _resolve_otlp_http_tls(signal: Literal["TRACES", "METRICS", "LOGS"]) -> _OtlpHttpTls:
|
||||
if os.getenv(f"OTEL_EXPORTER_OTLP_{signal}_CERTIFICATE") or os.getenv("OTEL_EXPORTER_OTLP_CERTIFICATE"):
|
||||
return _OtlpHttpTls(certificate_file=None, session=None)
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import get_ssl_verify
|
||||
|
||||
verify: Final = get_ssl_verify()
|
||||
if verify is False:
|
||||
session: Final = requests.Session()
|
||||
session.mount("https://", _NoVerifyAdapter())
|
||||
return _OtlpHttpTls(certificate_file=None, session=session)
|
||||
if isinstance(verify, str):
|
||||
return _OtlpHttpTls(certificate_file=verify, session=None)
|
||||
return _OtlpHttpTls(certificate_file=None, session=None)
|
||||
|
||||
|
||||
LITELLM_TRACER_NAME: Final = os.getenv("OTEL_TRACER_NAME", "litellm")
|
||||
LITELLM_METER_NAME: Final = os.getenv("LITELLM_METER_NAME", "litellm")
|
||||
LITELLM_LOGGER_NAME: Final = os.getenv("LITELLM_LOGGER_NAME", "litellm")
|
||||
|
|
@ -3084,8 +3123,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
otel_exporter,
|
||||
)
|
||||
normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces")
|
||||
tls: Final = _resolve_otlp_http_tls("TRACES")
|
||||
return BatchSpanProcessor(
|
||||
OTLPSpanExporterHTTP(endpoint=normalized_endpoint, headers=_split_otel_headers),
|
||||
OTLPSpanExporterHTTP(
|
||||
endpoint=normalized_endpoint,
|
||||
headers=_split_otel_headers,
|
||||
certificate_file=tls.certificate_file,
|
||||
session=tls.session,
|
||||
),
|
||||
)
|
||||
elif otel_exporter == "otlp_grpc" or otel_exporter == "grpc":
|
||||
try:
|
||||
|
|
@ -3166,7 +3211,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
self.OTEL_EXPORTER,
|
||||
normalized_endpoint,
|
||||
)
|
||||
return OTLPLogExporter(endpoint=normalized_endpoint, headers=_split_otel_headers)
|
||||
tls: Final = _resolve_otlp_http_tls("LOGS")
|
||||
return OTLPLogExporter(
|
||||
endpoint=normalized_endpoint,
|
||||
headers=_split_otel_headers,
|
||||
certificate_file=tls.certificate_file,
|
||||
session=tls.session,
|
||||
)
|
||||
elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc":
|
||||
try:
|
||||
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import (
|
||||
|
|
@ -3229,9 +3280,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
OTLPMetricExporter,
|
||||
)
|
||||
|
||||
tls: Final = _resolve_otlp_http_tls("METRICS")
|
||||
exporter = OTLPMetricExporter(
|
||||
endpoint=normalized_endpoint,
|
||||
headers=_split_otel_headers,
|
||||
certificate_file=tls.certificate_file,
|
||||
session=tls.session,
|
||||
)
|
||||
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
|
||||
|
||||
|
|
|
|||
|
|
@ -90,7 +90,7 @@ The suites run against a live proxy, so bring one up first by running the litell
|
|||
|
||||
They also need a proxy whose bundled UI contains the change under test, so run the proxy from your branch (an editable install serves the UI your checkout builds)
|
||||
|
||||
Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
|
||||
Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a Jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); the CI stack exports over TLS with its stack CA in `SSL_CERT_FILE`. Run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
|
||||
|
||||
The Presidio spend-log audit test also requires prompt storage on the proxy, because its assertions inspect the detected entities and their scores. Add the following to the proxy config before starting it:
|
||||
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ LINEAR_READONLY_TOOL: Final = "list_teams" # as listed by tools/list on mcp.lin
|
|||
# service in docker-compose.yml maps it to host 16686). Trace-completeness tests
|
||||
# read exported spans back through it.
|
||||
OTEL_QUERY_URL = os.environ.get("E2E_OTEL_QUERY_URL", "http://localhost:16686").rstrip("/")
|
||||
OTEL_EXPORTER_ENDPOINT = os.environ.get("E2E_OTEL_EXPORTER_ENDPOINT", "")
|
||||
|
||||
# Real-DataDog read-back (no local sink - destination fakes cannot be deployed
|
||||
# on the cluster): the proxy delivers with DD_API_KEY as in production, and the
|
||||
|
|
|
|||
|
|
@ -12,21 +12,24 @@ commit 1bd603d1ac).
|
|||
Both halves of the contract are asserted: the recorded state (the proxy reports
|
||||
the OTEL v2 logger active via /health/readiness/details) and the enforced
|
||||
behavior (the complete span tree at the destination, read back through the
|
||||
destination's own query API - never proxy-side "export succeeded" logs).
|
||||
destination's own query API - never proxy-side "export succeeded" logs). The
|
||||
TLS coverage requires the stack to export OTLP over HTTPS with a certificate
|
||||
signed by the CA in SSL_CERT_FILE, and treats a missing or plaintext endpoint
|
||||
as a stack misconfiguration rather than skipping the test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, OTEL_EXPORTER_ENDPOINT, unique_marker
|
||||
from lifecycle import ResourceManager
|
||||
from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient, first_ok, readiness_details_body
|
||||
from models import LiteLLMParamsBody
|
||||
from otel_client import JaegerSpan, JaegerTrace, OtelReader
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
|
@ -312,6 +315,34 @@ class TestOtelTraceCompleteness:
|
|||
)
|
||||
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
|
||||
|
||||
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
|
||||
def test_otel_export_over_tls_with_internal_ca_reaches_destination(
|
||||
self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager
|
||||
) -> None:
|
||||
_assert_otel_destination_configured(client)
|
||||
assert OTEL_EXPORTER_ENDPOINT.startswith("https://"), (
|
||||
"the stack must export OTLP over TLS signed by the CA in SSL_CERT_FILE "
|
||||
"(E2E_OTEL_EXPORTER_ENDPOINT) for this test to prove anything; a "
|
||||
"missing or plaintext value is a stack misconfiguration"
|
||||
)
|
||||
|
||||
route: Final = "/chat/completions"
|
||||
key: Final = client.key_with_alias(f"otel-trace-tls-{unique_marker()}", models=[MODEL])
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
||||
marker: Final = unique_marker()
|
||||
outcome: Final = first_ok(
|
||||
client, lambda: client.chat_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16)
|
||||
)
|
||||
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
|
||||
|
||||
hits: Final = otel_reader.poll_traces_for_call(
|
||||
call_id=outcome.call_id,
|
||||
settled_names=_settled_names(route=route, genai_span=f"chat {MODEL}"),
|
||||
settled_prefixes={DB_SPAN_PREFIX},
|
||||
)
|
||||
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
|
||||
|
||||
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
|
||||
def test_messages_exports_complete_trace(
|
||||
self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager
|
||||
|
|
|
|||
|
|
@ -9,10 +9,13 @@ import time
|
|||
import unittest
|
||||
import weakref
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from parameterized import parameterized
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# Adds the grandparent directory to sys.path to allow importing project modules
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk._logs import LogData
|
||||
|
|
@ -23,6 +26,7 @@ from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData
|
|||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from parameterized import parameterized
|
||||
|
||||
import litellm
|
||||
from litellm.integrations import opentelemetry as otel_module
|
||||
|
|
@ -2081,6 +2085,64 @@ class TestOpenTelemetryEndpointNormalization(unittest.TestCase):
|
|||
self.assertEqual(traces, "http://collector:4318/v1/traces")
|
||||
|
||||
|
||||
def test_resolve_otlp_http_tls_uses_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
certificate_file: Final = tmp_path / "ca.pem"
|
||||
certificate_file.write_text("certificate")
|
||||
monkeypatch.setenv("SSL_CERT_FILE", str(certificate_file))
|
||||
monkeypatch.delenv("SSL_VERIFY", raising=False)
|
||||
monkeypatch.delenv("OTEL_EXPORTER_OTLP_CERTIFICATE", raising=False)
|
||||
monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", raising=False)
|
||||
|
||||
tls: Final = otel_module._resolve_otlp_http_tls("TRACES")
|
||||
|
||||
assert tls.certificate_file == str(certificate_file)
|
||||
assert tls.session is None
|
||||
|
||||
|
||||
def test_resolve_otlp_http_tls_disables_verification(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("SSL_VERIFY", "false")
|
||||
monkeypatch.delenv("SSL_CERT_FILE", raising=False)
|
||||
monkeypatch.delenv("OTEL_EXPORTER_OTLP_CERTIFICATE", raising=False)
|
||||
monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", raising=False)
|
||||
|
||||
tls: Final = otel_module._resolve_otlp_http_tls("TRACES")
|
||||
assert tls.certificate_file is None
|
||||
assert tls.session is not None
|
||||
|
||||
adapter: Final = tls.session.get_adapter("https://example.invalid")
|
||||
connection: Final = adapter.poolmanager.connection_from_url("https://example.invalid")
|
||||
adapter.cert_verify(connection, "https://example.invalid", True, None)
|
||||
|
||||
assert connection.cert_reqs == "CERT_NONE"
|
||||
|
||||
|
||||
def test_resolve_otlp_http_tls_prefers_otel_certificate(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
certificate_file: Final = tmp_path / "ca.pem"
|
||||
certificate_file.write_text("certificate")
|
||||
monkeypatch.setenv("SSL_CERT_FILE", str(certificate_file))
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_CERTIFICATE", "/otel-ca.pem")
|
||||
|
||||
tls: Final = otel_module._resolve_otlp_http_tls("TRACES")
|
||||
|
||||
assert tls.certificate_file is None
|
||||
assert tls.session is None
|
||||
|
||||
|
||||
def test_http_span_exporter_uses_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
certificate_file: Final = tmp_path / "ca.pem"
|
||||
certificate_file.write_text("certificate")
|
||||
monkeypatch.setenv("SSL_CERT_FILE", str(certificate_file))
|
||||
monkeypatch.delenv("SSL_VERIFY", raising=False)
|
||||
monkeypatch.delenv("OTEL_EXPORTER_OTLP_CERTIFICATE", raising=False)
|
||||
monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", raising=False)
|
||||
|
||||
config: Final = OpenTelemetryConfig(exporter="otlp_http", endpoint="https://collector.invalid:4318")
|
||||
otel: Final = OpenTelemetry(config=config)
|
||||
processor: Final = otel._get_span_processor()
|
||||
|
||||
assert processor.span_exporter._certificate_file == str(certificate_file)
|
||||
|
||||
|
||||
class TestOpenTelemetryProtocolSelection(unittest.TestCase):
|
||||
"""Test suite for verifying correct exporter selection based on protocol"""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue