perf(otel): memoize per-request lazy import of otel runtime hooks (#31707)

The proxy auth path calls phase_span() and seed_request_identity() in
litellm/integrations/otel/runtime.py on every request, each doing a
try/except lazy import of litellm.integrations.otel.logger. When the
OpenTelemetry SDK is not installed (the default), that import raises, and
CPython never caches a failed import, so every request re-scanned sys.path
and contended on the import lock. At 750 concurrent users this cost about
12% throughput versus v1.85.0.

Resolve the hooks once and cache the outcome, absence included, with
functools.cache, so the import is attempted a single time instead of per
request. Throughput returns to the v1.85.0 baseline.

litellm/integrations/otel/runtime.py

@@ -8,7 +8,23 @@ identity unconditionally.
 """
 
 from contextlib import contextmanager
-from typing import Any, Iterator
+from functools import cache
+from typing import Any, Callable, Iterator, Optional
+
+
+@cache
+def _otel_runtime() -> "Optional[tuple[Callable[[str], Any], Callable[..., None]]]":
+    """Resolve the SDK-backed hooks once and cache the outcome, absence included.
+
+    CPython never caches a failed import, so without this memoization every call
+    site re-attempts the import on each request; when the OTel SDK is not installed
+    that re-scans ``sys.path`` and contends on the import lock on the hot path.
+    """
+    try:
+        from litellm.integrations.otel import logger
+    except Exception:
+        return None
+    return (logger.phase_span, logger.seed_request_identity)
 
 
 @contextmanager
@@ -18,21 +34,17 @@ def phase_span(name: str) -> "Iterator[Any]":
     Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not
     the active logger.
     """
-    try:
-        from litellm.integrations.otel.logger import phase_span as _phase_span
-    except Exception:
+    runtime = _otel_runtime()
+    if runtime is None:
         yield None
         return
-    with _phase_span(name) as span:
+    with runtime[0](name) as span:
         yield span
 
 
 def seed_request_identity(user_api_key_dict: Any, model: Any = None) -> None:
     """Seed request-identity Baggage at the auth boundary (no-op without V2)."""
-    try:
-        from litellm.integrations.otel.logger import (
-            seed_request_identity as _seed_request_identity,
-        )
-    except Exception:
+    runtime = _otel_runtime()
+    if runtime is None:
         return
-    _seed_request_identity(user_api_key_dict, model=model)
+    runtime[1](user_api_key_dict, model=model)

tests/test_litellm/integrations/otel/test_runtime.py

@@ -0,0 +1,64 @@
+"""Regression tests for the SDK-free OTel runtime shim.
+
+The proxy auth hot path calls ``phase_span`` and ``seed_request_identity`` on
+every request. These wrappers resolve the SDK-backed implementations with a
+lazy import. CPython never caches a failed import, so before memoization an
+absent OTel SDK made every request re-scan ``sys.path`` and contend on the
+import lock. These tests pin the import to a single resolution.
+"""
+
+import builtins
+
+import litellm.integrations.otel.runtime as runtime
+
+
+def test_logger_not_reimported_after_first_resolution(monkeypatch):
+    runtime._otel_runtime.cache_clear()
+
+    counts = {"n": 0}
+    real_import = builtins.__import__
+
+    def counting_import(name, globals=None, locals=None, fromlist=(), level=0):
+        if name == "litellm.integrations.otel" and fromlist and "logger" in fromlist:
+            counts["n"] += 1
+        return real_import(name, globals, locals, fromlist, level)
+
+    monkeypatch.setattr(builtins, "__import__", counting_import)
+
+    with runtime.phase_span("auth /v1/chat/completions"):
+        pass
+    after_first = counts["n"]
+
+    for _ in range(49):
+        with runtime.phase_span("auth /v1/chat/completions"):
+            pass
+
+    assert counts["n"] == after_first, (
+        f"otel.logger re-imported {counts['n'] - after_first} times after the first "
+        "resolution; it must be memoized so it does not re-scan sys.path per request"
+    )
+
+    runtime._otel_runtime.cache_clear()
+
+
+def test_resolution_is_memoized():
+    runtime._otel_runtime.cache_clear()
+
+    for _ in range(25):
+        with runtime.phase_span("p"):
+            pass
+
+    info = runtime._otel_runtime.cache_info()
+    assert info.misses == 1
+    assert info.hits >= 24
+
+    runtime._otel_runtime.cache_clear()
+
+
+def test_wrappers_no_op_when_runtime_absent(monkeypatch):
+    monkeypatch.setattr(runtime, "_otel_runtime", lambda: None)
+
+    with runtime.phase_span("auth") as span:
+        assert span is None
+
+    assert runtime.seed_request_identity({"token": "sk-x"}, model="gpt-4o") is None