mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
2e575d39f2
The proxy auth path calls phase_span() and seed_request_identity() in litellm/integrations/otel/runtime.py on every request, each doing a try/except lazy import of litellm.integrations.otel.logger. When the OpenTelemetry SDK is not installed (the default), that import raises, and CPython never caches a failed import, so every request re-scanned sys.path and contended on the import lock. At 750 concurrent users this cost about 12% throughput versus v1.85.0. Resolve the hooks once and cache the outcome, absence included, with functools.cache, so the import is attempted a single time instead of per request. Throughput returns to the v1.85.0 baseline.
64 lines
2 KiB
Python
64 lines
2 KiB
Python
"""Regression tests for the SDK-free OTel runtime shim.
|
|
|
|
The proxy auth hot path calls ``phase_span`` and ``seed_request_identity`` on
|
|
every request. These wrappers resolve the SDK-backed implementations with a
|
|
lazy import. CPython never caches a failed import, so before memoization an
|
|
absent OTel SDK made every request re-scan ``sys.path`` and contend on the
|
|
import lock. These tests pin the import to a single resolution.
|
|
"""
|
|
|
|
import builtins
|
|
|
|
import litellm.integrations.otel.runtime as runtime
|
|
|
|
|
|
def test_logger_not_reimported_after_first_resolution(monkeypatch):
|
|
runtime._otel_runtime.cache_clear()
|
|
|
|
counts = {"n": 0}
|
|
real_import = builtins.__import__
|
|
|
|
def counting_import(name, globals=None, locals=None, fromlist=(), level=0):
|
|
if name == "litellm.integrations.otel" and fromlist and "logger" in fromlist:
|
|
counts["n"] += 1
|
|
return real_import(name, globals, locals, fromlist, level)
|
|
|
|
monkeypatch.setattr(builtins, "__import__", counting_import)
|
|
|
|
with runtime.phase_span("auth /v1/chat/completions"):
|
|
pass
|
|
after_first = counts["n"]
|
|
|
|
for _ in range(49):
|
|
with runtime.phase_span("auth /v1/chat/completions"):
|
|
pass
|
|
|
|
assert counts["n"] == after_first, (
|
|
f"otel.logger re-imported {counts['n'] - after_first} times after the first "
|
|
"resolution; it must be memoized so it does not re-scan sys.path per request"
|
|
)
|
|
|
|
runtime._otel_runtime.cache_clear()
|
|
|
|
|
|
def test_resolution_is_memoized():
|
|
runtime._otel_runtime.cache_clear()
|
|
|
|
for _ in range(25):
|
|
with runtime.phase_span("p"):
|
|
pass
|
|
|
|
info = runtime._otel_runtime.cache_info()
|
|
assert info.misses == 1
|
|
assert info.hits >= 24
|
|
|
|
runtime._otel_runtime.cache_clear()
|
|
|
|
|
|
def test_wrappers_no_op_when_runtime_absent(monkeypatch):
|
|
monkeypatch.setattr(runtime, "_otel_runtime", lambda: None)
|
|
|
|
with runtime.phase_span("auth") as span:
|
|
assert span is None
|
|
|
|
assert runtime.seed_request_identity({"token": "sk-x"}, model="gpt-4o") is None
|