mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-11 03:38:38 +00:00
b4fcc5c1bc
* fix(otel): read the registered v2 logger without importing the proxy phase_span, which the router enters on every deployment pick since #44150, looked up the proxy's OTel logger by importing litellm.proxy.proxy_server. In an SDK process that import loads the whole proxy synchronously on the caller's event loop during its first request, which stalled litellm_router_unit_testing past its 5s wait. Read the module from sys.modules instead: when the proxy was never imported it has no registered logger. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): pin the no-proxy-import guarantee in-process Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo <mateo@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
95 lines
3.1 KiB
Python
95 lines
3.1 KiB
Python
"""Regression tests for the SDK-free OTel runtime shim.
|
|
|
|
The proxy auth hot path calls ``phase_span`` and ``seed_request_identity`` on
|
|
every request. These wrappers resolve the SDK-backed implementations with a
|
|
lazy import. CPython never caches a failed import, so before memoization an
|
|
absent OTel SDK made every request re-scan ``sys.path`` and contend on the
|
|
import lock. These tests pin the import to a single resolution.
|
|
"""
|
|
|
|
import builtins
|
|
import importlib.abc
|
|
import sys
|
|
|
|
import litellm.integrations.otel.runtime as runtime
|
|
|
|
|
|
def test_logger_not_reimported_after_first_resolution(monkeypatch):
|
|
runtime._otel_runtime.cache_clear()
|
|
|
|
counts = {"n": 0}
|
|
real_import = builtins.__import__
|
|
|
|
def counting_import(name, globals=None, locals=None, fromlist=(), level=0):
|
|
if name == "litellm.integrations.otel" and fromlist and "logger" in fromlist:
|
|
counts["n"] += 1
|
|
return real_import(name, globals, locals, fromlist, level)
|
|
|
|
monkeypatch.setattr(builtins, "__import__", counting_import)
|
|
|
|
with runtime.phase_span("auth /v1/chat/completions"):
|
|
pass
|
|
after_first = counts["n"]
|
|
|
|
for _ in range(49):
|
|
with runtime.phase_span("auth /v1/chat/completions"):
|
|
pass
|
|
|
|
assert counts["n"] == after_first, (
|
|
f"otel.logger re-imported {counts['n'] - after_first} times after the first "
|
|
"resolution; it must be memoized so it does not re-scan sys.path per request"
|
|
)
|
|
|
|
runtime._otel_runtime.cache_clear()
|
|
|
|
|
|
def test_resolution_is_memoized():
|
|
runtime._otel_runtime.cache_clear()
|
|
|
|
for _ in range(25):
|
|
with runtime.phase_span("p"):
|
|
pass
|
|
|
|
info = runtime._otel_runtime.cache_info()
|
|
assert info.misses == 1
|
|
assert info.hits >= 24
|
|
|
|
runtime._otel_runtime.cache_clear()
|
|
|
|
|
|
def test_wrappers_no_op_when_runtime_absent(monkeypatch):
|
|
monkeypatch.setattr(runtime, "_otel_runtime", lambda: None)
|
|
|
|
with runtime.phase_span("auth") as span:
|
|
assert span is None
|
|
|
|
assert runtime.seed_request_identity({"token": "sk-x"}, model="gpt-4o") is None
|
|
|
|
|
|
def test_phase_event_no_ops_when_runtime_absent(monkeypatch):
|
|
monkeypatch.setattr(runtime, "_otel_runtime", lambda: None)
|
|
|
|
assert runtime.phase_event("litellm.request.body_parsed") is None
|
|
assert runtime.phase_event("litellm.request.body_received", {"litellm.request.body_bytes": 3}) is None
|
|
|
|
|
|
def test_phase_span_does_not_import_the_proxy_in_an_sdk_process(monkeypatch):
|
|
import litellm.proxy
|
|
|
|
monkeypatch.delitem(sys.modules, "litellm.proxy.proxy_server", raising=False)
|
|
monkeypatch.delattr(litellm.proxy, "proxy_server", raising=False)
|
|
proxy_imports: list[str] = []
|
|
|
|
class _RefuseProxyImport(importlib.abc.MetaPathFinder):
|
|
def find_spec(self, fullname, path, target=None):
|
|
if fullname == "litellm.proxy.proxy_server":
|
|
proxy_imports.append(fullname)
|
|
raise ImportError(fullname)
|
|
return None
|
|
|
|
monkeypatch.setattr(sys, "meta_path", [_RefuseProxyImport(), *sys.meta_path])
|
|
|
|
with runtime.phase_span("route gpt-5-mini") as span:
|
|
assert span is None
|
|
|
|
assert proxy_imports == []
|