mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): harden usage telemetry exporter, instance id, and model labels
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
ecaadda80a
commit
16ad81e427
2 changed files with 218 additions and 92 deletions
|
|
@ -8,9 +8,10 @@ untouched. The OSS twin of litellm.proxy.enterprise_billing.billing_metrics:
|
|||
same exporter shape, no license key, no client certificate, no hostname.
|
||||
|
||||
Only aggregate counts leave the process: request totals by endpoint category
|
||||
and status class, plus per-call model, provider, call type, token counts, and
|
||||
cost from the standard logging payload. No API keys, teams, users, model
|
||||
groups, or api_base values are ever exported.
|
||||
and status class, plus per-call provider, call type, token counts, and cost
|
||||
from the standard logging payload. Model names are exported only when they are
|
||||
present in the public pricing map; anything else is reported as "other". No
|
||||
API keys, teams, users, model groups, or api_base values are ever exported.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
|
@ -19,7 +20,7 @@ import uuid
|
|||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
|
||||
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
|
|
@ -28,6 +29,7 @@ from opentelemetry.sdk.resources import Resource
|
|||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy.middleware.billable_request_metrics_middleware import (
|
||||
|
|
@ -95,11 +97,19 @@ def load_usage_telemetry_config(*, litellm_version: str, instance_id: str) -> Us
|
|||
)
|
||||
|
||||
|
||||
def build_usage_meter_provider(config: UsageTelemetryConfig) -> MeterProvider:
|
||||
exporter: Final = OTLPMetricExporter(
|
||||
def _build_exporter(config: UsageTelemetryConfig) -> OTLPMetricExporter:
|
||||
"""``headers`` must be non-empty: with a falsy value OTLPMetricExporter
|
||||
reads OTEL_EXPORTER_OTLP_*_HEADERS from the environment, which would
|
||||
forward a deployment's own collector auth header to telemetry.litellm.ai."""
|
||||
return OTLPMetricExporter(
|
||||
endpoint=_metrics_endpoint(config.endpoint),
|
||||
timeout=EXPORT_TIMEOUT_S,
|
||||
headers={"User-Agent": f"litellm-proxy/{config.litellm_version}"},
|
||||
)
|
||||
|
||||
|
||||
def build_usage_meter_provider(config: UsageTelemetryConfig) -> MeterProvider:
|
||||
exporter: Final = _build_exporter(config)
|
||||
reader: Final = PeriodicExportingMetricReader(exporter, export_interval_millis=config.export_interval_ms)
|
||||
resource: Final = Resource.create(
|
||||
MappingProxyType(
|
||||
|
|
@ -170,7 +180,13 @@ class UsageTelemetryRecorder(CustomLogger):
|
|||
),
|
||||
)
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
|
||||
async def async_log_success_event(
|
||||
self,
|
||||
kwargs: Mapping[str, object],
|
||||
response_obj: object,
|
||||
start_time: object,
|
||||
end_time: object,
|
||||
) -> None:
|
||||
raw_payload: Final = kwargs.get("standard_logging_object")
|
||||
if not isinstance(raw_payload, dict):
|
||||
return
|
||||
|
|
@ -181,7 +197,7 @@ class UsageTelemetryRecorder(CustomLogger):
|
|||
return
|
||||
attrs: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"litellm.model": payload.model,
|
||||
"litellm.model": _public_model_label(payload.model, payload.custom_llm_provider),
|
||||
"litellm.provider": payload.custom_llm_provider or "unknown",
|
||||
"litellm.call_type": payload.call_type,
|
||||
}
|
||||
|
|
@ -197,6 +213,17 @@ class UsageTelemetryRecorder(CustomLogger):
|
|||
self._provider.shutdown(timeout_millis=SHUTDOWN_FLUSH_TIMEOUT_MS)
|
||||
|
||||
|
||||
def _public_model_label(model: str, provider: str | None) -> str:
|
||||
"""Only public model names leave the process. A name absent from the
|
||||
public pricing map (fine-tunes, aliases, internal deployments) collapses
|
||||
to "other" so a private model identifier cannot leak."""
|
||||
if model in litellm.model_cost:
|
||||
return model
|
||||
if provider is not None and f"{provider}/{model}" in litellm.model_cost:
|
||||
return f"{provider}/{model}"
|
||||
return "other"
|
||||
|
||||
|
||||
class _ConfigParamWhere(TypedDict):
|
||||
param_name: ReadOnly[str]
|
||||
|
||||
|
|
@ -210,6 +237,18 @@ class _ConfigParamCreate(TypedDict):
|
|||
param_value: ReadOnly[str]
|
||||
|
||||
|
||||
class _ConfigParamRow(Protocol):
|
||||
param_value: object
|
||||
|
||||
|
||||
def _instance_id_from_row(row: "_ConfigParamRow | None") -> str | None:
|
||||
param_value: Final[object] = row.param_value if row is not None else None
|
||||
parsed: Final[object] = json.loads(param_value) if isinstance(param_value, str) else param_value
|
||||
if isinstance(parsed, dict) and isinstance(parsed.get("instance_id"), str):
|
||||
return parsed["instance_id"]
|
||||
return None
|
||||
|
||||
|
||||
async def resolve_instance_id(prisma_client: "PrismaClient | None") -> str:
|
||||
"""
|
||||
A stable anonymous deployment id persisted in ``LiteLLM_Config``, so counts
|
||||
|
|
@ -220,18 +259,22 @@ async def resolve_instance_id(prisma_client: "PrismaClient | None") -> str:
|
|||
return str(uuid.uuid4())
|
||||
try:
|
||||
where: Final[_ConfigParamWhere] = {"param_name": INSTANCE_ID_CONFIG_KEY}
|
||||
row: Final = await prisma_client.db.litellm_config.find_unique(where=where)
|
||||
param_value: Final[object] = row.param_value if row is not None else None
|
||||
parsed: Final[object] = json.loads(param_value) if isinstance(param_value, str) else param_value
|
||||
if isinstance(parsed, dict) and isinstance(parsed.get("instance_id"), str):
|
||||
return parsed["instance_id"]
|
||||
existing: Final = _instance_id_from_row(await prisma_client.db.litellm_config.find_unique(where=where))
|
||||
if existing is not None:
|
||||
return existing
|
||||
instance_id: Final = str(uuid.uuid4())
|
||||
instance_value: Final[_InstanceIdValue] = {"instance_id": instance_id}
|
||||
data: Final[_ConfigParamCreate] = {
|
||||
"param_name": INSTANCE_ID_CONFIG_KEY,
|
||||
"param_value": json.dumps(instance_value),
|
||||
}
|
||||
await prisma_client.db.litellm_config.create(data=data)
|
||||
try:
|
||||
await prisma_client.db.litellm_config.create(data=data)
|
||||
except Exception: # noqa: BLE001 -- a concurrent worker won the insert; read its row instead
|
||||
raced: Final = _instance_id_from_row(await prisma_client.db.litellm_config.find_unique(where=where))
|
||||
if raced is not None:
|
||||
return raced
|
||||
raise
|
||||
return instance_id
|
||||
except Exception as exc: # noqa: BLE001 -- telemetry must never break proxy startup
|
||||
verbose_proxy_logger.warning(
|
||||
|
|
|
|||
|
|
@ -8,34 +8,45 @@ and aggregated values. Nothing the proxy sends is mocked.
|
|||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader, Metric
|
||||
|
||||
import litellm
|
||||
from litellm.proxy import usage_telemetry as ut
|
||||
from litellm.proxy.middleware.billable_request_metrics_middleware import BillableCategory
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_env(monkeypatch):
|
||||
def clear_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for name in (ut.ENABLED_ENV, ut.ENDPOINT_ENV, ut.EXPORT_INTERVAL_ENV):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
yield
|
||||
ut.shutdown_usage_telemetry_recorder()
|
||||
|
||||
|
||||
def _data_points(reader: InMemoryMetricReader, metric_name: str):
|
||||
def _metric(reader: InMemoryMetricReader, metric_name: str) -> Metric | None:
|
||||
data = reader.get_metrics_data()
|
||||
if data is None:
|
||||
return None
|
||||
for resource_metric in data.resource_metrics:
|
||||
for scope_metric in resource_metric.scope_metrics:
|
||||
for metric in scope_metric.metrics:
|
||||
if metric.name == metric_name:
|
||||
return list(metric.data.data_points)
|
||||
return []
|
||||
return metric
|
||||
return None
|
||||
|
||||
|
||||
def _all_data_points(reader: InMemoryMetricReader):
|
||||
def _data_points(reader: InMemoryMetricReader, metric_name: str) -> list:
|
||||
metric: Final = _metric(reader, metric_name)
|
||||
return [] if metric is None else list(metric.data.data_points)
|
||||
|
||||
|
||||
def _all_data_points(reader: InMemoryMetricReader) -> list:
|
||||
data = reader.get_metrics_data()
|
||||
if data is None:
|
||||
return []
|
||||
|
|
@ -49,32 +60,39 @@ def _all_data_points(reader: InMemoryMetricReader):
|
|||
|
||||
|
||||
def _recorder_with_reader() -> tuple[ut.UsageTelemetryRecorder, InMemoryMetricReader]:
|
||||
reader = InMemoryMetricReader()
|
||||
reader: Final = InMemoryMetricReader()
|
||||
return ut.UsageTelemetryRecorder(MeterProvider(metric_readers=[reader])), reader
|
||||
|
||||
|
||||
# ── Middleware sink: litellm.usage.requests ────────────────────────────────────
|
||||
def _config(**overrides: object) -> ut.UsageTelemetryConfig:
|
||||
base: Final[dict[str, object]] = {
|
||||
"endpoint": "https://telemetry.litellm.ai/v1/metrics",
|
||||
"export_interval_ms": 60_000,
|
||||
"litellm_version": "1.2.3",
|
||||
"instance_id": "inst-1",
|
||||
}
|
||||
return ut.UsageTelemetryConfig(**{**base, **overrides})
|
||||
|
||||
|
||||
def test_record_aggregates_by_route_and_status_class():
|
||||
def test_record_aggregates_by_route_and_status_class() -> None:
|
||||
recorder, reader = _recorder_with_reader()
|
||||
|
||||
recorder.record(category=BillableCategory.LLM, route="/chat/completions", status_code=200)
|
||||
recorder.record(category=BillableCategory.LLM, route="/chat/completions", status_code=200)
|
||||
recorder.record(category=BillableCategory.LLM, route="/chat/completions", status_code=500)
|
||||
|
||||
points = _data_points(reader, "litellm.usage.requests")
|
||||
by_class = {point.attributes["http.response.status_class"]: point.value for point in points}
|
||||
points: Final = _data_points(reader, "litellm.usage.requests")
|
||||
by_class: Final = {point.attributes["http.response.status_class"]: point.value for point in points}
|
||||
assert by_class == {"2xx": 2, "5xx": 1}
|
||||
for point in points:
|
||||
assert point.attributes["http.route"] == "/chat/completions"
|
||||
assert point.attributes["litellm.endpoint.category"] == "llm"
|
||||
|
||||
|
||||
# ── Success callback: llm_requests / tokens / spend ────────────────────────────
|
||||
_PUBLIC_MODEL: Final = next(key for key in litellm.model_cost if "/" not in key)
|
||||
|
||||
_SUCCESS_PAYLOAD = {
|
||||
"model": "gpt-4.1",
|
||||
_SUCCESS_PAYLOAD: Final[dict] = {
|
||||
"model": _PUBLIC_MODEL,
|
||||
"custom_llm_provider": "openai",
|
||||
"call_type": "acompletion",
|
||||
"prompt_tokens": 10,
|
||||
|
|
@ -91,7 +109,7 @@ _SUCCESS_PAYLOAD = {
|
|||
}
|
||||
|
||||
|
||||
async def test_success_event_exports_only_anonymous_attributes():
|
||||
async def test_success_event_exports_only_anonymous_attributes() -> None:
|
||||
recorder, reader = _recorder_with_reader()
|
||||
|
||||
await recorder.async_log_success_event(
|
||||
|
|
@ -101,34 +119,64 @@ async def test_success_event_exports_only_anonymous_attributes():
|
|||
end_time=None,
|
||||
)
|
||||
|
||||
llm_points = _data_points(reader, "litellm.usage.llm_requests")
|
||||
llm_points: Final = _data_points(reader, "litellm.usage.llm_requests")
|
||||
assert len(llm_points) == 1
|
||||
assert llm_points[0].value == 1
|
||||
assert dict(llm_points[0].attributes) == {
|
||||
"litellm.model": "gpt-4.1",
|
||||
"litellm.model": _PUBLIC_MODEL,
|
||||
"litellm.provider": "openai",
|
||||
"litellm.call_type": "acompletion",
|
||||
}
|
||||
|
||||
token_points = _data_points(reader, "litellm.usage.tokens")
|
||||
by_kind = {point.attributes["litellm.token.kind"]: point.value for point in token_points}
|
||||
token_points: Final = _data_points(reader, "litellm.usage.tokens")
|
||||
by_kind: Final = {point.attributes["litellm.token.kind"]: point.value for point in token_points}
|
||||
assert by_kind == {"prompt": 10, "completion": 5}
|
||||
|
||||
spend_points = _data_points(reader, "litellm.usage.spend_usd")
|
||||
spend_points: Final = _data_points(reader, "litellm.usage.spend_usd")
|
||||
assert len(spend_points) == 1
|
||||
assert spend_points[0].value == pytest.approx(0.0123)
|
||||
|
||||
leaked = [
|
||||
leaked: Final = [
|
||||
value
|
||||
for point in _all_data_points(reader)
|
||||
for value in point.attributes.values()
|
||||
if value
|
||||
in ("sk-secret-key-value", "team-secret", "user-secret", "my-secret-alias", "https://customer-proxy.example.com")
|
||||
in (
|
||||
"sk-secret-key-value",
|
||||
"team-secret",
|
||||
"user-secret",
|
||||
"my-secret-alias",
|
||||
"https://customer-proxy.example.com",
|
||||
)
|
||||
]
|
||||
assert leaked == []
|
||||
|
||||
|
||||
async def test_success_event_with_missing_payload_records_nothing():
|
||||
async def test_private_model_name_collapses_to_other() -> None:
|
||||
"""A model name absent from the public pricing map could be an internal
|
||||
identifier (a fine-tune name, a private deployment alias), so it must not
|
||||
be exported; the label falls back to "other"."""
|
||||
recorder, reader = _recorder_with_reader()
|
||||
|
||||
await recorder.async_log_success_event(
|
||||
kwargs={
|
||||
"standard_logging_object": {
|
||||
**_SUCCESS_PAYLOAD,
|
||||
"model": "my-internal-finetune-v7",
|
||||
"custom_llm_provider": "openai",
|
||||
}
|
||||
},
|
||||
response_obj=None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
|
||||
llm_points: Final = _data_points(reader, "litellm.usage.llm_requests")
|
||||
assert len(llm_points) == 1
|
||||
assert llm_points[0].attributes["litellm.model"] == "other"
|
||||
|
||||
|
||||
async def test_success_event_with_missing_payload_records_nothing() -> None:
|
||||
recorder, reader = _recorder_with_reader()
|
||||
|
||||
await recorder.async_log_success_event(kwargs={}, response_obj=None, start_time=None, end_time=None)
|
||||
|
|
@ -148,27 +196,24 @@ async def test_success_event_with_missing_payload_records_nothing():
|
|||
assert _all_data_points(reader) == []
|
||||
|
||||
|
||||
# ── Env flag and factory ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["true", "TRUE", "1"])
|
||||
def test_usage_telemetry_enabled_true_values(monkeypatch, value):
|
||||
def test_usage_telemetry_enabled_true_values(monkeypatch: pytest.MonkeyPatch, value: str) -> None:
|
||||
monkeypatch.setenv(ut.ENABLED_ENV, value)
|
||||
assert ut.usage_telemetry_enabled() is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["false", "0", "yes", ""])
|
||||
def test_usage_telemetry_enabled_false_values(monkeypatch, value):
|
||||
def test_usage_telemetry_enabled_false_values(monkeypatch: pytest.MonkeyPatch, value: str) -> None:
|
||||
monkeypatch.setenv(ut.ENABLED_ENV, value)
|
||||
assert ut.usage_telemetry_enabled() is False
|
||||
|
||||
|
||||
def test_usage_telemetry_enabled_unset_is_false():
|
||||
def test_usage_telemetry_enabled_unset_is_false() -> None:
|
||||
assert ut.usage_telemetry_enabled() is False
|
||||
|
||||
|
||||
def test_build_recorder_returns_none_when_disabled_and_builds_no_provider():
|
||||
builds = []
|
||||
def test_build_recorder_returns_none_when_disabled_and_builds_no_provider() -> None:
|
||||
builds: Final[list] = []
|
||||
assert (
|
||||
ut.build_usage_telemetry_recorder(
|
||||
litellm_version="1.0", instance_id="i-1", provider_factory=lambda config: builds.append(config)
|
||||
|
|
@ -178,17 +223,17 @@ def test_build_recorder_returns_none_when_disabled_and_builds_no_provider():
|
|||
assert builds == []
|
||||
|
||||
|
||||
def test_build_recorder_uses_injected_provider_factory(monkeypatch):
|
||||
def test_build_recorder_uses_injected_provider_factory(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv(ut.ENABLED_ENV, "true")
|
||||
built: dict[str, ut.UsageTelemetryConfig] = {}
|
||||
built: Final[dict[str, ut.UsageTelemetryConfig]] = {}
|
||||
|
||||
reader = InMemoryMetricReader()
|
||||
reader: Final = InMemoryMetricReader()
|
||||
|
||||
def _factory(config: ut.UsageTelemetryConfig) -> MeterProvider:
|
||||
built["config"] = config
|
||||
return MeterProvider(metric_readers=[reader])
|
||||
|
||||
recorder = ut.build_usage_telemetry_recorder(
|
||||
recorder: Final = ut.build_usage_telemetry_recorder(
|
||||
litellm_version="9.9", instance_id="inst-1", provider_factory=_factory
|
||||
)
|
||||
assert isinstance(recorder, ut.UsageTelemetryRecorder)
|
||||
|
|
@ -197,112 +242,150 @@ def test_build_recorder_uses_injected_provider_factory(monkeypatch):
|
|||
ut.shutdown_usage_telemetry_recorder()
|
||||
|
||||
|
||||
def test_build_recorder_returns_none_when_provider_factory_raises(monkeypatch):
|
||||
def test_build_recorder_returns_none_when_provider_factory_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv(ut.ENABLED_ENV, "true")
|
||||
|
||||
def _explode(config: ut.UsageTelemetryConfig) -> MeterProvider:
|
||||
raise OSError("no network")
|
||||
|
||||
assert (
|
||||
ut.build_usage_telemetry_recorder(litellm_version="1.0", instance_id="i-1", provider_factory=_explode)
|
||||
is None
|
||||
ut.build_usage_telemetry_recorder(litellm_version="1.0", instance_id="i-1", provider_factory=_explode) is None
|
||||
)
|
||||
|
||||
|
||||
def test_export_interval_invalid_falls_back(monkeypatch):
|
||||
def test_export_interval_invalid_falls_back(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv(ut.ENABLED_ENV, "true")
|
||||
monkeypatch.setenv(ut.EXPORT_INTERVAL_ENV, "not-a-number")
|
||||
config = ut.load_usage_telemetry_config(litellm_version="1.0", instance_id="i-1")
|
||||
config: Final = ut.load_usage_telemetry_config(litellm_version="1.0", instance_id="i-1")
|
||||
assert config.export_interval_ms == ut.DEFAULT_EXPORT_INTERVAL_MS
|
||||
|
||||
|
||||
def test_metrics_endpoint_appends_signal_path():
|
||||
def test_metrics_endpoint_appends_signal_path() -> None:
|
||||
assert ut._metrics_endpoint("http://127.0.0.1:4318") == "http://127.0.0.1:4318/v1/metrics"
|
||||
assert ut._metrics_endpoint("http://127.0.0.1:4318/v1/metrics") == "http://127.0.0.1:4318/v1/metrics"
|
||||
|
||||
|
||||
# ── Sink composition ──────────────────────────────────────────────────────────
|
||||
def test_exporter_ignores_otlp_header_env_vars(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""With a falsy headers value OTLPMetricExporter falls back to the
|
||||
OTEL_EXPORTER_OTLP_*_HEADERS env vars, which would forward a deployment's
|
||||
own collector credentials to telemetry.litellm.ai."""
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer secret-token")
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_METRICS_HEADERS", "x-api-key=secret-key")
|
||||
|
||||
exporter: Final = ut._build_exporter(_config())
|
||||
headers: Final[Mapping[str, str]] = exporter._session.headers # pyright: ignore[reportPrivateUsage] # asserts the session's real header set
|
||||
|
||||
assert "authorization" not in {key.lower() for key in headers}
|
||||
assert "x-api-key" not in {key.lower() for key in headers}
|
||||
assert headers["User-Agent"] == "litellm-proxy/1.2.3"
|
||||
|
||||
|
||||
class _SpySink:
|
||||
def __init__(self, raises: bool = False) -> None:
|
||||
self.calls = []
|
||||
self._raises = raises
|
||||
self.calls: list[tuple[BillableCategory, str, int]] = []
|
||||
self._raises: Final = raises
|
||||
|
||||
def record(self, *, category, route, status_code) -> None:
|
||||
def record(self, *, category: BillableCategory, route: str, status_code: int) -> None:
|
||||
if self._raises:
|
||||
raise RuntimeError("boom")
|
||||
self.calls.append((category, route, status_code))
|
||||
|
||||
|
||||
def test_compose_sinks_empty_returns_none():
|
||||
def test_compose_sinks_empty_returns_none() -> None:
|
||||
assert ut.compose_gateway_request_sinks(None, None) is None
|
||||
|
||||
|
||||
def test_compose_sinks_single_returns_it():
|
||||
sink = _SpySink()
|
||||
def test_compose_sinks_single_returns_it() -> None:
|
||||
sink: Final = _SpySink()
|
||||
assert ut.compose_gateway_request_sinks(None, sink) is sink
|
||||
|
||||
|
||||
def test_compose_sinks_fans_out_and_isolates_failures():
|
||||
raising = _SpySink(raises=True)
|
||||
spy = _SpySink()
|
||||
sink = ut.compose_gateway_request_sinks(raising, spy)
|
||||
def test_compose_sinks_fans_out_and_isolates_failures() -> None:
|
||||
raising: Final = _SpySink(raises=True)
|
||||
spy: Final = _SpySink()
|
||||
sink: Final = ut.compose_gateway_request_sinks(raising, spy)
|
||||
assert sink is not None
|
||||
sink.record(category=BillableCategory.LLM, route="/chat/completions", status_code=200)
|
||||
assert spy.calls == [(BillableCategory.LLM, "/chat/completions", 200)]
|
||||
|
||||
|
||||
# ── Instance id resolution ────────────────────────────────────────────────────
|
||||
@dataclass
|
||||
class _FakeRow:
|
||||
param_value: object
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeConfigTable:
|
||||
def __init__(self, row=None) -> None:
|
||||
self._row = row
|
||||
self.created = []
|
||||
rows: list[_FakeRow] = field(default_factory=list)
|
||||
find_calls: int = 0
|
||||
created: list[dict] = field(default_factory=list)
|
||||
create_raises: bool = False
|
||||
|
||||
async def find_unique(self, *, where):
|
||||
return self._row
|
||||
async def find_unique(self, *, where: dict) -> _FakeRow | None:
|
||||
self.find_calls += 1
|
||||
return self.rows[0] if self.rows else None
|
||||
|
||||
async def create(self, *, data):
|
||||
async def create(self, *, data: dict) -> None:
|
||||
if self.create_raises:
|
||||
raise RuntimeError("duplicate key")
|
||||
self.created.append(data)
|
||||
self._row = type("Row", (), {"param_value": data["param_value"]})()
|
||||
self.rows.append(_FakeRow(data["param_value"]))
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeDb:
|
||||
litellm_config: _FakeConfigTable
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakePrisma:
|
||||
def __init__(self, row=None) -> None:
|
||||
self.db = type("DB", (), {"litellm_config": _FakeConfigTable(row)})()
|
||||
db: _FakeDb
|
||||
|
||||
|
||||
class _Row:
|
||||
def __init__(self, param_value) -> None:
|
||||
self.param_value = param_value
|
||||
|
||||
|
||||
async def test_resolve_instance_id_without_db_returns_uuid():
|
||||
resolved = await ut.resolve_instance_id(None)
|
||||
async def test_resolve_instance_id_without_db_returns_uuid() -> None:
|
||||
resolved: Final = await ut.resolve_instance_id(None)
|
||||
assert str(uuid.UUID(resolved)) == resolved
|
||||
|
||||
|
||||
async def test_resolve_instance_id_returns_existing():
|
||||
prisma = _FakePrisma(row=_Row({"instance_id": "existing-id"}))
|
||||
async def test_resolve_instance_id_returns_existing() -> None:
|
||||
prisma: Final = _FakePrisma(_FakeDb(_FakeConfigTable(rows=[_FakeRow({"instance_id": "existing-id"})])))
|
||||
assert await ut.resolve_instance_id(prisma) == "existing-id"
|
||||
|
||||
|
||||
async def test_resolve_instance_id_persists_new_id():
|
||||
prisma = _FakePrisma(row=None)
|
||||
resolved = await ut.resolve_instance_id(prisma)
|
||||
async def test_resolve_instance_id_persists_new_id() -> None:
|
||||
prisma: Final = _FakePrisma(_FakeDb(_FakeConfigTable()))
|
||||
resolved: Final = await ut.resolve_instance_id(prisma)
|
||||
assert str(uuid.UUID(resolved)) == resolved
|
||||
(created,) = prisma.db.litellm_config.created
|
||||
assert created["param_name"] == ut.INSTANCE_ID_CONFIG_KEY
|
||||
assert json.loads(created["param_value"]) == {"instance_id": resolved}
|
||||
|
||||
|
||||
async def test_resolve_instance_id_falls_back_on_db_error():
|
||||
async def test_resolve_instance_id_insert_race_returns_winner_row() -> None:
|
||||
"""When two workers miss the row and race the create, the loser must adopt
|
||||
the winner's persisted id rather than falling back to a per-process uuid."""
|
||||
winner: Final = _FakeRow({"instance_id": "winner-id"})
|
||||
|
||||
class _RaceTable:
|
||||
def __init__(self) -> None:
|
||||
self._find_calls: Final[list] = []
|
||||
|
||||
async def find_unique(self, *, where: dict) -> _FakeRow | None:
|
||||
self._find_calls.append(where)
|
||||
return None if len(self._find_calls) == 1 else winner
|
||||
|
||||
async def create(self, *, data: dict) -> None:
|
||||
raise RuntimeError("duplicate key")
|
||||
|
||||
prisma: Final = type("P", (), {"db": type("DB", (), {"litellm_config": _RaceTable()})()})()
|
||||
assert await ut.resolve_instance_id(prisma) == "winner-id"
|
||||
|
||||
|
||||
async def test_resolve_instance_id_falls_back_on_db_error() -> None:
|
||||
class _ExplodingTable:
|
||||
async def find_unique(self, *, where):
|
||||
async def find_unique(self, *, where: dict) -> None:
|
||||
raise RuntimeError("db down")
|
||||
|
||||
prisma = type("P", (), {"db": type("DB", (), {"litellm_config": _ExplodingTable()})()})()
|
||||
resolved = await ut.resolve_instance_id(prisma)
|
||||
prisma: Final = type("P", (), {"db": type("DB", (), {"litellm_config": _ExplodingTable()})()})()
|
||||
resolved: Final = await ut.resolve_instance_id(prisma)
|
||||
assert str(uuid.UUID(resolved)) == resolved
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue