diff --git a/litellm/proxy/usage_telemetry.py b/litellm/proxy/usage_telemetry.py index 305ffa324a6..a2223bc3d2e 100644 --- a/litellm/proxy/usage_telemetry.py +++ b/litellm/proxy/usage_telemetry.py @@ -8,9 +8,10 @@ untouched. The OSS twin of litellm.proxy.enterprise_billing.billing_metrics: same exporter shape, no license key, no client certificate, no hostname. Only aggregate counts leave the process: request totals by endpoint category -and status class, plus per-call model, provider, call type, token counts, and -cost from the standard logging payload. No API keys, teams, users, model -groups, or api_base values are ever exported. +and status class, plus per-call provider, call type, token counts, and cost +from the standard logging payload. Model names are exported only when they are +present in the public pricing map; anything else is reported as "other". No +API keys, teams, users, model groups, or api_base values are ever exported. """ import json @@ -19,7 +20,7 @@ import uuid from collections.abc import Callable, Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.sdk.metrics import MeterProvider @@ -28,6 +29,7 @@ from opentelemetry.sdk.resources import Resource from pydantic import BaseModel, ConfigDict, Field, ValidationError from typing_extensions import ReadOnly, TypedDict +import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger from litellm.proxy.middleware.billable_request_metrics_middleware import ( @@ -95,11 +97,19 @@ def load_usage_telemetry_config(*, litellm_version: str, instance_id: str) -> Us ) -def build_usage_meter_provider(config: UsageTelemetryConfig) -> MeterProvider: - exporter: Final = OTLPMetricExporter( +def _build_exporter(config: UsageTelemetryConfig) -> OTLPMetricExporter: + """``headers`` must be non-empty: with a falsy value OTLPMetricExporter + reads OTEL_EXPORTER_OTLP_*_HEADERS from the environment, which would + forward a deployment's own collector auth header to telemetry.litellm.ai.""" + return OTLPMetricExporter( endpoint=_metrics_endpoint(config.endpoint), timeout=EXPORT_TIMEOUT_S, + headers={"User-Agent": f"litellm-proxy/{config.litellm_version}"}, ) + + +def build_usage_meter_provider(config: UsageTelemetryConfig) -> MeterProvider: + exporter: Final = _build_exporter(config) reader: Final = PeriodicExportingMetricReader(exporter, export_interval_millis=config.export_interval_ms) resource: Final = Resource.create( MappingProxyType( @@ -170,7 +180,13 @@ class UsageTelemetryRecorder(CustomLogger): ), ) - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: raw_payload: Final = kwargs.get("standard_logging_object") if not isinstance(raw_payload, dict): return @@ -181,7 +197,7 @@ class UsageTelemetryRecorder(CustomLogger): return attrs: Final[Mapping[str, str]] = MappingProxyType( { - "litellm.model": payload.model, + "litellm.model": _public_model_label(payload.model, payload.custom_llm_provider), "litellm.provider": payload.custom_llm_provider or "unknown", "litellm.call_type": payload.call_type, } @@ -197,6 +213,17 @@ class UsageTelemetryRecorder(CustomLogger): self._provider.shutdown(timeout_millis=SHUTDOWN_FLUSH_TIMEOUT_MS) +def _public_model_label(model: str, provider: str | None) -> str: + """Only public model names leave the process. A name absent from the + public pricing map (fine-tunes, aliases, internal deployments) collapses + to "other" so a private model identifier cannot leak.""" + if model in litellm.model_cost: + return model + if provider is not None and f"{provider}/{model}" in litellm.model_cost: + return f"{provider}/{model}" + return "other" + + class _ConfigParamWhere(TypedDict): param_name: ReadOnly[str] @@ -210,6 +237,18 @@ class _ConfigParamCreate(TypedDict): param_value: ReadOnly[str] +class _ConfigParamRow(Protocol): + param_value: object + + +def _instance_id_from_row(row: "_ConfigParamRow | None") -> str | None: + param_value: Final[object] = row.param_value if row is not None else None + parsed: Final[object] = json.loads(param_value) if isinstance(param_value, str) else param_value + if isinstance(parsed, dict) and isinstance(parsed.get("instance_id"), str): + return parsed["instance_id"] + return None + + async def resolve_instance_id(prisma_client: "PrismaClient | None") -> str: """ A stable anonymous deployment id persisted in ``LiteLLM_Config``, so counts @@ -220,18 +259,22 @@ async def resolve_instance_id(prisma_client: "PrismaClient | None") -> str: return str(uuid.uuid4()) try: where: Final[_ConfigParamWhere] = {"param_name": INSTANCE_ID_CONFIG_KEY} - row: Final = await prisma_client.db.litellm_config.find_unique(where=where) - param_value: Final[object] = row.param_value if row is not None else None - parsed: Final[object] = json.loads(param_value) if isinstance(param_value, str) else param_value - if isinstance(parsed, dict) and isinstance(parsed.get("instance_id"), str): - return parsed["instance_id"] + existing: Final = _instance_id_from_row(await prisma_client.db.litellm_config.find_unique(where=where)) + if existing is not None: + return existing instance_id: Final = str(uuid.uuid4()) instance_value: Final[_InstanceIdValue] = {"instance_id": instance_id} data: Final[_ConfigParamCreate] = { "param_name": INSTANCE_ID_CONFIG_KEY, "param_value": json.dumps(instance_value), } - await prisma_client.db.litellm_config.create(data=data) + try: + await prisma_client.db.litellm_config.create(data=data) + except Exception: # noqa: BLE001 -- a concurrent worker won the insert; read its row instead + raced: Final = _instance_id_from_row(await prisma_client.db.litellm_config.find_unique(where=where)) + if raced is not None: + return raced + raise return instance_id except Exception as exc: # noqa: BLE001 -- telemetry must never break proxy startup verbose_proxy_logger.warning( diff --git a/tests/test_litellm/proxy/test_usage_telemetry.py b/tests/test_litellm/proxy/test_usage_telemetry.py index a5dc8fd0cb8..048546bf821 100644 --- a/tests/test_litellm/proxy/test_usage_telemetry.py +++ b/tests/test_litellm/proxy/test_usage_telemetry.py @@ -8,34 +8,45 @@ and aggregated values. Nothing the proxy sends is mocked. import json import uuid +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Final import pytest from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.metrics.export import InMemoryMetricReader, Metric +import litellm from litellm.proxy import usage_telemetry as ut from litellm.proxy.middleware.billable_request_metrics_middleware import BillableCategory @pytest.fixture(autouse=True) -def clear_env(monkeypatch): +def clear_env(monkeypatch: pytest.MonkeyPatch) -> None: for name in (ut.ENABLED_ENV, ut.ENDPOINT_ENV, ut.EXPORT_INTERVAL_ENV): monkeypatch.delenv(name, raising=False) yield ut.shutdown_usage_telemetry_recorder() -def _data_points(reader: InMemoryMetricReader, metric_name: str): +def _metric(reader: InMemoryMetricReader, metric_name: str) -> Metric | None: data = reader.get_metrics_data() + if data is None: + return None for resource_metric in data.resource_metrics: for scope_metric in resource_metric.scope_metrics: for metric in scope_metric.metrics: if metric.name == metric_name: - return list(metric.data.data_points) - return [] + return metric + return None -def _all_data_points(reader: InMemoryMetricReader): +def _data_points(reader: InMemoryMetricReader, metric_name: str) -> list: + metric: Final = _metric(reader, metric_name) + return [] if metric is None else list(metric.data.data_points) + + +def _all_data_points(reader: InMemoryMetricReader) -> list: data = reader.get_metrics_data() if data is None: return [] @@ -49,32 +60,39 @@ def _all_data_points(reader: InMemoryMetricReader): def _recorder_with_reader() -> tuple[ut.UsageTelemetryRecorder, InMemoryMetricReader]: - reader = InMemoryMetricReader() + reader: Final = InMemoryMetricReader() return ut.UsageTelemetryRecorder(MeterProvider(metric_readers=[reader])), reader -# ── Middleware sink: litellm.usage.requests ──────────────────────────────────── +def _config(**overrides: object) -> ut.UsageTelemetryConfig: + base: Final[dict[str, object]] = { + "endpoint": "https://telemetry.litellm.ai/v1/metrics", + "export_interval_ms": 60_000, + "litellm_version": "1.2.3", + "instance_id": "inst-1", + } + return ut.UsageTelemetryConfig(**{**base, **overrides}) -def test_record_aggregates_by_route_and_status_class(): +def test_record_aggregates_by_route_and_status_class() -> None: recorder, reader = _recorder_with_reader() recorder.record(category=BillableCategory.LLM, route="/chat/completions", status_code=200) recorder.record(category=BillableCategory.LLM, route="/chat/completions", status_code=200) recorder.record(category=BillableCategory.LLM, route="/chat/completions", status_code=500) - points = _data_points(reader, "litellm.usage.requests") - by_class = {point.attributes["http.response.status_class"]: point.value for point in points} + points: Final = _data_points(reader, "litellm.usage.requests") + by_class: Final = {point.attributes["http.response.status_class"]: point.value for point in points} assert by_class == {"2xx": 2, "5xx": 1} for point in points: assert point.attributes["http.route"] == "/chat/completions" assert point.attributes["litellm.endpoint.category"] == "llm" -# ── Success callback: llm_requests / tokens / spend ──────────────────────────── +_PUBLIC_MODEL: Final = next(key for key in litellm.model_cost if "/" not in key) -_SUCCESS_PAYLOAD = { - "model": "gpt-4.1", +_SUCCESS_PAYLOAD: Final[dict] = { + "model": _PUBLIC_MODEL, "custom_llm_provider": "openai", "call_type": "acompletion", "prompt_tokens": 10, @@ -91,7 +109,7 @@ _SUCCESS_PAYLOAD = { } -async def test_success_event_exports_only_anonymous_attributes(): +async def test_success_event_exports_only_anonymous_attributes() -> None: recorder, reader = _recorder_with_reader() await recorder.async_log_success_event( @@ -101,34 +119,64 @@ async def test_success_event_exports_only_anonymous_attributes(): end_time=None, ) - llm_points = _data_points(reader, "litellm.usage.llm_requests") + llm_points: Final = _data_points(reader, "litellm.usage.llm_requests") assert len(llm_points) == 1 assert llm_points[0].value == 1 assert dict(llm_points[0].attributes) == { - "litellm.model": "gpt-4.1", + "litellm.model": _PUBLIC_MODEL, "litellm.provider": "openai", "litellm.call_type": "acompletion", } - token_points = _data_points(reader, "litellm.usage.tokens") - by_kind = {point.attributes["litellm.token.kind"]: point.value for point in token_points} + token_points: Final = _data_points(reader, "litellm.usage.tokens") + by_kind: Final = {point.attributes["litellm.token.kind"]: point.value for point in token_points} assert by_kind == {"prompt": 10, "completion": 5} - spend_points = _data_points(reader, "litellm.usage.spend_usd") + spend_points: Final = _data_points(reader, "litellm.usage.spend_usd") assert len(spend_points) == 1 assert spend_points[0].value == pytest.approx(0.0123) - leaked = [ + leaked: Final = [ value for point in _all_data_points(reader) for value in point.attributes.values() if value - in ("sk-secret-key-value", "team-secret", "user-secret", "my-secret-alias", "https://customer-proxy.example.com") + in ( + "sk-secret-key-value", + "team-secret", + "user-secret", + "my-secret-alias", + "https://customer-proxy.example.com", + ) ] assert leaked == [] -async def test_success_event_with_missing_payload_records_nothing(): +async def test_private_model_name_collapses_to_other() -> None: + """A model name absent from the public pricing map could be an internal + identifier (a fine-tune name, a private deployment alias), so it must not + be exported; the label falls back to "other".""" + recorder, reader = _recorder_with_reader() + + await recorder.async_log_success_event( + kwargs={ + "standard_logging_object": { + **_SUCCESS_PAYLOAD, + "model": "my-internal-finetune-v7", + "custom_llm_provider": "openai", + } + }, + response_obj=None, + start_time=None, + end_time=None, + ) + + llm_points: Final = _data_points(reader, "litellm.usage.llm_requests") + assert len(llm_points) == 1 + assert llm_points[0].attributes["litellm.model"] == "other" + + +async def test_success_event_with_missing_payload_records_nothing() -> None: recorder, reader = _recorder_with_reader() await recorder.async_log_success_event(kwargs={}, response_obj=None, start_time=None, end_time=None) @@ -148,27 +196,24 @@ async def test_success_event_with_missing_payload_records_nothing(): assert _all_data_points(reader) == [] -# ── Env flag and factory ─────────────────────────────────────────────────────── - - @pytest.mark.parametrize("value", ["true", "TRUE", "1"]) -def test_usage_telemetry_enabled_true_values(monkeypatch, value): +def test_usage_telemetry_enabled_true_values(monkeypatch: pytest.MonkeyPatch, value: str) -> None: monkeypatch.setenv(ut.ENABLED_ENV, value) assert ut.usage_telemetry_enabled() is True @pytest.mark.parametrize("value", ["false", "0", "yes", ""]) -def test_usage_telemetry_enabled_false_values(monkeypatch, value): +def test_usage_telemetry_enabled_false_values(monkeypatch: pytest.MonkeyPatch, value: str) -> None: monkeypatch.setenv(ut.ENABLED_ENV, value) assert ut.usage_telemetry_enabled() is False -def test_usage_telemetry_enabled_unset_is_false(): +def test_usage_telemetry_enabled_unset_is_false() -> None: assert ut.usage_telemetry_enabled() is False -def test_build_recorder_returns_none_when_disabled_and_builds_no_provider(): - builds = [] +def test_build_recorder_returns_none_when_disabled_and_builds_no_provider() -> None: + builds: Final[list] = [] assert ( ut.build_usage_telemetry_recorder( litellm_version="1.0", instance_id="i-1", provider_factory=lambda config: builds.append(config) @@ -178,17 +223,17 @@ def test_build_recorder_returns_none_when_disabled_and_builds_no_provider(): assert builds == [] -def test_build_recorder_uses_injected_provider_factory(monkeypatch): +def test_build_recorder_uses_injected_provider_factory(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv(ut.ENABLED_ENV, "true") - built: dict[str, ut.UsageTelemetryConfig] = {} + built: Final[dict[str, ut.UsageTelemetryConfig]] = {} - reader = InMemoryMetricReader() + reader: Final = InMemoryMetricReader() def _factory(config: ut.UsageTelemetryConfig) -> MeterProvider: built["config"] = config return MeterProvider(metric_readers=[reader]) - recorder = ut.build_usage_telemetry_recorder( + recorder: Final = ut.build_usage_telemetry_recorder( litellm_version="9.9", instance_id="inst-1", provider_factory=_factory ) assert isinstance(recorder, ut.UsageTelemetryRecorder) @@ -197,112 +242,150 @@ def test_build_recorder_uses_injected_provider_factory(monkeypatch): ut.shutdown_usage_telemetry_recorder() -def test_build_recorder_returns_none_when_provider_factory_raises(monkeypatch): +def test_build_recorder_returns_none_when_provider_factory_raises(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv(ut.ENABLED_ENV, "true") def _explode(config: ut.UsageTelemetryConfig) -> MeterProvider: raise OSError("no network") assert ( - ut.build_usage_telemetry_recorder(litellm_version="1.0", instance_id="i-1", provider_factory=_explode) - is None + ut.build_usage_telemetry_recorder(litellm_version="1.0", instance_id="i-1", provider_factory=_explode) is None ) -def test_export_interval_invalid_falls_back(monkeypatch): +def test_export_interval_invalid_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv(ut.ENABLED_ENV, "true") monkeypatch.setenv(ut.EXPORT_INTERVAL_ENV, "not-a-number") - config = ut.load_usage_telemetry_config(litellm_version="1.0", instance_id="i-1") + config: Final = ut.load_usage_telemetry_config(litellm_version="1.0", instance_id="i-1") assert config.export_interval_ms == ut.DEFAULT_EXPORT_INTERVAL_MS -def test_metrics_endpoint_appends_signal_path(): +def test_metrics_endpoint_appends_signal_path() -> None: assert ut._metrics_endpoint("http://127.0.0.1:4318") == "http://127.0.0.1:4318/v1/metrics" assert ut._metrics_endpoint("http://127.0.0.1:4318/v1/metrics") == "http://127.0.0.1:4318/v1/metrics" -# ── Sink composition ────────────────────────────────────────────────────────── +def test_exporter_ignores_otlp_header_env_vars(monkeypatch: pytest.MonkeyPatch) -> None: + """With a falsy headers value OTLPMetricExporter falls back to the + OTEL_EXPORTER_OTLP_*_HEADERS env vars, which would forward a deployment's + own collector credentials to telemetry.litellm.ai.""" + monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer secret-token") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_METRICS_HEADERS", "x-api-key=secret-key") + + exporter: Final = ut._build_exporter(_config()) + headers: Final[Mapping[str, str]] = exporter._session.headers # pyright: ignore[reportPrivateUsage] # asserts the session's real header set + + assert "authorization" not in {key.lower() for key in headers} + assert "x-api-key" not in {key.lower() for key in headers} + assert headers["User-Agent"] == "litellm-proxy/1.2.3" class _SpySink: def __init__(self, raises: bool = False) -> None: - self.calls = [] - self._raises = raises + self.calls: list[tuple[BillableCategory, str, int]] = [] + self._raises: Final = raises - def record(self, *, category, route, status_code) -> None: + def record(self, *, category: BillableCategory, route: str, status_code: int) -> None: if self._raises: raise RuntimeError("boom") self.calls.append((category, route, status_code)) -def test_compose_sinks_empty_returns_none(): +def test_compose_sinks_empty_returns_none() -> None: assert ut.compose_gateway_request_sinks(None, None) is None -def test_compose_sinks_single_returns_it(): - sink = _SpySink() +def test_compose_sinks_single_returns_it() -> None: + sink: Final = _SpySink() assert ut.compose_gateway_request_sinks(None, sink) is sink -def test_compose_sinks_fans_out_and_isolates_failures(): - raising = _SpySink(raises=True) - spy = _SpySink() - sink = ut.compose_gateway_request_sinks(raising, spy) +def test_compose_sinks_fans_out_and_isolates_failures() -> None: + raising: Final = _SpySink(raises=True) + spy: Final = _SpySink() + sink: Final = ut.compose_gateway_request_sinks(raising, spy) assert sink is not None sink.record(category=BillableCategory.LLM, route="/chat/completions", status_code=200) assert spy.calls == [(BillableCategory.LLM, "/chat/completions", 200)] -# ── Instance id resolution ──────────────────────────────────────────────────── +@dataclass +class _FakeRow: + param_value: object +@dataclass class _FakeConfigTable: - def __init__(self, row=None) -> None: - self._row = row - self.created = [] + rows: list[_FakeRow] = field(default_factory=list) + find_calls: int = 0 + created: list[dict] = field(default_factory=list) + create_raises: bool = False - async def find_unique(self, *, where): - return self._row + async def find_unique(self, *, where: dict) -> _FakeRow | None: + self.find_calls += 1 + return self.rows[0] if self.rows else None - async def create(self, *, data): + async def create(self, *, data: dict) -> None: + if self.create_raises: + raise RuntimeError("duplicate key") self.created.append(data) - self._row = type("Row", (), {"param_value": data["param_value"]})() + self.rows.append(_FakeRow(data["param_value"])) +@dataclass +class _FakeDb: + litellm_config: _FakeConfigTable + + +@dataclass class _FakePrisma: - def __init__(self, row=None) -> None: - self.db = type("DB", (), {"litellm_config": _FakeConfigTable(row)})() + db: _FakeDb -class _Row: - def __init__(self, param_value) -> None: - self.param_value = param_value - - -async def test_resolve_instance_id_without_db_returns_uuid(): - resolved = await ut.resolve_instance_id(None) +async def test_resolve_instance_id_without_db_returns_uuid() -> None: + resolved: Final = await ut.resolve_instance_id(None) assert str(uuid.UUID(resolved)) == resolved -async def test_resolve_instance_id_returns_existing(): - prisma = _FakePrisma(row=_Row({"instance_id": "existing-id"})) +async def test_resolve_instance_id_returns_existing() -> None: + prisma: Final = _FakePrisma(_FakeDb(_FakeConfigTable(rows=[_FakeRow({"instance_id": "existing-id"})]))) assert await ut.resolve_instance_id(prisma) == "existing-id" -async def test_resolve_instance_id_persists_new_id(): - prisma = _FakePrisma(row=None) - resolved = await ut.resolve_instance_id(prisma) +async def test_resolve_instance_id_persists_new_id() -> None: + prisma: Final = _FakePrisma(_FakeDb(_FakeConfigTable())) + resolved: Final = await ut.resolve_instance_id(prisma) assert str(uuid.UUID(resolved)) == resolved (created,) = prisma.db.litellm_config.created assert created["param_name"] == ut.INSTANCE_ID_CONFIG_KEY assert json.loads(created["param_value"]) == {"instance_id": resolved} -async def test_resolve_instance_id_falls_back_on_db_error(): +async def test_resolve_instance_id_insert_race_returns_winner_row() -> None: + """When two workers miss the row and race the create, the loser must adopt + the winner's persisted id rather than falling back to a per-process uuid.""" + winner: Final = _FakeRow({"instance_id": "winner-id"}) + + class _RaceTable: + def __init__(self) -> None: + self._find_calls: Final[list] = [] + + async def find_unique(self, *, where: dict) -> _FakeRow | None: + self._find_calls.append(where) + return None if len(self._find_calls) == 1 else winner + + async def create(self, *, data: dict) -> None: + raise RuntimeError("duplicate key") + + prisma: Final = type("P", (), {"db": type("DB", (), {"litellm_config": _RaceTable()})()})() + assert await ut.resolve_instance_id(prisma) == "winner-id" + + +async def test_resolve_instance_id_falls_back_on_db_error() -> None: class _ExplodingTable: - async def find_unique(self, *, where): + async def find_unique(self, *, where: dict) -> None: raise RuntimeError("db down") - prisma = type("P", (), {"db": type("DB", (), {"litellm_config": _ExplodingTable()})()})() - resolved = await ut.resolve_instance_id(prisma) + prisma: Final = type("P", (), {"db": type("DB", (), {"litellm_config": _ExplodingTable()})()})() + resolved: Final = await ut.resolve_instance_id(prisma) assert str(uuid.UUID(resolved)) == resolved