From d447be15b965a052e49a2d3c0b9f538d85c9d865 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Sat, 22 Aug 2026 19:13:48 -0700 Subject: [PATCH 1/6] feat(newrelic): per-team New Relic trace routing via team callbacks (#37603) --- litellm/integrations/callback_configs.json | 14 +- litellm/integrations/otel/emitter.py | 6 + litellm/integrations/otel/logger.py | 36 +++- litellm/integrations/otel/model/config.py | 10 + litellm/integrations/otel/model/metadata.py | 6 + .../integrations/otel/plumbing/providers.py | 2 + litellm/integrations/otel/plumbing/routing.py | 29 ++- litellm/integrations/otel/presets/__init__.py | 70 +++++-- litellm/integrations/otel/presets/newrelic.py | 104 +++++++++++ .../initialize_dynamic_callback_params.py | 32 +++- litellm/litellm_core_utils/litellm_logging.py | 7 + .../callback_config_validation.py | 77 ++++++++ litellm/proxy/litellm_pre_call_utils.py | 6 + .../key_management_endpoints.py | 15 ++ .../team_callback_endpoints.py | 13 ++ litellm/types/utils.py | 5 + .../integrations/otel/test_otel_v2_dynamic.py | 164 +++++++++++----- .../integrations/otel/test_otel_v2_logger.py | 175 ++++++++++++++++++ .../integrations/otel/test_otel_v2_presets.py | 52 ++++++ ...test_initialize_dynamic_callback_params.py | 46 +++++ .../test_litellm_logging.py | 78 ++++++++ .../test_callback_management_endpoints.py | 145 +++++++++++++++ .../proxy/test_litellm_pre_call_utils.py | 89 +++++++++ .../src/components/callback_info_helpers.tsx | 12 ++ 24 files changed, 1111 insertions(+), 82 deletions(-) create mode 100644 litellm/integrations/otel/presets/newrelic.py create mode 100644 litellm/proxy/common_utils/callback_config_validation.py diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index 590c848767a..6d2bcea8bae 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -294,12 +294,18 @@ "id": "newrelic", "displayName": "New Relic", "logo": "newrelic.png", - "supports_key_team_logging": false, + "supports_key_team_logging": true, "dynamic_params": { - "NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED": { + "newrelic_api_key": { + "type": "password", + "ui_name": "New Relic Ingest License Key", + "description": "Per-team ingest (license) key. Team traces export to this key's New Relic account over OTLP.", + "required": false + }, + "newrelic_region": { "type": "text", - "ui_name": "Record AI Content (default: true)", - "description": "Whether to record AI message content. Set to false to disable.", + "ui_name": "New Relic Region (us or eu)", + "description": "Data center region for this team's account. Defaults to us.", "required": false } }, diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 0850d867c7b..244e58eddf3 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -156,6 +156,12 @@ class SpanEmitter: links=list(links) if links else None, ) + def mark_emitted(self, dedup_key: str | None, role: SpanRole) -> None: + """Register a span emitted outside :meth:`emit` (the boundary-opened + LLM-call span closed via :meth:`finish_span`) so a later :meth:`emit` + for the same ``(dedup_key, role)`` deduplicates against it.""" + self._seen(dedup_key, role) + def _seen(self, dedup_key: str | None, role: SpanRole) -> bool: """Return True once a ``(dedup_key, role)`` pair has been emitted. diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 53b9829023c..4359b222d06 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -484,10 +484,15 @@ class OpenTelemetryV2(CustomLogger): # ``pop`` is the dedup: this method runs from both the success and failure # paths, and whichever fires first removes the carrier and closes the span. carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None - if carrier is None: + # A missing carrier does not always mean nothing happened: a team/key-scoped + # logger is a success/failure callback only, so ``pre_call`` never reaches it + # and no carrier exists. The payload plus the request-level provider-handoff + # stamp (``upstream_started``) is the affirmative signal of a real call; a + # gate rejection carries ``is_no_upstream_call`` and gets no span. + if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None): return None try: - return self._finish_carrier(carrier, call, end_time) + return self._finish_carrier(carrier, call, start_time, end_time) finally: # After the span has ended, so a release-triggered provider shutdown # force-flushes it out rather than racing its enqueue. @@ -497,8 +502,11 @@ class OpenTelemetryV2(CustomLogger): """Remember an in-flight LLM call, evicting the oldest if over budget. A call that opens but never closes (a stream that only fires stream - events) would linger otherwise; the evicted span is simply dropped - (never exported). + events) would linger otherwise. Eviction only drops the boundary carrier, + not the call: if that call later closes as a real completed call, it still + emits through the deferred branch in ``_close_llm_call`` (the same path a + team/key-scoped logger uses, since it never opens a carrier), deduplicated + by call id. Only a call that is evicted and never closes goes unexported. """ self._open_llm_calls[call_id] = carrier if len(self._open_llm_calls) > _OPEN_CALLS_MAX: @@ -512,15 +520,20 @@ class OpenTelemetryV2(CustomLogger): def _finish_carrier( self, - carrier: _LLMCallSpan, + carrier: "_LLMCallSpan | None", call: LLMCallEvent, + start_time: datetime | float | None, end_time: datetime | float | None, ) -> Span | None: payload: Final = call.payload + call_id: Final = call.call_id if payload is None: - if carrier.span is not None: + if carrier is not None and carrier.span is not None: # Opened at the boundary but the payload never materialized — end - # it (named provisionally) so it isn't leaked as an open span. + # it (named provisionally) so it isn't leaked as an open span, and + # register the dedup marker so a later payload-carrying close for + # the same call id cannot re-emit through the deferred branch. + self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL) carrier.span.end(end_time=to_ns(end_time)) return None data: Final = LLMCallSpanData.from_standard_logging_payload( @@ -529,10 +542,13 @@ class OpenTelemetryV2(CustomLogger): time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, ) end_time_ns: Final = to_ns(end_time) - if carrier.span is not None: + if carrier is not None and carrier.span is not None: # Born at the boundary: stamp attributes from the typed payload, set # status, and end it. Its parent (the server span) was captured at - # creation from real ambient context. + # creation from real ambient context. Register the dedup marker so a + # second close for the same call id (success then failure on one + # logging object) cannot re-emit through the deferred branch. + self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL) self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns) return carrier.span # Deferred: ``pre_call`` saw no recordable parent, so create the span now. @@ -549,7 +565,7 @@ class OpenTelemetryV2(CustomLogger): SpanRole.LLM_CALL, data, parent_context=(set_span_in_context(INVALID_SPAN, parent_ctx) if route.detached else parent_ctx), - start_time_ns=carrier.start_time_ns, + start_time_ns=(carrier.start_time_ns if carrier is not None else to_ns(start_time)), end_time_ns=end_time_ns, tracer=route.tracer, links=_request_trace_links(parent_ctx) if route.detached else None, diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index 53178b48991..9e3064c2bff 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -39,6 +39,7 @@ class ExporterOwner(str, Enum): WEAVE_OTEL = "weave_otel" LEVO = "levo" AGENTOPS = "agentops" + NEWRELIC = "newrelic" class _OTelV2Flag(BaseSettings): @@ -97,6 +98,15 @@ class ExporterSpec(BaseModel): "auto (Simple for console/in_memory, Batch otherwise)." ), ) + requires_headers: bool = Field( + default=False, + description=( + "Skip this exporter when no headers are resolved. For destinations " + "that reject unauthenticated exports (e.g. New Relic), a spec kept " + "only as the per-request credential-stamping target would otherwise " + "export keyless traffic and produce a 4xx for every span batch." + ), + ) class OpenTelemetryV2Config(BaseSettings): diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index de6366e7dbd..062b2ca20b4 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -203,6 +203,11 @@ class LLMCallEvent: # True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire # the ``pre_call`` hook but never made an upstream call, so they get no span. is_no_upstream_call: bool + # True once the request handed off to a provider (``pre_call`` stamped + # ``api_call_start_time``). The affirmative signal that an LLM call was + # actually attempted — router pre-call rejections, SDK failures before the + # provider handoff, and standalone guardrail runs all lack it. + upstream_started: bool # A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The # span is renamed from the typed payload at close (``finish_span``); this only # needs to be reasonable for a span that never gets closed (a leak). @@ -221,6 +226,7 @@ class LLMCallEvent: dynamic_params=kwargs.get("standard_callback_dynamic_params"), auth_metadata=auth_metadata(payload, kwargs), is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)), + upstream_started=kwargs.get("api_call_start_time") is not None, provisional_span_name=f"{operation.value} {model}".strip(), time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs), ) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 80a8d01c061..81b00f788ee 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -436,6 +436,8 @@ def build_tracer_provider( # ``config._normalize`` guarantees at least one spec (it folds the top-level # ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty). for spec in config.exporters: + if spec.requires_headers and not spec.headers: + continue exp = _exporter_from_spec(spec) provider.add_span_processor( _processor_for( diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index f231df9e914..6a04dbb9bc8 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -28,6 +28,7 @@ from litellm.integrations.otel.plumbing.providers import ( get_tracer, ) from litellm.integrations.otel.presets import ( + dynamic_otlp_endpoint, dynamic_otlp_headers, project_routing_headers, ) @@ -129,7 +130,9 @@ class TenantTracerCache: # thread-pool workers concurrently with the event loop, so cache # updates, span counts, and retirement must be atomic. self._lock: Final = threading.Lock() - self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems], TracerProvider] = OrderedDict() + self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems, str | None], TracerProvider] = ( + OrderedDict() # mutable-ok: bounded LRU; eviction needs in-place ordered mutation + ) self._open_span_counts: dict[TracerProvider, int] = {} # mutable-ok: live refcount state # Oldest-first so an overflow of draining providers sheds the stalest. self._retired: OrderedDict[TracerProvider, None] = OrderedDict() # mutable-ok: draining evicted providers @@ -182,12 +185,16 @@ class TenantTracerCache: project_headers: Final = self._project_headers(auth_metadata) if not credential_headers and not project_headers: return TenantRoute(tracer=default, detached=False) + # A fixed per-integration region endpoint (New Relic us/eu), never a + # caller-supplied host; ``None`` keeps the preset's own endpoint. + endpoint: Final = dynamic_otlp_endpoint(self._callback_name, dynamic_params) cache_key: Final = ( tuple(sorted(credential_headers.items())), tuple(sorted(project_headers.items())), + endpoint, ) with self._lock: - provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers) + provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers, endpoint) self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1 evicted: Final = self._evicted_on_overflow_locked() if evicted is not None: @@ -200,15 +207,16 @@ class TenantTracerCache: def _cached_provider_locked( self, - cache_key: tuple[_HeaderItems, _HeaderItems], + cache_key: tuple[_HeaderItems, _HeaderItems, str | None], credential_headers: Mapping[str, str], project_headers: Mapping[str, str], + endpoint: str | None, ) -> TracerProvider: cached: Final = self._providers.get(cache_key) if cached is not None: self._providers.move_to_end(cache_key) return cached - built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers)) + built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers, endpoint)) self._providers[cache_key] = built return built @@ -257,6 +265,7 @@ class TenantTracerCache: self, credential_headers: Mapping[str, str], project_headers: Mapping[str, str], + endpoint: str | None = None, ) -> OpenTelemetryV2Config: """Clone the config, rewriting headers on the callback's own exporter. @@ -272,7 +281,8 @@ class TenantTracerCache: ``Authorization``), which must survive routing to a project. """ exporters: Final = [ - self._routed_exporter(spec, credential_headers, project_headers) for spec in self._config.exporters + self._routed_exporter(spec, credential_headers, project_headers, endpoint) + for spec in self._config.exporters ] return self._config.model_copy(update={"exporters": exporters}) @@ -281,6 +291,7 @@ class TenantTracerCache: spec: ExporterSpec, credential_headers: Mapping[str, str], project_headers: Mapping[str, str], + endpoint: str | None = None, ) -> ExporterSpec: kind: Final = spec.kind.lower() if spec.owner != self._callback_name or kind in _NON_OTLP_KINDS: @@ -291,4 +302,10 @@ class TenantTracerCache: if project_headers and kind not in _GRPC_KINDS else base ) - return spec if routed == spec.headers else spec.model_copy(update={"headers": routed}) + update: Final = { # mutable-ok: model_copy(update=...) requires a plain dict + field: value + for field, value in (("headers", routed), ("endpoint", endpoint)) + if (field == "headers" and routed != spec.headers) + or (field == "endpoint" and endpoint is not None and endpoint != spec.endpoint) + } + return spec if not update else spec.model_copy(update=update) diff --git a/litellm/integrations/otel/presets/__init__.py b/litellm/integrations/otel/presets/__init__.py index 35b0584c697..a0cd5b3fd98 100644 --- a/litellm/integrations/otel/presets/__init__.py +++ b/litellm/integrations/otel/presets/__init__.py @@ -21,6 +21,11 @@ from litellm.integrations.otel.presets.langfuse import ( ) from litellm.integrations.otel.presets.langtrace import langtrace_preset from litellm.integrations.otel.presets.levo import levo_preset +from litellm.integrations.otel.presets.newrelic import ( + newrelic_dynamic_endpoint, + newrelic_dynamic_headers, + newrelic_preset, +) from litellm.integrations.otel.presets.phoenix import ( phoenix_preset, phoenix_project_headers, @@ -30,25 +35,45 @@ from litellm.types.utils import StandardCallbackDynamicParams #: Callback name → preset. The ``Preset`` annotation makes mypy verify every #: registered value matches the preset interface. -PRESET_BY_CALLBACK: Final[dict[str, Preset]] = { - "agentops": agentops_preset, - "arize": arize_preset, - "arize_phoenix": phoenix_preset, - "langfuse_otel": langfuse_preset, - "langtrace": langtrace_preset, - "levo": levo_preset, - "weave_otel": weave_preset, -} +PRESET_BY_CALLBACK: Final[Mapping[str, Preset]] = MappingProxyType( + { + "agentops": agentops_preset, + "arize": arize_preset, + "arize_phoenix": phoenix_preset, + "langfuse_otel": langfuse_preset, + "langtrace": langtrace_preset, + "levo": levo_preset, + "newrelic": newrelic_preset, + "weave_otel": weave_preset, + } +) #: Callback name → per-request OTLP header builder (team/key multi-tenant #: routing). Only integrations that support dynamic credentials appear here — #: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's #: default tracer. -DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = { - "arize": arize_dynamic_headers, - "langfuse_otel": langfuse_dynamic_headers, - "weave_otel": weave_dynamic_headers, -} +DYNAMIC_HEADERS_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = ( + MappingProxyType( + { + "arize": arize_dynamic_headers, + "langfuse_otel": langfuse_dynamic_headers, + "newrelic": newrelic_dynamic_headers, + "weave_otel": weave_dynamic_headers, + } + ) +) + +#: Callback name → per-request OTLP endpoint resolver. Only integrations whose +#: destination host varies per tenant (from a fixed region table, never a +#: caller-supplied URL) appear here; for everyone else the preset's endpoint is +#: authoritative. +DYNAMIC_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = ( + MappingProxyType( + { + "newrelic": newrelic_dynamic_endpoint, + } + ) +) #: Callback name → per-request *routing* header builder, sourced from the key/team @@ -98,17 +123,34 @@ def project_routing_headers( return builder(auth_metadata) +def dynamic_otlp_endpoint( + callback_name: str | None, + dynamic_params: StandardCallbackDynamicParams | None, +) -> str | None: + """Per-request OTLP endpoint for ``callback_name``, or ``None`` if N/A. + + ``None`` means "keep the preset's own endpoint". + """ + resolver: Final = DYNAMIC_ENDPOINT_BY_CALLBACK.get(callback_name or "") + if resolver is None or not dynamic_params: + return None + return resolver(dynamic_params) + + __all__ = [ + "DYNAMIC_ENDPOINT_BY_CALLBACK", "DYNAMIC_HEADERS_BY_CALLBACK", "PRESET_BY_CALLBACK", "PROJECT_HEADERS_BY_CALLBACK", "Preset", "agentops_preset", "arize_preset", + "dynamic_otlp_endpoint", "dynamic_otlp_headers", "langfuse_preset", "langtrace_preset", "levo_preset", + "newrelic_preset", "phoenix_preset", "project_routing_headers", "weave_preset", diff --git a/litellm/integrations/otel/presets/newrelic.py b/litellm/integrations/otel/presets/newrelic.py new file mode 100644 index 00000000000..4660a707355 --- /dev/null +++ b/litellm/integrations/otel/presets/newrelic.py @@ -0,0 +1,104 @@ +"""New Relic preset — OTLP/HTTP exporter to New Relic + GenAI vocabulary.""" + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +from litellm._logging import verbose_logger +from litellm.integrations.otel.model.config import ( + ExporterOwner, + ExporterSpec, + OpenTelemetryV2Config, +) +from litellm.integrations.otel.presets.utils import ensure_mappers +from litellm.types.utils import StandardCallbackDynamicParams + +#: Region -> OTLP base endpoint. A fixed table by design: team config picks a +#: region enum rather than a free-form endpoint, so callback vars can never +#: redirect telemetry to an arbitrary host. +NEWRELIC_OTLP_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType( + { + "us": "https://otlp.nr-data.net", + "eu": "https://otlp.eu01.nr-data.net", + } +) + +_DEFAULT_REGION: Final = "us" + + +class _NewRelicSettings(BaseSettings): + model_config = SettingsConfigDict(case_sensitive=False, extra="ignore") + + # The same env vars the agent-based integration documents; the key is the + # operator-level fallback for traffic without team credentials, the region + # picks that fallback's data center, and the record-content flag keeps its + # documented meaning when the OTel path replaces the agent. + license_key: str | None = Field(default=None, validation_alias="NEW_RELIC_LICENSE_KEY") + region: str | None = Field(default=None, validation_alias="NEW_RELIC_REGION") + record_content: bool | None = Field(default=None, validation_alias="NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED") + + +def newrelic_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, +) -> OpenTelemetryV2Config: + settings: Final = _NewRelicSettings() + base: Final = config_overrides or OpenTelemetryV2Config() + endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get( + (settings.region or _DEFAULT_REGION).lower(), NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION] + ) + return base.model_copy( + update={ + "exporters": [ + *base.exporters, + ExporterSpec( + kind="otlp_http", + endpoint=endpoint, + headers=(f"api-key={settings.license_key}" if settings.license_key else None), + owner=ExporterOwner.NEWRELIC, + requires_headers=True, + ), + ], + # New Relic ingests the OTLP GenAI semantic conventions natively. + "mapper_names": ensure_mappers(base.mapper_names, "genai"), + **( + {"capture_message_content": ("span_only" if settings.record_content else "no_content")} + if settings.record_content is not None + else {} + ), + } + ) + + +def newrelic_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: + """Per-request New Relic OTLP headers from team/key dynamic params.""" + api_key: Final = params.get("newrelic_api_key") + return {header: value for header, value in (("api-key", api_key),) if value} + + +def newrelic_dynamic_endpoint(params: StandardCallbackDynamicParams) -> str: + """Per-request OTLP endpoint for the team's ``newrelic_region``. + + Always the team's own region endpoint, defaulting to US when the team left + the region unset. It never falls through to the preset's endpoint, which + follows the operator's ``NEW_RELIC_REGION`` env; a team that saved only its + ingest key must not inherit the operator's region and have its US-account + spans rejected by an EU-configured default (or vice versa). An unknown + region likewise resolves to the documented US default rather than a guess. + """ + region: Final = params.get("newrelic_region") + default_endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION] + if not region: + return default_endpoint + endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(region.lower()) + if endpoint is None: + verbose_logger.warning( + "New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.", + region, + ", ".join(sorted(NEWRELIC_OTLP_ENDPOINT_BY_REGION)), + ) + return default_endpoint + return endpoint diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 284989ab20f..3b42ca4eaaf 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -46,7 +46,7 @@ def validate_no_callback_env_reference(param: str, value: object, *, source: str # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict -_supported_callback_params: Final = [ +_supported_callback_params: Final[tuple[str, ...]] = ( "langfuse_public_key", "langfuse_secret", "langfuse_secret_key", @@ -72,8 +72,10 @@ _supported_callback_params: Final = [ "dd_site", "dd_agent_host", "dd_agent_port", + "newrelic_api_key", + "newrelic_region", "turn_off_message_logging", -] +) _request_blocked_callback_params: Final = frozenset( { @@ -83,6 +85,20 @@ _request_blocked_callback_params: Final = frozenset( "dd_site", "dd_agent_host", "dd_agent_port", + "newrelic_api_key", + "newrelic_region", + } +) + +# Request-blocked params that must still reach ``standard_callback_dynamic_params`` +# when the proxy itself stamped them from admin-configured team/key callback +# settings (the trusted-vars channel). The OTel per-tenant tracer routing reads +# ``standard_callback_dynamic_params``, so without this overlay a blocked param +# could never drive routing at all. +_trusted_overlay_callback_params: Final = frozenset( + { + "newrelic_api_key", + "newrelic_region", } ) @@ -121,7 +137,9 @@ def initialize_standard_callback_dynamic_params( if param in kwargs: _param_value = kwargs.get(param) validate_no_callback_env_reference(param, _param_value, source="request body") - standard_callback_dynamic_params[param] = _param_value + standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields + _param_value + ) for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs): for param in _supported_callback_params: @@ -130,6 +148,12 @@ def initialize_standard_callback_dynamic_params( if param not in standard_callback_dynamic_params and param in metadata: _param_value = metadata.get(param) validate_no_callback_env_reference(param, _param_value, source=slot_label) - standard_callback_dynamic_params[param] = _param_value + standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields + _param_value + ) + + for param, trusted_value in get_trusted_callback_params(kwargs): + if param in _trusted_overlay_callback_params: + standard_callback_dynamic_params[param] = trusted_value return standard_callback_dynamic_params diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c14dd6c3d8b..3ad4c187b6d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4503,6 +4503,9 @@ def _init_custom_logger_compatible_class( _in_memory_loggers.append(gitlab_logger) return gitlab_logger elif logging_integration == "newrelic": + _v2 = _maybe_construct_otel_v2("newrelic", _in_memory_loggers) + if _v2 is not None: + return _v2 for callback in _in_memory_loggers: if isinstance(callback, NewRelicLogger): return callback @@ -4789,7 +4792,11 @@ def get_custom_logger_compatible_class( if isinstance(callback, SMTPEmailLogger): return callback elif logging_integration == "newrelic": + from litellm.integrations.otel.logger import OpenTelemetryV2 + for callback in _in_memory_loggers: + if isinstance(callback, OpenTelemetryV2) and callback.callback_name == "newrelic": + return callback if isinstance(callback, NewRelicLogger): return callback return None diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py new file mode 100644 index 00000000000..680cc226d18 --- /dev/null +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -0,0 +1,77 @@ +"""Save-time validation of team/key logging configs the runtime cannot honor. + +Team callbacks arrive as a single ``AddTeamCallback``, key callbacks arrive as a +``logging`` list inside the key metadata, so both shapes funnel into the same +per-integration checks here. +""" + +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +_NEWRELIC_CALLBACK: Final = "newrelic" +_NEWRELIC_VAR_PREFIX: Final = "newrelic_" + + +def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, str] | None) -> str | None: + if callback_name != _NEWRELIC_CALLBACK or not callback_vars: + return None + return _newrelic_config_error(callback_vars) + + +def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str | None: + """Validate every ``logging`` entry of a team/key metadata payload.""" + if not metadata: + return None + entries: Final = metadata.get("logging") + if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)): + return None + return next( + (error for error in (_logging_entry_error(entry) for entry in entries) if error is not None), + None, + ) + + +def _logging_entry_error(entry: object) -> str | None: + if not isinstance(entry, Mapping): + return None + callback_name: Final = entry.get("callback_name") + callback_vars: Final = entry.get("callback_vars") + if not isinstance(callback_name, str) or not isinstance(callback_vars, Mapping): + return None + return callback_config_error( + callback_name, + MappingProxyType({str(key): str(value) for key, value in callback_vars.items()}), + ) + + +def _newrelic_config_error(callback_vars: Mapping[str, str]) -> str | None: + """Per-team New Relic routing runs on the OTel v2 path only. + + Accepting the config with the flag off would silently ship the team's traffic + through the operator's env-configured agent instead of the team's account. A + region outside the fixed table, or a region without a key, would likewise be + accepted and then silently ignored or misrouted at request time. + """ + if not any(key.startswith(_NEWRELIC_VAR_PREFIX) for key in callback_vars): + return None + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.integrations.otel.presets.newrelic import NEWRELIC_OTLP_ENDPOINT_BY_REGION + + if not is_otel_v2_enabled(): + return "Per-team New Relic routing requires the proxy to run with LITELLM_OTEL_V2=true." + + region: Final = callback_vars.get("newrelic_region") + if region is not None and region.lower() not in NEWRELIC_OTLP_ENDPOINT_BY_REGION: + return ( + f"Unknown newrelic_region {region!r}. " + f"Supported regions: {', '.join(sorted(NEWRELIC_OTLP_ENDPOINT_BY_REGION))}." + ) + + # ``callback_vars`` values are str()-coerced upstream, so a JSON ``null`` key + # arrives as the literal ``"None"``; treat that and the empty string as absent. + api_key: Final = callback_vars.get("newrelic_api_key") + if region is not None and (not api_key or api_key == "None"): + return "newrelic_region requires newrelic_api_key; the region rides the team's own key." + return None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 4794da05a3e..cb5002e431b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -754,6 +754,12 @@ def convert_key_logging_metadata_to_callback( team_callback_settings_obj.callbacks.append(data.callback_name) for var, value in data.callback_vars.items(): + # New Relic routing reads these from the trusted-vars overlay with no + # callback-name check, so scope them to the newrelic entry: a team that + # put newrelic_* under a different callback never asked for New Relic and + # must not export to it. + if var.startswith("newrelic_") and data.callback_name != "newrelic": + continue if team_callback_settings_obj.callback_vars is None: team_callback_settings_obj.callback_vars = {} team_callback_settings_obj.callback_vars[var] = str(value) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 34a91dc59da..54f567b7aa2 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -62,6 +62,7 @@ from litellm.proxy.auth.auth_utils import ( enforce_output_token_estimates_are_admin_only, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.callback_config_validation import logging_metadata_config_error from litellm.proxy.common_utils.callback_utils import ( decrypt_callback_vars, encrypt_callback_vars, @@ -553,6 +554,17 @@ def key_generation_check( return _personal_key_generation_check(user_api_key_dict=user_api_key_dict, data=data) +def raise_on_invalid_key_logging_config(metadata: Mapping[str, object] | None) -> None: + """Key-level logging writes go through key metadata, not /team/callback. + + Without this the same New Relic config the team endpoint rejects would be + accepted here and then silently ignored or misrouted at request time. + """ + error: Final = logging_metadata_config_error(metadata) + if error is not None: + raise HTTPException(status_code=400, detail={"error": error}) # mutable-ok: FastAPI detail contract + + def common_key_access_checks( user_api_key_dict: UserAPIKeyAuth, data: GenerateKeyRequest | UpdateKeyRequest, @@ -891,6 +903,7 @@ async def _common_key_generation_helper( ) validate_budget_duration(data.budget_duration) + raise_on_invalid_key_logging_config(data.metadata) if data.throttle_on_budget_exceeded is True and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value: raise HTTPException( @@ -1992,6 +2005,8 @@ def prepare_metadata_fields(data: BaseModel, non_default_values: dict, existing_ """ Check LiteLLM_ManagementEndpoint_MetadataFields (proxy/_types.py) for fields that are allowed to be updated """ + raise_on_invalid_key_logging_config(non_default_values.get("metadata")) + if "metadata" not in non_default_values: # allow user to set metadata to none non_default_values["metadata"] = existing_metadata.copy() diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 0e8c4e1825d..14a2a8a98a5 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -28,6 +28,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.callback_config_validation import callback_config_error from litellm.proxy.common_utils.callback_utils import ( _CALLBACK_VAR_ENCRYPTED_PREFIX, decrypt_callback_vars, @@ -51,6 +52,16 @@ router: Final = APIRouter() _CALLBACK_VARS_REDACTED: Final = "***REDACTED***" +def _callback_config_error(message: str) -> HTTPException: + return HTTPException(status_code=400, detail={"error": message}) # mutable-ok: FastAPI detail contract + + +def _validate_team_callback(data: "AddTeamCallback") -> None: + error: Final = callback_config_error(data.callback_name, data.callback_vars) + if error is not None: + raise _callback_config_error(error) + + def _redact_callback_secrets(metadata: Any) -> Any: """Strip secret values out of a team-metadata snapshot before audit logging. @@ -304,6 +315,8 @@ async def add_team_callbacks( user_api_key_dict=user_api_key_dict, ) + _validate_team_callback(data) + # store team callback settings in metadata team_metadata = _existing_team.metadata team_callback_settings: list[dict] = team_metadata.get("logging") # will be dict of type AddTeamCallback diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 94526de0757..67eae2b4f21 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3300,6 +3300,11 @@ class StandardCallbackDynamicParams(TypedDict, total=False): dd_agent_host: str | None dd_agent_port: str | None + # New Relic dynamic params (proxy-stamped team/key callback vars only; + # request-supplied values are blocked) + newrelic_api_key: str | None # writable-ok: initialize_standard_callback_dynamic_params assigns into the dict + newrelic_region: str | None # writable-ok: initialize_standard_callback_dynamic_params assigns into the dict + # Logging settings turn_off_message_logging: bool | None # when true will not log messages litellm_disabled_callbacks: list[str] | None diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py index a9a78dcb2f1..633be9f105f 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py @@ -7,6 +7,7 @@ from opentelemetry.trace import NoOpTracer from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config from litellm.integrations.otel.presets import ( + dynamic_otlp_endpoint, dynamic_otlp_headers, project_routing_headers, ) @@ -23,31 +24,23 @@ def _cache(callback_name, exporters=None): def test_arize_dynamic_headers(): - headers = dynamic_otlp_headers( - "arize", {"arize_space_id": "S", "arize_api_key": "K"} - ) + headers = dynamic_otlp_headers("arize", {"arize_space_id": "S", "arize_api_key": "K"}) assert headers == {"arize-space-id": "S", "api_key": "K"} def test_arize_space_key_overrides_space_id(): - headers = dynamic_otlp_headers( - "arize", {"arize_space_id": "S", "arize_space_key": "SK"} - ) + headers = dynamic_otlp_headers("arize", {"arize_space_id": "S", "arize_space_key": "SK"}) assert headers == {"arize-space-id": "SK"} def test_langfuse_dynamic_headers_need_both_keys(): assert dynamic_otlp_headers("langfuse_otel", {"langfuse_public_key": "pk"}) is None - headers = dynamic_otlp_headers( - "langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"} - ) + headers = dynamic_otlp_headers("langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}) assert headers is not None and "Authorization" in headers def test_langfuse_dynamic_headers_carry_v4_ingestion_version(): - headers = dynamic_otlp_headers( - "langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"} - ) + headers = dynamic_otlp_headers("langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}) expected_auth = "Basic " + base64.b64encode(b"pk:sk").decode() assert headers == { "Authorization": expected_auth, @@ -56,9 +49,7 @@ def test_langfuse_dynamic_headers_carry_v4_ingestion_version(): def test_weave_dynamic_headers(): - headers = dynamic_otlp_headers( - "weave_otel", {"wandb_api_key": "w", "weave_project_id": "p"} - ) + headers = dynamic_otlp_headers("weave_otel", {"wandb_api_key": "w", "weave_project_id": "p"}) assert headers is not None assert "Authorization" in headers and headers["project_id"] == "p" @@ -100,9 +91,7 @@ def test_provider_cache_is_bounded_and_evicts_lru(monkeypatch): monkeypatch.setattr(routing_mod, "_MAX_CACHED_PROVIDERS", 2) shut_down = [] - monkeypatch.setattr( - routing_mod, "_shutdown_provider", lambda p: shut_down.append(p) - ) + monkeypatch.setattr(routing_mod, "_shutdown_provider", lambda p: shut_down.append(p)) cache = _cache("arize") default = NoOpTracer() @@ -179,9 +168,7 @@ def test_dynamic_headers_do_not_leak_to_other_owners_exporter(): ), ], ) - new_cfg = cache._routed_config( - {"arize-space-id": "TEAMX", "api_key": "TEAMX_KEY"}, {} - ) + new_cfg = cache._routed_config({"arize-space-id": "TEAMX", "api_key": "TEAMX_KEY"}, {}) by_owner = {e.owner: e.headers for e in new_cfg.exporters} assert by_owner["arize"] == "arize-space-id=TEAMX,api_key=TEAMX_KEY" assert by_owner[None] == "x=base-collector" @@ -206,16 +193,14 @@ def _phoenix_cache(kind="otlp_http"): def test_phoenix_project_headers_precedence_and_blanks(): - assert project_routing_headers( - "arize_phoenix", {"phoenix_project_name": "team-proj"} - ) == {"x-project-name": "team-proj"} + assert project_routing_headers("arize_phoenix", {"phoenix_project_name": "team-proj"}) == { + "x-project-name": "team-proj" + } assert project_routing_headers( "arize_phoenix", {"phoenix_project_name_override": "override", "phoenix_project_name": "base"}, ) == {"x-project-name": "override"} - assert ( - project_routing_headers("arize_phoenix", {"phoenix_project_name": " "}) == {} - ) + assert project_routing_headers("arize_phoenix", {"phoenix_project_name": " "}) == {} assert project_routing_headers("arize_phoenix", None) == {} # Only Phoenix participates in project routing. assert project_routing_headers("arize", {"phoenix_project_name": "p"}) == {} @@ -286,10 +271,7 @@ def test_client_dynamic_params_cannot_choose_phoenix_project(): cache = _phoenix_cache() default = NoOpTracer() assert cache.route_for(default, {"phoenix_project_name": "attacker"}).tracer is default - assert ( - cache.route_for(default, {"phoenix_project_name_override": "attacker"}).tracer - is default - ) + assert cache.route_for(default, {"phoenix_project_name_override": "attacker"}).tracer is default assert cache._providers == {} @@ -321,9 +303,7 @@ def test_eviction_defers_shutdown_while_a_span_is_open(monkeypatch): monkeypatch.setattr(routing_mod, "_MAX_CACHED_PROVIDERS", 1) shut_down = [] - monkeypatch.setattr( - routing_mod, "_shutdown_provider", lambda p: shut_down.append(p) - ) + monkeypatch.setattr(routing_mod, "_shutdown_provider", lambda p: shut_down.append(p)) cache = _cache("arize") default = NoOpTracer() @@ -347,18 +327,13 @@ def test_retired_providers_are_capped(monkeypatch): monkeypatch.setattr(routing_mod, "_MAX_CACHED_PROVIDERS", 1) monkeypatch.setattr(routing_mod, "_MAX_RETIRED_PROVIDERS", 2) shut_down = [] - monkeypatch.setattr( - routing_mod, "_shutdown_provider", lambda p: shut_down.append(p) - ) + monkeypatch.setattr(routing_mod, "_shutdown_provider", lambda p: shut_down.append(p)) cache = _cache("arize") default = NoOpTracer() # Every route stays held (no release), so each one evicts and retires its # predecessor instead of shutting it down. - routes = [ - cache.route_for(default, {"arize_space_id": str(i), "arize_api_key": "K"}) - for i in range(5) - ] + routes = [cache.route_for(default, {"arize_space_id": str(i), "arize_api_key": "K"}) for i in range(5)] assert len(cache._providers) == 1 assert len(cache._retired) == 2 # capped, not one retiree per open call @@ -374,11 +349,112 @@ def test_release_without_eviction_keeps_provider_alive(monkeypatch): from litellm.integrations.otel.plumbing import routing as routing_mod shut_down = [] - monkeypatch.setattr( - routing_mod, "_shutdown_provider", lambda p: shut_down.append(p) - ) + monkeypatch.setattr(routing_mod, "_shutdown_provider", lambda p: shut_down.append(p)) cache = _cache("arize") route = cache.route_for(NoOpTracer(), {"arize_space_id": "A", "arize_api_key": "K"}) cache.release(route.provider) assert shut_down == [] # still cached, never retired cache.release(None) # default-route release is a no-op + + +# --- New Relic: per-team api-key header + fixed-table region endpoint --- # + + +def test_newrelic_dynamic_headers(): + assert dynamic_otlp_headers("newrelic", {"newrelic_api_key": "NRAL-KEY"}) == {"api-key": "NRAL-KEY"} + assert dynamic_otlp_headers("newrelic", {"newrelic_region": "eu"}) is None + + +def test_newrelic_dynamic_endpoint_resolves_from_fixed_table(): + from litellm.integrations.otel.presets import dynamic_otlp_endpoint + + assert dynamic_otlp_endpoint("newrelic", {"newrelic_region": "eu"}) == "https://otlp.eu01.nr-data.net" + assert dynamic_otlp_endpoint("newrelic", {"newrelic_region": "US"}) == "https://otlp.nr-data.net" + # A key-only team (no region) resolves to the fixed US default deterministically, + # never the operator's NEW_RELIC_REGION-configured preset endpoint. + assert dynamic_otlp_endpoint("newrelic", {"newrelic_api_key": "k"}) == "https://otlp.nr-data.net" + # An unknown region also resolves to the documented US default, not a guess. + assert dynamic_otlp_endpoint("newrelic", {"newrelic_region": "mars"}) == "https://otlp.nr-data.net" + # Callbacks without an endpoint resolver keep their preset endpoint. + assert dynamic_otlp_endpoint("arize", {"newrelic_region": "eu"}) is None + + +def test_newrelic_endpoint_stamped_onto_owned_exporter_only(): + cache = _cache( + "newrelic", + exporters=[ + ExporterSpec( + kind="otlp_http", + endpoint="http://self-hosted-collector:4318", + headers="x=base-collector", + owner=None, + ), + ExporterSpec( + kind="otlp_http", + endpoint="https://otlp.nr-data.net", + owner="newrelic", + requires_headers=True, + ), + ], + ) + new_cfg = cache._routed_config({"api-key": "TEAM-EU-KEY"}, {}, "https://otlp.eu01.nr-data.net") + by_owner = {e.owner: e for e in new_cfg.exporters} + assert by_owner["newrelic"].endpoint == "https://otlp.eu01.nr-data.net" + assert by_owner["newrelic"].headers == "api-key=TEAM-EU-KEY" + assert by_owner[None].endpoint == "http://self-hosted-collector:4318" + assert by_owner[None].headers == "x=base-collector" + + +def test_newrelic_provider_cached_per_key_and_region(): + cache = _cache( + "newrelic", + exporters=[ExporterSpec(kind="in_memory"), ExporterSpec(kind="otlp_http", owner="newrelic")], + ) + default = NoOpTracer() + cache.route_for(default, {"newrelic_api_key": "K1", "newrelic_region": "us"}) + cache.route_for(default, {"newrelic_api_key": "K1", "newrelic_region": "us"}) + assert len(cache._providers) == 1 + # Same key, different region → distinct provider (distinct endpoint). + cache.route_for(default, {"newrelic_api_key": "K1", "newrelic_region": "eu"}) + assert len(cache._providers) == 2 + cache.route_for(default, {"newrelic_api_key": "K2", "newrelic_region": "eu"}) + assert len(cache._providers) == 3 + + +def test_requires_headers_spec_skipped_without_headers(): + from litellm.integrations.otel.plumbing.providers import build_tracer_provider + + cfg = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="https://otlp.nr-data.net", requires_headers=True)] + ) + provider = build_tracer_provider(cfg) + processors = provider._active_span_processor._span_processors + # Only the baggage processor: the keyless spec must not export (New Relic + # rejects unauthenticated posts with a 4xx per span batch). + assert [type(p).__name__ for p in processors] == ["LiteLLMBaggageSpanProcessor"] + + keyed = OpenTelemetryV2Config( + exporters=[ + ExporterSpec( + kind="otlp_http", endpoint="https://otlp.nr-data.net", headers="api-key=k", requires_headers=True + ) + ] + ) + keyed_provider = build_tracer_provider(keyed) + assert len(keyed_provider._active_span_processor._span_processors) == 2 + + +def test_newrelic_key_only_team_routes_to_us_not_operator_region(monkeypatch): + """A team that saves a key but no region must export to the fixed US default, + independent of the operator's NEW_RELIC_REGION, so its spans are never + silently dropped by an operator-configured region its key does not match.""" + monkeypatch.setenv("NEW_RELIC_REGION", "eu") + cache = _cache( + "newrelic", + exporters=[ExporterSpec(kind="otlp_http", endpoint="https://otlp.eu01.nr-data.net", owner="newrelic")], + ) + new_cfg = cache._routed_config( + {"api-key": "US-KEY"}, {}, dynamic_otlp_endpoint("newrelic", {"newrelic_api_key": "US-KEY"}) + ) + owned = next(e for e in new_cfg.exporters if e.owner == "newrelic") + assert owned.endpoint == "https://otlp.nr-data.net" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index e5d5b62b856..704a1d3a7bb 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -341,6 +341,35 @@ def test_idempotent_on_repeat_callback(): assert len(exporter.get_finished_spans()) == 1 +def test_evicted_carrier_completed_call_emits_one_deferred_span(): + """Eviction over the concurrency budget drops only the boundary carrier, not + the call. When an evicted call later closes as a real completed call + (``upstream_started``, payload present) it still emits exactly one span + through the deferred branch, and a second close for the same id dedups. Only + an evicted call that never closes goes unexported.""" + logger, exporter = _logger() + kwargs = {**_kwargs(), "api_call_start_time": datetime(2026, 5, 26, 12, 0, 0, tzinfo=timezone.utc)} + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + assert "call_1" in logger._open_llm_calls + + # Evict exactly as ``_store_open_call`` does over budget: drop the oldest + # carrier and release its routed provider. + _, evicted = logger._open_llm_calls.popitem(last=False) + logger._release_carrier(evicted) + assert not logger._open_llm_calls + assert exporter.get_finished_spans() == () # the evicted boundary span is never exported + + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + spans = exporter.get_finished_spans() + assert len(spans) == 1, "the evicted call's real close re-emits one deferred span, not zero" + assert spans[0].name == "chat gpt-4o" + assert spans[0].attributes[LiteLLM.CALL_ID] == "call_1" + + # Success-then-failure on one logging object: the deferred branch dedups by id. + asyncio.run(logger.async_log_failure_event(kwargs, None, None, None)) + assert len(exporter.get_finished_spans()) == 1, "second close for the same id must not duplicate" + + # --------------------------------------------------------------------------- # # MCP tool-call spans # --------------------------------------------------------------------------- # @@ -2522,3 +2551,149 @@ def test_deferred_pre_call_does_not_churn_tenant_cache(monkeypatch): server.end() headers_b = next(h for h in captured if "proj-b" in h) assert [s.name for s in captured[headers_b].get_finished_spans()] == ["chat gpt-4o"] + + +# --- New Relic team-scoped deferred emit + dedup (no pre_call carrier) --- # + + +def test_no_span_when_request_never_reached_upstream(): + """A request rejected before the upstream call — at the auth/budget gate, or + blocked by a pre-call guardrail — carries the ``no upstream call`` marker + (stamped in ``proxy/utils.py`` before its handlers fire), so the failure log + produces no phantom CLIENT span even though a payload exists.""" + from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL + + logger, exporter = _logger() + payload = _payload( + status="failure", + error_information={"error_class": "ProxyException", "error_code": "401"}, + ) + kwargs = _kwargs(payload=payload) + kwargs[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True + # No log_pre_api_call: the call never started. + asyncio.run(logger.async_log_failure_event(kwargs, None, None, None)) + assert exporter.get_finished_spans() == () # no phantom LLM span + + +def test_success_without_pre_call_emits_deferred_span(): + """A team/key-scoped logger is registered as a success callback only, so + ``pre_call`` never reaches it and no carrier exists. A completed call (it has + its payload, no ``no upstream call`` marker) must still get its span — the + deferred branch — or team-scoped destinations receive nothing at all.""" + logger, exporter = _logger() + # No log_pre_api_call: this logger never receives the input hook. The + # request-level provider-handoff stamp is present (pre_call ran globally). + asyncio.run( + logger.async_log_success_event({**_kwargs(), "api_call_start_time": 100.0}, None, 100.0, 101.5) + ) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes.get("gen_ai.operation.name") + # Start time comes from the callback's start_time, not a bogus zero. + assert spans[0].start_time == 100_000_000_000 + assert spans[0].end_time == 101_500_000_000 + + +def test_no_carrier_and_no_payload_is_noop(): + logger, exporter = _logger() + asyncio.run( + logger.async_log_success_event({"litellm_params": {}}, None, None, None) + ) + assert exporter.get_finished_spans() == () + + +def test_second_close_for_same_call_does_not_duplicate_span(): + """Success and failure can both fire on one logging object for the same call + id. The first close pops the carrier and finishes the boundary span; the + second must dedup against it, not fabricate a duplicate through the + deferred branch.""" + logger, exporter = _logger() + kwargs = {**_kwargs(), "api_call_start_time": 100.0} + # Boundary open: the span is born at pre_call under a live server span and + # closed via finish_span, which never passes through emit()'s dedup. + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + _emit_llm(logger, kwargs, ambient=server) + server.end() + llm_before = [s for s in exporter.get_finished_spans() if s.name.startswith("chat")] + assert len(llm_before) == 1 + # Second close: carrier already popped, payload still present, handoff stamped. + asyncio.run(logger.async_log_failure_event(kwargs, None, None, None)) + llm_after = [s for s in exporter.get_finished_spans() if s.name.startswith("chat")] + assert len(llm_after) == 1 + + +def test_failure_without_pre_call_emits_deferred_error_span(): + """A team-scoped logger registered as a failure callback only still gets an + ERROR span for a real provider failure (payload present, no marker).""" + from opentelemetry.trace import StatusCode + + logger, exporter = _logger() + payload = _payload( + status="failure", + error_information={"error_class": "RateLimitError", "error_code": "429"}, + ) + asyncio.run( + logger.async_log_failure_event( + {**_kwargs(payload=payload), "api_call_start_time": 100.0}, None, None, None + ) + ) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + + +def test_boundary_open_with_no_payload_ends_provisional_span(): + """Opened at pre_call but the payload never materialized: the boundary span + is ended provisionally (no payload attributes) rather than leaked open.""" + logger, exporter = _logger() + kwargs = _kwargs() + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + with trace.use_span(server, end_on_exit=False): + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + asyncio.run( + logger.async_log_success_event( + {**kwargs, "standard_logging_object": None, "litellm_call_id": "call_1"}, None, None, None + ) + ) + server.end() + llm_spans = [s for s in exporter.get_finished_spans() if s.name.startswith("chat")] + assert len(llm_spans) == 1 + assert "gen_ai.usage.input_tokens" not in llm_spans[0].attributes + + +def test_failure_before_provider_handoff_emits_nothing(): + """A failure event whose request never handed off to a provider (router + pre-call rejection, SDK error before the call, standalone guardrail run) + has a payload but no ``api_call_start_time``; without a carrier it must not + fabricate an LLM-call span.""" + logger, exporter = _logger() + payload = _payload( + status="failure", + error_information={"error_class": "RateLimitError", "error_code": "429"}, + ) + asyncio.run(logger.async_log_failure_event(_kwargs(payload=payload), None, None, None)) + assert exporter.get_finished_spans() == () + + +def test_provisional_close_then_payload_close_does_not_duplicate(): + """Streaming shape: the success close arrives with no assembled payload (the + boundary span is ended provisionally), then the failure close arrives with a + payload for the same call id. Exactly one exported span.""" + logger, exporter = _logger() + kwargs = {**_kwargs(), "api_call_start_time": 100.0} + payload = kwargs["standard_logging_object"] + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + with trace.use_span(server, end_on_exit=False): + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + asyncio.run( + logger.async_log_success_event( + {**kwargs, "standard_logging_object": None, "litellm_call_id": payload["litellm_call_id"]}, + None, + None, + None, + ) + ) + asyncio.run(logger.async_log_failure_event(kwargs, None, None, None)) + server.end() + llm_spans = [s for s in exporter.get_finished_spans() if s.name.startswith("chat")] + assert len(llm_spans) == 1 diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_presets.py b/tests/test_litellm/integrations/otel/test_otel_v2_presets.py index add2caf9a48..58cfc1ceb3f 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_presets.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_presets.py @@ -160,3 +160,55 @@ def test_agentops_endpoint_points_at_live_host(): # so a typo or stale domain can never ship again. assert _AGENTOPS_ENDPOINT == "https://otlp.agentops.ai/v1/traces" assert "agentops.cloud" not in _AGENTOPS_ENDPOINT + + +def test_newrelic_preset_reads_env_license_key(monkeypatch): + monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "env-license-key") + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.newrelic import newrelic_preset + + cfg = newrelic_preset() + spec = next(e for e in cfg.exporters if e.owner == ExporterOwner.NEWRELIC) + assert spec.kind == "otlp_http" + assert spec.endpoint == "https://otlp.nr-data.net" + assert spec.headers == "api-key=env-license-key" + assert spec.requires_headers is True + assert "genai" in cfg.mapper_names + + +def test_newrelic_preset_without_key_still_contributes_owned_spec(monkeypatch): + # The owned spec is the stamping target for per-team credentials, so it must + # exist even with no operator env key; requires_headers keeps the keyless + # copy from ever exporting. + monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.newrelic import newrelic_preset + + cfg = newrelic_preset() + spec = next(e for e in cfg.exporters if e.owner == ExporterOwner.NEWRELIC) + assert spec.headers is None + assert spec.requires_headers is True + + +def test_newrelic_preset_operator_region_and_content_knob(monkeypatch): + monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "env-license-key") + monkeypatch.setenv("NEW_RELIC_REGION", "EU") + monkeypatch.setenv("NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED", "true") + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.newrelic import newrelic_preset + + cfg = newrelic_preset() + spec = next(e for e in cfg.exporters if e.owner == ExporterOwner.NEWRELIC) + assert spec.endpoint == "https://otlp.eu01.nr-data.net" + assert cfg.capture_span_content is True + + monkeypatch.setenv("NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED", "false") + assert newrelic_preset().capture_span_content is False + + +def test_newrelic_preset_unset_content_knob_keeps_default(monkeypatch): + monkeypatch.delenv("NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED", raising=False) + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + from litellm.integrations.otel.presets.newrelic import newrelic_preset + + assert newrelic_preset().capture_span_content is False diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index f9ddc47cc7c..9b2bd5e2585 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -187,3 +187,49 @@ def test_empty_kwargs_returns_empty_params(): params = initialize_standard_callback_dynamic_params({}) assert dict(params) == {} + + +def test_newrelic_callback_params_are_not_extracted_from_request_kwargs(): + kwargs = { + "newrelic_api_key": "caller-key", + "metadata": {"newrelic_api_key": "caller-key-2", "newrelic_region": "eu"}, + "litellm_params": {"metadata": {"newrelic_region": "eu"}}, + } + + params = initialize_standard_callback_dynamic_params(kwargs) + + assert params.get("newrelic_api_key") is None + assert params.get("newrelic_region") is None + + +def test_newrelic_trusted_vars_overlay_reaches_standard_params(): + from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD + + kwargs = { + # A caller-supplied copy must lose to the proxy-stamped trusted value. + "newrelic_api_key": "caller-key", + TRUSTED_CALLBACK_VARS_FIELD: { + "newrelic_api_key": "team-key", + "newrelic_region": "eu", + # Non-overlay trusted vars must not be copied by the overlay. + "langfuse_public_key": "pk-team", + }, + } + + params = initialize_standard_callback_dynamic_params(kwargs) + + assert params.get("newrelic_api_key") == "team-key" + assert params.get("newrelic_region") == "eu" + assert params.get("langfuse_public_key") is None + + +def test_trusted_vars_overlay_uses_shared_parser_semantics(): + # The overlay rides get_trusted_callback_params, the same parser the + # datadog handler consumes, so values are str()-coerced identically. + from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD + + params = initialize_standard_callback_dynamic_params( + {TRUSTED_CALLBACK_VARS_FIELD: {"newrelic_api_key": 12345}} + ) + + assert params.get("newrelic_api_key") == "12345" diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 873da28fc34..1c706be51fa 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -5200,3 +5200,81 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa ) for hook in [cb for cb in litellm.callbacks if isinstance(cb, VectorStorePreCallHook)]: litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, hook) +def test_newrelic_dispatch_prefers_otel_v2_when_flag_on(monkeypatch): + """With LITELLM_OTEL_V2 on, the "newrelic" callback builds the OTel v2 + logger (per-team credential routing); with the flag off (default) it keeps + the legacy agent-based logger, so existing deployments are untouched.""" + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils import litellm_logging as logging_module + + logging_module._in_memory_loggers.clear() + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + try: + v2_logger = logging_module._init_custom_logger_compatible_class( + logging_integration="newrelic", + internal_usage_cache=None, + llm_router=None, + custom_logger_init_args={}, + ) + assert isinstance(v2_logger, OpenTelemetryV2) + assert v2_logger.callback_name == "newrelic" + # Same name resolves to the same instance, not a second logger. + again = logging_module._init_custom_logger_compatible_class( + logging_integration="newrelic", + internal_usage_cache=None, + llm_router=None, + custom_logger_init_args={}, + ) + assert again is v2_logger + finally: + logging_module._in_memory_loggers.clear() + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + + +def test_newrelic_dispatch_keeps_legacy_agent_when_flag_off(monkeypatch): + from litellm.integrations.newrelic import NewRelicLogger + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils import litellm_logging as logging_module + + logging_module._in_memory_loggers.clear() + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + try: + legacy = logging_module._init_custom_logger_compatible_class( + logging_integration="newrelic", + internal_usage_cache=None, + llm_router=None, + custom_logger_init_args={}, + ) + assert isinstance(legacy, NewRelicLogger) + finally: + logging_module._in_memory_loggers.clear() + is_otel_v2_enabled.cache_clear() + + +def test_get_custom_logger_compatible_class_finds_v2_newrelic(monkeypatch): + """Under LITELLM_OTEL_V2 the "newrelic" instance is an OpenTelemetryV2; the + cached-lookup must find it or hook resolution (post-call failure/success + hooks) silently skips the callback.""" + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils import litellm_logging as logging_module + + logging_module._in_memory_loggers.clear() + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + try: + created = logging_module._init_custom_logger_compatible_class( + logging_integration="newrelic", + internal_usage_cache=None, + llm_router=None, + custom_logger_init_args={}, + ) + found = logging_module.get_custom_logger_compatible_class("newrelic") + assert found is created + finally: + logging_module._in_memory_loggers.clear() + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() diff --git a/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py index b2a242bf8f2..272a8ffa972 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py @@ -264,3 +264,148 @@ class TestCallbackManagementEndpoints: assert galileo_config["displayName"] == "Galileo" assert "GALILEO_API_KEY" in galileo_config["dynamic_params"] assert "GALILEO_PROJECT_ID" in galileo_config["dynamic_params"] + + +class TestNewRelicCallbackConfig: + def test_newrelic_entry_supports_team_logging_with_dynamic_params(self): + client = TestClient(app) + response = client.get("/callbacks/configs", headers={"Authorization": "Bearer sk-1234"}) + assert response.status_code == 200 + newrelic = next( + (config for config in response.json() if config.get("id") == "newrelic"), + None, + ) + assert newrelic is not None + assert newrelic["supports_key_team_logging"] is True + params = newrelic["dynamic_params"] + assert params["newrelic_api_key"]["type"] == "password" + assert "newrelic_region" in params + # The operator-only agent env flag must not appear as a team-configurable + # field: it is not a StandardCallbackDynamicParams key and would be rejected. + assert "NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED" not in params + + +class TestNewRelicTeamCallbackValidation: + def _data(self, callback_vars): + from litellm.proxy._types import AddTeamCallback + + return AddTeamCallback(callback_name="newrelic", callback_type="success", callback_vars=callback_vars) + + def test_rejects_when_otel_v2_off(self, monkeypatch): + from fastapi import HTTPException + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.proxy.management_endpoints.team_callback_endpoints import _validate_team_callback + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + try: + with pytest.raises(HTTPException) as exc: + _validate_team_callback(self._data({"newrelic_api_key": "k"})) + assert "LITELLM_OTEL_V2" in str(exc.value.detail) + finally: + is_otel_v2_enabled.cache_clear() + + def test_rejects_unknown_region_and_region_without_key(self, monkeypatch): + from fastapi import HTTPException + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.proxy.management_endpoints.team_callback_endpoints import _validate_team_callback + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + try: + with pytest.raises(HTTPException) as exc: + _validate_team_callback(self._data({"newrelic_api_key": "k", "newrelic_region": "mars"})) + assert "Unknown newrelic_region" in str(exc.value.detail) + with pytest.raises(HTTPException) as exc: + _validate_team_callback(self._data({"newrelic_region": "eu"})) + assert "requires newrelic_api_key" in str(exc.value.detail) + _validate_team_callback(self._data({"newrelic_api_key": "k", "newrelic_region": "EU"})) + # A JSON-null key is str()-coerced to "None" upstream; it must not + # slip past the region-requires-key guard. + with pytest.raises(HTTPException) as exc: + _validate_team_callback(self._data({"newrelic_api_key": None, "newrelic_region": "eu"})) + assert "requires newrelic_api_key" in str(exc.value.detail) + finally: + is_otel_v2_enabled.cache_clear() + + def test_ignores_other_callbacks_and_bare_newrelic(self, monkeypatch): + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.proxy._types import AddTeamCallback + from litellm.proxy.management_endpoints.team_callback_endpoints import _validate_team_callback + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + try: + assert _validate_team_callback(self._data({})) is None + assert ( + _validate_team_callback( + AddTeamCallback( + callback_name="langfuse", + callback_type="success", + callback_vars={"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, + ) + ) + is None + ) + finally: + is_otel_v2_enabled.cache_clear() + + +class TestNewRelicKeyLoggingValidation: + """Key-level logging is written through key metadata, not /team/callback.""" + + def _metadata(self, callback_vars): + return {"logging": [{"callback_name": "newrelic", "callback_type": "success", "callback_vars": callback_vars}]} + + def test_rejects_same_configs_as_team_endpoint(self, monkeypatch): + from fastapi import HTTPException + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.proxy.management_endpoints.key_management_endpoints import ( + raise_on_invalid_key_logging_config, + ) + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + try: + with pytest.raises(HTTPException) as exc: + raise_on_invalid_key_logging_config(self._metadata({"newrelic_api_key": "k"})) + assert "LITELLM_OTEL_V2" in str(exc.value.detail) + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + with pytest.raises(HTTPException) as exc: + raise_on_invalid_key_logging_config( + self._metadata({"newrelic_api_key": "k", "newrelic_region": "mars"}) + ) + assert "Unknown newrelic_region" in str(exc.value.detail) + with pytest.raises(HTTPException) as exc: + raise_on_invalid_key_logging_config(self._metadata({"newrelic_region": "eu"})) + assert "requires newrelic_api_key" in str(exc.value.detail) + raise_on_invalid_key_logging_config(self._metadata({"newrelic_api_key": "k", "newrelic_region": "EU"})) + finally: + is_otel_v2_enabled.cache_clear() + + def test_ignores_metadata_without_newrelic_logging(self, monkeypatch): + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.proxy.management_endpoints.key_management_endpoints import ( + raise_on_invalid_key_logging_config, + ) + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + try: + assert raise_on_invalid_key_logging_config(None) is None + assert raise_on_invalid_key_logging_config({"logging": "not-a-list"}) is None + assert raise_on_invalid_key_logging_config({"tags": ["a"]}) is None + assert raise_on_invalid_key_logging_config(self._metadata({})) is None + assert ( + raise_on_invalid_key_logging_config( + {"logging": [{"callback_name": "langfuse", "callback_vars": {"langfuse_public_key": "pk"}}]} + ) + is None + ) + finally: + is_otel_v2_enabled.cache_clear() diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 81a97a70efa..50ef6f29ec2 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -7329,3 +7329,92 @@ def test_vertex_sends_exactly_one_authorization_header(): vertex_request_headers.update(forwarded) assert _authorization_values(vertex_request_headers) == [GOOGLE_ACCESS_TOKEN] +@pytest.mark.asyncio +async def test_newrelic_team_callback_vars_reach_trusted_field(): + """A key with a newrelic team callback stamps its vars into the proxy-owned + trusted field, and a caller-supplied newrelic_api_key in the body is + stripped rather than merged.""" + key_with_newrelic_callback = UserAPIKeyAuth( + api_key="hashed-key", + metadata={ + "logging": [ + { + "callback_name": "newrelic", + "callback_type": "success", + "callback_vars": {"newrelic_api_key": "team-nr-key", "newrelic_region": "eu"}, + } + ] + }, + ) + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + "newrelic_api_key": "attacker-key", + } + + updated = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=key_with_newrelic_callback, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated[TRUSTED_CALLBACK_VARS_FIELD] == { + "newrelic_api_key": "team-nr-key", + "newrelic_region": "eu", + } + assert updated["success_callback"] == ["newrelic"] + + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params, + ) + + params = initialize_standard_callback_dynamic_params(updated) + assert params.get("newrelic_api_key") == "team-nr-key" + assert params.get("newrelic_region") == "eu" + + from litellm.integrations.otel.presets import dynamic_otlp_endpoint, dynamic_otlp_headers + + assert dynamic_otlp_headers("newrelic", params) == {"api-key": "team-nr-key"} + assert dynamic_otlp_endpoint("newrelic", params) == "https://otlp.eu01.nr-data.net" + + from litellm.utils import get_non_default_completion_params + + forwarded = get_non_default_completion_params(updated) + assert not any(param.startswith("newrelic_") for param in forwarded) + assert TRUSTED_CALLBACK_VARS_FIELD not in forwarded + + +def test_newrelic_vars_scoped_to_newrelic_callback_entry(): + """New Relic routing reads these vars from the trusted overlay with no + callback-name check, so a team that puts newrelic_* under a different + callback's vars must not have them enter the shared bag (and so never + exports to New Relic). Vars under a real newrelic entry are kept.""" + from litellm.proxy._types import AddTeamCallback + from litellm.proxy.litellm_pre_call_utils import convert_key_logging_metadata_to_callback + + smuggled = convert_key_logging_metadata_to_callback( + AddTeamCallback( + callback_name="langfuse", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "newrelic_api_key": "SMUGGLED", + "newrelic_region": "eu", + }, + ), + None, + ) + assert smuggled.callback_vars == {"langfuse_public_key": "pk"} + + legit = convert_key_logging_metadata_to_callback( + AddTeamCallback( + callback_name="newrelic", + callback_type="success", + callback_vars={"newrelic_api_key": "REAL", "newrelic_region": "us"}, + ), + None, + ) + assert legit.callback_vars == {"newrelic_api_key": "REAL", "newrelic_region": "us"} diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index 7aa121dcca5..3906aa744f7 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -6,6 +6,7 @@ import galileoLogo from "../../public/assets/logos/galileo.ico"; import lagoLogo from "../../public/assets/logos/lago.svg"; import langfuseLogo from "../../public/assets/logos/langfuse.png"; import langsmithLogo from "../../public/assets/logos/langsmith.png"; +import newrelicLogo from "../../public/assets/logos/newrelic.png"; import openmeterLogo from "../../public/assets/logos/openmeter.png"; import otelLogo from "../../public/assets/logos/otel.png"; @@ -77,6 +78,17 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ }, description: "Datadog Logging Integration", }, + { + id: "newrelic", + displayName: "New Relic", + logo: newrelicLogo.src, + supports_key_team_logging: true, + dynamic_params: { + newrelic_api_key: "password", + newrelic_region: "text", + }, + description: "New Relic Logging Integration", + }, { id: "lago", displayName: "Lago", From 01595d2fbf232f49c45177c4c0f2108041966f28 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 22 Aug 2026 22:44:32 -0700 Subject: [PATCH 2/6] perf(ci): cache uv dependencies in the lint job (#37783) The lint job installs its dependencies from scratch on every run. That step measures 2.8 minutes of a job whose p50 is 9.5, and lint is the slowest required check on 9 of the last 10 merged staging PRs, so it sets the critical path for the whole PR. _test-unit-base.yml already caches ~/.cache/uv and .venv keyed on uv.lock. This mirrors that block. The key carries its own `lint` namespace rather than sharing the unit tier's: the two jobs sync different group sets (proxy-dev + e2e-dev here, ci + proxy-dev + four extras there), so a shared .venv entry would be pruned and rebuilt on alternating runs. --- .github/workflows/test-linting.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index ccb58f5cc9c..9f1283da19e 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -67,6 +67,17 @@ jobs: with: version: "0.10.9" + - name: Cache uv dependencies + if: steps.changes.outputs.decision != 'skip' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv-lint- + - name: Clean Python cache if: steps.changes.outputs.decision != 'skip' run: | From 6e23288b47ecd89b658e6c5188f090dcf886a227 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 22 Aug 2026 22:44:58 -0700 Subject: [PATCH 3/6] perf(ci): fan the budget checkers out across cores (#37784) * perf(ci): fan the budget checkers out across cores check_type_discipline.py and check_test_quality.py each walk a few thousand files and parse every one, single-threaded. In the lint job those two steps measure 2.3 and 1.6 minutes, second and third behind dependency install, and lint is the slowest required check on 9 of the last 10 merged staging PRs. check_file is already pure per-file work, so the walk fans out over a process pool with no change to what either rule reports. Callers sort, which is what keeps output order stable when results land out of order. Runs below PARALLEL_MIN_PATHS stay serial rather than pay for process startup, and the worker count is capped so a large runner does not oversubscribe. Measured locally over the same trees, output byte-identical both times: type-discipline 17.8s -> 3.0s over litellm/ (78,768 report lines), test-quality 14.4s -> 2.3s over tests/ (6,321 report lines), per-rule counts unchanged. * test(ci): type the fan-out helpers and skip the comparison on one core --- scripts/check_test_quality.py | 27 +++++++- scripts/check_type_discipline.py | 27 +++++++- tests/test_litellm/test_check_test_quality.py | 62 +++++++++++++++++++ .../test_check_type_discipline.py | 62 +++++++++++++++++++ 4 files changed, 176 insertions(+), 2 deletions(-) diff --git a/scripts/check_test_quality.py b/scripts/check_test_quality.py index e3ffbac9808..6964aed56e4 100644 --- a/scripts/check_test_quality.py +++ b/scripts/check_test_quality.py @@ -102,11 +102,13 @@ from __future__ import annotations import ast import io +import os import re import sys import tokenize from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass +from multiprocessing import Pool from pathlib import Path from types import MappingProxyType from typing import Final, NamedTuple @@ -692,13 +694,36 @@ def collect_paths(raw: Iterable[str]) -> Iterator[Path]: yield candidate +PARALLEL_MIN_PATHS = 200 +MAX_WORKERS = 8 + + +def _worker_count(path_count: int) -> int: + """1 when the run is too small to repay process startup, else one worker per + core up to MAX_WORKERS.""" + if path_count < PARALLEL_MIN_PATHS: + return 1 + return max(1, min(os.cpu_count() or 1, MAX_WORKERS)) + + +def scan_paths(paths: Sequence[Path]) -> tuple[Violation, ...]: + """check_file over every path. Pure per-file work, so it fans out across + processes; callers sort, which is what keeps output order stable.""" + workers = _worker_count(len(paths)) + if workers == 1: + return tuple(v for path in paths for v in check_file(path)) + with Pool(workers) as pool: + return tuple(v for found in pool.imap_unordered(check_file, paths, chunksize=32) for v in found) + + def main(argv: Sequence[str]) -> int: paths: Final = tuple(a for a in argv if not a.startswith("-")) if not paths: print("usage: check_test_quality.py ...", file=sys.stderr) return 2 - violations: Final = sorted(v for path in collect_paths(paths) for v in check_file(path)) + targets: Final = tuple(collect_paths(paths)) + violations: Final = sorted(scan_paths(targets)) for violation in violations: print(violation.render()) diff --git a/scripts/check_type_discipline.py b/scripts/check_type_discipline.py index 0706c8a7bd8..a2ab4760c4f 100644 --- a/scripts/check_type_discipline.py +++ b/scripts/check_type_discipline.py @@ -114,10 +114,12 @@ from __future__ import annotations import ast import io +import os import re import sys import tokenize from dataclasses import dataclass +from multiprocessing import Pool from pathlib import Path from collections.abc import Iterable, Iterator, Mapping, Sequence from typing import NamedTuple @@ -1070,13 +1072,36 @@ def collect_paths(raw: Iterable[str]) -> Iterator[Path]: yield p +PARALLEL_MIN_PATHS = 200 +MAX_WORKERS = 8 + + +def _worker_count(path_count: int) -> int: + """1 when the run is too small to repay process startup, else one worker per + core up to MAX_WORKERS.""" + if path_count < PARALLEL_MIN_PATHS: + return 1 + return max(1, min(os.cpu_count() or 1, MAX_WORKERS)) + + +def scan_paths(paths: Sequence[Path]) -> tuple[Violation, ...]: + """check_file over every path. Pure per-file work, so it fans out across + processes; callers sort, which is what keeps output order stable.""" + workers = _worker_count(len(paths)) + if workers == 1: + return tuple(v for path in paths for v in check_file(path)) + with Pool(workers) as pool: + return tuple(v for found in pool.imap_unordered(check_file, paths, chunksize=32) for v in found) + + def main(argv: Sequence[str]) -> int: paths = tuple(a for a in argv if not a.startswith("-")) if not paths: print("usage: check_type_discipline.py ...", file=sys.stderr) return 2 - violations = sorted(v for path in collect_paths(paths) for v in check_file(path)) + targets = tuple(collect_paths(paths)) + violations = sorted(scan_paths(targets)) for v in violations: print(v.render()) diff --git a/tests/test_litellm/test_check_test_quality.py b/tests/test_litellm/test_check_test_quality.py index 4fea5761cc8..7d59e5a5dba 100644 --- a/tests/test_litellm/test_check_test_quality.py +++ b/tests/test_litellm/test_check_test_quality.py @@ -8,9 +8,13 @@ in the test body. """ import importlib.util +import os +import subprocess import sys from pathlib import Path +import pytest + _REPO_ROOT = Path(__file__).resolve().parents[2] _MODULE_PATH = _REPO_ROOT / "scripts" / "check_test_quality.py" _spec = importlib.util.spec_from_file_location("check_test_quality", _MODULE_PATH) @@ -548,3 +552,61 @@ def test_the_read_may_sit_a_statement_above_the_store(tmp_path): def test_a_loop_storing_under_a_key_that_is_not_the_loop_variable_is_not_an_inventory(tmp_path): source = _HELPER_DICT_CONFTEST.replace("state[attr] =", 'state["fixed"] =') assert [v.code for v in checker.check_file(_written(tmp_path, source))] == [] + + +_FANS_OUT = checker._worker_count(checker.PARALLEL_MIN_PATHS) > 1 +_SERIAL_ONLY = "one usable core, so scan_paths stays serial and there is no fan-out to compare" + + +def _corpus(tmp_path: Path, count: int) -> tuple[Path, ...]: + for index in range(count): + (tmp_path / f"test_gen_{index}.py").write_text( + f"def test_flagged_{index}():\n compute()\n\n\ndef test_clean_{index}():\n assert compute() == {index}\n", + encoding="utf-8", + ) + return tuple(sorted(tmp_path.rglob("*.py"))) + + +def _run_checker(target: Path) -> list[str]: + completed = subprocess.run( + [sys.executable, str(_MODULE_PATH), str(target)], + capture_output=True, text=True, timeout=300, + ) + return completed.stdout.splitlines() + + +def test_worker_count_stays_serial_below_the_threshold(): + assert checker._worker_count(checker.PARALLEL_MIN_PATHS - 1) == 1 + + +def test_worker_count_fans_out_at_the_threshold(): + assert checker._worker_count(checker.PARALLEL_MIN_PATHS) == max( + 1, min(os.cpu_count() or 1, checker.MAX_WORKERS) + ) + + +def test_worker_count_never_exceeds_the_cap(): + assert checker._worker_count(100_000) <= checker.MAX_WORKERS + + +def test_scan_paths_below_the_threshold_returns_every_violation(tmp_path): + paths = _corpus(tmp_path, 3) + assert checker._worker_count(len(paths)) == 1 + assert [v.code for v in checker.scan_paths(paths)] == ["TQ001"] * 3 + + +@pytest.mark.skipif(not _FANS_OUT, reason=_SERIAL_ONLY) +def test_a_fanned_out_run_reports_exactly_what_a_serial_run_reports(tmp_path): + paths = _corpus(tmp_path, checker.PARALLEL_MIN_PATHS + 5) + serial = [v.render() for v in sorted(v for path in paths for v in checker.check_file(path))] + assert serial, "corpus must produce violations or the comparison proves nothing" + assert _run_checker(tmp_path) == serial + + +@pytest.mark.skipif(not _FANS_OUT, reason=_SERIAL_ONLY) +def test_a_fanned_out_run_reports_each_generated_file_exactly_once(tmp_path): + paths = _corpus(tmp_path, checker.PARALLEL_MIN_PATHS + 5) + reported = _run_checker(tmp_path) + assert len(reported) == len(paths) + assert len({line.split(":")[0] for line in reported}) == len(paths) + assert all(" TQ001 " in line for line in reported) diff --git a/tests/test_litellm/test_check_type_discipline.py b/tests/test_litellm/test_check_type_discipline.py index 84dd547ad80..2d49332e687 100644 --- a/tests/test_litellm/test_check_type_discipline.py +++ b/tests/test_litellm/test_check_type_discipline.py @@ -8,10 +8,14 @@ a test fail. The comment-scanner cases are the regression for the readline path: import importlib.util import json +import os import re +import subprocess import sys from pathlib import Path +import pytest + _REPO_ROOT = Path(__file__).resolve().parents[2] _MODULE_PATH = _REPO_ROOT / "scripts" / "check_type_discipline.py" _spec = importlib.util.spec_from_file_location("check_type_discipline", _MODULE_PATH) @@ -695,3 +699,61 @@ def test_budget_covers_exactly_the_checker_rules(): for spec in budget.values(): assert isinstance(spec["limit"], int) assert spec["limit"] >= 0 + + +_FANS_OUT = checker._worker_count(checker.PARALLEL_MIN_PATHS) > 1 +_SERIAL_ONLY = "one usable core, so scan_paths stays serial and there is no fan-out to compare" + + +def _corpus(tmp_path: Path, count: int) -> tuple[Path, ...]: + for index in range(count): + (tmp_path / f"mod_{index}.py").write_text( + f"def build_{index}(items: list[int]) -> None:\n return None\n", + encoding="utf-8", + ) + return tuple(sorted(tmp_path.rglob("*.py"))) + + +def _run_checker(target: Path) -> list[str]: + completed = subprocess.run( + [sys.executable, str(_MODULE_PATH), str(target)], + capture_output=True, text=True, timeout=300, + ) + return completed.stdout.splitlines() + + +def test_worker_count_stays_serial_below_the_threshold(): + assert checker._worker_count(checker.PARALLEL_MIN_PATHS - 1) == 1 + + +def test_worker_count_fans_out_at_the_threshold(): + assert checker._worker_count(checker.PARALLEL_MIN_PATHS) == max( + 1, min(os.cpu_count() or 1, checker.MAX_WORKERS) + ) + + +def test_worker_count_never_exceeds_the_cap(): + assert checker._worker_count(100_000) <= checker.MAX_WORKERS + + +def test_scan_paths_below_the_threshold_returns_every_violation(tmp_path): + paths = _corpus(tmp_path, 3) + assert checker._worker_count(len(paths)) == 1 + found = checker.scan_paths(paths) + assert found and len({v.path for v in found}) == 3 + + +@pytest.mark.skipif(not _FANS_OUT, reason=_SERIAL_ONLY) +def test_a_fanned_out_run_reports_exactly_what_a_serial_run_reports(tmp_path): + paths = _corpus(tmp_path, checker.PARALLEL_MIN_PATHS + 5) + serial = [v.render() for v in sorted(v for path in paths for v in checker.check_file(path))] + assert serial, "corpus must produce violations or the comparison proves nothing" + assert _run_checker(tmp_path) == serial + + +@pytest.mark.skipif(not _FANS_OUT, reason=_SERIAL_ONLY) +def test_a_fanned_out_run_reports_each_generated_file_exactly_once(tmp_path): + paths = _corpus(tmp_path, checker.PARALLEL_MIN_PATHS + 5) + reported = _run_checker(tmp_path) + assert reported + assert len({line.split(":")[0] for line in reported}) == len(paths) From 346c69386026c245df93a3ef284e4c525df69640 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 22 Aug 2026 22:45:30 -0700 Subject: [PATCH 4/6] ci: port the Postgres suites off CircleCI onto service containers (#37785) * ci: port the Postgres suites off CircleCI onto service containers proxy_behavior_tests, proxy_security_tests and schema_migration_check were near-identical CircleCI jobs: a Postgres sidecar, a schema seed, and one pytest tree each. They ran nowhere else, and CircleCI holds none of the branch ruleset's required checks, so the signal they produced gated nothing. test-postgres.yml runs the same three trees on a Postgres service container as one matrix, keeping each suite's own seeding rather than normalising it: the behavior and security trees keep `prisma db push`, and the migration tree keeps an empty database, which is what it needs to apply every committed migration itself. Their CircleCI definitions and workflow entries go with them, taking the config from 47 jobs to 44. assert_ci_coverage.py stays green: dropping the new workflow fails the census on exactly these trees, so the coverage moved rather than went missing. auth_ui_unit_tests is deliberately left behind. Ported, two of its tests fail because prepare_metadata_fields refuses enterprise-only keys without LITELLM_LICENSE, which exists as a CircleCI project variable and has no GitHub Actions secret. Creating that secret is a human action, so the job stays on CircleCI until it exists rather than shipping a red shard or quietly deselecting the two tests. * chore(ci): drop the narrative header from test-postgres.yml --- .circleci/config.yml | 126 ------------------------ .github/workflows/test-postgres.yml | 145 ++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/test-postgres.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e77729df29..2586485e79c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -651,126 +651,6 @@ jobs: - auth_ui_unit_tests_coverage.xml - auth_ui_unit_tests_coverage - proxy_behavior_tests: - docker: - - *python312_image - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: litellm_test - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - steps: - - checkout - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Seed DB schema via prisma db push - command: | - uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Generate Prisma Client - command: uv run --no-sync python -m prisma generate - - run: - name: Run proxy management behavior tests - command: | - mkdir -p test-results - uv run --no-sync python -m pytest tests/proxy_behavior \ - -v --junitxml=test-results/junit.xml --durations=10 - no_output_timeout: 15m - - store_test_results: - path: test-results - - proxy_security_tests: - docker: - - *python312_image - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: litellm_test - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - steps: - - checkout - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Seed DB schema via prisma db push - command: | - uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Generate Prisma Client - command: uv run --no-sync python -m prisma generate - - run: - name: Run proxy security tests - command: | - mkdir -p test-results - uv run --no-sync python -m pytest tests/proxy_security_tests \ - -v --junitxml=test-results/junit.xml --durations=10 - no_output_timeout: 15m - - store_test_results: - path: test-results - - schema_migration_check: - docker: - - *python312_image - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: litellm_test - working_directory: ~/project - environment: - # An empty database; the test applies every committed migration itself. - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - steps: - - checkout - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Generate Prisma Client - command: uv run --no-sync python -m prisma generate - - run: - name: Check schema.prisma is in sync with committed migrations - command: | - mkdir -p test-results - uv run --no-sync python -m pytest tests/proxy_migration_tests \ - -v --junitxml=test-results/junit.xml --durations=10 - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_router_testing: # Runs all tests with the "router" keyword docker: - *python312_image @@ -3105,12 +2985,6 @@ workflows: filters: *main_branches - auth_ui_unit_tests: filters: *main_branches - - proxy_behavior_tests: - filters: *main_branches - - proxy_security_tests: - filters: *main_branches - - schema_migration_check: - filters: *main_branches - build_docker_database_image: filters: *main_branches - e2e_ui_testing: diff --git a/.github/workflows/test-postgres.yml b/.github/workflows/test-postgres.yml new file mode 100644 index 00000000000..96c514dff7c --- /dev/null +++ b/.github/workflows/test-postgres.yml @@ -0,0 +1,145 @@ +name: "Postgres Tests" + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_staging + - "litellm_**" + push: + branches: + - main + - litellm_internal_staging + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + postgres: + name: ${{ matrix.shard }} + runs-on: ubuntu-latest + timeout-minutes: ${{ matrix.job-timeout-minutes }} + permissions: + contents: read + + services: + postgres: + image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 10 + + strategy: + fail-fast: false + matrix: + include: + - shard: proxy-behavior + test-path: "tests/proxy_behavior" + seed: db-push + workers: 0 + timeout-minutes: 25 + job-timeout-minutes: 50 + + - shard: proxy-security + test-path: "tests/proxy_security_tests" + seed: db-push + workers: 0 + timeout-minutes: 15 + job-timeout-minutes: 40 + + - shard: schema-migration + test-path: "tests/proxy_migration_tests" + seed: none + workers: 0 + timeout-minutes: 20 + job-timeout-minutes: 45 + + env: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + timeout-minutes: 3 + with: + persist-credentials: false + + - name: Detect relevant changes + id: changes + timeout-minutes: 2 + uses: ./.github/actions/detect-changes + + - name: Set up Python + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 3 + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache uv dependencies + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 5 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv-postgres- + + - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 12 + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras + + - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 3 + uses: ./.github/actions/cache-prisma-binaries + + - name: Generate Prisma client + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 5 + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Seed database schema + if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none' + timeout-minutes: 10 + run: | + uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + + - name: Run tests + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: ${{ matrix.timeout-minutes }} + env: + TEST_PATH: ${{ matrix.test-path }} + WORKERS: ${{ matrix.workers }} + run: | + if [ "${WORKERS}" = "0" ]; then + uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 + else + uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}" + fi From a734afca322959557694734e179b095f2e6f1039 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 22 Aug 2026 22:54:30 -0700 Subject: [PATCH 5/6] feat(ci): gate patching of SDK internals in tests as TQ008 (#37787) * feat(ci): gate patching of SDK internals in tests as TQ008 TQ002 catches the narrowest symptom of the suite's dominant mocking idiom, patch X then assert only that X was called. The idiom itself is wider: tests reach for litellm's own functions instead of faking the wire, so they pin how the code is wired rather than what it does, and a test that patches internals but makes weak real assertions trips nothing today. TQ008 counts patch targets rooted at `litellm`, both the dotted string form and the attribute chain handed to patch.object, and ratchets like every other rule. Mocking anything outside the SDK is untouched: respx, httpx transports and third-party clients do not trip it, which is the point, since those are the patterns this is meant to move the suite toward. Seeded at 9,643, in line with the ~9.4k patch sites an independent grep found in the mirror. The burn-down horizon is long; the value here is stopping the flow rather than clearing the stock. Five existing rule tests patched `litellm.completion` incidentally and now report TQ008 alongside what they were pinning. Their expected values are updated to the accurate pair rather than loosened, so they keep failing on a regression in either rule. * test: add TQ008 to the shipped-budget rule canary * fix(ci): resolve imported SDK names in TQ008 patch.object(handler.OpenAIChatCompletion, ...) after a from-import reaches the same internal as the dotted string form, but the rule only saw the bare local name and let it through. Import bindings are now resolved to the path they stand for, so the aliased, renamed and from-imported forms all read alike and the reported target is the real one. That is 1,496 patches the ratchet could not see, so the TQ008 limit moves from 9,643 to 11,139. Third-party names and locals with no SDK import behind them stay unflagged. --- scripts/check_test_quality.py | 66 +++++++++ test-quality-budget.json | 3 + tests/test_litellm/test_check_test_quality.py | 137 +++++++++++++++++- tests/test_litellm/test_test_quality_gate.py | 2 +- 4 files changed, 202 insertions(+), 6 deletions(-) diff --git a/scripts/check_test_quality.py b/scripts/check_test_quality.py index 6964aed56e4..41342acd23a 100644 --- a/scripts/check_test_quality.py +++ b/scripts/check_test_quality.py @@ -48,6 +48,10 @@ TQ006 A `pytest.skip` reached only when a credential-shaped environment variab deliberate branch. The gate follows one local or module-level binding, which is the `key = os.getenv(...)` then `if not key: pytest.skip(...)` shape most of these use. +TQ008 A `patch(...)` whose target is a `litellm.` internal. Patching the SDK's own + functions pins the test to the current wiring instead of the behaviour, and it + is the idiom the suite reaches for instead of faking the HTTP boundary. Mocking + a third-party client, a transport, or anything outside `litellm.` is untouched. TQ007 A module global that a conftest saves before every test and restores after it. The save/restore list is a hand-maintained inventory of the leaks the suite already knows about, so it is allowed to shrink and never to grow: a new entry @@ -469,6 +473,67 @@ def iter_global_mutation_violations(path: Path, tree: ast.Module) -> Iterator[Vi ) +def _is_sdk_internal(dotted: str) -> bool: + return dotted == SDK_MODULE or dotted.startswith(f"{SDK_MODULE}.") + + +def _sdk_import_bindings(tree: ast.Module) -> Iterator[tuple[str, str]]: + """(local name, dotted path) for every import that binds something under `litellm`.""" + for node in ast.walk(tree): + if isinstance(node, ast.Import): + yield from ( + (alias.asname, alias.name) if alias.asname else (root, root) + for alias in node.names + if _is_sdk_internal(alias.name) + for root in (alias.name.partition(".")[0],) + ) + elif isinstance(node, ast.ImportFrom) and node.module and _is_sdk_internal(node.module): + yield from ((alias.asname or alias.name, f"{node.module}.{alias.name}") for alias in node.names) + + +def _sdk_aliases(tree: ast.Module) -> Mapping[str, str]: + """Local names bound to something under `litellm`, mapped to the path they stand for. + + `from litellm.llms.openai.chat import handler` then `patch.object(handler.X, ...)` + reaches the same internal as the dotted string form and has to read the same way. + """ + return MappingProxyType({name: dotted for name, dotted in _sdk_import_bindings(tree)}) + + +def _resolved(dotted: str, aliases: Mapping[str, str]) -> str: + root, _, rest = dotted.partition(".") + base: Final = aliases.get(root, root) + return f"{base}.{rest}" if rest else base + + +def _patch_targets(call: ast.Call, aliases: Mapping[str, str]) -> Iterator[str]: + """What a patch installer is replacing: the dotted string it names, or the + attribute chain handed to `patch.object` / `patch.dict`, resolved through the + module's imports so a locally bound SDK object reads as its full path.""" + for first in call.args[:1]: + if isinstance(first, ast.Constant) and isinstance(first.value, str): + yield first.value + elif dotted := _dotted_name(first): + yield _resolved(dotted, aliases) + + +def iter_internal_patch_violations(path: Path, tree: ast.Module) -> Iterator[Violation]: + aliases: Final = _sdk_aliases(tree) + for node in ast.walk(tree): + if not (isinstance(node, ast.Call) and _is_patch_installer(_dotted_name(node.func))): + continue + for target in _patch_targets(node, aliases): + if _is_sdk_internal(target): + yield Violation( + path, + node.lineno, + "TQ008", + f"patches `{target}`, an SDK internal, so the test is pinned to how the code is " + "wired rather than what it does; fake the HTTP boundary (respx / MockTransport) " + f"or inject the collaborator (suppress: `# {SUPPRESSION_TOKEN}: `)", + ) + + def _environ_keys(node: ast.AST) -> Iterator[str]: for inner in ast.walk(node): if isinstance(inner, ast.Call) and _dotted_name(inner.func) in ENVIRON_READERS: @@ -680,6 +745,7 @@ def check_file(path: Path) -> tuple[Violation, ...]: *iter_global_mutation_violations(path, tree), *iter_credential_skip_violations(path, tree), *iter_conftest_inventory_violations(path, tree), + *iter_internal_patch_violations(path, tree), ) if violation.line not in skip ) diff --git a/test-quality-budget.json b/test-quality-budget.json index 0dea4e8fe93..4a7bc7edff2 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -19,5 +19,8 @@ }, "TQ007": { "limit": 117 + }, + "TQ008": { + "limit": 11139 } } diff --git a/tests/test_litellm/test_check_test_quality.py b/tests/test_litellm/test_check_test_quality.py index 7d59e5a5dba..a75b1e43fb7 100644 --- a/tests/test_litellm/test_check_test_quality.py +++ b/tests/test_litellm/test_check_test_quality.py @@ -187,7 +187,7 @@ def test_mock_echo_is_flagged(tmp_path): " run()\n" " mock_completion.assert_called_once()\n" ) - assert _codes(tmp_path, source) == ["TQ002"] + assert _codes(tmp_path, source) == ["TQ002", "TQ008"] def test_call_args_inspection_is_mock_echo(tmp_path): @@ -200,7 +200,7 @@ def test_call_args_inspection_is_mock_echo(tmp_path): " run()\n" " assert mock_completion.call_args[1]['model'] == 'gpt-4o'\n" ) - assert _codes(tmp_path, source) == ["TQ002"] + assert _codes(tmp_path, source) == ["TQ002", "TQ008"] def test_patch_decorator_counts_as_installing_a_patch(tmp_path): @@ -213,7 +213,7 @@ def test_patch_decorator_counts_as_installing_a_patch(tmp_path): " run()\n" " mock_completion.assert_called_once()\n" ) - assert _codes(tmp_path, source) == ["TQ002"] + assert _codes(tmp_path, source) == ["TQ002", "TQ008"] def test_patching_but_asserting_the_output_is_not_mock_echo(tmp_path): @@ -227,7 +227,7 @@ def test_patching_but_asserting_the_output_is_not_mock_echo(tmp_path): " mock_completion.assert_called_once()\n" " assert result.choices[0].message.content == 'pong'\n" ) - assert _codes(tmp_path, source) == [] + assert _codes(tmp_path, source) == ["TQ008"] def test_asserting_without_patching_is_not_mock_echo(tmp_path): @@ -244,7 +244,7 @@ def test_a_test_with_no_assertions_is_tq001_not_tq002(tmp_path): " with patch('litellm.completion'):\n" " run()\n" ) - assert _codes(tmp_path, source) == ["TQ001"] + assert _codes(tmp_path, source) == ["TQ001", "TQ008"] def test_sys_path_insert_is_flagged(tmp_path): @@ -554,6 +554,133 @@ def test_a_loop_storing_under_a_key_that_is_not_the_loop_variable_is_not_an_inve assert [v.code for v in checker.check_file(_written(tmp_path, source))] == [] +def test_patching_an_sdk_function_by_string_is_flagged(tmp_path): + source = 'from unittest.mock import patch\n\n\n@patch("litellm.completion")\ndef test_x(m):\n assert m\n' + assert "TQ008" in _codes(tmp_path, source) + + +def test_patching_a_deep_sdk_path_is_flagged(tmp_path): + source = ( + "from unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch("litellm.llms.openai.chat.handler.OpenAIChatCompletion.completion"):\n' + " assert True\n" + ) + assert "TQ008" in _codes(tmp_path, source) + + +def test_patch_object_rooted_at_the_sdk_is_flagged(tmp_path): + source = ( + "import litellm\nfrom unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch.object(litellm, "api_key", "x"):\n' + " assert True\n" + ) + assert "TQ008" in _codes(tmp_path, source) + + +def test_patch_object_on_a_from_imported_sdk_module_is_flagged(tmp_path): + source = ( + "from litellm.llms.openai.chat import handler\nfrom unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch.object(handler.OpenAIChatCompletion, "completion"):\n' + " assert True\n" + ) + assert "TQ008" in _codes(tmp_path, source) + + +def test_patch_object_on_an_aliased_sdk_module_is_flagged(tmp_path): + source = ( + "import litellm.llms.openai.chat.handler as oai\nfrom unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch.object(oai.OpenAIChatCompletion, "completion"):\n' + " assert True\n" + ) + assert "TQ008" in _codes(tmp_path, source) + + +def test_patch_object_on_a_renamed_sdk_symbol_is_flagged(tmp_path): + source = ( + "from litellm.utils import get_llm_provider as glp\nfrom unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch.object(glp, "__wrapped__"):\n' + " assert True\n" + ) + assert "TQ008" in _codes(tmp_path, source) + + +def test_the_reported_target_is_the_resolved_sdk_path(tmp_path): + source = ( + "from litellm.llms.openai.chat import handler\nfrom unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch.object(handler.OpenAIChatCompletion, "completion"):\n' + " assert True\n" + ) + reported = [v.message for v in checker.check_file(_written(tmp_path, source)) if v.code == "TQ008"] + assert reported + assert "litellm.llms.openai.chat.handler.OpenAIChatCompletion" in reported[0] + + +def test_patch_object_on_a_from_imported_third_party_is_not_flagged(tmp_path): + source = ( + "from openai import OpenAI\nfrom unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch.object(OpenAI, "chat"):\n' + " assert True\n" + ) + assert "TQ008" not in _codes(tmp_path, source) + + +def test_a_local_name_with_no_sdk_import_behind_it_is_not_flagged(tmp_path): + source = ( + "from unittest.mock import patch\n\n\n" + "def test_x(handler):\n" + ' with patch.object(handler, "completion"):\n' + " assert True\n" + ) + assert "TQ008" not in _codes(tmp_path, source) + + +def test_mocking_a_third_party_client_is_not_flagged(tmp_path): + source = ( + "from unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch("openai.OpenAI.chat"):\n' + " assert True\n" + ) + assert "TQ008" not in _codes(tmp_path, source) + + +def test_mocking_the_http_transport_is_not_flagged(tmp_path): + source = ( + "from unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch("httpx.AsyncClient.send"):\n' + " assert True\n" + ) + assert "TQ008" not in _codes(tmp_path, source) + + +def test_a_name_merely_starting_with_litellm_is_not_the_sdk(tmp_path): + source = ( + "from unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch("litellm_enterprise.thing.go"):\n' + " assert True\n" + ) + assert "TQ008" not in _codes(tmp_path, source) + + +def test_an_sdk_patch_can_be_suppressed(tmp_path): + source = ( + "from unittest.mock import patch\n\n\n" + "def test_x():\n" + ' with patch("litellm.completion"): # test-quality-ok: pinning the router seam\n' + " assert True\n" + ) + assert "TQ008" not in _codes(tmp_path, source) + + _FANS_OUT = checker._worker_count(checker.PARALLEL_MIN_PATHS) > 1 _SERIAL_ONLY = "one usable core, so scan_paths stays serial and there is no fan-out to compare" diff --git a/tests/test_litellm/test_test_quality_gate.py b/tests/test_litellm/test_test_quality_gate.py index 3bf4b89ac4e..8cce6bc735a 100644 --- a/tests/test_litellm/test_test_quality_gate.py +++ b/tests/test_litellm/test_test_quality_gate.py @@ -144,5 +144,5 @@ def test_the_shipped_budget_covers_every_rule_the_checker_can_emit(): import json budget = json.loads((_REPO_ROOT / "test-quality-budget.json").read_text()) - assert set(budget) == {"TQ001", "TQ002", "TQ003", "TQ004", "TQ005", "TQ006", "TQ007"} + assert set(budget) == {"TQ001", "TQ002", "TQ003", "TQ004", "TQ005", "TQ006", "TQ007", "TQ008"} assert all(spec["limit"] >= 0 for spec in budget.values()) From 6c30b4331d2d9b5571efcb0ff94fb731651d500d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 22 Aug 2026 22:55:01 -0700 Subject: [PATCH 6/6] ci: measure enterprise/ coverage (#37788) codecov.yaml has carried an `Enterprise` component scoped to `enterprise/**` since it was written, and it has never received a line of data. Every one of the 19 coverage invocations across the unit base, the MCP workflow and the CircleCI config passes `--cov=./litellm` and nothing else, so 11,203 lines of paid-customer code sat outside the measured universe while the reported number described only the rest. litellm-enterprise is a uv workspace member and a direct dependency, so every job that syncs already has it installed and importable; only the measurement was missing. Measured on tests/test_litellm/enterprise, the shard that exercises this code: 0 enterprise files in the report before, 142 after, at `enterprise/...` paths that match the component's existing glob. That shard alone puts enterprise at 30.8%, which nudged its own total from 24.09% to 24.20% rather than down. The aggregate direction across every shard is not knowable until they all report, and a drop there is the instrument working, not a regression. --- .circleci/config.yml | 32 +++++++++++++-------------- .github/workflows/_test-unit-base.yml | 4 ++-- .github/workflows/test-mcp.yml | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2586485e79c..a261b7c78fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -430,7 +430,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ - --cov=./litellm \ + --cov=./litellm --cov=./enterprise/litellm_enterprise \ --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ @@ -504,7 +504,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ - --cov=./litellm \ + --cov=./litellm --cov=./enterprise/litellm_enterprise \ --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ @@ -631,7 +631,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" @@ -738,7 +738,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" @@ -865,7 +865,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ -n 4 \ @@ -910,7 +910,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m @@ -954,7 +954,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2 \ @@ -1000,7 +1000,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ --retries 3 --retry-delay 5" @@ -1091,7 +1091,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" @@ -1135,7 +1135,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" @@ -1213,7 +1213,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" @@ -1257,7 +1257,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" @@ -1302,7 +1302,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" @@ -1381,7 +1381,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ -n 4 \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -1426,7 +1426,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m @@ -1479,7 +1479,7 @@ jobs: --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ - --cov=./litellm --cov-report=xml \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 -n 2 \ --reruns 2 --reruns-delay 1" diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index b7d185bd0b9..c4045a08ffb 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -149,7 +149,7 @@ jobs: --reruns "${RERUNS}" \ --reruns-delay 1 \ --durations=20 \ - --cov=./litellm \ + --cov=./litellm --cov=./enterprise/litellm_enterprise \ --cov-report=xml:coverage.xml \ --cov-config=pyproject.toml else @@ -161,7 +161,7 @@ jobs: --reruns-delay 1 \ --dist="${DIST}" \ --durations=20 \ - --cov=./litellm \ + --cov=./litellm --cov=./enterprise/litellm_enterprise \ --cov-report=xml:coverage.xml \ --cov-config=pyproject.toml fi diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 6ea814dc2de..93ffcbe0586 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -60,4 +60,4 @@ jobs: - name: Run MCP tests if: steps.changes.outputs.decision != 'skip' run: | - uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5 + uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5