From 1f565426fa7ed01a50b1dddbea02de00e26dc3e6 Mon Sep 17 00:00:00 2001 From: Jesshuan Date: Thu, 25 Jun 2026 07:26:59 +0200 Subject: [PATCH] fix(ecologits): lazy module import for prometheus --- ISSUE_ecologits.md | 40 +++++++++++++++++++++++++++++++ litellm/integrations/ecologits.py | 31 ++++++++++++++++++++---- 2 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 ISSUE_ecologits.md diff --git a/ISSUE_ecologits.md b/ISSUE_ecologits.md new file mode 100644 index 00000000000..0d5dacca5b1 --- /dev/null +++ b/ISSUE_ecologits.md @@ -0,0 +1,40 @@ +# Feature: EcoLogits callback — enrich LLM-call metrics with green-cost data + +## The feature + +We propose to enrich LiteLLM's existing LLM-call metrics with **green-cost metrics**, via an optional EcoLogits callback plugin. On every successful call, the plugin queries the [EcoLogits API](https://api.ecologits.ai/docs) and makes the resulting environmental impacts available to **all downstream observability callbacks** (Langfuse, Datadog, Prometheus, …). + +For this first "proxy mode" version, the principle is simple: + +- **Per-model zone config.** When registering a model, admins can set a `model_info` argument `ecologits_electricity_mix_zone` (e.g. `FRA`, `DEU`) reflecting where the model physically runs. If omitted, the plugin falls back to the `ECOLOGITS_ELECTRICITY_MIX_ZONE` env var, and finally to EcoLogits' own default (`WOR` — world average). +- **Per-call enrichment.** On each successful call to a provider, the plugin calls the EcoLogits API (public, or an enterprise self-hosted instance via `ECOLOGITS_API_BASE`) and attaches the impacts to the call payload. It runs in the `async_logging_hook`, *before* other callbacks see the data — so any downstream integration the user enables gets the green metrics for free. The enrichment is written where each consumer looks: `litellm_params.metadata` (Langfuse) and `standard_logging_object.metadata` (Datadog, SpendLogs). For Prometheus, dedicated green-cost Counters (energy kWh, GWP kgCO2eq, …) are exposed alongside. +- **Never fatal.** Any failure (timeout, non-200, bad payload) is swallowed — at worst the call is logged without enrichment. + +## Motivation + +Today LiteLLM lets organizations make deployment and quota decisions on cost and billing, but not on the **ecological cost** of those decisions. For a growing number of organizations this is no longer optional: incoming regulations will require Data/AI departments to report the environmental impact of their LLM usage. + +[EcoLogits](https://ecologits.ai/latest/) is a non-profit project backed by institutional and non-institutional partners and hosted by [CodeCarbon](https://codecarbon.io/). Its [LLM-inference methodology](https://ecologits.ai/latest/methodology/llm_inference/) sits at the crossing of state-of-the-art research estimates, real data-center measurements, and common MLOps practice — a credible approximation in a world where major providers don't disclose their model architectures (with acknowledged limitations). + +By delegating the calculation to EcoLogits, the green-cost estimation is **decoupled from LiteLLM itself**: LiteLLM developers don't have to maintain a state-of-the-art calculator, and users automatically benefit from future EcoLogits improvements. This integration can of course coexist with any native LiteLLM green-cost evaluation. + +## Example configuration + +```yaml +litellm_settings: + # EcoLogits must be registered first so it enriches the payload + # before any downstream callback observes it. + callbacks: ["ecologits", "langfuse"] + +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + model_info: + ecologits_electricity_mix_zone: "FRA" # optional; defaults to "WOR" +``` + +## Scope & limitations + +- Proxy mode only for this first version. +- Impacts are **estimates** produced by the EcoLogits methodology, not measured values; accuracy is bounded by what providers disclose about their models. diff --git a/litellm/integrations/ecologits.py b/litellm/integrations/ecologits.py index 6ebde37a1ac..0a3bc2f2257 100644 --- a/litellm/integrations/ecologits.py +++ b/litellm/integrations/ecologits.py @@ -29,9 +29,7 @@ is logged without ecologits enrichment. """ import os -from typing import Any, Dict, Optional, Tuple - -from prometheus_client import Counter +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple import litellm from litellm._logging import verbose_logger @@ -41,6 +39,9 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) +if TYPE_CHECKING: + from prometheus_client import Counter + ECOLOGITS_DEFAULT_API_BASE = "https://api.ecologits.ai" ECOLOGITS_ESTIMATIONS_PATH = "/v1beta/estimations" ECOLOGITS_DEFAULT_TIMEOUT_S = 2.0 @@ -74,8 +75,26 @@ ECOLOGITS_IMPACTS = ( ) -def _build_ecologits_counters() -> Dict[str, Counter]: - counters: Dict[str, Counter] = {} +def _build_ecologits_counters() -> Optional[Dict[str, "Counter"]]: + """Build one Prometheus Counter per impact, or ``None`` if prometheus_client + is not installed. + + Guarding the import (rather than importing it at module level) means users + who only want the Langfuse/Datadog enrichment can enable the ``ecologits`` + callback without installing ``prometheus-client``; they simply get no + Prometheus metrics. + """ + try: + from prometheus_client import Counter + except ImportError: + verbose_logger.warning( + "Missing prometheus_client. Run `pip install prometheus-client` to " + "emit EcoLogits Prometheus metrics. EcoLogits enrichment " + "(Langfuse/Datadog/SpendLogs) still works without it." + ) + return None + + counters: Dict[str, "Counter"] = {} for json_key, suffix, unit, description in ECOLOGITS_IMPACTS: counters[json_key] = Counter( name=f"litellm_ecologits_{suffix}_total", @@ -177,6 +196,8 @@ class EcoLogitsLogger(CustomLogger): custom_llm_provider: Optional[str], zone: str, ) -> None: + if _ECOLOGITS_COUNTERS is None: + return if not isinstance(ecologits_data, dict): return impacts = ecologits_data.get("impacts")