From 2e2f2cbfb7f25824e4fc31b785f0298db5a244e1 Mon Sep 17 00:00:00 2001 From: Antoine Tielbeke Date: Sun, 20 Sep 2026 00:57:05 +0200 Subject: [PATCH 1/2] feat(logging): record estimated environmental impact per request The gateway already counts cost for every request, and nothing else about what the request consumed. Teams asked to report on AI energy use have no data to report, and the existing EcoLogits integration instruments the Python SDK only, so it sees nothing that goes through the proxy. This adds `response_impact` next to `response_cost` on StandardLoggingPayload, so every logger that already serializes the payload picks it up with no change of its own. Estimation is delegated to EcoLogits rather than implemented here: LiteLLM decides when an estimate is taken and where it is recorded, and owns none of the methodology. Core carries no third-party import. `impact_calculator` holds the types and an estimator protocol, and the EcoLogits-backed implementation sits behind that protocol in integrations, resolved lazily so importing LiteLLM never requires the package. Off by default behind `litellm.track_impact`. Figures keep the interval the estimator modelled rather than a bare number. Closed models are sized from an inferred parameter count, so most figures come back as a range, and reporting only the midpoint would present an inference as a measurement. Cache hits are skipped, since a cached response ran no inference and attributing generation energy to it would double count. --- litellm/__init__.py | 1 + litellm/impact_calculator.py | 90 +++++++++ litellm/integrations/ecologits_impact.py | 121 ++++++++++++ litellm/litellm_core_utils/litellm_logging.py | 45 +++++ litellm/types/utils.py | 44 +++++ .../integrations/test_ecologits_impact.py | 174 ++++++++++++++++++ .../test_litellm_logging.py | 88 +++++++++ tests/test_litellm/test_impact_calculator.py | 115 ++++++++++++ 8 files changed, 678 insertions(+) create mode 100644 litellm/impact_calculator.py create mode 100644 litellm/integrations/ecologits_impact.py create mode 100644 tests/test_litellm/integrations/test_ecologits_impact.py create mode 100644 tests/test_litellm/test_impact_calculator.py diff --git a/litellm/__init__.py b/litellm/__init__.py index e17ab613dac..cbf596d3f98 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -335,6 +335,7 @@ disable_token_counter: bool = False disable_add_transform_inline_image_block: bool = False disable_add_user_agent_to_request_tags: bool = False disable_anthropic_gemini_context_caching_transform: bool = False +track_impact: bool = False enable_anthropic_prompt_caching: bool = os.getenv("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", "false").lower() == "true" _anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL") anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = ( diff --git a/litellm/impact_calculator.py b/litellm/impact_calculator.py new file mode 100644 index 00000000000..b25f0f7630c --- /dev/null +++ b/litellm/impact_calculator.py @@ -0,0 +1,90 @@ +# What is this? +## File for 'response_impact' estimation in Logging +from dataclasses import dataclass +from functools import lru_cache +from typing import Final, Protocol + +import litellm +from litellm._logging import verbose_logger +from litellm.types.utils import ImpactInformation, ImpactValue + + +@dataclass(frozen=True, slots=True) +class ImpactRequest: + model: str + custom_llm_provider: str | None + completion_tokens: int + response_time: float | None + + +class ImpactEstimator(Protocol): + """Produces an environmental impact estimate for a request.""" + + @property + def name(self) -> str: ... + + def estimate(self, request: ImpactRequest) -> ImpactInformation | None: ... + + +def point_impact_value(unit: str, value: float) -> ImpactValue: + return ImpactValue(value=value, min=None, max=None, unit=unit) + + +def interval_impact_value(unit: str, minimum: float, maximum: float) -> ImpactValue: + """``value`` is the midpoint, so a consumer summing it stays inside the modelled interval.""" + return ImpactValue(value=(minimum + maximum) / 2, min=minimum, max=maximum, unit=unit) + + +@dataclass(slots=True) +class _EstimatorRegistry: + estimator: ImpactEstimator | None = None + + +_registry: Final = _EstimatorRegistry() + + +def register_impact_estimator(estimator: ImpactEstimator | None) -> None: + _registry.estimator = estimator + + +@lru_cache(maxsize=1) +def _resolve_default_estimator() -> ImpactEstimator | None: + from litellm.integrations.ecologits_impact import EcoLogitsImpactEstimator + + try: + return EcoLogitsImpactEstimator() + except Exception as e: # noqa: BLE001 # a missing backend must never fail request logging + verbose_logger.warning( + "litellm.track_impact is on but no impact estimator is available, so no impact is recorded: %s", + str(e), + ) + return None + + +def get_impact_estimator() -> ImpactEstimator | None: + if _registry.estimator is not None: + return _registry.estimator + return _resolve_default_estimator() + + +def current_estimator_name() -> str | None: + estimator: Final = get_impact_estimator() + return None if estimator is None else estimator.name + + +def reset_default_estimator_cache() -> None: + _resolve_default_estimator.cache_clear() + + +def calculate_impact(request: ImpactRequest) -> ImpactInformation | None: + """ + Estimate the environmental impact of a request. + + Raises whatever the estimator raises, so the caller can record the failure. + """ + if litellm.track_impact is not True: + return None + estimator: Final = get_impact_estimator() + if estimator is None: + return None + return estimator.estimate(request) diff --git a/litellm/integrations/ecologits_impact.py b/litellm/integrations/ecologits_impact.py new file mode 100644 index 00000000000..536df844114 --- /dev/null +++ b/litellm/integrations/ecologits_impact.py @@ -0,0 +1,121 @@ +from collections.abc import Sequence +from types import MappingProxyType +from typing import Final, Protocol + +from litellm._logging import verbose_logger +from litellm.impact_calculator import ImpactRequest, interval_impact_value, point_impact_value +from litellm.types.utils import ImpactInformation, ImpactValue + +ESTIMATOR_NAME: Final = "ecologits" + +# ecologits totals cover the usage phase plus embodied manufacturing, which is boundary B. +# Source: https://ecologits.ai/latest/methodology/llm_inference/ read 2026-09-20 +IMPACT_BOUNDARY: Final = "B" + +# litellm names a provider by the API it speaks to; ecologits names one by who trained the +# model, and keys its registry on that. Providers absent here have no ecologits data. +PROVIDER_NAMES: Final = MappingProxyType( + { + "anthropic": "anthropic", + "azure": "openai", + "azure_ai": "openai", + "cohere": "cohere", + "cohere_chat": "cohere", + "gemini": "google_genai", + "huggingface": "huggingface_hub", + "mistral": "mistralai", + "openai": "openai", + "text-completion-openai": "openai", + "vertex_ai": "google_genai", + } +) + + +class _RangeLike(Protocol): + min: float + max: float + + +class _CriterionLike(Protocol): + value: float | _RangeLike + unit: str + + +class _ErrorLike(Protocol): + message: str + + +class _ImpactsLike(Protocol): + energy: _CriterionLike | None + gwp: _CriterionLike | None + adpe: _CriterionLike | None + pe: _CriterionLike | None + wcf: _CriterionLike | None + errors: Sequence[_ErrorLike] + + +def _to_impact_value(criterion: _CriterionLike | None) -> ImpactValue | None: + if criterion is None: + return None + raw: Final = criterion.value + if isinstance(raw, (int, float)): + return point_impact_value(unit=criterion.unit, value=float(raw)) + return interval_impact_value(unit=criterion.unit, minimum=float(raw.min), maximum=float(raw.max)) + + +def _bare_model_name(model: str) -> str: + """Strip the routing prefix litellm carries, leaving the name ecologits registers.""" + return model.rsplit("/", 1)[-1] + + +class EcoLogitsImpactEstimator: + """ + Estimates request impact with EcoLogits. + + Constructing this requires the ``ecologits`` package. The import is deferred to + ``__init__`` so that importing LiteLLM, and this module, stays free of that + dependency and only enabling the feature can fail. + """ + + def __init__(self) -> None: + from ecologits.tracers.utils import llm_impacts + + self._llm_impacts: Final = llm_impacts + + @property + def name(self) -> str: + return ESTIMATOR_NAME + + def estimate(self, request: ImpactRequest) -> ImpactInformation | None: + if request.custom_llm_provider is None or request.response_time is None: + return None + if request.completion_tokens <= 0: + return None + provider: Final = PROVIDER_NAMES.get(request.custom_llm_provider) + if provider is None: + return None + + impacts: Final[_ImpactsLike | None] = self._llm_impacts( + provider=provider, + model_name=_bare_model_name(request.model), + output_token_count=request.completion_tokens, + request_latency=request.response_time, + ) + if impacts is None or impacts.errors: + verbose_logger.debug( + "ecologits has no impact data for %s/%s: %s", + provider, + request.model, + tuple(error.message for error in impacts.errors) if impacts is not None else "no result", + ) + return None + + return ImpactInformation( + energy=_to_impact_value(impacts.energy), + gwp=_to_impact_value(impacts.gwp), + adpe=_to_impact_value(impacts.adpe), + pe=_to_impact_value(impacts.pe), + water=_to_impact_value(impacts.wcf), + boundary=IMPACT_BOUNDARY, + estimator=ESTIMATOR_NAME, + ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f7679b31f69..7d4fcab8e21 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -132,6 +132,7 @@ from litellm.types.utils import ( EmbeddingResponse, GuardrailStatus, ImageResponse, + ImpactInformation, LiteLLMBatch, LiteLLMLoggingBaseClass, LiteLLMRealtimeStreamLoggingObject, @@ -143,6 +144,7 @@ from litellm.types.utils import ( StandardCallbackDynamicParams, StandardLoggingAdditionalHeaders, StandardLoggingHiddenParams, + StandardLoggingImpactFailureDebugInformation, StandardLoggingMCPToolCall, StandardLoggingMetadata, StandardLoggingModelCostFailureDebugInformation, @@ -6173,6 +6175,37 @@ def _autorouter_savings_for_payload( return None +def _response_impact_for_payload( + model: str, + custom_llm_provider: str | None, + completion_tokens: int, + response_time: float, +) -> tuple[ImpactInformation | None, StandardLoggingImpactFailureDebugInformation | None]: + """Estimate impact for one request, recording any estimator failure the way a cost failure is.""" + from litellm.impact_calculator import ImpactRequest, calculate_impact, current_estimator_name + + try: + impact: Final = calculate_impact( + ImpactRequest( + model=model, + custom_llm_provider=custom_llm_provider, + completion_tokens=completion_tokens, + response_time=response_time, + ) + ) + return impact, None + except Exception as e: # noqa: BLE001 # an estimator must never fail request logging + debug_info: Final = StandardLoggingImpactFailureDebugInformation( + error_str=str(e), + traceback_str=_get_traceback_str_for_error(str(e)), + model=model, + custom_llm_provider=custom_llm_provider, + estimator=current_estimator_name(), + ) + verbose_logger.debug("response_impact_failure_debug_information: %s", debug_info) + return None, debug_info + + def get_standard_logging_object_payload( kwargs: dict | None, init_response_obj: Any | BaseModel | dict, @@ -6343,6 +6376,16 @@ def get_standard_logging_object_payload( model_name = response_model_name request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost) + response_impact, response_impact_failure_debug_info = ( + (None, None) + if cache_hit is True + else _response_impact_for_payload( + model=model_name, + custom_llm_provider=custom_llm_provider, + completion_tokens=usage_dict.get("completion_tokens", 0), + response_time=end_time_float - start_time_float, + ) + ) autorouter_savings: Final = _autorouter_savings_for_payload( request_metadata=metadata, model=model_name, @@ -6395,6 +6438,8 @@ def get_standard_logging_object_payload( cache_key=clean_hidden_params["cache_key"], response_cost=response_cost, cost_breakdown=request_cost_breakdown, + response_impact=response_impact, + response_impact_failure_debug_info=response_impact_failure_debug_info, autorouter_savings=autorouter_savings, total_tokens=usage_dict.get("total_tokens", 0), prompt_tokens=usage_dict.get("prompt_tokens", 0), diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d416e2af33a..dbbd0b24eb9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3406,6 +3406,48 @@ class CostBreakdown(TypedDict, total=False): margin_total_amount: float # Total margin added in USD (optional) +class ImpactValue(TypedDict, total=False): + """One environmental impact figure.""" + + value: ReadOnly[float] # Always set; the midpoint when the estimator models an interval + min: ReadOnly[float | None] # Lower bound, set only when the figure is modelled as an interval + max: ReadOnly[float | None] + unit: ReadOnly[str] + + +class ImpactInformation(TypedDict, total=False): + """ + Estimated environmental impact of a request. + + Figures are only comparable when their ``boundary`` matches, and the boundary cannot + be recovered after the fact, so it is recorded per request: ``A`` is operational + electricity only, ``B`` adds allocated embodied hardware, ``C`` adds amortized + training. See https://arxiv.org/pdf/2606.10660 read 2026-09-20 + """ + + energy: ReadOnly[ImpactValue | None] + gwp: ReadOnly[ImpactValue | None] + adpe: ReadOnly[ImpactValue | None] + pe: ReadOnly[ImpactValue | None] + water: ReadOnly[ImpactValue | None] + boundary: ReadOnly[Literal["A", "B", "C"] | None] + estimator: ReadOnly[str | None] + + +class StandardLoggingImpactFailureDebugInformation(TypedDict, total=False): + """ + Debug information, if impact estimation fails. + + Avoid logging sensitive information like response or optional params + """ + + error_str: Required[ReadOnly[str]] + traceback_str: Required[ReadOnly[str]] + model: ReadOnly[str | None] + custom_llm_provider: ReadOnly[str | None] + estimator: ReadOnly[str | None] + + class StandardLoggingPayloadStatusFields(TypedDict, total=False): """Status fields for easy filtering and analytics""" @@ -3484,6 +3526,8 @@ class StandardLoggingPayload(ClassifierAudit): hidden_params: StandardLoggingHiddenParams guardrail_information: list[StandardLoggingGuardrailInformation] | None standard_built_in_tools_params: StandardBuiltInToolsParams | None + response_impact: NotRequired[ReadOnly[ImpactInformation | None]] + response_impact_failure_debug_info: NotRequired[ReadOnly[StandardLoggingImpactFailureDebugInformation | None]] from collections.abc import AsyncIterator, Iterator diff --git a/tests/test_litellm/integrations/test_ecologits_impact.py b/tests/test_litellm/integrations/test_ecologits_impact.py new file mode 100644 index 00000000000..b3f87ddb9e4 --- /dev/null +++ b/tests/test_litellm/integrations/test_ecologits_impact.py @@ -0,0 +1,174 @@ +import sys +from dataclasses import dataclass, field +from types import ModuleType +from unittest.mock import patch + +import pytest + +from litellm.impact_calculator import ImpactRequest + + +@dataclass +class _Range: + min: float + max: float + + +@dataclass +class _Criterion: + value: float | _Range + unit: str + + +@dataclass +class _Error: + message: str + + +@dataclass +class _Impacts: + energy: _Criterion | None = None + gwp: _Criterion | None = None + adpe: _Criterion | None = None + pe: _Criterion | None = None + wcf: _Criterion | None = None + errors: list = field(default_factory=list) + + +def _build_estimator(impacts): + calls = [] + + def llm_impacts(provider, model_name, output_token_count, request_latency, electricity_mix_zone=None): + calls.append( + { + "provider": provider, + "model_name": model_name, + "output_token_count": output_token_count, + "request_latency": request_latency, + } + ) + return impacts + + utils = ModuleType("ecologits.tracers.utils") + utils.llm_impacts = llm_impacts + modules = { + "ecologits": ModuleType("ecologits"), + "ecologits.tracers": ModuleType("ecologits.tracers"), + "ecologits.tracers.utils": utils, + } + with patch.dict(sys.modules, modules): + from litellm.integrations.ecologits_impact import EcoLogitsImpactEstimator + + return EcoLogitsImpactEstimator(), calls + + +def _request(**overrides) -> ImpactRequest: + defaults = { + "model": "gpt-4o", + "custom_llm_provider": "openai", + "completion_tokens": 250, + "response_time": 3.5, + } + return ImpactRequest(**{**defaults, **overrides}) + + +@pytest.fixture +def energy_only() -> _Impacts: + return _Impacts(energy=_Criterion(value=1.0, unit="Wh")) + + +def test_a_modelled_interval_survives_the_conversion(): + """An interval is the estimator saying it does not know precisely, so it must not be flattened.""" + estimator, _ = _build_estimator(_Impacts(energy=_Criterion(value=_Range(min=2.0, max=6.0), unit="Wh"))) + + energy = estimator.estimate(_request())["energy"] + + assert energy["min"] == 2.0 + assert energy["max"] == 6.0 + assert energy["min"] < energy["value"] < energy["max"] + + +def test_a_point_estimate_stays_a_point(): + estimator, _ = _build_estimator(_Impacts(energy=_Criterion(value=1.25, unit="Wh"))) + + energy = estimator.estimate(_request())["energy"] + + assert energy["value"] == 1.25 + assert energy["min"] is None + assert energy["max"] is None + + +def test_water_comes_from_the_same_total_as_the_other_figures(): + """Water read from a narrower phase than the rest would not match the boundary they all claim.""" + estimator, _ = _build_estimator(_Impacts(wcf=_Criterion(value=0.4, unit="L"))) + + result = estimator.estimate(_request()) + + assert result["water"]["value"] == 0.4 + assert result["water"]["unit"] == "L" + + +def test_the_boundary_and_estimator_are_recorded(energy_only): + estimator, _ = _build_estimator(energy_only) + + result = estimator.estimate(_request()) + + assert result["boundary"] == "B" + assert result["estimator"] == "ecologits" + + +def test_a_result_carrying_errors_is_not_an_estimate(): + """ecologits reports an unknown model as an error on a filled-in result, not as a missing one.""" + estimator, _ = _build_estimator(_Impacts(errors=[_Error(message="Could not find model")])) + + assert estimator.estimate(_request()) is None + + +def test_generated_tokens_and_latency_drive_the_estimate(energy_only): + estimator, calls = _build_estimator(energy_only) + request = _request() + + estimator.estimate(request) + + assert calls[0]["output_token_count"] == request.completion_tokens + assert calls[0]["request_latency"] == request.response_time + + +def test_the_routing_prefix_is_stripped_from_the_model_name(energy_only): + """Under the router a model arrives provider-prefixed, which no estimator registry knows.""" + estimator, calls = _build_estimator(energy_only) + + estimator.estimate(_request(model="openai/gpt-4o")) + + assert calls[0]["model_name"] == "gpt-4o" + + +def test_provider_names_are_translated(energy_only): + """litellm names a provider by the API it calls, the estimator by who trained the model.""" + estimator, calls = _build_estimator(energy_only) + + estimator.estimate(_request(custom_llm_provider="mistral")) + + assert calls[0]["provider"] == "mistralai" + + +def test_an_untranslatable_provider_yields_no_estimate(energy_only): + estimator, calls = _build_estimator(energy_only) + + assert estimator.estimate(_request(custom_llm_provider="some_new_provider")) is None + assert calls == [] + + +def test_no_estimate_without_a_latency_measurement(energy_only): + """Energy is modelled from how long generation ran, so an unmeasured request has no estimate.""" + estimator, calls = _build_estimator(energy_only) + + assert estimator.estimate(_request(response_time=None)) is None + assert calls == [] + + +def test_no_estimate_when_nothing_was_generated(energy_only): + estimator, calls = _build_estimator(energy_only) + + assert estimator.estimate(_request(completion_tokens=0)) is None + assert calls == [] diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 999adbdd935..d1979e30002 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -7397,3 +7397,91 @@ class TestAzurePTUSpilloverCost: finally: litellm.model_cost.pop(custom_model_id, None) self._unregister_models() + + +# ── response_impact ─────────────────────────────────────────────────────────── + + +class _StubImpactEstimator: + name = "stub" + + def __init__(self, impact=None, raises=False): + from litellm.impact_calculator import point_impact_value + from litellm.types.utils import ImpactInformation + + self.impact = impact if impact is not None else ImpactInformation( + energy=point_impact_value(unit="Wh", value=1.5), + boundary="B", + estimator="stub", + ) + self.raises = raises + self.calls = 0 + + def estimate(self, request): + self.calls += 1 + if self.raises: + raise ValueError("no model data") + return self.impact + + +@pytest.fixture +def impact_estimator(monkeypatch): + import litellm + from litellm.impact_calculator import register_impact_estimator, reset_default_estimator_cache + + monkeypatch.setattr(litellm, "track_impact", True) + estimator = _StubImpactEstimator() + register_impact_estimator(estimator) + yield estimator + register_impact_estimator(None) + reset_default_estimator_cache() + + +def _build_payload(logging_obj, **extra_kwargs): + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import ( + get_standard_logging_object_payload, + ) + + now = datetime.now() + return get_standard_logging_object_payload( + kwargs={"model": "gpt-4o", "messages": [], **extra_kwargs}, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + +def test_standard_logging_payload_carries_the_impact_estimate(logging_obj, impact_estimator): + """The estimate has to reach the payload, since that is the only thing loggers ever see.""" + payload = _build_payload(logging_obj) + + assert payload is not None + assert payload["response_impact"] == impact_estimator.impact + assert payload["response_impact_failure_debug_info"] is None + + +def test_standard_logging_payload_records_an_impact_failure_without_losing_the_payload( + logging_obj, impact_estimator +): + """A broken estimator must cost the impact figure only, never the whole log record.""" + impact_estimator.raises = True + + payload = _build_payload(logging_obj) + + assert payload is not None + assert payload["response_cost"] is not None + assert payload["response_impact"] is None + assert "no model data" in payload["response_impact_failure_debug_info"]["error_str"] + + +def test_a_cache_hit_is_not_attributed_inference_impact(logging_obj, impact_estimator): + """A cached response ran no inference, so attributing energy to it would double count.""" + payload = _build_payload(logging_obj, cache_hit=True) + + assert payload is not None + assert payload["response_impact"] is None + assert impact_estimator.calls == 0 diff --git a/tests/test_litellm/test_impact_calculator.py b/tests/test_litellm/test_impact_calculator.py new file mode 100644 index 00000000000..0d8e0b8c0c4 --- /dev/null +++ b/tests/test_litellm/test_impact_calculator.py @@ -0,0 +1,115 @@ +import pytest + +import litellm +from litellm.impact_calculator import ( + ImpactRequest, + calculate_impact, + current_estimator_name, + get_impact_estimator, + interval_impact_value, + point_impact_value, + register_impact_estimator, + reset_default_estimator_cache, +) +from litellm.types.utils import ImpactInformation + + +@pytest.fixture +def impact_request() -> ImpactRequest: + return ImpactRequest( + model="gpt-4o", + custom_llm_provider="openai", + completion_tokens=100, + response_time=2.0, + ) + + +@pytest.fixture(autouse=True) +def clear_registration(): + yield + register_impact_estimator(None) + reset_default_estimator_cache() + + +class _StubEstimator: + name = "stub" + + def __init__(self) -> None: + self.impact = ImpactInformation(estimator="stub") + self.calls = 0 + + def estimate(self, request: ImpactRequest) -> ImpactInformation | None: + self.calls += 1 + return self.impact + + +class _RaisingEstimator: + name = "raising" + + def estimate(self, request: ImpactRequest) -> ImpactInformation | None: + raise ValueError("no model data") + + +def test_tracking_off_never_calls_the_estimator(monkeypatch, impact_request): + """The flag gates estimation itself, so an opted-out deployment pays nothing for it.""" + estimator = _StubEstimator() + register_impact_estimator(estimator) + monkeypatch.setattr(litellm, "track_impact", False) + + assert calculate_impact(impact_request) is None + assert estimator.calls == 0 + + +def test_tracking_on_returns_the_registered_estimators_result(monkeypatch, impact_request): + estimator = _StubEstimator() + register_impact_estimator(estimator) + monkeypatch.setattr(litellm, "track_impact", True) + + result = calculate_impact(impact_request) + + assert estimator.calls == 1 + assert result is estimator.impact + + +def test_registration_takes_precedence_over_default_resolution(): + estimator = _StubEstimator() + register_impact_estimator(estimator) + + assert get_impact_estimator() is estimator + assert current_estimator_name() == "stub" + + +def test_clearing_registration_stops_routing_to_the_cleared_estimator(monkeypatch, impact_request): + estimator = _StubEstimator() + register_impact_estimator(estimator) + monkeypatch.setattr(litellm, "track_impact", True) + + register_impact_estimator(None) + calculate_impact(impact_request) + + assert estimator.calls == 0 + + +def test_estimator_failure_reaches_the_caller(monkeypatch, impact_request): + """calculate_impact does not swallow, so the logging layer can record the failure.""" + register_impact_estimator(_RaisingEstimator()) + monkeypatch.setattr(litellm, "track_impact", True) + + with pytest.raises(ValueError, match="no model data"): + calculate_impact(impact_request) + + +def test_a_point_value_carries_no_interval(): + value = point_impact_value(unit="Wh", value=1.5) + + assert value["value"] == 1.5 + assert value["min"] is None + assert value["max"] is None + + +def test_an_interval_value_reports_its_midpoint(): + """A consumer summing `value` must read a figure inside the modelled interval.""" + value = interval_impact_value(unit="Wh", minimum=2.0, maximum=6.0) + + assert value["min"] < value["value"] < value["max"] + assert value["value"] == 4.0 From f32ac5815fac30db25534662821d37215c59cd15 Mon Sep 17 00:00:00 2001 From: Antoine Tielbeke Date: Sun, 20 Sep 2026 11:40:13 +0200 Subject: [PATCH 2/2] fix(logging): estimate the routed deployment, not the model-group alias Greptile flagged that an unprefixed routed deployment reaches the payload builder as its model-group alias, which names a routing target rather than a model, so the estimator would size something that does not exist. The cost path already resolves this through base_model, with a comment saying lookup via the alias always misses; the impact path now uses the same identifier. Also drops the file header comment on impact_calculator, which described the module rather than explaining anything. --- litellm/impact_calculator.py | 2 -- litellm/litellm_core_utils/litellm_logging.py | 2 +- .../litellm_core_utils/test_litellm_logging.py | 17 +++++++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/litellm/impact_calculator.py b/litellm/impact_calculator.py index b25f0f7630c..6a56f7ce181 100644 --- a/litellm/impact_calculator.py +++ b/litellm/impact_calculator.py @@ -1,5 +1,3 @@ -# What is this? -## File for 'response_impact' estimation in Logging from dataclasses import dataclass from functools import lru_cache from typing import Final, Protocol diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7d4fcab8e21..04c539e8d5c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6380,7 +6380,7 @@ def get_standard_logging_object_payload( (None, None) if cache_hit is True else _response_impact_for_payload( - model=model_name, + model=base_model or model_name, custom_llm_provider=custom_llm_provider, completion_tokens=usage_dict.get("completion_tokens", 0), response_time=end_time_float - start_time_float, diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index d1979e30002..bbdfb5283bc 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -7416,9 +7416,11 @@ class _StubImpactEstimator: ) self.raises = raises self.calls = 0 + self.requests = [] def estimate(self, request): self.calls += 1 + self.requests.append(request) if self.raises: raise ValueError("no model data") return self.impact @@ -7485,3 +7487,18 @@ def test_a_cache_hit_is_not_attributed_inference_impact(logging_obj, impact_esti assert payload is not None assert payload["response_impact"] is None assert impact_estimator.calls == 0 + + +def test_the_routed_deployment_is_estimated_rather_than_the_group_alias(logging_obj, impact_estimator): + """A model group is a routing label with no hardware behind it, so estimating it would size the wrong model.""" + payload = _build_payload( + logging_obj, + model="my-model-group", + litellm_params={ + "metadata": {"deployment": "gpt-4o"}, + "proxy_server_request": {"body": {}}, + }, + ) + + assert payload is not None + assert impact_estimator.requests[0].model == "gpt-4o"