This commit is contained in:
antoinetielbeke 2026-09-21 04:54:36 -04:00 • committed by GitHub
commit 1c33c90434
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 693 additions and 0 deletions

View file

@ -334,6 +334,7 @@ disable_token_counter: bool = False
disable_add_transform_inline_image_block: bool = False
disable_add_user_agent_to_request_tags: bool = False
disable_anthropic_gemini_context_caching_transform: bool = False
track_impact: bool = False
enable_anthropic_prompt_caching: bool = os.getenv("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", "false").lower() == "true"
_anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL")
anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (

View file

@ -0,0 +1,88 @@
from dataclasses import dataclass
from functools import lru_cache
from typing import Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import ImpactInformation, ImpactValue
@dataclass(frozen=True, slots=True)
class ImpactRequest:
model: str
custom_llm_provider: str | None
completion_tokens: int
response_time: float | None
class ImpactEstimator(Protocol):
"""Produces an environmental impact estimate for a request."""
@property
def name(self) -> str: ...
def estimate(self, request: ImpactRequest) -> ImpactInformation | None: ...
def point_impact_value(unit: str, value: float) -> ImpactValue:
return ImpactValue(value=value, min=None, max=None, unit=unit)
def interval_impact_value(unit: str, minimum: float, maximum: float) -> ImpactValue:
"""``value`` is the midpoint, so a consumer summing it stays inside the modelled interval."""
return ImpactValue(value=(minimum + maximum) / 2, min=minimum, max=maximum, unit=unit)
@dataclass(slots=True)
class _EstimatorRegistry:
estimator: ImpactEstimator | None = None
_registry: Final = _EstimatorRegistry()
def register_impact_estimator(estimator: ImpactEstimator | None) -> None:
_registry.estimator = estimator
@lru_cache(maxsize=1)
def _resolve_default_estimator() -> ImpactEstimator | None:
from litellm.integrations.ecologits_impact import EcoLogitsImpactEstimator
try:
return EcoLogitsImpactEstimator()
except Exception as e: # noqa: BLE001 # a missing backend must never fail request logging
verbose_logger.warning(
"litellm.track_impact is on but no impact estimator is available, so no impact is recorded: %s",
str(e),
)
return None
def get_impact_estimator() -> ImpactEstimator | None:
if _registry.estimator is not None:
return _registry.estimator
return _resolve_default_estimator()
def current_estimator_name() -> str | None:
estimator: Final = get_impact_estimator()
return None if estimator is None else estimator.name
def reset_default_estimator_cache() -> None:
_resolve_default_estimator.cache_clear()
def calculate_impact(request: ImpactRequest) -> ImpactInformation | None:
"""
Estimate the environmental impact of a request.
Raises whatever the estimator raises, so the caller can record the failure.
"""
if litellm.track_impact is not True:
return None
estimator: Final = get_impact_estimator()
if estimator is None:
return None
return estimator.estimate(request)

View file

@ -0,0 +1,121 @@
from collections.abc import Sequence
from types import MappingProxyType
from typing import Final, Protocol
from litellm._logging import verbose_logger
from litellm.impact_calculator import ImpactRequest, interval_impact_value, point_impact_value
from litellm.types.utils import ImpactInformation, ImpactValue
ESTIMATOR_NAME: Final = "ecologits"
# ecologits totals cover the usage phase plus embodied manufacturing, which is boundary B.
# Source: https://ecologits.ai/latest/methodology/llm_inference/ read 2026-09-20
IMPACT_BOUNDARY: Final = "B"
# litellm names a provider by the API it speaks to; ecologits names one by who trained the
# model, and keys its registry on that. Providers absent here have no ecologits data.
PROVIDER_NAMES: Final = MappingProxyType(
{
"anthropic": "anthropic",
"azure": "openai",
"azure_ai": "openai",
"cohere": "cohere",
"cohere_chat": "cohere",
"gemini": "google_genai",
"huggingface": "huggingface_hub",
"mistral": "mistralai",
"openai": "openai",
"text-completion-openai": "openai",
"vertex_ai": "google_genai",
}
)
class _RangeLike(Protocol):
min: float
max: float
class _CriterionLike(Protocol):
value: float | _RangeLike
unit: str
class _ErrorLike(Protocol):
message: str
class _ImpactsLike(Protocol):
energy: _CriterionLike | None
gwp: _CriterionLike | None
adpe: _CriterionLike | None
pe: _CriterionLike | None
wcf: _CriterionLike | None
errors: Sequence[_ErrorLike]
def _to_impact_value(criterion: _CriterionLike | None) -> ImpactValue | None:
if criterion is None:
return None
raw: Final = criterion.value
if isinstance(raw, (int, float)):
return point_impact_value(unit=criterion.unit, value=float(raw))
return interval_impact_value(unit=criterion.unit, minimum=float(raw.min), maximum=float(raw.max))
def _bare_model_name(model: str) -> str:
"""Strip the routing prefix litellm carries, leaving the name ecologits registers."""
return model.rsplit("/", 1)[-1]
class EcoLogitsImpactEstimator:
"""
Estimates request impact with EcoLogits.
Constructing this requires the ``ecologits`` package. The import is deferred to
``__init__`` so that importing LiteLLM, and this module, stays free of that
dependency and only enabling the feature can fail.
"""
def __init__(self) -> None:
from ecologits.tracers.utils import llm_impacts
self._llm_impacts: Final = llm_impacts
@property
def name(self) -> str:
return ESTIMATOR_NAME
def estimate(self, request: ImpactRequest) -> ImpactInformation | None:
if request.custom_llm_provider is None or request.response_time is None:
return None
if request.completion_tokens <= 0:
return None
provider: Final = PROVIDER_NAMES.get(request.custom_llm_provider)
if provider is None:
return None
impacts: Final[_ImpactsLike | None] = self._llm_impacts(
provider=provider,
model_name=_bare_model_name(request.model),
output_token_count=request.completion_tokens,
request_latency=request.response_time,
)
if impacts is None or impacts.errors:
verbose_logger.debug(
"ecologits has no impact data for %s/%s: %s",
provider,
request.model,
tuple(error.message for error in impacts.errors) if impacts is not None else "no result",
)
return None
return ImpactInformation(
energy=_to_impact_value(impacts.energy),
gwp=_to_impact_value(impacts.gwp),
adpe=_to_impact_value(impacts.adpe),
pe=_to_impact_value(impacts.pe),
water=_to_impact_value(impacts.wcf),
boundary=IMPACT_BOUNDARY,
estimator=ESTIMATOR_NAME,
)

View file

@ -132,6 +132,7 @@ from litellm.types.utils import (
EmbeddingResponse,
GuardrailStatus,
ImageResponse,
ImpactInformation,
LiteLLMBatch,
LiteLLMLoggingBaseClass,
LiteLLMRealtimeStreamLoggingObject,
@ -143,6 +144,7 @@ from litellm.types.utils import (
StandardCallbackDynamicParams,
StandardLoggingAdditionalHeaders,
StandardLoggingHiddenParams,
StandardLoggingImpactFailureDebugInformation,
StandardLoggingMCPToolCall,
StandardLoggingMetadata,
StandardLoggingModelCostFailureDebugInformation,
@ -6217,6 +6219,37 @@ def _autorouter_savings_for_payload(
return None
def _response_impact_for_payload(
model: str,
custom_llm_provider: str | None,
completion_tokens: int,
response_time: float,
) -> tuple[ImpactInformation | None, StandardLoggingImpactFailureDebugInformation | None]:
"""Estimate impact for one request, recording any estimator failure the way a cost failure is."""
from litellm.impact_calculator import ImpactRequest, calculate_impact, current_estimator_name
try:
impact: Final = calculate_impact(
ImpactRequest(
model=model,
custom_llm_provider=custom_llm_provider,
completion_tokens=completion_tokens,
response_time=response_time,
)
)
return impact, None
except Exception as e: # noqa: BLE001 # an estimator must never fail request logging
debug_info: Final = StandardLoggingImpactFailureDebugInformation(
error_str=str(e),
traceback_str=_get_traceback_str_for_error(str(e)),
model=model,
custom_llm_provider=custom_llm_provider,
estimator=current_estimator_name(),
)
verbose_logger.debug("response_impact_failure_debug_information: %s", debug_info)
return None, debug_info
def get_standard_logging_object_payload(
kwargs: dict | None,
init_response_obj: Any | BaseModel | dict,
@ -6387,6 +6420,16 @@ def get_standard_logging_object_payload(
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
response_impact, response_impact_failure_debug_info = (
(None, None)
if cache_hit is True
else _response_impact_for_payload(
model=base_model or model_name,
custom_llm_provider=custom_llm_provider,
completion_tokens=usage_dict.get("completion_tokens", 0),
response_time=end_time_float - start_time_float,
)
)
captured_baseline: Final = logging_obj.baseline_observation
autorouter_savings: Final = (
None
@ -6444,6 +6487,8 @@ def get_standard_logging_object_payload(
cache_key=clean_hidden_params["cache_key"],
response_cost=response_cost,
cost_breakdown=request_cost_breakdown,
response_impact=response_impact,
response_impact_failure_debug_info=response_impact_failure_debug_info,
autorouter_savings=autorouter_savings,
autorouter_savings_estimate=(
{

View file

@ -3416,6 +3416,48 @@ class CostBreakdown(TypedDict, total=False):
margin_total_amount: float # Total margin added in USD (optional)
class ImpactValue(TypedDict, total=False):
"""One environmental impact figure."""
value: ReadOnly[float] # Always set; the midpoint when the estimator models an interval
min: ReadOnly[float | None] # Lower bound, set only when the figure is modelled as an interval
max: ReadOnly[float | None]
unit: ReadOnly[str]
class ImpactInformation(TypedDict, total=False):
"""
Estimated environmental impact of a request.
Figures are only comparable when their ``boundary`` matches, and the boundary cannot
be recovered after the fact, so it is recorded per request: ``A`` is operational
electricity only, ``B`` adds allocated embodied hardware, ``C`` adds amortized
training. See https://arxiv.org/pdf/2606.10660 read 2026-09-20
"""
energy: ReadOnly[ImpactValue | None]
gwp: ReadOnly[ImpactValue | None]
adpe: ReadOnly[ImpactValue | None]
pe: ReadOnly[ImpactValue | None]
water: ReadOnly[ImpactValue | None]
boundary: ReadOnly[Literal["A", "B", "C"] | None]
estimator: ReadOnly[str | None]
class StandardLoggingImpactFailureDebugInformation(TypedDict, total=False):
"""
Debug information, if impact estimation fails.
Avoid logging sensitive information like response or optional params
"""
error_str: Required[ReadOnly[str]]
traceback_str: Required[ReadOnly[str]]
model: ReadOnly[str | None]
custom_llm_provider: ReadOnly[str | None]
estimator: ReadOnly[str | None]
class StandardLoggingPayloadStatusFields(TypedDict, total=False):
"""Status fields for easy filtering and analytics"""
@ -3496,6 +3538,8 @@ class StandardLoggingPayload(ClassifierAudit):
hidden_params: StandardLoggingHiddenParams
guardrail_information: list[StandardLoggingGuardrailInformation] | None
standard_built_in_tools_params: StandardBuiltInToolsParams | None
response_impact: NotRequired[ReadOnly[ImpactInformation | None]]
response_impact_failure_debug_info: NotRequired[ReadOnly[StandardLoggingImpactFailureDebugInformation | None]]
from collections.abc import AsyncIterator, Iterator

View file

@ -0,0 +1,174 @@
import sys
from dataclasses import dataclass, field
from types import ModuleType
from unittest.mock import patch
import pytest
from litellm.impact_calculator import ImpactRequest
@dataclass
class _Range:
min: float
max: float
@dataclass
class _Criterion:
value: float | _Range
unit: str
@dataclass
class _Error:
message: str
@dataclass
class _Impacts:
energy: _Criterion | None = None
gwp: _Criterion | None = None
adpe: _Criterion | None = None
pe: _Criterion | None = None
wcf: _Criterion | None = None
errors: list = field(default_factory=list)
def _build_estimator(impacts):
calls = []
def llm_impacts(provider, model_name, output_token_count, request_latency, electricity_mix_zone=None):
calls.append(
{
"provider": provider,
"model_name": model_name,
"output_token_count": output_token_count,
"request_latency": request_latency,
}
)
return impacts
utils = ModuleType("ecologits.tracers.utils")
utils.llm_impacts = llm_impacts
modules = {
"ecologits": ModuleType("ecologits"),
"ecologits.tracers": ModuleType("ecologits.tracers"),
"ecologits.tracers.utils": utils,
}
with patch.dict(sys.modules, modules):
from litellm.integrations.ecologits_impact import EcoLogitsImpactEstimator
return EcoLogitsImpactEstimator(), calls
def _request(**overrides) -> ImpactRequest:
defaults = {
"model": "gpt-4o",
"custom_llm_provider": "openai",
"completion_tokens": 250,
"response_time": 3.5,
}
return ImpactRequest(**{**defaults, **overrides})
@pytest.fixture
def energy_only() -> _Impacts:
return _Impacts(energy=_Criterion(value=1.0, unit="Wh"))
def test_a_modelled_interval_survives_the_conversion():
"""An interval is the estimator saying it does not know precisely, so it must not be flattened."""
estimator, _ = _build_estimator(_Impacts(energy=_Criterion(value=_Range(min=2.0, max=6.0), unit="Wh")))
energy = estimator.estimate(_request())["energy"]
assert energy["min"] == 2.0
assert energy["max"] == 6.0
assert energy["min"] < energy["value"] < energy["max"]
def test_a_point_estimate_stays_a_point():
estimator, _ = _build_estimator(_Impacts(energy=_Criterion(value=1.25, unit="Wh")))
energy = estimator.estimate(_request())["energy"]
assert energy["value"] == 1.25
assert energy["min"] is None
assert energy["max"] is None
def test_water_comes_from_the_same_total_as_the_other_figures():
"""Water read from a narrower phase than the rest would not match the boundary they all claim."""
estimator, _ = _build_estimator(_Impacts(wcf=_Criterion(value=0.4, unit="L")))
result = estimator.estimate(_request())
assert result["water"]["value"] == 0.4
assert result["water"]["unit"] == "L"
def test_the_boundary_and_estimator_are_recorded(energy_only):
estimator, _ = _build_estimator(energy_only)
result = estimator.estimate(_request())
assert result["boundary"] == "B"
assert result["estimator"] == "ecologits"
def test_a_result_carrying_errors_is_not_an_estimate():
"""ecologits reports an unknown model as an error on a filled-in result, not as a missing one."""
estimator, _ = _build_estimator(_Impacts(errors=[_Error(message="Could not find model")]))
assert estimator.estimate(_request()) is None
def test_generated_tokens_and_latency_drive_the_estimate(energy_only):
estimator, calls = _build_estimator(energy_only)
request = _request()
estimator.estimate(request)
assert calls[0]["output_token_count"] == request.completion_tokens
assert calls[0]["request_latency"] == request.response_time
def test_the_routing_prefix_is_stripped_from_the_model_name(energy_only):
"""Under the router a model arrives provider-prefixed, which no estimator registry knows."""
estimator, calls = _build_estimator(energy_only)
estimator.estimate(_request(model="openai/gpt-4o"))
assert calls[0]["model_name"] == "gpt-4o"
def test_provider_names_are_translated(energy_only):
"""litellm names a provider by the API it calls, the estimator by who trained the model."""
estimator, calls = _build_estimator(energy_only)
estimator.estimate(_request(custom_llm_provider="mistral"))
assert calls[0]["provider"] == "mistralai"
def test_an_untranslatable_provider_yields_no_estimate(energy_only):
estimator, calls = _build_estimator(energy_only)
assert estimator.estimate(_request(custom_llm_provider="some_new_provider")) is None
assert calls == []
def test_no_estimate_without_a_latency_measurement(energy_only):
"""Energy is modelled from how long generation ran, so an unmeasured request has no estimate."""
estimator, calls = _build_estimator(energy_only)
assert estimator.estimate(_request(response_time=None)) is None
assert calls == []
def test_no_estimate_when_nothing_was_generated(energy_only):
estimator, calls = _build_estimator(energy_only)
assert estimator.estimate(_request(completion_tokens=0)) is None
assert calls == []

View file

@ -7415,3 +7415,108 @@ class TestAzurePTUSpilloverCost:
finally:
litellm.model_cost.pop(custom_model_id, None)
self._unregister_models()
# ── response_impact ───────────────────────────────────────────────────────────
class _StubImpactEstimator:
name = "stub"
def __init__(self, impact=None, raises=False):
from litellm.impact_calculator import point_impact_value
from litellm.types.utils import ImpactInformation
self.impact = impact if impact is not None else ImpactInformation(
energy=point_impact_value(unit="Wh", value=1.5),
boundary="B",
estimator="stub",
)
self.raises = raises
self.calls = 0
self.requests = []
def estimate(self, request):
self.calls += 1
self.requests.append(request)
if self.raises:
raise ValueError("no model data")
return self.impact
@pytest.fixture
def impact_estimator(monkeypatch):
import litellm
from litellm.impact_calculator import register_impact_estimator, reset_default_estimator_cache
monkeypatch.setattr(litellm, "track_impact", True)
estimator = _StubImpactEstimator()
register_impact_estimator(estimator)
yield estimator
register_impact_estimator(None)
reset_default_estimator_cache()
def _build_payload(logging_obj, **extra_kwargs):
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload,
)
now = datetime.now()
return get_standard_logging_object_payload(
kwargs={"model": "gpt-4o", "messages": [], **extra_kwargs},
init_response_obj={},
start_time=now,
end_time=now,
logging_obj=logging_obj,
status="success",
)
def test_standard_logging_payload_carries_the_impact_estimate(logging_obj, impact_estimator):
"""The estimate has to reach the payload, since that is the only thing loggers ever see."""
payload = _build_payload(logging_obj)
assert payload is not None
assert payload["response_impact"] == impact_estimator.impact
assert payload["response_impact_failure_debug_info"] is None
def test_standard_logging_payload_records_an_impact_failure_without_losing_the_payload(
logging_obj, impact_estimator
):
"""A broken estimator must cost the impact figure only, never the whole log record."""
impact_estimator.raises = True
payload = _build_payload(logging_obj)
assert payload is not None
assert payload["response_cost"] is not None
assert payload["response_impact"] is None
assert "no model data" in payload["response_impact_failure_debug_info"]["error_str"]
def test_a_cache_hit_is_not_attributed_inference_impact(logging_obj, impact_estimator):
"""A cached response ran no inference, so attributing energy to it would double count."""
payload = _build_payload(logging_obj, cache_hit=True)
assert payload is not None
assert payload["response_impact"] is None
assert impact_estimator.calls == 0
def test_the_routed_deployment_is_estimated_rather_than_the_group_alias(logging_obj, impact_estimator):
"""A model group is a routing label with no hardware behind it, so estimating it would size the wrong model."""
payload = _build_payload(
logging_obj,
model="my-model-group",
litellm_params={
"metadata": {"deployment": "gpt-4o"},
"proxy_server_request": {"body": {}},
},
)
assert payload is not None
assert impact_estimator.requests[0].model == "gpt-4o"

View file

@ -0,0 +1,115 @@
import pytest
import litellm
from litellm.impact_calculator import (
ImpactRequest,
calculate_impact,
current_estimator_name,
get_impact_estimator,
interval_impact_value,
point_impact_value,
register_impact_estimator,
reset_default_estimator_cache,
)
from litellm.types.utils import ImpactInformation
@pytest.fixture
def impact_request() -> ImpactRequest:
return ImpactRequest(
model="gpt-4o",
custom_llm_provider="openai",
completion_tokens=100,
response_time=2.0,
)
@pytest.fixture(autouse=True)
def clear_registration():
yield
register_impact_estimator(None)
reset_default_estimator_cache()
class _StubEstimator:
name = "stub"
def __init__(self) -> None:
self.impact = ImpactInformation(estimator="stub")
self.calls = 0
def estimate(self, request: ImpactRequest) -> ImpactInformation | None:
self.calls += 1
return self.impact
class _RaisingEstimator:
name = "raising"
def estimate(self, request: ImpactRequest) -> ImpactInformation | None:
raise ValueError("no model data")
def test_tracking_off_never_calls_the_estimator(monkeypatch, impact_request):
"""The flag gates estimation itself, so an opted-out deployment pays nothing for it."""
estimator = _StubEstimator()
register_impact_estimator(estimator)
monkeypatch.setattr(litellm, "track_impact", False)
assert calculate_impact(impact_request) is None
assert estimator.calls == 0
def test_tracking_on_returns_the_registered_estimators_result(monkeypatch, impact_request):
estimator = _StubEstimator()
register_impact_estimator(estimator)
monkeypatch.setattr(litellm, "track_impact", True)
result = calculate_impact(impact_request)
assert estimator.calls == 1
assert result is estimator.impact
def test_registration_takes_precedence_over_default_resolution():
estimator = _StubEstimator()
register_impact_estimator(estimator)
assert get_impact_estimator() is estimator
assert current_estimator_name() == "stub"
def test_clearing_registration_stops_routing_to_the_cleared_estimator(monkeypatch, impact_request):
estimator = _StubEstimator()
register_impact_estimator(estimator)
monkeypatch.setattr(litellm, "track_impact", True)
register_impact_estimator(None)
calculate_impact(impact_request)
assert estimator.calls == 0
def test_estimator_failure_reaches_the_caller(monkeypatch, impact_request):
"""calculate_impact does not swallow, so the logging layer can record the failure."""
register_impact_estimator(_RaisingEstimator())
monkeypatch.setattr(litellm, "track_impact", True)
with pytest.raises(ValueError, match="no model data"):
calculate_impact(impact_request)
def test_a_point_value_carries_no_interval():
value = point_impact_value(unit="Wh", value=1.5)
assert value["value"] == 1.5
assert value["min"] is None
assert value["max"] is None
def test_an_interval_value_reports_its_midpoint():
"""A consumer summing `value` must read a figure inside the modelled interval."""
value = interval_impact_value(unit="Wh", minimum=2.0, maximum=6.0)
assert value["min"] < value["value"] < value["max"]
assert value["value"] == 4.0