mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 24212e7cca into 1cac8bd9ab
This commit is contained in:
commit
1c33c90434
8 changed files with 693 additions and 0 deletions
|
|
@ -334,6 +334,7 @@ disable_token_counter: bool = False
|
|||
disable_add_transform_inline_image_block: bool = False
|
||||
disable_add_user_agent_to_request_tags: bool = False
|
||||
disable_anthropic_gemini_context_caching_transform: bool = False
|
||||
track_impact: bool = False
|
||||
enable_anthropic_prompt_caching: bool = os.getenv("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", "false").lower() == "true"
|
||||
_anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL")
|
||||
anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
|
||||
|
|
|
|||
88
litellm/impact_calculator.py
Normal file
88
litellm/impact_calculator.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import Final, Protocol
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.utils import ImpactInformation, ImpactValue
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ImpactRequest:
|
||||
model: str
|
||||
custom_llm_provider: str | None
|
||||
completion_tokens: int
|
||||
response_time: float | None
|
||||
|
||||
|
||||
class ImpactEstimator(Protocol):
|
||||
"""Produces an environmental impact estimate for a request."""
|
||||
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
|
||||
def estimate(self, request: ImpactRequest) -> ImpactInformation | None: ...
|
||||
|
||||
|
||||
def point_impact_value(unit: str, value: float) -> ImpactValue:
|
||||
return ImpactValue(value=value, min=None, max=None, unit=unit)
|
||||
|
||||
|
||||
def interval_impact_value(unit: str, minimum: float, maximum: float) -> ImpactValue:
|
||||
"""``value`` is the midpoint, so a consumer summing it stays inside the modelled interval."""
|
||||
return ImpactValue(value=(minimum + maximum) / 2, min=minimum, max=maximum, unit=unit)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _EstimatorRegistry:
|
||||
estimator: ImpactEstimator | None = None
|
||||
|
||||
|
||||
_registry: Final = _EstimatorRegistry()
|
||||
|
||||
|
||||
def register_impact_estimator(estimator: ImpactEstimator | None) -> None:
|
||||
_registry.estimator = estimator
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _resolve_default_estimator() -> ImpactEstimator | None:
|
||||
from litellm.integrations.ecologits_impact import EcoLogitsImpactEstimator
|
||||
|
||||
try:
|
||||
return EcoLogitsImpactEstimator()
|
||||
except Exception as e: # noqa: BLE001 # a missing backend must never fail request logging
|
||||
verbose_logger.warning(
|
||||
"litellm.track_impact is on but no impact estimator is available, so no impact is recorded: %s",
|
||||
str(e),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def get_impact_estimator() -> ImpactEstimator | None:
|
||||
if _registry.estimator is not None:
|
||||
return _registry.estimator
|
||||
return _resolve_default_estimator()
|
||||
|
||||
|
||||
def current_estimator_name() -> str | None:
|
||||
estimator: Final = get_impact_estimator()
|
||||
return None if estimator is None else estimator.name
|
||||
|
||||
|
||||
def reset_default_estimator_cache() -> None:
|
||||
_resolve_default_estimator.cache_clear()
|
||||
|
||||
|
||||
def calculate_impact(request: ImpactRequest) -> ImpactInformation | None:
|
||||
"""
|
||||
Estimate the environmental impact of a request.
|
||||
|
||||
Raises whatever the estimator raises, so the caller can record the failure.
|
||||
"""
|
||||
if litellm.track_impact is not True:
|
||||
return None
|
||||
estimator: Final = get_impact_estimator()
|
||||
if estimator is None:
|
||||
return None
|
||||
return estimator.estimate(request)
|
||||
121
litellm/integrations/ecologits_impact.py
Normal file
121
litellm/integrations/ecologits_impact.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
from collections.abc import Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.impact_calculator import ImpactRequest, interval_impact_value, point_impact_value
|
||||
from litellm.types.utils import ImpactInformation, ImpactValue
|
||||
|
||||
ESTIMATOR_NAME: Final = "ecologits"
|
||||
|
||||
# ecologits totals cover the usage phase plus embodied manufacturing, which is boundary B.
|
||||
# Source: https://ecologits.ai/latest/methodology/llm_inference/ read 2026-09-20
|
||||
IMPACT_BOUNDARY: Final = "B"
|
||||
|
||||
# litellm names a provider by the API it speaks to; ecologits names one by who trained the
|
||||
# model, and keys its registry on that. Providers absent here have no ecologits data.
|
||||
PROVIDER_NAMES: Final = MappingProxyType(
|
||||
{
|
||||
"anthropic": "anthropic",
|
||||
"azure": "openai",
|
||||
"azure_ai": "openai",
|
||||
"cohere": "cohere",
|
||||
"cohere_chat": "cohere",
|
||||
"gemini": "google_genai",
|
||||
"huggingface": "huggingface_hub",
|
||||
"mistral": "mistralai",
|
||||
"openai": "openai",
|
||||
"text-completion-openai": "openai",
|
||||
"vertex_ai": "google_genai",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _RangeLike(Protocol):
|
||||
min: float
|
||||
max: float
|
||||
|
||||
|
||||
class _CriterionLike(Protocol):
|
||||
value: float | _RangeLike
|
||||
unit: str
|
||||
|
||||
|
||||
class _ErrorLike(Protocol):
|
||||
message: str
|
||||
|
||||
|
||||
class _ImpactsLike(Protocol):
|
||||
energy: _CriterionLike | None
|
||||
gwp: _CriterionLike | None
|
||||
adpe: _CriterionLike | None
|
||||
pe: _CriterionLike | None
|
||||
wcf: _CriterionLike | None
|
||||
errors: Sequence[_ErrorLike]
|
||||
|
||||
|
||||
def _to_impact_value(criterion: _CriterionLike | None) -> ImpactValue | None:
|
||||
if criterion is None:
|
||||
return None
|
||||
raw: Final = criterion.value
|
||||
if isinstance(raw, (int, float)):
|
||||
return point_impact_value(unit=criterion.unit, value=float(raw))
|
||||
return interval_impact_value(unit=criterion.unit, minimum=float(raw.min), maximum=float(raw.max))
|
||||
|
||||
|
||||
def _bare_model_name(model: str) -> str:
|
||||
"""Strip the routing prefix litellm carries, leaving the name ecologits registers."""
|
||||
return model.rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
class EcoLogitsImpactEstimator:
|
||||
"""
|
||||
Estimates request impact with EcoLogits.
|
||||
|
||||
Constructing this requires the ``ecologits`` package. The import is deferred to
|
||||
``__init__`` so that importing LiteLLM, and this module, stays free of that
|
||||
dependency and only enabling the feature can fail.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
from ecologits.tracers.utils import llm_impacts
|
||||
|
||||
self._llm_impacts: Final = llm_impacts
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return ESTIMATOR_NAME
|
||||
|
||||
def estimate(self, request: ImpactRequest) -> ImpactInformation | None:
|
||||
if request.custom_llm_provider is None or request.response_time is None:
|
||||
return None
|
||||
if request.completion_tokens <= 0:
|
||||
return None
|
||||
provider: Final = PROVIDER_NAMES.get(request.custom_llm_provider)
|
||||
if provider is None:
|
||||
return None
|
||||
|
||||
impacts: Final[_ImpactsLike | None] = self._llm_impacts(
|
||||
provider=provider,
|
||||
model_name=_bare_model_name(request.model),
|
||||
output_token_count=request.completion_tokens,
|
||||
request_latency=request.response_time,
|
||||
)
|
||||
if impacts is None or impacts.errors:
|
||||
verbose_logger.debug(
|
||||
"ecologits has no impact data for %s/%s: %s",
|
||||
provider,
|
||||
request.model,
|
||||
tuple(error.message for error in impacts.errors) if impacts is not None else "no result",
|
||||
)
|
||||
return None
|
||||
|
||||
return ImpactInformation(
|
||||
energy=_to_impact_value(impacts.energy),
|
||||
gwp=_to_impact_value(impacts.gwp),
|
||||
adpe=_to_impact_value(impacts.adpe),
|
||||
pe=_to_impact_value(impacts.pe),
|
||||
water=_to_impact_value(impacts.wcf),
|
||||
boundary=IMPACT_BOUNDARY,
|
||||
estimator=ESTIMATOR_NAME,
|
||||
)
|
||||
|
|
@ -132,6 +132,7 @@ from litellm.types.utils import (
|
|||
EmbeddingResponse,
|
||||
GuardrailStatus,
|
||||
ImageResponse,
|
||||
ImpactInformation,
|
||||
LiteLLMBatch,
|
||||
LiteLLMLoggingBaseClass,
|
||||
LiteLLMRealtimeStreamLoggingObject,
|
||||
|
|
@ -143,6 +144,7 @@ from litellm.types.utils import (
|
|||
StandardCallbackDynamicParams,
|
||||
StandardLoggingAdditionalHeaders,
|
||||
StandardLoggingHiddenParams,
|
||||
StandardLoggingImpactFailureDebugInformation,
|
||||
StandardLoggingMCPToolCall,
|
||||
StandardLoggingMetadata,
|
||||
StandardLoggingModelCostFailureDebugInformation,
|
||||
|
|
@ -6217,6 +6219,37 @@ def _autorouter_savings_for_payload(
|
|||
return None
|
||||
|
||||
|
||||
def _response_impact_for_payload(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
completion_tokens: int,
|
||||
response_time: float,
|
||||
) -> tuple[ImpactInformation | None, StandardLoggingImpactFailureDebugInformation | None]:
|
||||
"""Estimate impact for one request, recording any estimator failure the way a cost failure is."""
|
||||
from litellm.impact_calculator import ImpactRequest, calculate_impact, current_estimator_name
|
||||
|
||||
try:
|
||||
impact: Final = calculate_impact(
|
||||
ImpactRequest(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
completion_tokens=completion_tokens,
|
||||
response_time=response_time,
|
||||
)
|
||||
)
|
||||
return impact, None
|
||||
except Exception as e: # noqa: BLE001 # an estimator must never fail request logging
|
||||
debug_info: Final = StandardLoggingImpactFailureDebugInformation(
|
||||
error_str=str(e),
|
||||
traceback_str=_get_traceback_str_for_error(str(e)),
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
estimator=current_estimator_name(),
|
||||
)
|
||||
verbose_logger.debug("response_impact_failure_debug_information: %s", debug_info)
|
||||
return None, debug_info
|
||||
|
||||
|
||||
def get_standard_logging_object_payload(
|
||||
kwargs: dict | None,
|
||||
init_response_obj: Any | BaseModel | dict,
|
||||
|
|
@ -6387,6 +6420,16 @@ def get_standard_logging_object_payload(
|
|||
model_name = response_model_name
|
||||
|
||||
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
|
||||
response_impact, response_impact_failure_debug_info = (
|
||||
(None, None)
|
||||
if cache_hit is True
|
||||
else _response_impact_for_payload(
|
||||
model=base_model or model_name,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
completion_tokens=usage_dict.get("completion_tokens", 0),
|
||||
response_time=end_time_float - start_time_float,
|
||||
)
|
||||
)
|
||||
captured_baseline: Final = logging_obj.baseline_observation
|
||||
autorouter_savings: Final = (
|
||||
None
|
||||
|
|
@ -6444,6 +6487,8 @@ def get_standard_logging_object_payload(
|
|||
cache_key=clean_hidden_params["cache_key"],
|
||||
response_cost=response_cost,
|
||||
cost_breakdown=request_cost_breakdown,
|
||||
response_impact=response_impact,
|
||||
response_impact_failure_debug_info=response_impact_failure_debug_info,
|
||||
autorouter_savings=autorouter_savings,
|
||||
autorouter_savings_estimate=(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -3416,6 +3416,48 @@ class CostBreakdown(TypedDict, total=False):
|
|||
margin_total_amount: float # Total margin added in USD (optional)
|
||||
|
||||
|
||||
class ImpactValue(TypedDict, total=False):
|
||||
"""One environmental impact figure."""
|
||||
|
||||
value: ReadOnly[float] # Always set; the midpoint when the estimator models an interval
|
||||
min: ReadOnly[float | None] # Lower bound, set only when the figure is modelled as an interval
|
||||
max: ReadOnly[float | None]
|
||||
unit: ReadOnly[str]
|
||||
|
||||
|
||||
class ImpactInformation(TypedDict, total=False):
|
||||
"""
|
||||
Estimated environmental impact of a request.
|
||||
|
||||
Figures are only comparable when their ``boundary`` matches, and the boundary cannot
|
||||
be recovered after the fact, so it is recorded per request: ``A`` is operational
|
||||
electricity only, ``B`` adds allocated embodied hardware, ``C`` adds amortized
|
||||
training. See https://arxiv.org/pdf/2606.10660 read 2026-09-20
|
||||
"""
|
||||
|
||||
energy: ReadOnly[ImpactValue | None]
|
||||
gwp: ReadOnly[ImpactValue | None]
|
||||
adpe: ReadOnly[ImpactValue | None]
|
||||
pe: ReadOnly[ImpactValue | None]
|
||||
water: ReadOnly[ImpactValue | None]
|
||||
boundary: ReadOnly[Literal["A", "B", "C"] | None]
|
||||
estimator: ReadOnly[str | None]
|
||||
|
||||
|
||||
class StandardLoggingImpactFailureDebugInformation(TypedDict, total=False):
|
||||
"""
|
||||
Debug information, if impact estimation fails.
|
||||
|
||||
Avoid logging sensitive information like response or optional params
|
||||
"""
|
||||
|
||||
error_str: Required[ReadOnly[str]]
|
||||
traceback_str: Required[ReadOnly[str]]
|
||||
model: ReadOnly[str | None]
|
||||
custom_llm_provider: ReadOnly[str | None]
|
||||
estimator: ReadOnly[str | None]
|
||||
|
||||
|
||||
class StandardLoggingPayloadStatusFields(TypedDict, total=False):
|
||||
"""Status fields for easy filtering and analytics"""
|
||||
|
||||
|
|
@ -3496,6 +3538,8 @@ class StandardLoggingPayload(ClassifierAudit):
|
|||
hidden_params: StandardLoggingHiddenParams
|
||||
guardrail_information: list[StandardLoggingGuardrailInformation] | None
|
||||
standard_built_in_tools_params: StandardBuiltInToolsParams | None
|
||||
response_impact: NotRequired[ReadOnly[ImpactInformation | None]]
|
||||
response_impact_failure_debug_info: NotRequired[ReadOnly[StandardLoggingImpactFailureDebugInformation | None]]
|
||||
|
||||
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
|
|
|
|||
174
tests/test_litellm/integrations/test_ecologits_impact.py
Normal file
174
tests/test_litellm/integrations/test_ecologits_impact.py
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from types import ModuleType
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.impact_calculator import ImpactRequest
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Range:
|
||||
min: float
|
||||
max: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Criterion:
|
||||
value: float | _Range
|
||||
unit: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Error:
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Impacts:
|
||||
energy: _Criterion | None = None
|
||||
gwp: _Criterion | None = None
|
||||
adpe: _Criterion | None = None
|
||||
pe: _Criterion | None = None
|
||||
wcf: _Criterion | None = None
|
||||
errors: list = field(default_factory=list)
|
||||
|
||||
|
||||
def _build_estimator(impacts):
|
||||
calls = []
|
||||
|
||||
def llm_impacts(provider, model_name, output_token_count, request_latency, electricity_mix_zone=None):
|
||||
calls.append(
|
||||
{
|
||||
"provider": provider,
|
||||
"model_name": model_name,
|
||||
"output_token_count": output_token_count,
|
||||
"request_latency": request_latency,
|
||||
}
|
||||
)
|
||||
return impacts
|
||||
|
||||
utils = ModuleType("ecologits.tracers.utils")
|
||||
utils.llm_impacts = llm_impacts
|
||||
modules = {
|
||||
"ecologits": ModuleType("ecologits"),
|
||||
"ecologits.tracers": ModuleType("ecologits.tracers"),
|
||||
"ecologits.tracers.utils": utils,
|
||||
}
|
||||
with patch.dict(sys.modules, modules):
|
||||
from litellm.integrations.ecologits_impact import EcoLogitsImpactEstimator
|
||||
|
||||
return EcoLogitsImpactEstimator(), calls
|
||||
|
||||
|
||||
def _request(**overrides) -> ImpactRequest:
|
||||
defaults = {
|
||||
"model": "gpt-4o",
|
||||
"custom_llm_provider": "openai",
|
||||
"completion_tokens": 250,
|
||||
"response_time": 3.5,
|
||||
}
|
||||
return ImpactRequest(**{**defaults, **overrides})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def energy_only() -> _Impacts:
|
||||
return _Impacts(energy=_Criterion(value=1.0, unit="Wh"))
|
||||
|
||||
|
||||
def test_a_modelled_interval_survives_the_conversion():
|
||||
"""An interval is the estimator saying it does not know precisely, so it must not be flattened."""
|
||||
estimator, _ = _build_estimator(_Impacts(energy=_Criterion(value=_Range(min=2.0, max=6.0), unit="Wh")))
|
||||
|
||||
energy = estimator.estimate(_request())["energy"]
|
||||
|
||||
assert energy["min"] == 2.0
|
||||
assert energy["max"] == 6.0
|
||||
assert energy["min"] < energy["value"] < energy["max"]
|
||||
|
||||
|
||||
def test_a_point_estimate_stays_a_point():
|
||||
estimator, _ = _build_estimator(_Impacts(energy=_Criterion(value=1.25, unit="Wh")))
|
||||
|
||||
energy = estimator.estimate(_request())["energy"]
|
||||
|
||||
assert energy["value"] == 1.25
|
||||
assert energy["min"] is None
|
||||
assert energy["max"] is None
|
||||
|
||||
|
||||
def test_water_comes_from_the_same_total_as_the_other_figures():
|
||||
"""Water read from a narrower phase than the rest would not match the boundary they all claim."""
|
||||
estimator, _ = _build_estimator(_Impacts(wcf=_Criterion(value=0.4, unit="L")))
|
||||
|
||||
result = estimator.estimate(_request())
|
||||
|
||||
assert result["water"]["value"] == 0.4
|
||||
assert result["water"]["unit"] == "L"
|
||||
|
||||
|
||||
def test_the_boundary_and_estimator_are_recorded(energy_only):
|
||||
estimator, _ = _build_estimator(energy_only)
|
||||
|
||||
result = estimator.estimate(_request())
|
||||
|
||||
assert result["boundary"] == "B"
|
||||
assert result["estimator"] == "ecologits"
|
||||
|
||||
|
||||
def test_a_result_carrying_errors_is_not_an_estimate():
|
||||
"""ecologits reports an unknown model as an error on a filled-in result, not as a missing one."""
|
||||
estimator, _ = _build_estimator(_Impacts(errors=[_Error(message="Could not find model")]))
|
||||
|
||||
assert estimator.estimate(_request()) is None
|
||||
|
||||
|
||||
def test_generated_tokens_and_latency_drive_the_estimate(energy_only):
|
||||
estimator, calls = _build_estimator(energy_only)
|
||||
request = _request()
|
||||
|
||||
estimator.estimate(request)
|
||||
|
||||
assert calls[0]["output_token_count"] == request.completion_tokens
|
||||
assert calls[0]["request_latency"] == request.response_time
|
||||
|
||||
|
||||
def test_the_routing_prefix_is_stripped_from_the_model_name(energy_only):
|
||||
"""Under the router a model arrives provider-prefixed, which no estimator registry knows."""
|
||||
estimator, calls = _build_estimator(energy_only)
|
||||
|
||||
estimator.estimate(_request(model="openai/gpt-4o"))
|
||||
|
||||
assert calls[0]["model_name"] == "gpt-4o"
|
||||
|
||||
|
||||
def test_provider_names_are_translated(energy_only):
|
||||
"""litellm names a provider by the API it calls, the estimator by who trained the model."""
|
||||
estimator, calls = _build_estimator(energy_only)
|
||||
|
||||
estimator.estimate(_request(custom_llm_provider="mistral"))
|
||||
|
||||
assert calls[0]["provider"] == "mistralai"
|
||||
|
||||
|
||||
def test_an_untranslatable_provider_yields_no_estimate(energy_only):
|
||||
estimator, calls = _build_estimator(energy_only)
|
||||
|
||||
assert estimator.estimate(_request(custom_llm_provider="some_new_provider")) is None
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_no_estimate_without_a_latency_measurement(energy_only):
|
||||
"""Energy is modelled from how long generation ran, so an unmeasured request has no estimate."""
|
||||
estimator, calls = _build_estimator(energy_only)
|
||||
|
||||
assert estimator.estimate(_request(response_time=None)) is None
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_no_estimate_when_nothing_was_generated(energy_only):
|
||||
estimator, calls = _build_estimator(energy_only)
|
||||
|
||||
assert estimator.estimate(_request(completion_tokens=0)) is None
|
||||
assert calls == []
|
||||
|
|
@ -7415,3 +7415,108 @@ class TestAzurePTUSpilloverCost:
|
|||
finally:
|
||||
litellm.model_cost.pop(custom_model_id, None)
|
||||
self._unregister_models()
|
||||
|
||||
|
||||
# ── response_impact ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _StubImpactEstimator:
|
||||
name = "stub"
|
||||
|
||||
def __init__(self, impact=None, raises=False):
|
||||
from litellm.impact_calculator import point_impact_value
|
||||
from litellm.types.utils import ImpactInformation
|
||||
|
||||
self.impact = impact if impact is not None else ImpactInformation(
|
||||
energy=point_impact_value(unit="Wh", value=1.5),
|
||||
boundary="B",
|
||||
estimator="stub",
|
||||
)
|
||||
self.raises = raises
|
||||
self.calls = 0
|
||||
self.requests = []
|
||||
|
||||
def estimate(self, request):
|
||||
self.calls += 1
|
||||
self.requests.append(request)
|
||||
if self.raises:
|
||||
raise ValueError("no model data")
|
||||
return self.impact
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def impact_estimator(monkeypatch):
|
||||
import litellm
|
||||
from litellm.impact_calculator import register_impact_estimator, reset_default_estimator_cache
|
||||
|
||||
monkeypatch.setattr(litellm, "track_impact", True)
|
||||
estimator = _StubImpactEstimator()
|
||||
register_impact_estimator(estimator)
|
||||
yield estimator
|
||||
register_impact_estimator(None)
|
||||
reset_default_estimator_cache()
|
||||
|
||||
|
||||
def _build_payload(logging_obj, **extra_kwargs):
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
get_standard_logging_object_payload,
|
||||
)
|
||||
|
||||
now = datetime.now()
|
||||
return get_standard_logging_object_payload(
|
||||
kwargs={"model": "gpt-4o", "messages": [], **extra_kwargs},
|
||||
init_response_obj={},
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
logging_obj=logging_obj,
|
||||
status="success",
|
||||
)
|
||||
|
||||
|
||||
def test_standard_logging_payload_carries_the_impact_estimate(logging_obj, impact_estimator):
|
||||
"""The estimate has to reach the payload, since that is the only thing loggers ever see."""
|
||||
payload = _build_payload(logging_obj)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["response_impact"] == impact_estimator.impact
|
||||
assert payload["response_impact_failure_debug_info"] is None
|
||||
|
||||
|
||||
def test_standard_logging_payload_records_an_impact_failure_without_losing_the_payload(
|
||||
logging_obj, impact_estimator
|
||||
):
|
||||
"""A broken estimator must cost the impact figure only, never the whole log record."""
|
||||
impact_estimator.raises = True
|
||||
|
||||
payload = _build_payload(logging_obj)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["response_cost"] is not None
|
||||
assert payload["response_impact"] is None
|
||||
assert "no model data" in payload["response_impact_failure_debug_info"]["error_str"]
|
||||
|
||||
|
||||
def test_a_cache_hit_is_not_attributed_inference_impact(logging_obj, impact_estimator):
|
||||
"""A cached response ran no inference, so attributing energy to it would double count."""
|
||||
payload = _build_payload(logging_obj, cache_hit=True)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["response_impact"] is None
|
||||
assert impact_estimator.calls == 0
|
||||
|
||||
|
||||
def test_the_routed_deployment_is_estimated_rather_than_the_group_alias(logging_obj, impact_estimator):
|
||||
"""A model group is a routing label with no hardware behind it, so estimating it would size the wrong model."""
|
||||
payload = _build_payload(
|
||||
logging_obj,
|
||||
model="my-model-group",
|
||||
litellm_params={
|
||||
"metadata": {"deployment": "gpt-4o"},
|
||||
"proxy_server_request": {"body": {}},
|
||||
},
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert impact_estimator.requests[0].model == "gpt-4o"
|
||||
|
|
|
|||
115
tests/test_litellm/test_impact_calculator.py
Normal file
115
tests/test_litellm/test_impact_calculator.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.impact_calculator import (
|
||||
ImpactRequest,
|
||||
calculate_impact,
|
||||
current_estimator_name,
|
||||
get_impact_estimator,
|
||||
interval_impact_value,
|
||||
point_impact_value,
|
||||
register_impact_estimator,
|
||||
reset_default_estimator_cache,
|
||||
)
|
||||
from litellm.types.utils import ImpactInformation
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def impact_request() -> ImpactRequest:
|
||||
return ImpactRequest(
|
||||
model="gpt-4o",
|
||||
custom_llm_provider="openai",
|
||||
completion_tokens=100,
|
||||
response_time=2.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_registration():
|
||||
yield
|
||||
register_impact_estimator(None)
|
||||
reset_default_estimator_cache()
|
||||
|
||||
|
||||
class _StubEstimator:
|
||||
name = "stub"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.impact = ImpactInformation(estimator="stub")
|
||||
self.calls = 0
|
||||
|
||||
def estimate(self, request: ImpactRequest) -> ImpactInformation | None:
|
||||
self.calls += 1
|
||||
return self.impact
|
||||
|
||||
|
||||
class _RaisingEstimator:
|
||||
name = "raising"
|
||||
|
||||
def estimate(self, request: ImpactRequest) -> ImpactInformation | None:
|
||||
raise ValueError("no model data")
|
||||
|
||||
|
||||
def test_tracking_off_never_calls_the_estimator(monkeypatch, impact_request):
|
||||
"""The flag gates estimation itself, so an opted-out deployment pays nothing for it."""
|
||||
estimator = _StubEstimator()
|
||||
register_impact_estimator(estimator)
|
||||
monkeypatch.setattr(litellm, "track_impact", False)
|
||||
|
||||
assert calculate_impact(impact_request) is None
|
||||
assert estimator.calls == 0
|
||||
|
||||
|
||||
def test_tracking_on_returns_the_registered_estimators_result(monkeypatch, impact_request):
|
||||
estimator = _StubEstimator()
|
||||
register_impact_estimator(estimator)
|
||||
monkeypatch.setattr(litellm, "track_impact", True)
|
||||
|
||||
result = calculate_impact(impact_request)
|
||||
|
||||
assert estimator.calls == 1
|
||||
assert result is estimator.impact
|
||||
|
||||
|
||||
def test_registration_takes_precedence_over_default_resolution():
|
||||
estimator = _StubEstimator()
|
||||
register_impact_estimator(estimator)
|
||||
|
||||
assert get_impact_estimator() is estimator
|
||||
assert current_estimator_name() == "stub"
|
||||
|
||||
|
||||
def test_clearing_registration_stops_routing_to_the_cleared_estimator(monkeypatch, impact_request):
|
||||
estimator = _StubEstimator()
|
||||
register_impact_estimator(estimator)
|
||||
monkeypatch.setattr(litellm, "track_impact", True)
|
||||
|
||||
register_impact_estimator(None)
|
||||
calculate_impact(impact_request)
|
||||
|
||||
assert estimator.calls == 0
|
||||
|
||||
|
||||
def test_estimator_failure_reaches_the_caller(monkeypatch, impact_request):
|
||||
"""calculate_impact does not swallow, so the logging layer can record the failure."""
|
||||
register_impact_estimator(_RaisingEstimator())
|
||||
monkeypatch.setattr(litellm, "track_impact", True)
|
||||
|
||||
with pytest.raises(ValueError, match="no model data"):
|
||||
calculate_impact(impact_request)
|
||||
|
||||
|
||||
def test_a_point_value_carries_no_interval():
|
||||
value = point_impact_value(unit="Wh", value=1.5)
|
||||
|
||||
assert value["value"] == 1.5
|
||||
assert value["min"] is None
|
||||
assert value["max"] is None
|
||||
|
||||
|
||||
def test_an_interval_value_reports_its_midpoint():
|
||||
"""A consumer summing `value` must read a figure inside the modelled interval."""
|
||||
value = interval_impact_value(unit="Wh", minimum=2.0, maximum=6.0)
|
||||
|
||||
assert value["min"] < value["value"] < value["max"]
|
||||
assert value["value"] == 4.0
|
||||
Loading…
Add table
Reference in a new issue