mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
style: bring the savings changes under the strict and type-discipline budgets
`Optional[PreRoutingHookResponse]` becomes the union form the codebase is moving to, the recorded metadata is built in one shot rather than seeded empty and mutated, and the spend log's usage object is typed as a Mapping of object rather than a bare mutable dict, which also drops the Any the strict gate bans. The local run compared against a stale base and read clean, so these only surfaced once CI diffed against the real merge base.
This commit is contained in:
parent
10a2a29c66
commit
54103063a7
2 changed files with 19 additions and 10 deletions
|
|
@ -8,6 +8,7 @@ are known) and summed into the daily tables; tokens cannot be priced after they
|
|||
have been aggregated across models.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import NamedTuple
|
||||
|
||||
import litellm
|
||||
|
|
@ -172,7 +173,7 @@ def compute_autorouter_savings(
|
|||
return baseline_cost - selected_cost
|
||||
|
||||
|
||||
def _usage_from_spend_log(usage_object: dict | None) -> Usage | None:
|
||||
def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage | None:
|
||||
"""Rebuild the request's ``Usage`` from the copy the spend log recorded."""
|
||||
if not usage_object:
|
||||
return None
|
||||
|
|
@ -192,7 +193,7 @@ def compute_savings_spend(
|
|||
compression_saved_tokens: int,
|
||||
cache_read_input_tokens: int,
|
||||
baseline_model: str | None = None,
|
||||
usage_object: dict | None = None,
|
||||
usage_object: Mapping[str, object] | None = None,
|
||||
) -> SavingsSpend:
|
||||
"""
|
||||
Dollar savings for one request, split by optimization driver.
|
||||
|
|
|
|||
|
|
@ -11191,7 +11191,7 @@ class Router:
|
|||
@staticmethod
|
||||
def _record_routing_decision(
|
||||
request_kwargs: dict,
|
||||
pre_routing_hook_response: Optional[PreRoutingHookResponse],
|
||||
pre_routing_hook_response: PreRoutingHookResponse | None,
|
||||
) -> None:
|
||||
"""Make the request's metadata describe THIS routing attempt, and only this one.
|
||||
|
||||
|
|
@ -11210,15 +11210,23 @@ class Router:
|
|||
routing_decision = pre_routing_hook_response.routing_decision if pre_routing_hook_response else None
|
||||
baseline_model = pre_routing_hook_response.savings_baseline_model if pre_routing_hook_response else None
|
||||
|
||||
recorded: dict[str, Any] = {}
|
||||
if routing_decision is not None:
|
||||
recorded["routing_decision"] = Router._redact_prompt_text_if_needed(
|
||||
request_kwargs=request_kwargs, routing_decision=routing_decision
|
||||
recorded = {
|
||||
key: value
|
||||
for key, value in (
|
||||
(
|
||||
"routing_decision",
|
||||
Router._redact_prompt_text_if_needed(
|
||||
request_kwargs=request_kwargs, routing_decision=routing_decision
|
||||
)
|
||||
if routing_decision is not None
|
||||
else None,
|
||||
),
|
||||
("auto_router_savings_baseline_model", baseline_model),
|
||||
)
|
||||
if baseline_model is not None:
|
||||
recorded["auto_router_savings_baseline_model"] = baseline_model
|
||||
if value is not None
|
||||
}
|
||||
|
||||
cleared = {"routing_decision", "auto_router_savings_baseline_model"} - recorded.keys()
|
||||
cleared = frozenset({"routing_decision", "auto_router_savings_baseline_model"}) - recorded.keys()
|
||||
for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")):
|
||||
if isinstance(bucket, dict):
|
||||
for key in cleared:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue