mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
Merge e4b33bc8a9 into 4ece6c9fb8
This commit is contained in:
commit
407326a0bd
32 changed files with 2982 additions and 126 deletions
|
|
@ -6,13 +6,16 @@ together because they have to agree: a deployment the rollup declines to charge
|
|||
router prices at zero serves its traffic for free.
|
||||
"""
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, time, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.types.router import ModelInfo
|
||||
from litellm.types.utils import AzureSpillover, CustomPricingLiteLLMParams, MirroredPricingParams
|
||||
|
||||
|
|
@ -22,8 +25,12 @@ AZURE_SPILLOVER_FROM_HEADER: Final = "x-ms-spillover-from-deployment"
|
|||
|
||||
|
||||
def is_ptu_cost_attribution_enabled() -> bool:
|
||||
"""Whether PTU flat-cost attribution is turned on for this process."""
|
||||
return get_secret_bool(PTU_COST_ATTRIBUTION_ENV_VAR, False) is True
|
||||
"""Whether PTU flat-cost attribution is turned on for this process.
|
||||
|
||||
Read from the environment alone: the router and the rate limiter ask on every request,
|
||||
and ``get_secret`` would forward each of those reads to a configured secret manager.
|
||||
"""
|
||||
return str_to_bool(os.environ.get(PTU_COST_ATTRIBUTION_ENV_VAR)) is True
|
||||
|
||||
|
||||
PTU_ZEROED_PRICING_FIELDS: Final = tuple(f for f in MirroredPricingParams.model_fields if f != "tiered_pricing") + (
|
||||
|
|
@ -56,9 +63,13 @@ PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()] | Mapping[str, float]]]
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PTUTerms:
|
||||
"""The reservation a deployment declares, once every field has been validated."""
|
||||
"""The reservation a deployment declares, once every field has been validated.
|
||||
|
||||
team_id: str
|
||||
``shares`` maps every team the capacity is attributed to onto its PTUs and adds up to
|
||||
``ptu_count``: a deployment declaring a single ``team_id`` holds the whole count under it.
|
||||
"""
|
||||
|
||||
shares: Mapping[str, int]
|
||||
ptu_count: int
|
||||
cost_per_ptu_per_hour: float
|
||||
effective_from: datetime
|
||||
|
|
@ -127,7 +138,72 @@ def ptu_identity_error(
|
|||
return None
|
||||
|
||||
|
||||
PTU_MODEL_INFO_FIELDS: Final = ("ptu_count", "cost_per_ptu_per_hour", "ptu_effective_from", "ptu_effective_to")
|
||||
PTU_MODEL_INFO_FIELDS: Final = (
|
||||
"ptu_count",
|
||||
"cost_per_ptu_per_hour",
|
||||
"ptu_effective_from",
|
||||
"ptu_effective_to",
|
||||
"ptu_shares",
|
||||
)
|
||||
|
||||
|
||||
def _is_mapping(
|
||||
value: object,
|
||||
) -> TypeIs[Mapping[object, object]]: # guard-ok: isinstance decides it, keys and values stay object
|
||||
return isinstance(value, Mapping)
|
||||
|
||||
|
||||
def is_model_info_mapping(
|
||||
value: object,
|
||||
) -> TypeIs[Mapping[str, object]]: # guard-ok: model_info is a str-keyed JSON object from config.yaml or the db
|
||||
return isinstance(value, Mapping)
|
||||
|
||||
|
||||
def parsed_ptu_shares(raw: object) -> Mapping[str, int] | None:
|
||||
"""``ptu_shares`` as team id -> whole PTUs, else None when empty or any entry is unusable.
|
||||
|
||||
A share is a count of reserved units, so it has to be a positive integer; ``bool`` is
|
||||
excluded because it is an ``int`` subclass and ``True`` would read as one PTU.
|
||||
"""
|
||||
if not _is_mapping(raw) or not raw:
|
||||
return None
|
||||
entries: Final = tuple(
|
||||
(team_id, share)
|
||||
for team_id, share in raw.items()
|
||||
if isinstance(team_id, str) and team_id and isinstance(share, int) and not isinstance(share, bool) and share > 0
|
||||
)
|
||||
if len(entries) != len(raw):
|
||||
return None
|
||||
return MappingProxyType(dict(entries))
|
||||
|
||||
|
||||
def _parsed_ptu_count(model_info: Mapping[str, object]) -> int | None:
|
||||
"""``ptu_count`` as the whole number of reserved units within bounds, else None."""
|
||||
raw: Final = model_info.get("ptu_count")
|
||||
if isinstance(raw, bool) or not isinstance(raw, (int, float, str)):
|
||||
return None
|
||||
if isinstance(raw, float) and not raw.is_integer():
|
||||
return None
|
||||
try:
|
||||
count: Final = int(raw)
|
||||
except (ValueError, OverflowError):
|
||||
return None
|
||||
return count if 0 < count <= ModelInfo.MAX_PTU_COUNT else None
|
||||
|
||||
|
||||
def _declared_shares(model_info: Mapping[str, object], ptu_count: int) -> Mapping[str, int] | None:
|
||||
"""Who holds the capacity: the single ``team_id`` holding all of it, or the ``ptu_shares``
|
||||
that add up to it, else None."""
|
||||
team_id: Final = model_info.get("team_id")
|
||||
raw_shares: Final = model_info.get("ptu_shares")
|
||||
if team_id and raw_shares is None:
|
||||
return MappingProxyType({str(team_id): ptu_count})
|
||||
if team_id or raw_shares is None:
|
||||
return None
|
||||
shares: Final = parsed_ptu_shares(raw_shares)
|
||||
if shares is None or sum(shares.values()) != ptu_count:
|
||||
return None
|
||||
return shares
|
||||
|
||||
|
||||
def declares_ptu(model_info: Mapping[str, object]) -> bool:
|
||||
|
|
@ -153,8 +229,11 @@ def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None
|
|||
|
||||
has_count: Final = model_info.get("ptu_count") is not None
|
||||
has_rate: Final = model_info.get("cost_per_ptu_per_hour") is not None
|
||||
if not has_count and not has_rate:
|
||||
has_shares: Final = model_info.get("ptu_shares") is not None
|
||||
if not has_count and not has_rate and not has_shares:
|
||||
return None
|
||||
if not has_count and not has_rate:
|
||||
return _named("ptu_count and cost_per_ptu_per_hour are required when ptu_shares is set", model_name)
|
||||
if has_count != has_rate:
|
||||
return _named("ptu_count and cost_per_ptu_per_hour must be set together", model_name)
|
||||
if effective_from is None:
|
||||
|
|
@ -164,8 +243,36 @@ def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None
|
|||
"today could be billed for days it did not exist",
|
||||
model_name,
|
||||
)
|
||||
if not model_info.get("team_id"):
|
||||
return _named("team_id is required when PTU fields are set (one model maps to one team)", model_name)
|
||||
return _ptu_holder_error(model_info, model_name)
|
||||
|
||||
|
||||
def _ptu_holder_error(model_info: Mapping[str, object], model_name: str | None) -> str | None:
|
||||
"""Why the teams this capacity is attributed to cannot be read, else None.
|
||||
|
||||
The shares have to add up to the count exactly: a shortfall would leave PTU-hours the
|
||||
provider bills attributed to nobody, and a surplus would attribute capacity that was
|
||||
never reserved.
|
||||
"""
|
||||
team_id: Final = model_info.get("team_id")
|
||||
raw_shares: Final = model_info.get("ptu_shares")
|
||||
if team_id and raw_shares is not None:
|
||||
return _named(
|
||||
"team_id and ptu_shares cannot both be set; ptu_shares lists every team the capacity is split across",
|
||||
model_name,
|
||||
)
|
||||
if not team_id and raw_shares is None:
|
||||
return _named("team_id or ptu_shares is required when PTU fields are set", model_name)
|
||||
if raw_shares is None:
|
||||
return None
|
||||
shares: Final = parsed_ptu_shares(raw_shares)
|
||||
if shares is None:
|
||||
return _named("ptu_shares must map at least one team_id to a positive whole number of PTUs", model_name)
|
||||
ptu_count: Final = _parsed_ptu_count(model_info)
|
||||
if ptu_count is None:
|
||||
return None
|
||||
allocated: Final = sum(shares.values())
|
||||
if allocated != ptu_count:
|
||||
return _named(f"ptu_shares must add up to ptu_count ({allocated} of {ptu_count} allocated)", model_name)
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -176,17 +283,13 @@ def ptu_terms(model_info: Mapping[str, object]) -> PTUTerms | None:
|
|||
present but unparseable bound would read as no bound and widen the window to the whole
|
||||
day, so either one leaves the deployment unpriced until the config is fixed.
|
||||
"""
|
||||
ptu_count: Final = model_info.get("ptu_count")
|
||||
ptu_count_int: Final = _parsed_ptu_count(model_info)
|
||||
cost_per_hour: Final = model_info.get("cost_per_ptu_per_hour")
|
||||
team_id: Final = model_info.get("team_id")
|
||||
if ptu_count is None or cost_per_hour is None or not team_id:
|
||||
if ptu_count_int is None or isinstance(cost_per_hour, bool) or not isinstance(cost_per_hour, (int, float, str)):
|
||||
return None
|
||||
try:
|
||||
ptu_count_int: Final = int(ptu_count)
|
||||
cost_per_hour_float: Final = float(cost_per_hour)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
if not 0 < ptu_count_int <= ModelInfo.MAX_PTU_COUNT:
|
||||
except (ValueError, OverflowError):
|
||||
return None
|
||||
if not 0 <= cost_per_hour_float <= ModelInfo.MAX_COST_PER_PTU_PER_HOUR:
|
||||
return None
|
||||
|
|
@ -199,8 +302,11 @@ def ptu_terms(model_info: Mapping[str, object]) -> PTUTerms | None:
|
|||
return None
|
||||
if effective_to is not None and effective_to <= effective_from:
|
||||
return None
|
||||
shares: Final = _declared_shares(model_info, ptu_count_int)
|
||||
if shares is None:
|
||||
return None
|
||||
return PTUTerms(
|
||||
team_id=str(team_id),
|
||||
shares=shares,
|
||||
ptu_count=ptu_count_int,
|
||||
cost_per_ptu_per_hour=cost_per_hour_float,
|
||||
effective_from=effective_from,
|
||||
|
|
|
|||
131
litellm/llms/azure/ptu_capacity.py
Normal file
131
litellm/llms/azure/ptu_capacity.py
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
"""How much throughput one Azure OpenAI provisioned throughput unit (PTU) serves per model.
|
||||
|
||||
Azure sizes a provisioned deployment in normalized tokens per minute:
|
||||
``input TPM x (1 - cache hit rate) + output-to-input ratio x output TPM``, divided by the
|
||||
model's "Input TPM per PTU" to get the PTUs required. The same two numbers turn a team's
|
||||
PTU share into a per-minute token ceiling and a request's usage back into PTU-hours.
|
||||
|
||||
Table read from
|
||||
https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/provisioned-throughput-sizing#deployment-parameters-and-throughput-values-by-model
|
||||
on 2026-09-24 (page dated 2026-09-23). Azure deducts cached input tokens in full for every
|
||||
model except the GPT-6 family, where a cached input token costs a tenth of an uncached one.
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol
|
||||
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
class NormalizedTokenWeights(Protocol):
|
||||
@property
|
||||
def output_to_input_ratio(self) -> float: ...
|
||||
|
||||
@property
|
||||
def cached_input_ratio(self) -> float: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PTUCapacity:
|
||||
input_tpm_per_ptu: int
|
||||
output_to_input_ratio: float
|
||||
cached_input_ratio: float = 0.0
|
||||
|
||||
@property
|
||||
def normalized_tokens_per_ptu_hour(self) -> int:
|
||||
return self.input_tpm_per_ptu * 60
|
||||
|
||||
def input_tpm_for(self, ptus: int) -> int:
|
||||
return ptus * self.input_tpm_per_ptu
|
||||
|
||||
|
||||
AZURE_PTU_CAPACITY: Final[Mapping[str, PTUCapacity]] = MappingProxyType(
|
||||
{
|
||||
"gpt-6-sol": PTUCapacity(3_000, 5.0, cached_input_ratio=0.1),
|
||||
"gpt-6-astra": PTUCapacity(600, 5.0, cached_input_ratio=0.1),
|
||||
"gpt-5.6-luna": PTUCapacity(30_000, 6.0),
|
||||
"gpt-5.6-terra": PTUCapacity(3_000, 6.0),
|
||||
"gpt-5.6-sol": PTUCapacity(1_200, 6.0),
|
||||
"gpt-5.5": PTUCapacity(1_200, 6.0),
|
||||
"gpt-5.4": PTUCapacity(2_400, 6.0),
|
||||
"gpt-5.4-mini": PTUCapacity(7_900, 6.0),
|
||||
"gpt-5.3-codex": PTUCapacity(3_400, 8.0),
|
||||
"gpt-5.2": PTUCapacity(3_400, 8.0),
|
||||
"gpt-5.2-codex": PTUCapacity(3_400, 8.0),
|
||||
"gpt-5.1": PTUCapacity(4_750, 8.0),
|
||||
"gpt-5.1-codex": PTUCapacity(4_750, 8.0),
|
||||
"gpt-5": PTUCapacity(4_750, 8.0),
|
||||
"gpt-5-mini": PTUCapacity(23_750, 8.0),
|
||||
"gpt-4.1": PTUCapacity(3_000, 4.0),
|
||||
"gpt-4.1-mini": PTUCapacity(14_900, 4.0),
|
||||
"gpt-4.1-nano": PTUCapacity(59_400, 4.0),
|
||||
"o3": PTUCapacity(3_000, 4.0),
|
||||
"o4-mini": PTUCapacity(5_400, 4.0),
|
||||
"gpt-4o": PTUCapacity(2_500, 4.0),
|
||||
"gpt-4o-mini": PTUCapacity(37_000, 4.0),
|
||||
"o3-mini": PTUCapacity(2_500, 4.0),
|
||||
"o1": PTUCapacity(230, 4.0),
|
||||
"llama-3.3-70b-instruct": PTUCapacity(8_450, 4.0),
|
||||
}
|
||||
)
|
||||
|
||||
_VERSION_SUFFIX: Final = re.compile(r"-\d{4}-\d{2}-\d{2}$")
|
||||
_AZURE_PROVIDERS: Final = frozenset({LlmProviders.AZURE.value, LlmProviders.AZURE_AI.value})
|
||||
|
||||
|
||||
def azure_ptu_capacity(model: str) -> PTUCapacity | None:
|
||||
"""The sizing row for ``model``, read as its last path segment with a dated version dropped.
|
||||
|
||||
``azure/gpt-4.1-2025-04-14`` and ``gpt-4.1`` both resolve to the ``gpt-4.1`` row; a
|
||||
deployment name that is not a model name resolves to nothing, which is why callers
|
||||
prefer ``model_info.base_model``.
|
||||
"""
|
||||
name: Final = model.rsplit("/", 1)[-1].strip().lower()
|
||||
return AZURE_PTU_CAPACITY.get(name) or AZURE_PTU_CAPACITY.get(_VERSION_SUFFIX.sub("", name))
|
||||
|
||||
|
||||
def deployment_ptu_capacity(deployment: Mapping[str, object]) -> PTUCapacity | None:
|
||||
"""The sizing row a deployment resolves to: ``model_info.base_model`` first, since an
|
||||
Azure deployment name is arbitrary, then ``litellm_params.model``."""
|
||||
model_info: Final = deployment.get("model_info")
|
||||
litellm_params: Final = deployment.get("litellm_params")
|
||||
candidates: Final = tuple(
|
||||
value
|
||||
for value in (
|
||||
model_info.get("base_model") if isinstance(model_info, Mapping) else None,
|
||||
litellm_params.get("model") if isinstance(litellm_params, Mapping) else None,
|
||||
)
|
||||
if isinstance(value, str) and value
|
||||
)
|
||||
return next((capacity for capacity in map(azure_ptu_capacity, candidates) if capacity is not None), None)
|
||||
|
||||
|
||||
def is_azure_deployment(deployment: Mapping[str, object]) -> bool:
|
||||
"""Whether ``litellm_params`` route this deployment to Azure OpenAI or Azure AI, by
|
||||
``custom_llm_provider`` first and the ``model`` prefix otherwise."""
|
||||
litellm_params: Final = deployment.get("litellm_params")
|
||||
if not isinstance(litellm_params, Mapping):
|
||||
return False
|
||||
provider: Final = litellm_params.get("custom_llm_provider")
|
||||
if isinstance(provider, str):
|
||||
return provider in _AZURE_PROVIDERS
|
||||
model: Final = litellm_params.get("model")
|
||||
return isinstance(model, str) and model.partition("/")[0] in _AZURE_PROVIDERS
|
||||
|
||||
|
||||
def normalized_tokens(
|
||||
weights: NormalizedTokenWeights, *, prompt_tokens: int, completion_tokens: int, cache_read_tokens: int = 0
|
||||
) -> float:
|
||||
"""Azure's normalized token count for one request: uncached input in full, cached input
|
||||
at the model's cached ratio, output weighted by the output-to-input ratio."""
|
||||
cached: Final = min(max(cache_read_tokens, 0), max(prompt_tokens, 0))
|
||||
uncached: Final = max(prompt_tokens, 0) - cached
|
||||
return uncached + weights.cached_input_ratio * cached + weights.output_to_input_ratio * max(completion_tokens, 0)
|
||||
|
||||
|
||||
def ptu_hours(capacity: PTUCapacity, normalized: float) -> float:
|
||||
"""PTU-hours ``normalized`` tokens amount to: one PTU serves its input TPM for sixty minutes."""
|
||||
return normalized / capacity.normalized_tokens_per_ptu_hour
|
||||
|
|
@ -3616,6 +3616,11 @@
|
|||
"title": "Total Prompt Tokens",
|
||||
"type": "integer"
|
||||
},
|
||||
"total_ptu_hours": {
|
||||
"default": 0.0,
|
||||
"title": "Total Ptu Hours",
|
||||
"type": "number"
|
||||
},
|
||||
"total_response_time_ms": {
|
||||
"default": 0,
|
||||
"title": "Total Response Time Ms",
|
||||
|
|
@ -4424,6 +4429,11 @@
|
|||
"title": "Prompt Tokens",
|
||||
"type": "integer"
|
||||
},
|
||||
"ptu_hours": {
|
||||
"default": 0.0,
|
||||
"title": "Ptu Hours",
|
||||
"type": "number"
|
||||
},
|
||||
"spend": {
|
||||
"default": 0.0,
|
||||
"title": "Spend",
|
||||
|
|
|
|||
|
|
@ -45,7 +45,9 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
)
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.llms.azure.ptu_capacity import normalized_tokens
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
ESTIMATED_OUTPUT_TOKENS_FIELD,
|
||||
|
|
@ -73,6 +75,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
|
|||
response_has_hidden_params,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling, team_ptu_ceiling
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
|
||||
|
|
@ -102,6 +105,17 @@ else:
|
|||
_REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ReconciledUsage:
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
cached_tokens: int
|
||||
|
||||
@property
|
||||
def billable_input_tokens(self) -> int:
|
||||
return max(0, self.prompt_tokens - self.cached_tokens)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RateLimitedModel:
|
||||
requested: str
|
||||
|
|
@ -122,6 +136,19 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
|
|||
return resolve_model_group_alias(llm_router.model_group_alias, model)
|
||||
|
||||
|
||||
def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
if llm_router is None or not is_ptu_cost_attribution_enabled():
|
||||
return None
|
||||
return team_ptu_ceiling(
|
||||
llm_router.get_model_list() or (),
|
||||
llm_router.model_list, # pyright: ignore[reportUnknownArgumentType] # Router.model_list is a bare list
|
||||
team_id,
|
||||
model_group,
|
||||
)
|
||||
|
||||
|
||||
FAIL_CLOSED_RATE_LIMIT_ENFORCEMENT_SETTING: Final = "fail_closed_rate_limit_enforcement"
|
||||
RATE_LIMIT_UNVERIFIABLE_MESSAGE: Final = (
|
||||
"Rate limit enforcement unavailable: request counters could not be verified against Redis, and "
|
||||
|
|
@ -470,6 +497,7 @@ _AUDIO_BYTES_PER_TOKEN: Final = 1600
|
|||
# on the same project+model simultaneously without colliding on cache keys.
|
||||
PROJECT_ITPM_DESCRIPTOR_KEY: Final = "model_per_project_itpm"
|
||||
PROJECT_OTPM_DESCRIPTOR_KEY: Final = "model_per_project_otpm"
|
||||
PTU_TEAM_DESCRIPTOR_KEY: Final = "model_per_team_ptu"
|
||||
# How long an acquired slot counts toward the in-flight total before it is
|
||||
# considered leaked (worker crashed without any release callback firing) and
|
||||
# pruned. Also the longest request duration the gauge can track: a request
|
||||
|
|
@ -634,6 +662,8 @@ class RequestRateLimiterStash:
|
|||
reserved_tokens: int = 0
|
||||
reserved_model: RateLimitedModel | None = None
|
||||
reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset)
|
||||
ptu_ceiling: PTUTeamCeiling | None = None
|
||||
ptu_reserved_tokens: int = 0
|
||||
itpm_reserved_tokens: int = 0
|
||||
itpm_reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset)
|
||||
itpm_reserved_window_identities: frozenset[tuple[str, str, Literal["redis", "local"]]] = field(
|
||||
|
|
@ -764,12 +794,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
time_provider: Callable[[], datetime] | None = None,
|
||||
tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db,
|
||||
model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router,
|
||||
ptu_team_ceiling_resolver: Callable[
|
||||
[str, str], PTUTeamCeiling | None
|
||||
] = _resolve_ptu_team_ceiling_via_proxy_router,
|
||||
fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings,
|
||||
):
|
||||
self.internal_usage_cache = internal_usage_cache
|
||||
self._time_provider = time_provider or datetime.now
|
||||
self._tag_rate_limit_resolver = tag_rate_limit_resolver
|
||||
self._model_group_resolver = model_group_resolver
|
||||
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
|
||||
self._fail_closed_resolver = fail_closed_resolver
|
||||
if self.internal_usage_cache.dual_cache.redis_cache is not None:
|
||||
self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
|
||||
|
|
@ -1071,6 +1105,30 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
|
||||
return total_estimated
|
||||
|
||||
def _estimate_ptu_tokens_for_request(
|
||||
self,
|
||||
ceiling: PTUTeamCeiling | None,
|
||||
data: Mapping[str, object],
|
||||
min_configured_tpm_limit: int | None,
|
||||
call_type: str | None,
|
||||
configured_output_tokens: int | None,
|
||||
raw_estimate: int,
|
||||
) -> int:
|
||||
"""The team PTU ceiling counts Azure normalized tokens, so its reservation weighs the
|
||||
output budget the way the ceiling does instead of the raw sum the other scopes reserve."""
|
||||
if ceiling is None:
|
||||
return raw_estimate
|
||||
estimated_input_tokens, max_tokens_estimate = self._estimate_input_and_output_tokens(
|
||||
data=data,
|
||||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
)
|
||||
normalized: Final = normalized_tokens(
|
||||
ceiling, prompt_tokens=estimated_input_tokens, completion_tokens=max_tokens_estimate
|
||||
)
|
||||
return max(round(normalized), 1)
|
||||
|
||||
def _estimate_input_and_output_tokens(
|
||||
self,
|
||||
data: object,
|
||||
|
|
@ -2538,11 +2596,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
descriptors: list[RateLimitDescriptor],
|
||||
estimated_tokens: int,
|
||||
parent_otel_span: Span | None = None,
|
||||
scope_estimates: Mapping[str, int] = MappingProxyType({}),
|
||||
) -> RateLimitResponse:
|
||||
"""
|
||||
Reserve ``estimated_tokens`` against every TPM-bearing descriptor
|
||||
BEFORE the upstream call, so concurrent requests cannot all observe
|
||||
"under limit" before any of them increments the counter.
|
||||
``scope_estimates`` replaces that amount per descriptor key for a
|
||||
scope counted in other units, such as the team PTU ceiling.
|
||||
|
||||
Thin wrapper around ``atomic_check_and_increment_by_n``: builds a
|
||||
TPM-only descriptor/increment list and delegates the all-or-nothing
|
||||
|
|
@ -2569,7 +2630,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
return RateLimitResponse(overall_code="OK", statuses=[])
|
||||
|
||||
increments: Final[list[dict[Literal["requests", "tokens"], int]]] = [
|
||||
{"tokens": estimated_tokens} for _ in tpm_descriptors
|
||||
{"tokens": scope_estimates.get(d["key"], estimated_tokens)} for d in tpm_descriptors
|
||||
]
|
||||
return await self.atomic_check_and_increment_by_n(
|
||||
descriptors=tpm_descriptors,
|
||||
|
|
@ -3260,6 +3321,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
requested_model=requested_model if isinstance(requested_model, str) else None,
|
||||
descriptors=descriptors,
|
||||
)
|
||||
self._add_team_ptu_rate_limit_descriptor(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
requested_model=requested_model if isinstance(requested_model, str) else None,
|
||||
descriptors=descriptors,
|
||||
)
|
||||
|
||||
resolved_agent_id: Final = self._get_resolved_agent_id(user_api_key_dict, data)
|
||||
for agent_id in dict.fromkeys((resolved_agent_id, user_api_key_dict.invoked_agent_id)):
|
||||
|
|
@ -3403,6 +3469,31 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
)
|
||||
|
||||
def _add_team_ptu_rate_limit_descriptor(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
requested_model: str | None,
|
||||
descriptors: list[RateLimitDescriptor],
|
||||
) -> None:
|
||||
model: Final = self._rate_limited_model(requested_model)
|
||||
if model is None or not user_api_key_dict.team_id:
|
||||
return
|
||||
ceiling: Final = self._ptu_team_ceiling_resolver(user_api_key_dict.team_id, model.group)
|
||||
if ceiling is None:
|
||||
return
|
||||
stash: Final = get_request_stash()
|
||||
if stash is not None:
|
||||
stash.ptu_ceiling = ceiling
|
||||
descriptors.append(
|
||||
RateLimitDescriptor(
|
||||
key=PTU_TEAM_DESCRIPTOR_KEY,
|
||||
value=f"{user_api_key_dict.team_id}:{ceiling.model_group}",
|
||||
rate_limit=RateLimitDescriptorRateLimitObject(
|
||||
requests_per_unit=None, tokens_per_unit=ceiling.tpm_limit, window_size=self.window_size
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def _add_project_model_rate_limit_descriptor_from_metadata(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
@ -3815,10 +3906,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# (still-stashed) reservation and refunds it again.
|
||||
if tpm_reservation_amount > 0:
|
||||
await self._refund_reserved_tokens(
|
||||
scopes=tpm_reservation_scopes,
|
||||
scopes=tuple(scope for scope in tpm_reservation_scopes if scope[0] != PTU_TEAM_DESCRIPTOR_KEY),
|
||||
amount=tpm_reservation_amount,
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
)
|
||||
await self._refund_reserved_tokens(
|
||||
scopes=tuple(scope for scope in tpm_reservation_scopes if scope[0] == PTU_TEAM_DESCRIPTOR_KEY),
|
||||
amount=stash.ptu_reserved_tokens,
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
)
|
||||
stash.reservation_released = True
|
||||
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
|
||||
self._handle_rate_limit_error(
|
||||
|
|
@ -4068,13 +4164,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# in-memory check otherwise — single-worker protection still holds
|
||||
# even without Redis.
|
||||
# ----------------------------------------------------------------
|
||||
configured_tpm_limits: Final = [
|
||||
int(v)
|
||||
ptu_raw_output_limit: Final = (
|
||||
stash.ptu_ceiling.raw_output_token_limit() if stash.ptu_ceiling is not None else None
|
||||
)
|
||||
configured_tpm_limits: Final = tuple(
|
||||
ptu_raw_output_limit
|
||||
if d["key"] == PTU_TEAM_DESCRIPTOR_KEY and ptu_raw_output_limit is not None
|
||||
else int(v)
|
||||
for d in descriptors
|
||||
if d["key"] not in (PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY)
|
||||
for v in [(d.get("rate_limit") or {}).get("tokens_per_unit")]
|
||||
if v is not None
|
||||
]
|
||||
and (v := (d.get("rate_limit") or {}).get("tokens_per_unit")) is not None
|
||||
)
|
||||
has_tpm_limits: Final = bool(configured_tpm_limits)
|
||||
|
||||
# Populated on a successful combined-TPM reservation below, so the
|
||||
|
|
@ -4136,10 +4236,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit,
|
||||
)
|
||||
|
||||
capped_request: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(data)
|
||||
ptu_estimated_tokens: Final = self._estimate_ptu_tokens_for_request(
|
||||
ceiling=stash.ptu_ceiling,
|
||||
data=capped_request,
|
||||
min_configured_tpm_limit=min_configured_tpm_limit,
|
||||
call_type=call_type,
|
||||
configured_output_tokens=configured_output_tokens,
|
||||
raw_estimate=estimated_tokens,
|
||||
)
|
||||
tpm_response: Final = await self.reserve_tpm_tokens(
|
||||
descriptors=descriptors,
|
||||
estimated_tokens=estimated_tokens,
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
scope_estimates=MappingProxyType({PTU_TEAM_DESCRIPTOR_KEY: ptu_estimated_tokens}),
|
||||
)
|
||||
|
||||
if tpm_response["overall_code"] == "OVER_LIMIT":
|
||||
|
|
@ -4155,6 +4265,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# the (actual - reserved) delta to those — unreserved
|
||||
# scopes get charged the full actual usage instead.
|
||||
stash.reserved_tokens = estimated_tokens
|
||||
stash.ptu_reserved_tokens = ptu_estimated_tokens
|
||||
stash.reserved_model = self._rate_limited_model(requested_model)
|
||||
stash.reserved_scopes = frozenset(
|
||||
(d["key"], d["value"])
|
||||
|
|
@ -4580,20 +4691,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
return max(0, total_tokens), 0, True
|
||||
return None
|
||||
|
||||
def _resolve_io_token_reconcile_usage(
|
||||
self,
|
||||
response_obj: object,
|
||||
) -> tuple[int, int, bool]:
|
||||
"""
|
||||
Resolve ``(billable_input_tokens, completion_tokens, usage_resolved)``
|
||||
for ITPM/OTPM reconciliation. Cache-read tokens are excluded from
|
||||
billable input -- Bedrock Mantle doesn't count them toward ITPM --
|
||||
but they're untouched everywhere else (cost/usage logging still sees
|
||||
the full prompt token count).
|
||||
"""
|
||||
def _resolve_reconciled_usage(self, response_obj: object) -> _ReconciledUsage | None:
|
||||
"""The prompt, completion, and cache-read token counts a response reports, else None
|
||||
when it reports no usage at all. Cache-read tokens stay inside ``prompt_tokens`` here;
|
||||
each consumer decides what they cost it."""
|
||||
rerank_usage: Final = self._resolve_rerank_token_usage(response_obj)
|
||||
if rerank_usage is not None:
|
||||
return rerank_usage
|
||||
return _ReconciledUsage(prompt_tokens=rerank_usage[0], completion_tokens=rerank_usage[1], cached_tokens=0)
|
||||
|
||||
usage: Final = self._response_usage(response_obj)
|
||||
|
||||
|
|
@ -4606,8 +4710,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
else 0
|
||||
)
|
||||
if prompt_tokens == 0 and completion_tokens == 0:
|
||||
return 0, 0, False
|
||||
return max(0, prompt_tokens - cached_tokens), completion_tokens, True
|
||||
return None
|
||||
return _ReconciledUsage(
|
||||
prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, cached_tokens=cached_tokens
|
||||
)
|
||||
|
||||
if isinstance(usage, ResponseAPIUsage):
|
||||
response_input_tokens: Final = usage.input_tokens or 0
|
||||
|
|
@ -4616,8 +4722,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
usage.input_tokens_details.cached_tokens or 0 if usage.input_tokens_details is not None else 0
|
||||
)
|
||||
if response_input_tokens == 0 and response_output_tokens == 0:
|
||||
return 0, 0, False
|
||||
return max(0, response_input_tokens - response_cached_tokens), response_output_tokens, True
|
||||
return None
|
||||
return _ReconciledUsage(
|
||||
prompt_tokens=response_input_tokens,
|
||||
completion_tokens=response_output_tokens,
|
||||
cached_tokens=response_cached_tokens,
|
||||
)
|
||||
|
||||
if isinstance(usage, Mapping):
|
||||
raw_prompt_tokens: Final = usage.get("prompt_tokens") or usage.get("input_tokens") or 0
|
||||
|
|
@ -4632,10 +4742,30 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
mapped_cached_tokens: Final = raw_cached_tokens if isinstance(raw_cached_tokens, int) else 0
|
||||
if mapped_prompt_tokens == 0 and mapped_completion_tokens == 0:
|
||||
return 0, 0, False
|
||||
return max(0, mapped_prompt_tokens - mapped_cached_tokens), mapped_completion_tokens, True
|
||||
return None
|
||||
return _ReconciledUsage(
|
||||
prompt_tokens=mapped_prompt_tokens,
|
||||
completion_tokens=mapped_completion_tokens,
|
||||
cached_tokens=mapped_cached_tokens,
|
||||
)
|
||||
|
||||
return 0, 0, False
|
||||
return None
|
||||
|
||||
def _resolve_io_token_reconcile_usage(
|
||||
self,
|
||||
response_obj: object,
|
||||
) -> tuple[int, int, bool]:
|
||||
"""
|
||||
Resolve ``(billable_input_tokens, completion_tokens, usage_resolved)``
|
||||
for ITPM/OTPM reconciliation. Cache-read tokens are excluded from
|
||||
billable input -- Bedrock Mantle doesn't count them toward ITPM --
|
||||
but they're untouched everywhere else (cost/usage logging still sees
|
||||
the full prompt token count).
|
||||
"""
|
||||
usage: Final = self._resolve_reconciled_usage(response_obj)
|
||||
if usage is None:
|
||||
return 0, 0, False
|
||||
return usage.billable_input_tokens, usage.completion_tokens, True
|
||||
|
||||
def _build_io_token_reservation_ops(
|
||||
self,
|
||||
|
|
@ -4983,9 +5113,75 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
reserved_tokens=reserved_tokens,
|
||||
)
|
||||
)
|
||||
pipeline_operations.extend(
|
||||
self._build_team_ptu_tpm_ops(
|
||||
standard_logging_metadata, # pyright: ignore[reportUnknownArgumentType] # untyped logging metadata
|
||||
response_obj=response_obj,
|
||||
reconcile_model=reconcile_model,
|
||||
reserved_scopes=reserved_scopes,
|
||||
reserved_ceiling=stash.ptu_ceiling if stash is not None else None,
|
||||
reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
)
|
||||
|
||||
return pipeline_operations
|
||||
|
||||
def _build_team_ptu_tpm_ops(
|
||||
self,
|
||||
standard_logging_metadata: Mapping[str, object],
|
||||
response_obj: object,
|
||||
reconcile_model: RateLimitedModel | None,
|
||||
reserved_scopes: Set[tuple[str, str]],
|
||||
reserved_ceiling: PTUTeamCeiling | None,
|
||||
reserved_tokens: int,
|
||||
total_tokens: int,
|
||||
) -> Sequence[RedisPipelineIncrementOperation]:
|
||||
"""Settle the team's PTU counter in Azure normalized tokens: uncached input in full,
|
||||
cached input at the model's cached ratio, output weighted by its output-to-input
|
||||
ratio, the way Azure sizes a PTU.
|
||||
|
||||
The reservation was taken in those units against the ceiling admission resolved, so that
|
||||
ceiling settles it even after the share changed or went away; when usage cannot be
|
||||
resolved it charges the raw total the other scopes charge so the reservation is never
|
||||
left standing.
|
||||
"""
|
||||
team_id: Final = standard_logging_metadata.get("user_api_key_team_id")
|
||||
if reconcile_model is None or not isinstance(team_id, str) or not team_id:
|
||||
return ()
|
||||
ceiling: Final = (
|
||||
reserved_ceiling
|
||||
if reserved_ceiling is not None
|
||||
else self._ptu_team_ceiling_resolver(team_id, reconcile_model.group)
|
||||
)
|
||||
reserved_ptu_scopes: Final = tuple(scope for scope in reserved_scopes if scope[0] == PTU_TEAM_DESCRIPTOR_KEY)
|
||||
targets: Final = reserved_ptu_scopes or (
|
||||
((PTU_TEAM_DESCRIPTOR_KEY, f"{team_id}:{ceiling.model_group}"),) if ceiling is not None else ()
|
||||
)
|
||||
if not targets:
|
||||
return ()
|
||||
return self._build_reservation_aware_tpm_ops(
|
||||
targets=targets,
|
||||
reserved_scopes=reserved_scopes,
|
||||
actual_tokens=self._ptu_settlement_tokens(
|
||||
ceiling, self._resolve_reconciled_usage(response_obj), total_tokens
|
||||
),
|
||||
reserved_tokens=reserved_tokens,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _ptu_settlement_tokens(ceiling: PTUTeamCeiling | None, usage: _ReconciledUsage | None, raw_total: int) -> int:
|
||||
if ceiling is None or usage is None:
|
||||
return raw_total
|
||||
return round(
|
||||
normalized_tokens(
|
||||
ceiling,
|
||||
prompt_tokens=usage.prompt_tokens,
|
||||
completion_tokens=usage.completion_tokens,
|
||||
cache_read_tokens=usage.cached_tokens,
|
||||
)
|
||||
)
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""
|
||||
Update TPM usage on successful API calls by incrementing counters using pipeline
|
||||
|
|
@ -5091,9 +5287,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
statuses=statuses,
|
||||
)
|
||||
|
||||
def _recovered_partial_usage_tokens(self, source: Mapping[str, object]) -> tuple[int, int, int]:
|
||||
def _recovered_partial_usage(self, source: Mapping[str, object]) -> Usage | None:
|
||||
usage: Final = source.get("combined_usage_object")
|
||||
if not isinstance(usage, Usage) or (usage.completion_tokens or 0) <= 0:
|
||||
return None
|
||||
return usage
|
||||
|
||||
def _recovered_partial_usage_tokens(self, source: Mapping[str, object]) -> tuple[int, int, int]:
|
||||
usage: Final = self._recovered_partial_usage(source)
|
||||
if usage is None:
|
||||
return 0, 0, 0
|
||||
billable_input, completion_tokens, _ = self._resolve_io_token_reconcile_usage(usage)
|
||||
return (
|
||||
|
|
@ -5102,6 +5304,26 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
completion_tokens,
|
||||
)
|
||||
|
||||
def _build_ptu_failure_settlement_ops(
|
||||
self, stash: RequestRateLimiterStash, source: Mapping[str, object], raw_actual_tokens: int
|
||||
) -> Sequence[RedisPipelineIncrementOperation]:
|
||||
"""Settle the team PTU reservation on failure in the normalized tokens it was taken in:
|
||||
at the recovered partial usage when there is one, else a full refund."""
|
||||
ptu_scopes: Final = tuple(scope for scope in stash.reserved_scopes if scope[0] == PTU_TEAM_DESCRIPTOR_KEY)
|
||||
if not ptu_scopes:
|
||||
return ()
|
||||
usage: Final = self._recovered_partial_usage(source)
|
||||
return self._build_reservation_aware_tpm_ops(
|
||||
targets=ptu_scopes,
|
||||
reserved_scopes=stash.reserved_scopes,
|
||||
actual_tokens=self._ptu_settlement_tokens(
|
||||
stash.ptu_ceiling,
|
||||
self._resolve_reconciled_usage(usage) if usage is not None else None,
|
||||
raw_actual_tokens,
|
||||
),
|
||||
reserved_tokens=stash.ptu_reserved_tokens,
|
||||
)
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""
|
||||
On failure: decrement max_parallel_requests and refund the upfront
|
||||
|
|
@ -5143,12 +5365,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# refund there would drive their counter negative.
|
||||
pipeline_operations.extend(
|
||||
self._build_reservation_aware_tpm_ops(
|
||||
targets=list(stash.reserved_scopes),
|
||||
targets=tuple(scope for scope in stash.reserved_scopes if scope[0] != PTU_TEAM_DESCRIPTOR_KEY),
|
||||
reserved_scopes=stash.reserved_scopes,
|
||||
actual_tokens=tpm_actual,
|
||||
reserved_tokens=reserved_tokens,
|
||||
)
|
||||
)
|
||||
pipeline_operations.extend(
|
||||
self._build_ptu_failure_settlement_ops(
|
||||
stash,
|
||||
kwargs, # pyright: ignore[reportUnknownArgumentType] # hook kwargs are unannotated
|
||||
tpm_actual,
|
||||
)
|
||||
)
|
||||
|
||||
# Settle project ITPM/OTPM reservations the same way: at the
|
||||
# recovered partial usage, or a full refund when there is none.
|
||||
|
|
@ -5337,11 +5566,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(request_data)
|
||||
|
||||
combined_ops: Final = (
|
||||
self._build_reservation_aware_tpm_ops(
|
||||
targets=tuple(stash.reserved_scopes),
|
||||
reserved_scopes=stash.reserved_scopes,
|
||||
actual_tokens=tpm_actual,
|
||||
reserved_tokens=reserved_tokens,
|
||||
(
|
||||
*self._build_reservation_aware_tpm_ops(
|
||||
targets=tuple(scope for scope in stash.reserved_scopes if scope[0] != PTU_TEAM_DESCRIPTOR_KEY),
|
||||
reserved_scopes=stash.reserved_scopes,
|
||||
actual_tokens=tpm_actual,
|
||||
reserved_tokens=reserved_tokens,
|
||||
),
|
||||
*self._build_ptu_failure_settlement_ops(
|
||||
stash,
|
||||
request_data, # pyright: ignore[reportUnknownArgumentType] # request_data is a bare dict
|
||||
tpm_actual,
|
||||
),
|
||||
)
|
||||
if reserved_tokens > 0
|
||||
else ()
|
||||
|
|
|
|||
|
|
@ -34,10 +34,12 @@ from litellm.proxy.management_endpoints.daily_activity_scopes import (
|
|||
EntityScopeResolver,
|
||||
ResolvedScope,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, with_ptu_consumption
|
||||
from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error
|
||||
from litellm.proxy.management_helpers.utils import management_endpoint_wrapper
|
||||
from litellm.proxy.utils import PrismaClient, get_prisma_client_or_throw
|
||||
from litellm.repositories.daily_activity_repository import DailyActivityRepository
|
||||
from litellm.router import Router
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
CacheLeakageKeysResponse,
|
||||
DailyActivityKeyPageResponse,
|
||||
|
|
@ -68,6 +70,12 @@ def get_daily_activity_repository() -> DailyActivityRepository:
|
|||
return daily_activity_repository(get_daily_activity_prisma_client())
|
||||
|
||||
|
||||
def get_daily_activity_llm_router() -> Router | None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return llm_router
|
||||
|
||||
|
||||
def _date_range_error(query: EntityQuery, *, user_aggregated: bool) -> InvalidDateRange | None:
|
||||
if user_aggregated:
|
||||
date_range: Final = parse_canonical_date_range(query.start_date, query.end_date)
|
||||
|
|
@ -244,6 +252,7 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver,
|
|||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
repository: Annotated[DailyActivityRepository, Depends(get_daily_activity_repository)],
|
||||
prisma_client: Annotated[PrismaClient, Depends(get_daily_activity_prisma_client)],
|
||||
llm_router: Annotated[Router | None, Depends(get_daily_activity_llm_router)],
|
||||
api_key_limit: Annotated[
|
||||
int, Query(ge=1, le=constants.USAGE_TOP_API_KEYS_MAX)
|
||||
] = constants.USAGE_TOP_API_KEYS_DEFAULT,
|
||||
|
|
@ -256,13 +265,16 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver,
|
|||
prisma_client,
|
||||
user_aggregated=resolver.entity == "user",
|
||||
)
|
||||
return await get_daily_activity_aggregated(
|
||||
activity: Final = await get_daily_activity_aggregated(
|
||||
repository,
|
||||
resolved.scope,
|
||||
entity_metadata_field=resolved.entity_metadata,
|
||||
include_entity_breakdown=resolver.include_entity_breakdown,
|
||||
api_key_limit=api_key_limit,
|
||||
)
|
||||
if resolver.entity != "team":
|
||||
return activity
|
||||
return with_ptu_consumption(activity, llm_router, single_team_id(resolved.scope.entity_ids))
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequen
|
|||
from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress
|
||||
from dataclasses import dataclass
|
||||
from fnmatch import fnmatchcase
|
||||
from functools import partial
|
||||
from json import JSONDecodeError
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
|
||||
|
|
@ -37,6 +38,7 @@ from litellm.litellm_core_utils.ptu_pricing import (
|
|||
PTU_ZEROED_PRICING_FIELDS,
|
||||
PTU_ZEROED_TABLE_FIELDS,
|
||||
SEARCH_CONTEXT_SIZES,
|
||||
parsed_ptu_shares,
|
||||
ptu_config_error,
|
||||
)
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
|
||||
|
|
@ -98,6 +100,7 @@ from litellm.proxy.spend_tracking.ptu_feature_flag import (
|
|||
is_ptu_cost_attribution_enabled,
|
||||
)
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.repositories.chunked_in import find_many_in
|
||||
from litellm.repositories.credentials_repository import CredentialsRepository
|
||||
from litellm.repositories.model_repository import ModelRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
|
|
@ -257,6 +260,9 @@ class _ExistingModelRow(Protocol):
|
|||
|
||||
|
||||
class _TeamRow(Protocol):
|
||||
@property
|
||||
def team_id(self) -> str: ...
|
||||
|
||||
@property
|
||||
def models(self) -> Sequence[str]: ...
|
||||
|
||||
|
|
@ -266,6 +272,8 @@ class _TeamRow(Protocol):
|
|||
class _TeamLookupTable(Protocol):
|
||||
def find_unique(self, *, where: Mapping[str, object]) -> Awaitable[_TeamRow | None]: ...
|
||||
|
||||
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_TeamRow]: ...
|
||||
|
||||
|
||||
class _TeamTable(_TeamLookupTable, Protocol):
|
||||
def update(
|
||||
|
|
@ -780,6 +788,21 @@ def _validate_ptu_model_info(model_info: Mapping[str, object]) -> None:
|
|||
raise HTTPException(status_code=400, detail=error)
|
||||
|
||||
|
||||
async def _raise_if_ptu_share_teams_missing(
|
||||
model_info: Mapping[str, object], team_table: Callable[[], _TeamLookupTable]
|
||||
) -> None:
|
||||
"""Hold every team named in ``ptu_shares`` to the existence check ``team_id`` already gets."""
|
||||
shares: Final = parsed_ptu_shares(model_info.get("ptu_shares"))
|
||||
if shares is None:
|
||||
return
|
||||
rows: Final = await find_many_in(team_table(), "team_id", shares.keys())
|
||||
found: Final = frozenset(row.team_id for row in rows)
|
||||
missing: Final = tuple(team_id for team_id in shares if team_id not in found)
|
||||
if not missing:
|
||||
return
|
||||
raise HTTPException(status_code=400, detail={"error": f"Team id={', '.join(missing)} does not exist in db"})
|
||||
|
||||
|
||||
# The mirrored per-token pricing fields plus the remaining rates the public cost map or a
|
||||
# provider default would otherwise supply (the cache back-fills, the Maps grounding rate). An
|
||||
# unset field falls back to those sources, so a field left out here is one a PTU deployment
|
||||
|
|
@ -1623,8 +1646,10 @@ async def _update_team_model_in_db(
|
|||
# raising the rate on a configured model carries no ptu_effective_from, which the
|
||||
# stored row supplies.
|
||||
if patch_data.model_info is not None:
|
||||
_raise_if_ptu_cost_attribution_disabled(patch_data.model_info.model_dump(exclude_none=True))
|
||||
incoming_model_info: Final = patch_data.model_info.model_dump(exclude_none=True)
|
||||
_raise_if_ptu_cost_attribution_disabled(incoming_model_info)
|
||||
_validate_ptu_model_info(_merged_ptu_model_info(db_model=db_model, patch_data=patch_data))
|
||||
await _raise_if_ptu_share_teams_missing(incoming_model_info, partial(_repo_team_table, prisma_client))
|
||||
_raise_if_ptu_deployment_is_priced(
|
||||
model_info=_merged_ptu_model_info(db_model=db_model, patch_data=patch_data),
|
||||
supplied=(
|
||||
|
|
@ -2462,6 +2487,7 @@ async def add_new_model(
|
|||
incoming_model_info: Final = model_params.model_info.model_dump(exclude_none=True)
|
||||
_raise_if_ptu_cost_attribution_disabled(incoming_model_info)
|
||||
_validate_ptu_model_info(incoming_model_info)
|
||||
await _raise_if_ptu_share_teams_missing(incoming_model_info, partial(_repo_team_table, prisma_client))
|
||||
priced_model_params: Final = _ptu_priced_deployment(model_params)
|
||||
|
||||
if store_model_in_db is True:
|
||||
|
|
|
|||
140
litellm/proxy/management_endpoints/ptu_consumption.py
Normal file
140
litellm/proxy/management_endpoints/ptu_consumption.py
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
"""PTU-hours a team's tokens amount to, attached to a daily activity response.
|
||||
|
||||
Azure sizes a provisioned deployment in normalized tokens per minute per PTU, so the prompt,
|
||||
cached, and completion tokens a team sent to a PTU model group convert back to the share of
|
||||
a PTU-hour it consumed, reported next to the raw token counts.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
from litellm.llms.azure.ptu_capacity import PTUCapacity, normalized_tokens, ptu_hours
|
||||
from litellm.router import Router
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.ptu_shares import (
|
||||
model_group_ptu_capacity,
|
||||
routed_deployments,
|
||||
team_servable_deployments,
|
||||
)
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
DailySpendData,
|
||||
MetricWithMetadata,
|
||||
SpendAnalyticsPaginatedResponse,
|
||||
SpendMetrics,
|
||||
)
|
||||
|
||||
|
||||
def _with_ptu_hours(metrics: SpendMetrics, capacity: PTUCapacity) -> SpendMetrics:
|
||||
consumed: Final = ptu_hours(
|
||||
capacity,
|
||||
normalized_tokens(
|
||||
capacity,
|
||||
prompt_tokens=metrics.prompt_tokens,
|
||||
completion_tokens=metrics.completion_tokens,
|
||||
cache_read_tokens=metrics.cache_read_input_tokens,
|
||||
),
|
||||
)
|
||||
return metrics.model_copy(update=MappingProxyType({"ptu_hours": consumed}))
|
||||
|
||||
|
||||
def _model_group_with_ptu_hours(bucket: MetricWithMetadata, capacity: PTUCapacity) -> MetricWithMetadata:
|
||||
api_key_breakdown: Final = {
|
||||
api_key: key_bucket.model_copy(
|
||||
update=MappingProxyType({"metrics": _with_ptu_hours(key_bucket.metrics, capacity)})
|
||||
)
|
||||
for api_key, key_bucket in bucket.api_key_breakdown.items()
|
||||
}
|
||||
return bucket.model_copy(
|
||||
update=MappingProxyType(
|
||||
{"metrics": _with_ptu_hours(bucket.metrics, capacity), "api_key_breakdown": api_key_breakdown}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _day_with_ptu_hours(
|
||||
day: DailySpendData, capacity_for_model_group: Callable[[str], PTUCapacity | None]
|
||||
) -> DailySpendData:
|
||||
priced: Final = MappingProxyType(
|
||||
{
|
||||
model_group: _model_group_with_ptu_hours(bucket, capacity)
|
||||
for model_group, bucket in day.breakdown.model_groups.items()
|
||||
if (capacity := capacity_for_model_group(model_group)) is not None
|
||||
}
|
||||
)
|
||||
if not priced:
|
||||
return day
|
||||
model_groups: Final = {
|
||||
**day.breakdown.model_groups,
|
||||
**priced,
|
||||
}
|
||||
return day.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"metrics": day.metrics.model_copy(
|
||||
update=MappingProxyType({"ptu_hours": sum(bucket.metrics.ptu_hours for bucket in priced.values())})
|
||||
),
|
||||
"breakdown": day.breakdown.model_copy(update=MappingProxyType({"model_groups": model_groups})),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def attach_ptu_hours(
|
||||
response: SpendAnalyticsPaginatedResponse, capacity_for_model_group: Callable[[str], PTUCapacity | None]
|
||||
) -> SpendAnalyticsPaginatedResponse:
|
||||
"""The response with ``ptu_hours`` filled in on every PTU model group and its api keys,
|
||||
on each day, and on the total, from the tokens already on the page.
|
||||
|
||||
A model group the resolver has no sizing row for keeps ``ptu_hours`` at zero.
|
||||
"""
|
||||
days: Final = tuple(_day_with_ptu_hours(day, capacity_for_model_group) for day in response.results)
|
||||
results: Final = list(days)
|
||||
return response.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"results": results,
|
||||
"metadata": response.metadata.model_copy(
|
||||
update=MappingProxyType({"total_ptu_hours": sum(day.metrics.ptu_hours for day in days)})
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def capacity_by_requested_name(llm_router: Router, team_id: str | None) -> Callable[[str], PTUCapacity | None]:
|
||||
"""The sizing row behind the name a usage row is keyed by, resolved the way the ceiling
|
||||
resolves a request: an alias to its group, then a group, a routing group, a deployment id,
|
||||
or a provider model to the deployments it is served from. For one team those are narrowed
|
||||
to the deployments it can be served from, its shared one first, so a team with no share on
|
||||
a name reads no PTU-hours for it; a page spanning teams keeps every deployment behind the name."""
|
||||
listed_rows: Final = llm_router.get_model_list() or ()
|
||||
aliases: Final = llm_router.model_group_alias
|
||||
|
||||
def capacity_for(requested_model: str) -> PTUCapacity | None:
|
||||
model_group: Final = resolve_model_group_alias(aliases, requested_model) or requested_model
|
||||
routed: Final = routed_deployments(
|
||||
listed_rows,
|
||||
llm_router.model_list, # pyright: ignore[reportUnknownArgumentType] # Router.model_list is a bare list
|
||||
model_group,
|
||||
)
|
||||
return model_group_ptu_capacity(routed if team_id is None else team_servable_deployments(routed, team_id))
|
||||
|
||||
return capacity_for
|
||||
|
||||
|
||||
def single_team_id(team_ids: Sequence[str] | None) -> str | None:
|
||||
"""The one team a usage page is scoped to, else None for a page spanning several or all teams."""
|
||||
return team_ids[0] if team_ids is not None and len(team_ids) == 1 else None
|
||||
|
||||
|
||||
def with_ptu_consumption(
|
||||
activity: SpendAnalyticsPaginatedResponse, llm_router: Router | None, team_id: str | None
|
||||
) -> SpendAnalyticsPaginatedResponse:
|
||||
"""``activity`` with PTU-hours attached from the deployments ``team_id`` is served from (every
|
||||
sized deployment when the page spans teams), untouched while PTU cost attribution is off or no
|
||||
router is loaded."""
|
||||
if llm_router is None or not is_ptu_cost_attribution_enabled():
|
||||
return activity
|
||||
return attach_ptu_hours(activity, capacity_by_requested_name(llm_router, team_id))
|
||||
|
|
@ -143,6 +143,7 @@ from litellm.proxy.management_endpoints.common_utils import (
|
|||
from litellm.proxy.management_endpoints.organization_endpoints import (
|
||||
add_member_to_organization,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, with_ptu_consumption
|
||||
from litellm.proxy.management_endpoints.router_weights import validate_router_settings_weights
|
||||
from litellm.proxy.management_endpoints.tag_management_endpoints import (
|
||||
get_daily_activity,
|
||||
|
|
@ -6655,6 +6656,7 @@ async def get_team_daily_activity(
|
|||
SpendAnalyticsPaginatedResponse: Paginated response containing daily activity data.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
llm_router,
|
||||
prisma_client,
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
|
|
@ -6673,7 +6675,7 @@ async def get_team_daily_activity(
|
|||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
return await get_daily_activity(
|
||||
activity: Final = await get_daily_activity(
|
||||
prisma_client=prisma_client,
|
||||
table_name="litellm_dailyteamspend",
|
||||
entity_id_field="team_id",
|
||||
|
|
@ -6687,6 +6689,7 @@ async def get_team_daily_activity(
|
|||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
return with_ptu_consumption(activity, llm_router, single_team_id(scope.team_ids))
|
||||
|
||||
|
||||
_MAX_AGGREGATED_RANGE_DAYS: Final = 400
|
||||
|
|
|
|||
|
|
@ -14,9 +14,10 @@ and share the existing unique constraint.
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
|
|
@ -82,7 +83,8 @@ class BackfillResult:
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PTUModel:
|
||||
"""A model deployment carrying valid manual PTU config."""
|
||||
"""One team's holding on a deployment carrying valid manual PTU config: the whole count
|
||||
under a single team_id, or that team's share of a split deployment."""
|
||||
|
||||
model_id: str
|
||||
model_name: str
|
||||
|
|
@ -98,10 +100,10 @@ def _public_model_name(row: object, model_info: Mapping[str, object]) -> str:
|
|||
|
||||
Creating a team-scoped deployment rewrites model_name to a synthetic routing key
|
||||
(``model_name_<team_id>_<uuid4>``) and keeps the chosen name in
|
||||
``model_info.team_public_model_name``. PTU config is only accepted alongside a
|
||||
team_id, so every PTU deployment carries that synthetic name; keying the sentinel
|
||||
row on it would file each charge under a UUID that no usage view can resolve and
|
||||
that never lines up with the same model's request rows.
|
||||
``model_info.team_public_model_name``. A PTU deployment held by one team_id carries
|
||||
that synthetic name; keying the sentinel row on it would file each charge under a
|
||||
UUID that no usage view can resolve and that never lines up with the same model's
|
||||
request rows. A deployment split into ptu_shares keeps the name it was declared with.
|
||||
"""
|
||||
public_name: Final = model_info.get("team_public_model_name")
|
||||
if isinstance(public_name, str) and public_name:
|
||||
|
|
@ -128,7 +130,7 @@ def _decode_model_info(raw: object) -> "Mapping[str, object] | None":
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PTUDeployment:
|
||||
"""A deployment in the shape ``_parse_ptu_model`` reads, whatever declared it.
|
||||
"""A deployment in the shape ``_parse_ptu_models`` reads, whatever declared it.
|
||||
|
||||
A ``LiteLLM_ProxyModelTable`` row already has it. A router entry does not: its id
|
||||
lives in ``model_info.id`` rather than on the entry itself.
|
||||
|
|
@ -160,26 +162,30 @@ def _router_deployment(deployment: Mapping[str, object]) -> _PTUDeployment | Non
|
|||
)
|
||||
|
||||
|
||||
def _parse_ptu_model(row: object) -> PTUModel | None:
|
||||
"""Return a PTUModel when the deployment carries valid manual PTU config, else None.
|
||||
def _parse_ptu_models(row: object) -> tuple[PTUModel, ...]:
|
||||
"""One PTUModel per team the deployment's valid PTU config attributes capacity to, else empty.
|
||||
|
||||
Valid means model_info has a positive ptu_count, a non-negative
|
||||
cost_per_ptu_per_hour, and a team_id (1 model -> 1 team).
|
||||
A deployment held by one team_id yields one entry carrying the whole count; a deployment
|
||||
split into ptu_shares yields one per team carrying that team's share, so each team's row
|
||||
accrues its share of the flat cost.
|
||||
"""
|
||||
model_info: Final = _decode_model_info(getattr(row, "model_info", None))
|
||||
if model_info is None:
|
||||
return None
|
||||
return ()
|
||||
terms: Final = ptu_terms(model_info)
|
||||
if terms is None:
|
||||
return None
|
||||
return PTUModel(
|
||||
model_id=str(getattr(row, "model_id", "") or ""),
|
||||
model_name=_public_model_name(row, model_info),
|
||||
team_id=terms.team_id,
|
||||
ptu_count=terms.ptu_count,
|
||||
cost_per_ptu_per_hour=terms.cost_per_ptu_per_hour,
|
||||
effective_from=terms.effective_from,
|
||||
effective_to=terms.effective_to,
|
||||
return ()
|
||||
return tuple(
|
||||
PTUModel(
|
||||
model_id=str(getattr(row, "model_id", "") or ""),
|
||||
model_name=_public_model_name(row, model_info),
|
||||
team_id=team_id,
|
||||
ptu_count=share,
|
||||
cost_per_ptu_per_hour=terms.cost_per_ptu_per_hour,
|
||||
effective_from=terms.effective_from,
|
||||
effective_to=terms.effective_to,
|
||||
)
|
||||
for team_id, share in sorted(terms.shares.items())
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -376,9 +382,7 @@ async def _load_ptu_models(prisma_client: "PrismaClient", *, router: object | No
|
|||
rows: Final = await _proxy_model_table(prisma_client).find_many()
|
||||
db_ids: Final = frozenset(model_id for row in rows if (model_id := str(getattr(row, "model_id", "") or "")))
|
||||
config_records: Final = _config_deployments(router, owned_by_db=db_ids)
|
||||
models: Final = tuple(
|
||||
parsed for parsed in (_parse_ptu_model(row) for row in (*rows, *config_records)) if parsed is not None
|
||||
)
|
||||
models: Final = tuple(chain.from_iterable(_parse_ptu_models(row) for row in (*rows, *config_records)))
|
||||
return _LoadedDeployments(
|
||||
models=models,
|
||||
scanned_ids=db_ids
|
||||
|
|
@ -451,16 +455,17 @@ async def run_ptu_flat_cost_rollup(
|
|||
scanned_ids=loaded.scanned_ids,
|
||||
)
|
||||
|
||||
models_processed: Final = len(frozenset(model.model_id for model in ptu_models))
|
||||
verbose_proxy_logger.info(
|
||||
"PTU rollup for %s: %d PTU models processed, %d rows written, %d rows failed",
|
||||
date_str,
|
||||
len(ptu_models),
|
||||
models_processed,
|
||||
rows_written,
|
||||
rows_failed,
|
||||
)
|
||||
return RollupResult(
|
||||
day=day,
|
||||
models_processed=len(ptu_models),
|
||||
models_processed=models_processed,
|
||||
rows_written=rows_written,
|
||||
rows_failed=rows_failed,
|
||||
lapsed=_lapsed_models(ptu_models, run_started),
|
||||
|
|
@ -485,11 +490,13 @@ def _lapsed_models(ptu_models: tuple[PTUModel, ...], now: datetime) -> tuple[str
|
|||
charge the provider does not make for reserved capacity.
|
||||
"""
|
||||
return tuple(
|
||||
_slack_safe(model.model_name)
|
||||
for model in sorted(
|
||||
(m for m in ptu_models if m.effective_to is not None and m.effective_to <= now),
|
||||
key=lambda m: m.effective_to,
|
||||
reverse=True,
|
||||
dict.fromkeys(
|
||||
_slack_safe(model.model_name)
|
||||
for model in sorted(
|
||||
(m for m in ptu_models if m.effective_to is not None and m.effective_to <= now),
|
||||
key=lambda m: m.effective_to or now,
|
||||
reverse=True,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -511,6 +518,15 @@ def _backfill_window(ptu_models: tuple[PTUModel, ...], end: date) -> tuple[date,
|
|||
return tuple(start + timedelta(days=offset) for offset in range((end - start).days + 1))
|
||||
|
||||
|
||||
def _unpriced_charges(
|
||||
ptu_models: tuple[PTUModel, ...], days: tuple[date, ...], priced: frozenset[tuple[str, str, str]]
|
||||
) -> Iterator[tuple[str, _PTUCharge]]:
|
||||
for day in days:
|
||||
for charge in _aggregate_charges(ptu_models, day):
|
||||
if (charge.team_id, charge.model_id, day.isoformat()) not in priced:
|
||||
yield (day.isoformat(), charge)
|
||||
|
||||
|
||||
async def _existing_sentinel_keys(
|
||||
prisma_client: "PrismaClient",
|
||||
*,
|
||||
|
|
@ -566,12 +582,7 @@ async def run_ptu_flat_cost_backfill(
|
|||
return BackfillResult(start=end, end=end, days_scanned=0, rows_written=0)
|
||||
|
||||
priced: Final = await _existing_sentinel_keys(prisma_client, start=days[0], end=days[-1])
|
||||
missing: Final = tuple(
|
||||
(day.isoformat(), charge)
|
||||
for day in days
|
||||
for charge in _aggregate_charges(ptu_models, day)
|
||||
if (charge.team_id, charge.model_id, day.isoformat()) not in priced
|
||||
)
|
||||
missing: Final = tuple(_unpriced_charges(ptu_models, days, priced))
|
||||
if not missing:
|
||||
return BackfillResult(start=days[0], end=days[-1], days_scanned=len(days), rows_written=0)
|
||||
|
||||
|
|
|
|||
|
|
@ -185,6 +185,7 @@ from litellm.router_utils.common_utils import (
|
|||
get_request_team_id,
|
||||
provider_for_generic_call,
|
||||
resolve_model_group_alias,
|
||||
team_may_use_deployment,
|
||||
truncate_fallback_error_detail,
|
||||
warn_on_provider_credential_mismatch,
|
||||
)
|
||||
|
|
@ -244,6 +245,7 @@ from litellm.router_utils.pre_call_checks.model_rate_limit_check import (
|
|||
from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import (
|
||||
PromptCachingDeploymentCheck,
|
||||
)
|
||||
from litellm.router_utils.ptu_shares import filter_ptu_shared_deployments, ptu_capacity_warning
|
||||
from litellm.router_utils.reasoning_effort_capability import (
|
||||
deployment_is_catalog_mapped,
|
||||
intersect_supported_reasoning_efforts,
|
||||
|
|
@ -8955,11 +8957,27 @@ class Router:
|
|||
ptu_error: Final = (
|
||||
(ptu_config_error(_model_info, model_name=_model_name) or identity_error) if config_sourced else None
|
||||
)
|
||||
if ptu_error is not None and is_ptu_cost_attribution_enabled():
|
||||
declares_split: Final = _model_info.get("ptu_shares") is not None
|
||||
if ptu_error is not None and (declares_split or is_ptu_cost_attribution_enabled()):
|
||||
raise ValueError(ptu_error)
|
||||
access_windows_error: Final = access_windows_config_error(_model_info, model_name=_model_name)
|
||||
if access_windows_error is not None:
|
||||
raise ValueError(access_windows_error)
|
||||
capacity_warning: Final = (
|
||||
ptu_capacity_warning(
|
||||
_model_name,
|
||||
MappingProxyType(
|
||||
{ # pyright: ignore[reportUnknownArgumentType] # router deployment dicts are untyped
|
||||
"model_info": _model_info,
|
||||
"litellm_params": _litellm_params,
|
||||
}
|
||||
),
|
||||
)
|
||||
if is_ptu_cost_attribution_enabled()
|
||||
else None
|
||||
)
|
||||
if capacity_warning is not None:
|
||||
verbose_router_logger.warning(capacity_warning)
|
||||
zeroed_pricing: Final = zeroed_ptu_pricing(_model_info, _litellm_params) if config_sourced else None
|
||||
litellm_params: Final[LiteLLM_Params] = LiteLLM_Params(
|
||||
**( # pyright: ignore[reportArgumentType] # untyped merged dict; already true for every field here
|
||||
|
|
@ -10309,8 +10327,7 @@ class Router:
|
|||
callers from that same team; deployments without a team owner are shared.
|
||||
"""
|
||||
model_info: Final = model.get("model_info") if isinstance(model, dict) else model.model_info
|
||||
owner_team_id: Final = model_info.get("team_id") if model_info is not None else None
|
||||
return owner_team_id is None or owner_team_id == team_id
|
||||
return team_may_use_deployment(model_info.get("team_id") if model_info is not None else None, team_id)
|
||||
|
||||
def _get_model_group_deployment_usable_by_team(
|
||||
self, model_group_name: str, team_id: str | None
|
||||
|
|
@ -12918,7 +12935,14 @@ class Router:
|
|||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
return result.deployments
|
||||
shared: Final = filter_ptu_shared_deployments(result.deployments, request_team_id)
|
||||
if shared.withheld and len(shared.deployments) == 0:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Deployment {model} is reserved for the teams holding a PTU share of it",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
return shared.deployments
|
||||
|
||||
def _filter_deployments_by_model_access_groups(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -28,6 +28,11 @@ def _is_proxy_admin_request(request_kwargs: Mapping[str, object] | None) -> bool
|
|||
return getattr(user_api_key_auth, "user_role", None) == "proxy_admin"
|
||||
|
||||
|
||||
def team_may_use_deployment(owner_team_id: object, team_id: str | None) -> bool:
|
||||
"""A deployment owned by a team (``model_info.team_id``) serves only that team; one with no owner serves every team."""
|
||||
return owner_team_id is None or owner_team_id == team_id
|
||||
|
||||
|
||||
def get_request_team_id(request_kwargs: Mapping[str, object] | None) -> str | None:
|
||||
"""The caller's team id, from whichever metadata bucket this surface writes to."""
|
||||
if request_kwargs is None:
|
||||
|
|
|
|||
219
litellm/router_utils/ptu_shares.py
Normal file
219
litellm/router_utils/ptu_shares.py
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
"""Per-team PTU shares on a shared Azure provisioned deployment.
|
||||
|
||||
A deployment declaring ``model_info.ptu_shares`` is served only to the teams named in it,
|
||||
and each team's share converts to a normalized-tokens-per-minute ceiling through the
|
||||
model's Azure sizing row, so the proxy enforces the split instead of an operator hand
|
||||
converting PTUs to ``model_tpm_limit``.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Generic, TypeVar
|
||||
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_model_info_mapping, parsed_ptu_shares, ptu_terms
|
||||
from litellm.llms.azure.ptu_capacity import PTUCapacity, deployment_ptu_capacity, is_azure_deployment
|
||||
from litellm.router_utils.common_utils import team_may_use_deployment
|
||||
|
||||
_DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PTUTeamCeiling:
|
||||
model_group: str
|
||||
tpm_limit: int
|
||||
output_to_input_ratio: float
|
||||
cached_input_ratio: float
|
||||
|
||||
def raw_output_token_limit(self) -> int:
|
||||
"""The ceiling expressed in unweighted tokens: what fits under it when every token is output."""
|
||||
return max(1, int(self.tpm_limit / max(self.output_to_input_ratio, 1.0)))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PTUShareFilterResult(Generic[_DeploymentT]):
|
||||
deployments: tuple[_DeploymentT, ...]
|
||||
withheld: bool
|
||||
|
||||
|
||||
def _deployment_shares(deployment: Mapping[str, object]) -> Mapping[str, int] | None:
|
||||
"""The teams a deployment is declared split across, else None.
|
||||
|
||||
The declared map is the access rule on its own; the pricing terms are read separately, so a
|
||||
split whose terms are missing still serves only the teams it names rather than everyone.
|
||||
"""
|
||||
model_info: Final = deployment.get("model_info")
|
||||
if not is_model_info_mapping(model_info):
|
||||
return None
|
||||
return parsed_ptu_shares(model_info.get("ptu_shares"))
|
||||
|
||||
|
||||
def filter_ptu_shared_deployments(
|
||||
healthy_deployments: Sequence[_DeploymentT], request_team_id: str | None
|
||||
) -> PTUShareFilterResult[_DeploymentT]:
|
||||
"""Drop every deployment split into PTU shares that ``request_team_id`` holds none of.
|
||||
|
||||
A caller with no team, the master key included, holds no share, the same way an access
|
||||
window reserves a deployment away from it.
|
||||
"""
|
||||
checks: Final = tuple((deployment, _deployment_shares(deployment)) for deployment in healthy_deployments)
|
||||
kept: Final = tuple(
|
||||
deployment for deployment, shares in checks if shares is None or (request_team_id or "") in shares
|
||||
)
|
||||
return PTUShareFilterResult(deployments=kept, withheld=len(kept) < len(checks))
|
||||
|
||||
|
||||
def team_ptu_ceiling(
|
||||
listed_rows: Sequence[Mapping[str, object]],
|
||||
deployments: Sequence[Mapping[str, object]],
|
||||
team_id: str,
|
||||
requested_model: str,
|
||||
) -> PTUTeamCeiling | None:
|
||||
"""The per-minute normalized-token ceiling ``team_id``'s shares on the group serving
|
||||
``requested_model`` add up to, else None when the team holds no share on a deployment with
|
||||
a known sizing row.
|
||||
|
||||
``listed_rows`` is every row the router lists a name under, alias and routing-group copies
|
||||
included, and ``deployments`` is the router's own deployments. A name resolves to the
|
||||
deployments behind it, so a group, a routing group, a deployment id, and a provider model
|
||||
all count against the one ceiling of the group whose shared deployment the team can be
|
||||
served from.
|
||||
|
||||
Two shared deployments of different models in one group are weighted by the larger
|
||||
output and cached-input ratios, which over-counts those tokens on the cheaper one rather
|
||||
than under-counting them on the dearer one.
|
||||
"""
|
||||
model_group: Final = _model_group_of(listed_rows, deployments, team_id, requested_model)
|
||||
priced: Final = tuple(
|
||||
(shares[team_id], capacity)
|
||||
for deployment in model_group_deployments(deployments, model_group)
|
||||
if (shares := _deployment_shares(deployment)) is not None
|
||||
and team_id in shares
|
||||
and (capacity := deployment_ptu_capacity(deployment)) is not None
|
||||
)
|
||||
if not priced:
|
||||
return None
|
||||
return PTUTeamCeiling(
|
||||
model_group=model_group,
|
||||
tpm_limit=sum(capacity.input_tpm_for(share) for share, capacity in priced),
|
||||
output_to_input_ratio=max(capacity.output_to_input_ratio for _, capacity in priced),
|
||||
cached_input_ratio=max(capacity.cached_input_ratio for _, capacity in priced),
|
||||
)
|
||||
|
||||
|
||||
def model_group_deployments(deployments: Sequence[_DeploymentT], model_group: str) -> tuple[_DeploymentT, ...]:
|
||||
"""Every deployment serving ``model_group``: by its own name, or by the public name a
|
||||
team-scoped deployment keeps in ``model_info.team_public_model_name`` after the router
|
||||
renames it to a unique internal one."""
|
||||
return tuple(
|
||||
deployment
|
||||
for deployment in deployments
|
||||
if deployment.get("model_name") == model_group
|
||||
or (
|
||||
isinstance(model_info := deployment.get("model_info"), Mapping)
|
||||
and model_info.get("team_public_model_name") == model_group
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _deployment_id(deployment: Mapping[str, object]) -> object:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
return model_info.get("id") if isinstance(model_info, Mapping) else None
|
||||
|
||||
|
||||
def _names_deployment(deployment: Mapping[str, object], name: str) -> bool:
|
||||
litellm_params: Final = deployment.get("litellm_params")
|
||||
return _deployment_id(deployment) == name or (
|
||||
isinstance(litellm_params, Mapping) and litellm_params.get("model") == name
|
||||
)
|
||||
|
||||
|
||||
def _deployment_model_group(deployment: Mapping[str, object]) -> str | None:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
public_name: Final = model_info.get("team_public_model_name") if isinstance(model_info, Mapping) else None
|
||||
if isinstance(public_name, str):
|
||||
return public_name
|
||||
model_name: Final = deployment.get("model_name")
|
||||
return model_name if isinstance(model_name, str) else None
|
||||
|
||||
|
||||
def routed_deployments(
|
||||
listed_rows: Sequence[Mapping[str, object]], deployments: Sequence[Mapping[str, object]], requested_model: str
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""The router's own deployments behind ``requested_model``: those of the rows listed under it
|
||||
when it names a group, else the one it names by id or by provider model, the way the router
|
||||
falls back to them."""
|
||||
listed_ids: Final = frozenset(_deployment_id(row) for row in model_group_deployments(listed_rows, requested_model))
|
||||
if listed_ids:
|
||||
return tuple(deployment for deployment in deployments if _deployment_id(deployment) in listed_ids)
|
||||
return tuple(deployment for deployment in deployments if _names_deployment(deployment, requested_model))
|
||||
|
||||
|
||||
def _model_group_of(
|
||||
listed_rows: Sequence[Mapping[str, object]],
|
||||
deployments: Sequence[Mapping[str, object]],
|
||||
team_id: str,
|
||||
requested_model: str,
|
||||
) -> str:
|
||||
"""The group of the deployment behind ``requested_model`` this team can be served from, one
|
||||
holding its share first."""
|
||||
servable: Final = team_servable_deployments(routed_deployments(listed_rows, deployments, requested_model), team_id)
|
||||
return next(
|
||||
(group for deployment in servable if (group := _deployment_model_group(deployment)) is not None),
|
||||
requested_model,
|
||||
)
|
||||
|
||||
|
||||
def _deployment_owner(deployment: Mapping[str, object]) -> object:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
return model_info.get("team_id") if isinstance(model_info, Mapping) else None
|
||||
|
||||
|
||||
def team_servable_deployments(deployments: Sequence[_DeploymentT], team_id: str) -> tuple[_DeploymentT, ...]:
|
||||
"""The deployments ``team_id`` can be served from, the ones it holds a share on first: never one
|
||||
reserved for another team, nor one split into shares it holds none of."""
|
||||
shareable: Final = filter_ptu_shared_deployments(deployments, team_id).deployments
|
||||
servable: Final = tuple(
|
||||
deployment for deployment in shareable if team_may_use_deployment(_deployment_owner(deployment), team_id)
|
||||
)
|
||||
return tuple(sorted(servable, key=lambda deployment: _deployment_shares(deployment) is None))
|
||||
|
||||
|
||||
def model_group_ptu_capacity(deployments: Sequence[Mapping[str, object]]) -> PTUCapacity | None:
|
||||
"""The sizing row of the group's first reserved deployment, single-team or shared, so a
|
||||
team's tokens on that group convert to PTU-hours."""
|
||||
return next(
|
||||
(
|
||||
capacity
|
||||
for deployment in deployments
|
||||
if is_model_info_mapping(model_info := deployment.get("model_info"))
|
||||
and ptu_terms(model_info) is not None
|
||||
and (capacity := deployment_ptu_capacity(deployment)) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def ptu_capacity_warning(model_name: str, deployment: Mapping[str, object]) -> str | None:
|
||||
"""Why this reserved deployment's tokens cannot be converted to PTUs, else None.
|
||||
|
||||
Only a deployment that declares shares (whose ceilings need a sizing row) or one served by
|
||||
Azure (whose PTU hours need one) is worth warning about; a single-team reservation on another
|
||||
provider only ever used the flat-cost rollup, which needs no sizing.
|
||||
"""
|
||||
model_info: Final = deployment.get("model_info")
|
||||
if not is_model_info_mapping(model_info) or ptu_terms(model_info) is None:
|
||||
return None
|
||||
if deployment_ptu_capacity(deployment) is not None:
|
||||
return None
|
||||
has_shares: Final = model_info.get("ptu_shares") is not None
|
||||
if has_shares:
|
||||
return (
|
||||
f"PTU deployment '{model_name}' has no Azure sizing row for its model, so its PTU shares set no "
|
||||
"team ceiling and its usage reports no PTU hours; set model_info.base_model to the Azure model name"
|
||||
)
|
||||
if is_azure_deployment(deployment):
|
||||
return (
|
||||
f"PTU deployment '{model_name}' has no Azure sizing row for its model, so its usage reports no "
|
||||
"PTU hours; set model_info.base_model to the Azure model name"
|
||||
)
|
||||
return None
|
||||
|
|
@ -19,6 +19,7 @@ class GroupByDimension(str, Enum):
|
|||
class SpendMetrics(BaseModel):
|
||||
spend: float = Field(default=0.0)
|
||||
flat_cost: float = Field(default=0.0)
|
||||
ptu_hours: float = Field(default=0.0)
|
||||
prompt_tokens: int = Field(default=0)
|
||||
completion_tokens: int = Field(default=0)
|
||||
cache_read_input_tokens: int = Field(default=0)
|
||||
|
|
@ -83,6 +84,7 @@ class DailySpendData(BaseModel):
|
|||
class DailySpendMetadata(BaseModel):
|
||||
total_spend: float = Field(default=0.0)
|
||||
total_flat_cost: float = Field(default=0.0)
|
||||
total_ptu_hours: float = Field(default=0.0)
|
||||
total_prompt_tokens: int = Field(default=0)
|
||||
total_completion_tokens: int = Field(default=0)
|
||||
total_tokens: int = Field(default=0)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, Generic, Lite
|
|||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictInt, field_validator, model_validator
|
||||
from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -256,6 +256,7 @@ class ModelInfo(MirroredPricingParams):
|
|||
cost_per_ptu_per_hour: float | None = None
|
||||
ptu_effective_from: datetime.datetime | None = None
|
||||
ptu_effective_to: datetime.datetime | None = None
|
||||
ptu_shares: Mapping[str, StrictInt] | None = None
|
||||
|
||||
# when tag-based routing's "!" or "&" constraints eliminate every deployment
|
||||
# in this model group, fall back to the default-tagged pool instead of
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ def test_a_complete_reservation_is_accepted():
|
|||
terms = ptu_terms(_VALID)
|
||||
|
||||
assert terms is not None
|
||||
assert terms.team_id == "team-alpha"
|
||||
assert terms.shares == {"team-alpha": 100}
|
||||
assert terms.ptu_count == 100
|
||||
assert terms.effective_from == datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
assert terms.effective_to is None
|
||||
|
|
@ -193,8 +193,8 @@ def test_a_deployment_with_no_ptu_fields_is_not_a_ptu_deployment():
|
|||
@pytest.mark.parametrize(
|
||||
"override, expected",
|
||||
[
|
||||
({"team_id": None}, "team_id is required when PTU fields are set (one model maps to one team)"),
|
||||
({"team_id": ""}, "team_id is required when PTU fields are set (one model maps to one team)"),
|
||||
({"team_id": None}, "team_id or ptu_shares is required when PTU fields are set"),
|
||||
({"team_id": ""}, "team_id or ptu_shares is required when PTU fields are set"),
|
||||
({"cost_per_ptu_per_hour": None}, "ptu_count and cost_per_ptu_per_hour must be set together"),
|
||||
({"ptu_count": None}, "ptu_count and cost_per_ptu_per_hour must be set together"),
|
||||
({"ptu_effective_to": "2025-01-01T00:00:00Z"}, "ptu_effective_to must be after ptu_effective_from"),
|
||||
|
|
@ -356,3 +356,114 @@ def test_no_spillover_marker_returns_none():
|
|||
is None
|
||||
)
|
||||
assert azure_spillover(response_headers=None, additional_headers=None) is None
|
||||
|
||||
|
||||
# --- a reservation split into per-team shares ------------------------------------------
|
||||
|
||||
_SHARED = {**{k: v for k, v in _VALID.items() if k != "team_id"}, "ptu_shares": {"team-a": 60, "team-b": 40}}
|
||||
|
||||
|
||||
def test_a_split_reservation_attributes_each_share_to_its_team():
|
||||
terms = ptu_terms(_SHARED)
|
||||
|
||||
assert terms is not None
|
||||
assert terms.shares == {"team-a": 60, "team-b": 40}
|
||||
assert terms.ptu_count == 100
|
||||
assert ptu_config_error(_SHARED) is None
|
||||
|
||||
|
||||
def test_a_single_team_reservation_holds_the_whole_count_under_that_team():
|
||||
terms = ptu_terms(_VALID)
|
||||
|
||||
assert terms is not None
|
||||
assert sum(terms.shares.values()) == terms.ptu_count
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override, expected",
|
||||
[
|
||||
(
|
||||
{"team_id": "team-alpha"},
|
||||
"team_id and ptu_shares cannot both be set; ptu_shares lists every team the capacity is split across",
|
||||
),
|
||||
({"ptu_shares": {}}, "ptu_shares must map at least one team_id to a positive whole number of PTUs"),
|
||||
(
|
||||
{"ptu_shares": {"team-a": 0, "team-b": 100}},
|
||||
"ptu_shares must map at least one team_id to a positive whole number of PTUs",
|
||||
),
|
||||
(
|
||||
{"ptu_shares": {"team-a": 50.5, "team-b": 49.5}},
|
||||
"ptu_shares must map at least one team_id to a positive whole number of PTUs",
|
||||
),
|
||||
(
|
||||
{"ptu_shares": {"team-a": True}},
|
||||
"ptu_shares must map at least one team_id to a positive whole number of PTUs",
|
||||
),
|
||||
({"ptu_shares": {"": 100}}, "ptu_shares must map at least one team_id to a positive whole number of PTUs"),
|
||||
({"ptu_shares": {None: 100}}, "ptu_shares must map at least one team_id to a positive whole number of PTUs"),
|
||||
({"ptu_shares": {1: 100}}, "ptu_shares must map at least one team_id to a positive whole number of PTUs"),
|
||||
({"ptu_shares": ["team-a"]}, "ptu_shares must map at least one team_id to a positive whole number of PTUs"),
|
||||
({"ptu_shares": {"team-a": 60, "team-b": 30}}, "ptu_shares must add up to ptu_count (90 of 100 allocated)"),
|
||||
({"ptu_shares": {"team-a": 60, "team-b": 50}}, "ptu_shares must add up to ptu_count (110 of 100 allocated)"),
|
||||
],
|
||||
ids=[
|
||||
"team_id and shares",
|
||||
"empty shares",
|
||||
"zero share",
|
||||
"fractional share",
|
||||
"boolean share",
|
||||
"blank team",
|
||||
"null team",
|
||||
"numeric team",
|
||||
"not a mapping",
|
||||
"shares short of the count",
|
||||
"shares over the count",
|
||||
],
|
||||
)
|
||||
def test_an_incoherent_split_names_its_reason_and_reserves_nothing(override, expected):
|
||||
assert ptu_config_error({**_SHARED, **override}) == expected
|
||||
assert ptu_terms({**_SHARED, **override}) is None
|
||||
|
||||
|
||||
def test_a_whole_count_written_as_a_float_is_checked_against_the_shares_all_the_same():
|
||||
assert ptu_config_error({**_SHARED, "ptu_count": 100.0, "ptu_shares": {"team-a": 60, "team-b": 30}}) == (
|
||||
"ptu_shares must add up to ptu_count (90 of 100 allocated)"
|
||||
)
|
||||
terms = ptu_terms({**_SHARED, "ptu_count": 100.0})
|
||||
assert terms is not None
|
||||
assert terms.ptu_count == 100
|
||||
|
||||
|
||||
def test_a_fractional_count_reserves_nothing():
|
||||
assert ptu_terms({**_VALID, "ptu_count": 100.5}) is None
|
||||
assert ptu_terms({**_SHARED, "ptu_count": 100.5}) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shares_only",
|
||||
[
|
||||
{"ptu_shares": {"team-a": 60, "team-b": 40}},
|
||||
{"ptu_shares": {"team-a": 60, "team-b": 40}, "ptu_effective_from": "2026-01-01T00:00:00Z"},
|
||||
],
|
||||
ids=["shares alone", "shares with a window"],
|
||||
)
|
||||
def test_shares_without_the_count_and_rate_are_refused_instead_of_registering_unpriced(shares_only):
|
||||
"""Shares split a count, so a split with no count has nothing to add up to and would register a
|
||||
deployment the share filter and ceiling honour while flat cost never rolls up."""
|
||||
assert ptu_config_error(shares_only) == "ptu_count and cost_per_ptu_per_hour are required when ptu_shares is set"
|
||||
assert ptu_terms(shares_only) is None
|
||||
|
||||
|
||||
def test_the_split_is_named_after_the_deployment_when_the_caller_supplies_one():
|
||||
error = ptu_config_error({**_SHARED, "ptu_shares": {"team-a": 1}}, model_name="gpt-4.1-ptu")
|
||||
|
||||
assert error is not None
|
||||
assert "'gpt-4.1-ptu'" in error
|
||||
assert "1 of 100 allocated" in error
|
||||
|
||||
|
||||
def test_a_split_reservation_is_zeroed_like_a_single_team_one():
|
||||
zeroed = _with_flag(_SHARED)
|
||||
|
||||
assert zeroed["input_cost_per_token"] == 0.0
|
||||
assert zeroed["output_cost_per_token"] == 0.0
|
||||
|
|
|
|||
89
tests/unit/llms/azure/test_azure_ptu_capacity.py
Normal file
89
tests/unit/llms/azure/test_azure_ptu_capacity.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
"""Tests for the Azure PTU sizing table and the conversions built on it."""
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.azure.ptu_capacity import (
|
||||
AZURE_PTU_CAPACITY,
|
||||
PTUCapacity,
|
||||
azure_ptu_capacity,
|
||||
deployment_ptu_capacity,
|
||||
normalized_tokens,
|
||||
ptu_hours,
|
||||
)
|
||||
|
||||
_GPT41: Final = AZURE_PTU_CAPACITY["gpt-4.1"]
|
||||
_ROW: Final = PTUCapacity(input_tpm_per_ptu=1_000, output_to_input_ratio=4.0)
|
||||
_CACHED_ROW: Final = PTUCapacity(input_tpm_per_ptu=1_000, output_to_input_ratio=5.0, cached_input_ratio=0.1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["gpt-4.1", "azure/gpt-4.1", "azure/gpt-4.1-2025-04-14", "GPT-4.1", "azure/eastus/gpt-4.1-2025-04-14"],
|
||||
)
|
||||
def test_a_model_resolves_to_its_row_through_a_provider_prefix_and_a_dated_version(model):
|
||||
assert azure_ptu_capacity(model) is _GPT41
|
||||
|
||||
|
||||
def test_a_deployment_name_that_is_not_a_model_has_no_row():
|
||||
assert azure_ptu_capacity("azure/my-ptu-deployment") is None
|
||||
assert azure_ptu_capacity("") is None
|
||||
|
||||
|
||||
def test_a_dated_version_only_drops_a_full_date_suffix():
|
||||
"""``gpt-4o-mini`` must not lose its ``-mini`` the way ``-2024-07-18`` is dropped."""
|
||||
assert azure_ptu_capacity("azure/gpt-4o-mini-2024-07-18") is AZURE_PTU_CAPACITY["gpt-4o-mini"]
|
||||
assert azure_ptu_capacity("gpt-4o-mini") is not AZURE_PTU_CAPACITY["gpt-4o"]
|
||||
|
||||
|
||||
def test_every_row_serves_its_input_tpm_for_an_hour():
|
||||
for capacity in AZURE_PTU_CAPACITY.values():
|
||||
assert capacity.normalized_tokens_per_ptu_hour == capacity.input_tpm_per_ptu * 60
|
||||
assert capacity.input_tpm_per_ptu > 0
|
||||
assert capacity.output_to_input_ratio >= 1.0
|
||||
assert 0.0 <= capacity.cached_input_ratio < 1.0
|
||||
|
||||
|
||||
def test_a_deployment_prefers_its_declared_base_model_over_its_deployment_name():
|
||||
deployment: Final = {
|
||||
"model_info": {"base_model": "azure/gpt-4.1"},
|
||||
"litellm_params": {"model": "azure/gpt-4o"},
|
||||
}
|
||||
assert deployment_ptu_capacity(deployment) is _GPT41
|
||||
|
||||
|
||||
def test_a_deployment_falls_back_to_its_litellm_model_when_no_base_model_is_declared():
|
||||
assert deployment_ptu_capacity({"litellm_params": {"model": "azure/gpt-4o"}}) is AZURE_PTU_CAPACITY["gpt-4o"]
|
||||
assert (
|
||||
deployment_ptu_capacity({"model_info": {"base_model": ""}, "litellm_params": {"model": "azure/gpt-4o"}})
|
||||
is (AZURE_PTU_CAPACITY["gpt-4o"])
|
||||
)
|
||||
|
||||
|
||||
def test_a_deployment_with_no_recognisable_model_has_no_row():
|
||||
assert deployment_ptu_capacity({"litellm_params": {"model": "azure/team-a-ptu"}}) is None
|
||||
assert deployment_ptu_capacity({"model_info": None, "litellm_params": None}) is None
|
||||
assert deployment_ptu_capacity({}) is None
|
||||
|
||||
|
||||
def test_output_is_weighted_by_the_models_ratio_and_uncached_input_counts_in_full():
|
||||
assert normalized_tokens(_ROW, prompt_tokens=100, completion_tokens=10) == pytest.approx(140.0)
|
||||
|
||||
|
||||
def test_cached_input_is_free_unless_the_row_prices_it():
|
||||
assert normalized_tokens(_ROW, prompt_tokens=100, completion_tokens=0, cache_read_tokens=60) == pytest.approx(40.0)
|
||||
assert normalized_tokens(
|
||||
_CACHED_ROW, prompt_tokens=100, completion_tokens=0, cache_read_tokens=60
|
||||
) == pytest.approx(46.0)
|
||||
|
||||
|
||||
def test_cached_input_never_exceeds_the_prompt_and_negatives_count_as_zero():
|
||||
assert normalized_tokens(_ROW, prompt_tokens=100, completion_tokens=0, cache_read_tokens=250) == 0.0
|
||||
assert normalized_tokens(_ROW, prompt_tokens=-5, completion_tokens=-5, cache_read_tokens=-5) == 0.0
|
||||
|
||||
|
||||
def test_one_ptu_hour_is_one_ptus_input_tpm_served_for_sixty_minutes():
|
||||
assert ptu_hours(_ROW, _ROW.input_tpm_per_ptu * 60) == pytest.approx(1.0)
|
||||
assert ptu_hours(_ROW, _ROW.input_tpm_per_ptu * 30) == pytest.approx(0.5)
|
||||
assert ptu_hours(_ROW, 0.0) == 0.0
|
||||
|
|
@ -10,6 +10,7 @@ from collections.abc import Iterator, Sequence
|
|||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, Final, List, Optional
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -19,6 +20,7 @@ from litellm import Router
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.llms.azure.ptu_capacity import AZURE_PTU_CAPACITY
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
|
|
@ -36,12 +38,15 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
|||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
_PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR
|
||||
from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import (
|
||||
EmbeddingResponse,
|
||||
ModelResponse,
|
||||
PromptTokensDetailsWrapper,
|
||||
TextCompletionResponse,
|
||||
Usage,
|
||||
)
|
||||
|
|
@ -4301,7 +4306,7 @@ async def test_tpm_over_limit_rejection_releases_parallel_slot_v3(monkeypatch):
|
|||
)
|
||||
counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests"
|
||||
|
||||
async def over_limit_reservation(descriptors, estimated_tokens, parent_otel_span=None):
|
||||
async def over_limit_reservation(descriptors, estimated_tokens, parent_otel_span=None, **_kwargs):
|
||||
return {
|
||||
"overall_code": "OVER_LIMIT",
|
||||
"statuses": [
|
||||
|
|
@ -4530,8 +4535,8 @@ async def test_async_data_generator_releases_counter_when_wrapped_v3():
|
|||
from the outer generator: the counter returns to 0 (not -1), proving the
|
||||
nested hook does not also refund and there is no double decrement.
|
||||
"""
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
class _PassthroughIteratorOverride(CustomLogger):
|
||||
async def async_post_call_streaming_iterator_hook(
|
||||
|
|
@ -7399,6 +7404,511 @@ async def test_success_tpm_accounting_keeps_the_admission_target_after_an_alias_
|
|||
assert not any(":target-b" in key for key in charged)
|
||||
|
||||
|
||||
# --- a team's PTU share on a shared Azure provisioned deployment ---------------------------
|
||||
|
||||
|
||||
def _ptu_ceiling_for(team_id: str, model_group: str, tpm_limit: int, ratio: float, cached_ratio: float = 0.0):
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
def resolve(requested_team: str, requested_group: str) -> PTUTeamCeiling | None:
|
||||
calls.append((requested_team, requested_group))
|
||||
if (requested_team, requested_group) != (team_id, model_group):
|
||||
return None
|
||||
return PTUTeamCeiling(
|
||||
model_group=model_group, tpm_limit=tpm_limit, output_to_input_ratio=ratio, cached_input_ratio=cached_ratio
|
||||
)
|
||||
|
||||
return resolve, calls
|
||||
|
||||
|
||||
def _ptu_request(model: str = "test-model") -> dict:
|
||||
return {"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 300}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_teams_ptu_share_is_a_hard_tpm_ceiling_on_the_shared_model():
|
||||
cache = DualCache()
|
||||
resolve, _ = _ptu_ceiling_for("t", "test-model", tpm_limit=2000, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
|
||||
await handler.async_pre_call_hook(user_api_key_dict=key, cache=cache, data=_ptu_request(), call_type="acompletion")
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=cache, data=_ptu_request(), call_type="acompletion"
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 429
|
||||
assert "model_per_team_ptu" in str(exc.value.detail)
|
||||
assert exc.value.headers["rate_limit_type"] == "tokens"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_ptu_ceiling_on_one_model_leaves_the_teams_other_models_alone():
|
||||
cache = DualCache()
|
||||
resolve, calls = _ptu_ceiling_for("t", "test-model", tpm_limit=500, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
|
||||
for _ in range(3):
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=cache, data=_ptu_request("other-model"), call_type="acompletion"
|
||||
)
|
||||
|
||||
assert calls == [("t", "other-model")] * 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_team_without_a_share_and_a_key_without_a_team_get_no_ptu_ceiling():
|
||||
cache = DualCache()
|
||||
resolve, calls = _ptu_ceiling_for("t", "test-model", tpm_limit=500, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
other_team = UserAPIKeyAuth(api_key=hash_token("sk-other"), team_id="u")
|
||||
no_team = UserAPIKeyAuth(api_key=hash_token("sk-no-team"))
|
||||
|
||||
for _ in range(3):
|
||||
for caller in (other_team, no_team):
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=caller, cache=cache, data=_ptu_request(), call_type="acompletion"
|
||||
)
|
||||
|
||||
assert set(calls) == {("u", "test-model")}
|
||||
|
||||
|
||||
def _shared_ptu_router(model_group: str) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model_group,
|
||||
"litellm_params": {"model": "azure/gpt-4.1", "api_key": "sk-ptu", "api_base": "https://ptu.example"},
|
||||
"model_info": {
|
||||
"id": "shared-ptu",
|
||||
"base_model": "azure/gpt-4.1",
|
||||
"ptu_count": 1,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
"ptu_shares": {"t": 1},
|
||||
},
|
||||
}
|
||||
],
|
||||
model_group_alias={f"{model_group}-alias": model_group},
|
||||
)
|
||||
|
||||
|
||||
def _two_thirds_of_a_ptu_minute() -> dict:
|
||||
"""An output budget worth two thirds of a gpt-4.1 PTU minute once weighted at the model's
|
||||
output-to-input ratio, the way the ceiling counts it."""
|
||||
capacity = AZURE_PTU_CAPACITY["gpt-4.1"]
|
||||
return {**_ptu_request(), "max_tokens": int(capacity.input_tpm_per_ptu * 2 / 3 / capacity.output_to_input_ratio)}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_reservation_weighs_output_the_way_the_ceiling_does():
|
||||
"""300 output tokens are 1200 normalized tokens at 4:1, over a 1000-token ceiling the raw
|
||||
301-token estimate would clear; the same request at 1:1 is admitted."""
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
unweighted_cache = DualCache()
|
||||
unweighted = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(unweighted_cache),
|
||||
ptu_team_ceiling_resolver=_ptu_ceiling_for("t", "test-model", tpm_limit=1000, ratio=1.0)[0],
|
||||
)
|
||||
await unweighted.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=unweighted_cache, data=_ptu_request(), call_type="acompletion"
|
||||
)
|
||||
|
||||
weighted_cache = DualCache()
|
||||
weighted = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(weighted_cache),
|
||||
ptu_team_ceiling_resolver=_ptu_ceiling_for("t", "test-model", tpm_limit=1000, ratio=4.0)[0],
|
||||
)
|
||||
with pytest.raises(HTTPException) as rejected:
|
||||
await weighted.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=weighted_cache, data=_ptu_request(), call_type="acompletion"
|
||||
)
|
||||
assert rejected.value.status_code == 429
|
||||
assert "model_per_team_ptu" in str(rejected.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_one_ptu_share_admits_four_uncapped_requests_a_minute_and_rejects_the_fifth():
|
||||
"""One PTU of gpt-4.1 is 3,000 normalized tokens a minute. A request without max_tokens gets the
|
||||
proxy's output floor measured against the share in output tokens (750), so its cap costs at most
|
||||
a quarter of the share once weighted 4:1 and four such requests fit before the fifth is refused."""
|
||||
cache = DualCache()
|
||||
resolve, _ = _ptu_ceiling_for("t", "test-model", tpm_limit=3000, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
uncapped = {"model": "test-model", "messages": [{"role": "user", "content": "hi"}]}
|
||||
|
||||
admitted = [dict(uncapped) for _ in range(4)]
|
||||
for data in admitted:
|
||||
await handler.async_pre_call_hook(user_api_key_dict=key, cache=cache, data=data, call_type="acompletion")
|
||||
with pytest.raises(HTTPException) as fifth:
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=cache, data=dict(uncapped), call_type="acompletion"
|
||||
)
|
||||
|
||||
assert fifth.value.status_code == 429
|
||||
assert "model_per_team_ptu" in str(fifth.value.detail)
|
||||
assert all(data["max_tokens"] * 4 <= 3000 // 4 for data in admitted)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_uncapped_prompt_longer_than_the_output_floor_reserves_the_cap_the_proxy_writes():
|
||||
"""Without a cap the output budget defaults to the prompt's own length, so a long prompt would be
|
||||
reserved twice, weighted 4:1, and refused on an empty window. The ceiling counts the cap the proxy
|
||||
writes into the request instead, the same output the deployment can produce."""
|
||||
cache = DualCache()
|
||||
resolve, _ = _ptu_ceiling_for("t", "test-model", tpm_limit=3000, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
data = {"model": "test-model", "messages": [{"role": "user", "content": "word " * 1200}]}
|
||||
|
||||
await handler.async_pre_call_hook(user_api_key_dict=key, cache=cache, data=data, call_type="acompletion")
|
||||
|
||||
stash = get_request_stash()
|
||||
assert stash is not None
|
||||
assert data["max_tokens"] * 4 <= 3000 // 4
|
||||
assert stash.ptu_reserved_tokens == stash.reserved_tokens + 3 * data["max_tokens"]
|
||||
assert stash.ptu_reserved_tokens <= 3000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_ptu_counter_holds_the_normalized_reservation_beside_the_raw_one():
|
||||
cache = DualCache()
|
||||
resolve, _ = _ptu_ceiling_for("t", "test-model", tpm_limit=2000, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
|
||||
await handler.async_pre_call_hook(user_api_key_dict=key, cache=cache, data=_ptu_request(), call_type="acompletion")
|
||||
|
||||
stash = get_request_stash()
|
||||
assert stash is not None
|
||||
assert stash.ptu_reserved_tokens == stash.reserved_tokens + 3 * 300
|
||||
ptu_key = handler.create_rate_limit_keys("model_per_team_ptu", "t:test-model", "tokens")
|
||||
assert int(await cache.async_get_cache(key=ptu_key) or 0) == stash.ptu_reserved_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_proxy_router_turns_a_teams_share_into_its_ceiling_when_attribution_is_on(monkeypatch):
|
||||
"""With no resolver injected the ceiling comes from the proxy router's own deployments: one
|
||||
PTU of gpt-4.1 a minute, so two requests each reserving two thirds of it are one too many."""
|
||||
monkeypatch.setenv(PTU_COST_ATTRIBUTION_ENV_VAR, "true")
|
||||
cache = DualCache()
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache))
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _shared_ptu_router("test-model")):
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=cache, data=_two_thirds_of_a_ptu_minute(), call_type="acompletion"
|
||||
)
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=cache, data=_two_thirds_of_a_ptu_minute(), call_type="acompletion"
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 429
|
||||
assert "model_per_team_ptu" in str(exc.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("deployment_name", ["shared-ptu", "azure/gpt-4.1"])
|
||||
async def test_naming_the_shared_deployment_directly_draws_on_the_same_ceiling_as_its_group(
|
||||
monkeypatch, deployment_name
|
||||
):
|
||||
"""The router also serves a deployment named by its id or its provider model, so a team that
|
||||
spent its share by group name cannot keep going under the deployment's other names, even
|
||||
though the router lists the deployment's alias copy ahead of it."""
|
||||
monkeypatch.setenv(PTU_COST_ATTRIBUTION_ENV_VAR, "true")
|
||||
cache = DualCache()
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache))
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _shared_ptu_router("test-model")):
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=cache, data=_two_thirds_of_a_ptu_minute(), call_type="acompletion"
|
||||
)
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key,
|
||||
cache=cache,
|
||||
data={**_two_thirds_of_a_ptu_minute(), "model": deployment_name},
|
||||
call_type="acompletion",
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 429
|
||||
assert "model_per_team_ptu" in str(exc.value.detail)
|
||||
ptu_keys = [cache_key for cache_key in cache.in_memory_cache.cache_dict if "model_per_team_ptu" in cache_key]
|
||||
assert handler.create_rate_limit_keys("model_per_team_ptu", "t:test-model", "tokens") in ptu_keys
|
||||
assert all(cache_key.startswith("{model_per_team_ptu:t:test-model}") for cache_key in ptu_keys)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_proxy_router_sets_no_ceiling_while_attribution_is_off(monkeypatch):
|
||||
monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR, raising=False)
|
||||
cache = DualCache()
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache))
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _shared_ptu_router("test-model")):
|
||||
for _ in range(3):
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=cache, data=_two_thirds_of_a_ptu_minute(), call_type="acompletion"
|
||||
)
|
||||
|
||||
assert not any("model_per_team_ptu" in cache_key for cache_key in cache.in_memory_cache.cache_dict)
|
||||
|
||||
|
||||
def _ptu_success_kwargs() -> dict:
|
||||
return {
|
||||
"standard_logging_object": {
|
||||
"metadata": {"user_api_key_hash": hash_token("sk-ptu"), "user_api_key_team_id": "t"}
|
||||
},
|
||||
"litellm_params": {
|
||||
"metadata": {"model_group": "test-model", "user_api_key_metadata": {}, "user_api_key_team_metadata": {}}
|
||||
},
|
||||
"model": "test-model",
|
||||
}
|
||||
|
||||
|
||||
def _ptu_response(usage: Usage) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
id="ptu-share",
|
||||
object="chat.completion",
|
||||
created=int(datetime.now().timestamp()),
|
||||
model="test-model",
|
||||
usage=usage,
|
||||
choices=[],
|
||||
)
|
||||
|
||||
|
||||
def _ptu_increment(handler: _PROXY_MaxParallelRequestsHandler, ops: list) -> float | None:
|
||||
ptu_key = handler.create_rate_limit_keys("model_per_team_ptu", "t:test-model", "tokens")
|
||||
return next((op["increment_value"] for op in ops if op["key"] == ptu_key), None)
|
||||
|
||||
|
||||
def test_success_accounting_settles_the_ptu_counter_in_azure_normalized_tokens():
|
||||
"""100 uncached input tokens plus 50 output tokens at a 4:1 output ratio are 300
|
||||
normalized tokens, which is what the share was sized in."""
|
||||
resolve, _ = _ptu_ceiling_for("t", "test-model", tpm_limit=500, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(DualCache()), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
response = _ptu_response(Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150))
|
||||
|
||||
ops = handler._build_success_event_pipeline_operations(
|
||||
kwargs=_ptu_success_kwargs(), response_obj=response, rate_limit_type="output"
|
||||
)
|
||||
|
||||
assert _ptu_increment(handler, ops) == 300
|
||||
|
||||
|
||||
def test_cached_input_is_not_charged_to_the_ptu_counter():
|
||||
resolve, _ = _ptu_ceiling_for("t", "test-model", tpm_limit=500, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(DualCache()), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
response = _ptu_response(
|
||||
Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=50,
|
||||
total_tokens=150,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40),
|
||||
)
|
||||
)
|
||||
|
||||
ops = handler._build_success_event_pipeline_operations(
|
||||
kwargs=_ptu_success_kwargs(), response_obj=response, rate_limit_type="output"
|
||||
)
|
||||
|
||||
assert _ptu_increment(handler, ops) == 60 + 4 * 50
|
||||
|
||||
|
||||
def test_cached_input_is_charged_at_the_models_cached_ratio():
|
||||
"""40 of the 100 input tokens were cache reads; at a tenth each they are 4 normalized tokens
|
||||
beside the 60 uncached ones and the 200 for 50 outputs at 4:1."""
|
||||
resolve, _ = _ptu_ceiling_for("t", "test-model", tpm_limit=500, ratio=4.0, cached_ratio=0.1)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(DualCache()), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
response = _ptu_response(
|
||||
Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=50,
|
||||
total_tokens=150,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40),
|
||||
)
|
||||
)
|
||||
|
||||
ops = handler._build_success_event_pipeline_operations(
|
||||
kwargs=_ptu_success_kwargs(), response_obj=response, rate_limit_type="output"
|
||||
)
|
||||
|
||||
assert _ptu_increment(handler, ops) == 60 + 4 + 4 * 50
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_reservation_is_settled_even_after_the_teams_share_is_gone():
|
||||
"""The share can be removed between admission and completion; the reserved tokens still
|
||||
come off the counter instead of standing in the window."""
|
||||
ceiling: dict[str, PTUTeamCeiling | None] = {
|
||||
"current": PTUTeamCeiling(
|
||||
model_group="test-model", tpm_limit=2000, output_to_input_ratio=4.0, cached_input_ratio=0.0
|
||||
)
|
||||
}
|
||||
cache = DualCache()
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache),
|
||||
ptu_team_ceiling_resolver=lambda _team, _group: ceiling["current"],
|
||||
)
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
|
||||
await handler.async_pre_call_hook(user_api_key_dict=key, cache=cache, data=_ptu_request(), call_type="acompletion")
|
||||
stash = get_request_stash()
|
||||
assert stash is not None
|
||||
assert ("model_per_team_ptu", "t:test-model") in stash.reserved_scopes
|
||||
assert stash.ptu_reserved_tokens > stash.reserved_tokens > 150
|
||||
|
||||
ceiling["current"] = None
|
||||
ops = handler._build_success_event_pipeline_operations(
|
||||
kwargs=_ptu_success_kwargs(),
|
||||
response_obj=_ptu_response(Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)),
|
||||
rate_limit_type="total",
|
||||
)
|
||||
|
||||
assert _ptu_increment(handler, ops) == 300 - stash.ptu_reserved_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_settles_the_scope_the_reservation_was_taken_on():
|
||||
"""Settlement credits the scope admission reserved, not one rebuilt from the name the
|
||||
request used, so a request naming the deployment by id cannot leave its reservation standing."""
|
||||
ceiling: dict[str, PTUTeamCeiling | None] = {
|
||||
"current": PTUTeamCeiling(
|
||||
model_group="test-model", tpm_limit=2000, output_to_input_ratio=4.0, cached_input_ratio=0.0
|
||||
)
|
||||
}
|
||||
cache = DualCache()
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache),
|
||||
ptu_team_ceiling_resolver=lambda _team, _group: ceiling["current"],
|
||||
)
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key, cache=cache, data=_ptu_request("shared-ptu"), call_type="acompletion"
|
||||
)
|
||||
stash = get_request_stash()
|
||||
assert stash is not None
|
||||
assert ("model_per_team_ptu", "t:test-model") in stash.reserved_scopes
|
||||
|
||||
ceiling["current"] = None
|
||||
stash.ptu_ceiling = None
|
||||
kwargs = _ptu_success_kwargs()
|
||||
kwargs["litellm_params"]["metadata"]["model_group"] = "shared-ptu"
|
||||
ops = handler._build_success_event_pipeline_operations(
|
||||
kwargs=kwargs,
|
||||
response_obj=_ptu_response(Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)),
|
||||
rate_limit_type="total",
|
||||
)
|
||||
|
||||
assert _ptu_increment(handler, ops) == 150 - stash.ptu_reserved_tokens
|
||||
|
||||
|
||||
async def _reserve_a_ptu_minute(cache: DualCache, call_id: str) -> tuple[_PROXY_MaxParallelRequestsHandler, str]:
|
||||
resolve, _ = _ptu_ceiling_for("t", "test-model", tpm_limit=2000, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
key = UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t")
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=key,
|
||||
cache=cache,
|
||||
data={**_ptu_request(), "litellm_call_id": call_id},
|
||||
call_type="acompletion",
|
||||
)
|
||||
stash = get_request_stash()
|
||||
assert stash is not None and stash.ptu_reserved_tokens > 0
|
||||
return handler, handler.create_rate_limit_keys("model_per_team_ptu", "t:test-model", "tokens")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_failed_stream_settles_the_ptu_counter_in_normalized_tokens():
|
||||
"""The partial usage a failed stream recovered is 20 input and 7 output tokens: 48 normalized
|
||||
at 4:1, which is what stays in the window instead of the raw 27 or the whole reservation."""
|
||||
cache = DualCache()
|
||||
handler, ptu_key = await _reserve_a_ptu_minute(cache, "ptu-partial")
|
||||
|
||||
await handler.async_log_failure_event(
|
||||
kwargs={
|
||||
"litellm_call_id": "ptu-partial",
|
||||
"standard_logging_object": {
|
||||
"metadata": {"user_api_key_hash": hash_token("sk-ptu"), "user_api_key_team_id": "t"}
|
||||
},
|
||||
"combined_usage_object": Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27),
|
||||
},
|
||||
response_obj=None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
|
||||
assert int(await cache.async_get_cache(key=ptu_key) or 0) == 20 + 4 * 7
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_proxy_side_rejection_refunds_the_whole_normalized_ptu_reservation():
|
||||
cache = DualCache()
|
||||
handler, ptu_key = await _reserve_a_ptu_minute(cache, "ptu-rejected")
|
||||
|
||||
await handler.async_post_call_failure_hook(
|
||||
request_data={**_ptu_request(), "litellm_call_id": "ptu-rejected"},
|
||||
original_exception=Exception("guardrail rejected the request"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key=hash_token("sk-ptu"), team_id="t"),
|
||||
)
|
||||
|
||||
assert int(await cache.async_get_cache(key=ptu_key) or 0) == 0
|
||||
|
||||
|
||||
def test_usage_that_only_reports_a_total_charges_that_total_to_the_ptu_counter():
|
||||
resolve, _ = _ptu_ceiling_for("t", "test-model", tpm_limit=500, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(DualCache()), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
response = _ptu_response(Usage(prompt_tokens=0, completion_tokens=0, total_tokens=150))
|
||||
|
||||
ops = handler._build_success_event_pipeline_operations(
|
||||
kwargs=_ptu_success_kwargs(), response_obj=response, rate_limit_type="output"
|
||||
)
|
||||
|
||||
assert _ptu_increment(handler, ops) == 150
|
||||
|
||||
|
||||
def test_success_accounting_charges_no_ptu_counter_without_a_ceiling():
|
||||
resolve, _ = _ptu_ceiling_for("u", "test-model", tpm_limit=500, ratio=4.0)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(DualCache()), ptu_team_ceiling_resolver=resolve
|
||||
)
|
||||
response = _ptu_response(Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150))
|
||||
|
||||
ops = handler._build_success_event_pipeline_operations(
|
||||
kwargs=_ptu_success_kwargs(), response_obj=response, rate_limit_type="output"
|
||||
)
|
||||
|
||||
assert _ptu_increment(handler, ops) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("self_call", [False, True])
|
||||
async def test_managed_invocations_enforce_actor_and_target_rate_policies(
|
||||
monkeypatch: pytest.MonkeyPatch, self_call: bool
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import io
|
|||
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass, fields
|
||||
from itertools import chain
|
||||
from types import SimpleNamespace
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
|
|
@ -12,14 +12,17 @@ from fastapi import FastAPI, Request
|
|||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm import constants
|
||||
from litellm.llms.azure.ptu_capacity import AZURE_PTU_CAPACITY
|
||||
from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable, LitellmUserRoles, Member, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.management_endpoints.daily_activity_routes import (
|
||||
_csv_cell,
|
||||
get_daily_activity_llm_router,
|
||||
get_daily_activity_prisma_client,
|
||||
get_daily_activity_repository,
|
||||
router,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import KeySpendMetrics, SpendMetrics
|
||||
from litellm.types.repositories.daily_activity import (
|
||||
AggregatedRows,
|
||||
|
|
@ -190,6 +193,7 @@ def _grouping_row(
|
|||
api_key: str | None,
|
||||
group_level: int,
|
||||
distinct_api_keys: int | None,
|
||||
model_group: str | None = None,
|
||||
) -> GroupingSetsRow:
|
||||
metric_values: Final = _metrics(rows)
|
||||
return GroupingSetsRow(
|
||||
|
|
@ -197,7 +201,7 @@ def _grouping_row(
|
|||
api_key=api_key,
|
||||
**metric_values,
|
||||
model=None,
|
||||
model_group=None,
|
||||
model_group=model_group,
|
||||
custom_llm_provider=None,
|
||||
mcp_namespaced_tool_name=None,
|
||||
endpoint=None,
|
||||
|
|
@ -211,17 +215,29 @@ def _grouping_rows_for_day(
|
|||
) -> tuple[GroupingSetsRow, ...]:
|
||||
date_rows: Final = tuple(row for row in rows if row.date == date)
|
||||
return (
|
||||
_grouping_row(date_rows, date=date, api_key=None, group_level=63, distinct_api_keys=distinct_api_keys),
|
||||
) + tuple(
|
||||
_grouping_row(
|
||||
tuple(row for row in date_rows if row.api_key == api_key),
|
||||
date=date,
|
||||
api_key=api_key,
|
||||
group_level=31,
|
||||
distinct_api_keys=None,
|
||||
(_grouping_row(date_rows, date=date, api_key=None, group_level=63, distinct_api_keys=distinct_api_keys),)
|
||||
+ tuple(
|
||||
_grouping_row(
|
||||
tuple(row for row in date_rows if row.api_key == api_key),
|
||||
date=date,
|
||||
api_key=api_key,
|
||||
group_level=31,
|
||||
distinct_api_keys=None,
|
||||
)
|
||||
for api_key in top_keys
|
||||
if any(row.api_key == api_key for row in date_rows)
|
||||
)
|
||||
+ tuple(
|
||||
_grouping_row(
|
||||
tuple(row for row in date_rows if row.model_group == model_group),
|
||||
date=date,
|
||||
api_key=None,
|
||||
group_level=55,
|
||||
distinct_api_keys=None,
|
||||
model_group=model_group,
|
||||
)
|
||||
for model_group in sorted({row.model_group for row in date_rows})
|
||||
)
|
||||
for api_key in top_keys
|
||||
if any(row.api_key == api_key for row in date_rows)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -562,6 +578,103 @@ def test_aggregated_routes_return_scoped_results(
|
|||
assert len(body["results"]) == 2, response.text
|
||||
|
||||
|
||||
def _ptu_sized_router(model_group: str, shares: Mapping[str, int] = MappingProxyType({"team-a": 30, "team-b": 20})) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model_group,
|
||||
"litellm_params": {"model": "azure/gpt-4.1", "api_key": "sk-ptu", "api_base": "https://ptu.example"},
|
||||
"model_info": {
|
||||
"id": "shared-ptu",
|
||||
"base_model": "azure/gpt-4.1",
|
||||
"ptu_count": 50,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2024-01-01T00:00:00Z",
|
||||
"ptu_shares": dict(shares),
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _inject_llm_router(client: TestClient, llm_router: Router) -> None:
|
||||
app: Final = client.app
|
||||
assert isinstance(app, FastAPI)
|
||||
app.dependency_overrides[get_daily_activity_llm_router] = lambda: llm_router
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attribution_enabled", [True, False])
|
||||
def test_team_aggregated_route_reports_ptu_hours_for_the_sized_model_group_only_while_attribution_is_on(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
attribution_enabled: bool,
|
||||
) -> None:
|
||||
"""team-a sent one rare-group request of 10 input and 5 output tokens on 2025-01-01, so with
|
||||
gpt-4.1's sizing that day, its model group, and the total carry those tokens normalized
|
||||
(output at the model's output ratio) over one PTU-hour of input; the unsized popular-group,
|
||||
the day without rare-group traffic, and the flag-off run all report zero."""
|
||||
client, _ = daily_activity_client
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True" if attribution_enabled else "False")
|
||||
_inject_llm_router(client, _ptu_sized_router("rare-group"))
|
||||
|
||||
response: Final = client.get("/team/daily/activity/aggregated", params=_entity_params("team_ids", "team-a"))
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = response.json()
|
||||
capacity: Final = AZURE_PTU_CAPACITY["gpt-4.1"]
|
||||
expected: Final = (
|
||||
(10 + 5 * capacity.output_to_input_ratio) / capacity.normalized_tokens_per_ptu_hour
|
||||
if attribution_enabled
|
||||
else 0.0
|
||||
)
|
||||
by_date: Final = {day["date"]: day for day in body["results"]}
|
||||
rare_day: Final = by_date["2025-01-01"]
|
||||
assert body["metadata"]["total_ptu_hours"] == pytest.approx(expected), response.text
|
||||
assert rare_day["metrics"]["ptu_hours"] == pytest.approx(expected), response.text
|
||||
assert rare_day["breakdown"]["model_groups"]["rare-group"]["metrics"]["ptu_hours"] == pytest.approx(expected)
|
||||
assert rare_day["breakdown"]["model_groups"]["popular-group"]["metrics"]["ptu_hours"] == 0, response.text
|
||||
assert by_date["2025-01-02"]["metrics"]["ptu_hours"] == 0, response.text
|
||||
assert body["metadata"]["total_tokens"] == 90, response.text
|
||||
|
||||
|
||||
def test_team_aggregated_route_reports_no_ptu_hours_to_a_team_holding_no_share(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""rare-group is split between team-b and team-c, so team-a's page alone converts none of its
|
||||
rare-group tokens even though the group is sized."""
|
||||
client, _ = daily_activity_client
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
_inject_llm_router(client, _ptu_sized_router("rare-group", shares=MappingProxyType({"team-b": 30, "team-c": 20})))
|
||||
|
||||
response: Final = client.get("/team/daily/activity/aggregated", params=_entity_params("team_ids", "team-a"))
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = response.json()
|
||||
rare_day: Final = {day["date"]: day for day in body["results"]}["2025-01-01"]
|
||||
assert body["metadata"]["total_ptu_hours"] == 0, response.text
|
||||
assert rare_day["breakdown"]["model_groups"]["rare-group"]["metrics"]["ptu_hours"] == 0, response.text
|
||||
assert body["metadata"]["total_tokens"] == 90, response.text
|
||||
|
||||
|
||||
def test_ptu_hours_stay_zero_outside_the_team_routes(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""PTU reservations are split per team, so only the team routes convert tokens to PTU-hours;
|
||||
a user's own activity on the same sized model group keeps the field at zero."""
|
||||
client, _ = daily_activity_client
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
_inject_llm_router(client, _ptu_sized_router("rare-group"))
|
||||
|
||||
response: Final = client.get("/user/daily/activity/aggregated", params=_entity_params("user_id", "user-a"))
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = response.json()
|
||||
rare_day: Final = next(day for day in body["results"] if day["date"] == "2025-01-01")
|
||||
assert body["metadata"]["total_ptu_hours"] == 0, response.text
|
||||
assert rare_day["breakdown"]["model_groups"]["rare-group"]["metrics"]["ptu_hours"] == 0, response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("prefix", "query_name", "entity_id"), _ENTITY_CASES)
|
||||
def test_admin_aggregates_all_entities_when_filter_is_omitted(
|
||||
daily_activity_client: tuple[TestClient, _FakeRepository], prefix: str, query_name: str, entity_id: str
|
||||
|
|
@ -593,6 +706,7 @@ def test_search_folds_each_entity_key_across_days(
|
|||
"metrics": {
|
||||
"spend": 3.0,
|
||||
"flat_cost": 0.0,
|
||||
"ptu_hours": 0.0,
|
||||
"prompt_tokens": 20,
|
||||
"completion_tokens": 10,
|
||||
"cache_read_input_tokens": 0,
|
||||
|
|
|
|||
279
tests/unit/proxy/management_endpoints/test_ptu_consumption.py
Normal file
279
tests/unit/proxy/management_endpoints/test_ptu_consumption.py
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
"""Tests for attaching PTU-hours to a daily activity response."""
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm import Router
|
||||
from litellm.llms.azure.ptu_capacity import AZURE_PTU_CAPACITY, PTUCapacity
|
||||
from litellm.proxy.management_endpoints.ptu_consumption import attach_ptu_hours, with_ptu_consumption
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
BreakdownMetrics,
|
||||
DailySpendData,
|
||||
DailySpendMetadata,
|
||||
KeyMetricWithMetadata,
|
||||
MetricWithMetadata,
|
||||
SpendAnalyticsPaginatedResponse,
|
||||
SpendMetrics,
|
||||
)
|
||||
|
||||
_ROW: Final = PTUCapacity(input_tpm_per_ptu=1_000, output_to_input_ratio=4.0)
|
||||
_CACHED_ROW: Final = PTUCapacity(input_tpm_per_ptu=1_000, output_to_input_ratio=4.0, cached_input_ratio=0.1)
|
||||
_CAPACITY: Final = {"gpt-4.1-ptu": _ROW, "gpt-6-ptu": _CACHED_ROW}
|
||||
_ONE_PTU_HOUR_OF_INPUT: Final = AZURE_PTU_CAPACITY["gpt-4.1"].normalized_tokens_per_ptu_hour
|
||||
|
||||
|
||||
def _metrics(prompt: int, completion: int, cached: int = 0) -> SpendMetrics:
|
||||
return SpendMetrics(
|
||||
prompt_tokens=prompt,
|
||||
completion_tokens=completion,
|
||||
cache_read_input_tokens=cached,
|
||||
total_tokens=prompt + completion,
|
||||
api_requests=1,
|
||||
successful_requests=1,
|
||||
)
|
||||
|
||||
|
||||
def _bucket(metrics: SpendMetrics, keys: dict[str, SpendMetrics] | None = None) -> MetricWithMetadata:
|
||||
return MetricWithMetadata(
|
||||
metrics=metrics,
|
||||
metadata={},
|
||||
api_key_breakdown={key: KeyMetricWithMetadata(metrics=m, metadata={}) for key, m in (keys or {}).items()},
|
||||
)
|
||||
|
||||
|
||||
def _day(date: str, model_groups: dict[str, MetricWithMetadata]) -> DailySpendData:
|
||||
total: Final = _metrics(
|
||||
sum(b.metrics.prompt_tokens for b in model_groups.values()),
|
||||
sum(b.metrics.completion_tokens for b in model_groups.values()),
|
||||
)
|
||||
return DailySpendData(date=date, metrics=total, breakdown=BreakdownMetrics(model_groups=model_groups))
|
||||
|
||||
|
||||
def _response(*days: DailySpendData) -> SpendAnalyticsPaginatedResponse:
|
||||
return SpendAnalyticsPaginatedResponse(results=list(days), metadata=DailySpendMetadata())
|
||||
|
||||
|
||||
def test_a_ptu_model_groups_tokens_become_ptu_hours_on_the_group_the_day_and_the_total():
|
||||
"""60,000 normalized tokens on a 1,000 input-TPM row is one PTU-hour."""
|
||||
day: Final = _day("2026-09-23", {"gpt-4.1-ptu": _bucket(_metrics(prompt=40_000, completion=5_000))})
|
||||
|
||||
attached: Final = attach_ptu_hours(_response(day), _CAPACITY.get)
|
||||
|
||||
group: Final = attached.results[0].breakdown.model_groups["gpt-4.1-ptu"]
|
||||
assert group.metrics.ptu_hours == pytest.approx(1.0)
|
||||
assert attached.results[0].metrics.ptu_hours == pytest.approx(1.0)
|
||||
assert attached.metadata.total_ptu_hours == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_a_model_group_without_a_sizing_row_keeps_zero_ptu_hours_and_the_rest_of_the_day_intact():
|
||||
day: Final = _day(
|
||||
"2026-09-23",
|
||||
{
|
||||
"gpt-4.1-ptu": _bucket(_metrics(prompt=30_000, completion=0)),
|
||||
"gpt-4o-mini": _bucket(_metrics(prompt=1_000_000, completion=1_000_000)),
|
||||
},
|
||||
)
|
||||
|
||||
attached: Final = attach_ptu_hours(_response(day), _CAPACITY.get)
|
||||
|
||||
groups: Final = attached.results[0].breakdown.model_groups
|
||||
assert groups["gpt-4o-mini"].metrics.ptu_hours == 0.0
|
||||
assert groups["gpt-4o-mini"].metrics.prompt_tokens == 1_000_000
|
||||
assert groups["gpt-4.1-ptu"].metrics.ptu_hours == pytest.approx(0.5)
|
||||
assert attached.results[0].metrics.ptu_hours == pytest.approx(0.5)
|
||||
assert attached.results[0].metrics.total_tokens == day.metrics.total_tokens
|
||||
|
||||
|
||||
def test_cached_input_is_charged_at_the_rows_cached_ratio():
|
||||
day: Final = _day("2026-09-23", {"gpt-6-ptu": _bucket(_metrics(prompt=60_000, completion=0, cached=60_000))})
|
||||
|
||||
attached: Final = attach_ptu_hours(_response(day), _CAPACITY.get)
|
||||
|
||||
assert attached.results[0].breakdown.model_groups["gpt-6-ptu"].metrics.ptu_hours == pytest.approx(0.1)
|
||||
|
||||
|
||||
def test_each_api_key_under_a_ptu_group_gets_its_own_ptu_hours():
|
||||
day: Final = _day(
|
||||
"2026-09-23",
|
||||
{
|
||||
"gpt-4.1-ptu": _bucket(
|
||||
_metrics(prompt=60_000, completion=0),
|
||||
keys={"key-a": _metrics(prompt=45_000, completion=0), "key-b": _metrics(prompt=15_000, completion=0)},
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
attached: Final = attach_ptu_hours(_response(day), _CAPACITY.get)
|
||||
|
||||
breakdown: Final = attached.results[0].breakdown.model_groups["gpt-4.1-ptu"].api_key_breakdown
|
||||
assert breakdown["key-a"].metrics.ptu_hours == pytest.approx(0.75)
|
||||
assert breakdown["key-b"].metrics.ptu_hours == pytest.approx(0.25)
|
||||
|
||||
|
||||
def test_the_total_sums_every_day_on_the_page():
|
||||
first: Final = _day("2026-09-22", {"gpt-4.1-ptu": _bucket(_metrics(prompt=60_000, completion=0))})
|
||||
second: Final = _day("2026-09-23", {"gpt-4.1-ptu": _bucket(_metrics(prompt=0, completion=15_000))})
|
||||
|
||||
attached: Final = attach_ptu_hours(_response(first, second), _CAPACITY.get)
|
||||
|
||||
assert [day.metrics.ptu_hours for day in attached.results] == [pytest.approx(1.0), pytest.approx(1.0)]
|
||||
assert attached.metadata.total_ptu_hours == pytest.approx(2.0)
|
||||
|
||||
|
||||
def test_a_page_with_no_ptu_group_is_returned_unchanged():
|
||||
day: Final = _day("2026-09-23", {"gpt-4o-mini": _bucket(_metrics(prompt=100, completion=100))})
|
||||
response: Final = _response(day)
|
||||
|
||||
attached: Final = attach_ptu_hours(response, _CAPACITY.get)
|
||||
|
||||
assert attached.results[0] is day
|
||||
assert attached.metadata.total_ptu_hours == 0.0
|
||||
assert response.metadata.total_ptu_hours == 0.0
|
||||
|
||||
|
||||
def _shared_ptu_router() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4.1-ptu",
|
||||
"litellm_params": {"model": "azure/gpt-4.1", "api_key": "sk-ptu", "api_base": "https://ptu.example"},
|
||||
"model_info": {
|
||||
"id": "shared-ptu",
|
||||
"base_model": "azure/gpt-4.1",
|
||||
"ptu_count": 50,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
"ptu_shares": {"team-a": 30, "team-b": 20},
|
||||
},
|
||||
}
|
||||
],
|
||||
model_group_alias={"ptu-alias": {"model": "gpt-4.1-ptu", "hidden": True}},
|
||||
)
|
||||
|
||||
|
||||
def test_rows_keyed_by_an_alias_a_deployment_id_or_a_provider_model_are_sized_like_the_group(monkeypatch):
|
||||
"""The ceiling charges a request however it names the shared deployment, so the usage row
|
||||
that request lands in, keyed by the name it used, reports the same PTU-hours as the group."""
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
names: Final = ("gpt-4.1-ptu", "ptu-alias", "shared-ptu", "azure/gpt-4.1")
|
||||
one_hour_each: Final = {name: _bucket(_metrics(prompt=_ONE_PTU_HOUR_OF_INPUT, completion=0)) for name in names}
|
||||
|
||||
attached: Final = with_ptu_consumption(_response(_day("2026-09-23", one_hour_each)), _shared_ptu_router(), "team-a")
|
||||
|
||||
groups: Final = attached.results[0].breakdown.model_groups
|
||||
assert [groups[name].metrics.ptu_hours for name in names] == [pytest.approx(1.0)] * len(names)
|
||||
assert attached.results[0].metrics.ptu_hours == pytest.approx(float(len(names)))
|
||||
assert attached.metadata.total_ptu_hours == pytest.approx(float(len(names)))
|
||||
|
||||
|
||||
def _mixed_ptu_router() -> Router:
|
||||
"""One group split between team-a on a gpt-4.1 PTU deployment and team-b on a gpt-5.5 one,
|
||||
beside an open pay-as-you-go deployment of gpt-4.1 in its own group."""
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ptu",
|
||||
"litellm_params": {"model": "azure/gpt-4.1", "api_key": "sk-ptu", "api_base": "https://ptu.example"},
|
||||
"model_info": {
|
||||
"id": "ptu-41",
|
||||
"base_model": "azure/gpt-4.1",
|
||||
"ptu_count": 50,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
"ptu_shares": {"team-a": 50},
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "ptu",
|
||||
"litellm_params": {"model": "azure/gpt-5.5", "api_key": "sk-ptu", "api_base": "https://ptu.example"},
|
||||
"model_info": {
|
||||
"id": "ptu-55",
|
||||
"base_model": "azure/gpt-5.5",
|
||||
"ptu_count": 50,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
"ptu_shares": {"team-b": 50},
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4.1",
|
||||
"litellm_params": {"model": "azure/gpt-4.1", "api_key": "sk-payg", "api_base": "https://payg.example"},
|
||||
"model_info": {"id": "payg"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _reserved_ptu_router() -> Router:
|
||||
"""A gpt-4.1 PTU deployment reserved for team-x alone the single-team way, beside an open
|
||||
pay-as-you-go deployment of gpt-4.1 that serves every other team."""
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "reserved",
|
||||
"litellm_params": {"model": "azure/gpt-4.1", "api_key": "sk-ptu", "api_base": "https://ptu.example"},
|
||||
"model_info": {
|
||||
"id": "reserved-41",
|
||||
"team_id": "team-x",
|
||||
"ptu_count": 50,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4.1",
|
||||
"litellm_params": {"model": "azure/gpt-4.1", "api_key": "sk-payg", "api_base": "https://payg.example"},
|
||||
"model_info": {"id": "payg"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("team_id", "expected_on_the_provider_model"), [("team-x", 1.0), ("team-y", 0.0), (None, 1.0)])
|
||||
def test_another_teams_single_team_reservation_sizes_nothing_on_a_teams_page(
|
||||
monkeypatch, team_id: str | None, expected_on_the_provider_model: float
|
||||
):
|
||||
"""A provider-model row reaches the reserved deployment only for the team it is reserved for: any
|
||||
other team was served by the open deployment, so its page counts no PTU-hours on that row, while
|
||||
a page spanning teams keeps the reserved deployment's sizing."""
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
one_hour: Final = {"azure/gpt-4.1": _bucket(_metrics(prompt=_ONE_PTU_HOUR_OF_INPUT, completion=0))}
|
||||
|
||||
attached: Final = with_ptu_consumption(_response(_day("2026-09-23", one_hour)), _reserved_ptu_router(), team_id)
|
||||
|
||||
row: Final = attached.results[0].breakdown.model_groups["azure/gpt-4.1"]
|
||||
assert row.metrics.ptu_hours == pytest.approx(expected_on_the_provider_model)
|
||||
assert attached.metadata.total_ptu_hours == pytest.approx(expected_on_the_provider_model)
|
||||
|
||||
|
||||
_ONE_GPT_41_PTU_HOUR_ON_GPT_55: Final = _ONE_PTU_HOUR_OF_INPUT / AZURE_PTU_CAPACITY["gpt-5.5"].normalized_tokens_per_ptu_hour
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("team_id", "expected_on_the_group", "expected_on_the_provider_model"),
|
||||
[
|
||||
("team-a", 1.0, 1.0),
|
||||
("team-b", _ONE_GPT_41_PTU_HOUR_ON_GPT_55, 0.0),
|
||||
("team-c", 0.0, 0.0),
|
||||
(None, 1.0, 1.0),
|
||||
],
|
||||
)
|
||||
def test_a_teams_rows_are_sized_by_the_deployment_it_is_served_from(
|
||||
monkeypatch, team_id: str | None, expected_on_the_group: float, expected_on_the_provider_model: float
|
||||
):
|
||||
"""A team's page converts its tokens through the deployment the ceiling served it from: team-b's
|
||||
share is on the gpt-5.5 deployment so its group row counts at that rate, its provider-model row
|
||||
reached only the open deployment so it counts nothing, team-c holds no share so it counts nothing,
|
||||
and a page spanning teams keeps the group's first sized deployment."""
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
one_hour_each: Final = {
|
||||
name: _bucket(_metrics(prompt=_ONE_PTU_HOUR_OF_INPUT, completion=0)) for name in ("ptu", "azure/gpt-4.1")
|
||||
}
|
||||
|
||||
attached: Final = with_ptu_consumption(_response(_day("2026-09-23", one_hour_each)), _mixed_ptu_router(), team_id)
|
||||
|
||||
groups: Final = attached.results[0].breakdown.model_groups
|
||||
assert groups["ptu"].metrics.ptu_hours == pytest.approx(expected_on_the_group)
|
||||
assert groups["azure/gpt-4.1"].metrics.ptu_hours == pytest.approx(expected_on_the_provider_model)
|
||||
assert attached.metadata.total_ptu_hours == pytest.approx(expected_on_the_group + expected_on_the_provider_model)
|
||||
|
|
@ -2,37 +2,43 @@
|
|||
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from contextlib import ExitStack
|
||||
from typing import Final, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import patch as patch_ctx
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.ptu_pricing import ptu_terms
|
||||
from litellm.llms.gemini.cost_calculator import cost_per_web_search_request
|
||||
from litellm.proxy._types import (
|
||||
LiteLLM_ProxyModelTable,
|
||||
LiteLLM_TeamTable,
|
||||
LitellmUserRoles,
|
||||
ProxyException,
|
||||
ReconcileOutcome,
|
||||
UserAPIKeyAuth,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.proxy.auth.auth_checks import _is_model_cost_zero
|
||||
from litellm.llms.gemini.cost_calculator import cost_per_web_search_request
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
_PTU_ZEROED_PRICING_FIELDS,
|
||||
_SEARCH_CONTEXT_SIZES,
|
||||
_is_nonzero_price,
|
||||
_merged_ptu_model_info,
|
||||
_update_team_model_in_db,
|
||||
_ptu_priced_deployment,
|
||||
_ptu_zeroed_pricing,
|
||||
_raise_if_ptu_cost_attribution_disabled,
|
||||
_raise_if_ptu_share_teams_missing,
|
||||
_update_team_model_in_db,
|
||||
_validate_ptu_model_info,
|
||||
add_new_model,
|
||||
update_db_model,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper
|
||||
from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE
|
||||
from litellm.router import Router
|
||||
from litellm.types.router import (
|
||||
SPECIAL_MODEL_INFO_PARAMS,
|
||||
|
|
@ -42,7 +48,7 @@ from litellm.types.router import (
|
|||
updateDeployment,
|
||||
updateLiteLLMParams,
|
||||
)
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
|
||||
async def _passthrough_row(update_data):
|
||||
|
|
@ -115,6 +121,18 @@ def test_model_info_allows_partial_delta_for_patch():
|
|||
assert info.cost_per_ptu_per_hour is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("share", [True, "2", 2.0])
|
||||
def test_model_info_rejects_a_share_that_is_not_a_whole_number(share):
|
||||
with pytest.raises(ValueError, match="ptu_shares"):
|
||||
ModelInfo(id="x", ptu_shares={"team-a": share})
|
||||
|
||||
|
||||
def test_model_info_keeps_whole_number_shares_and_refuses_a_fractional_count():
|
||||
assert ModelInfo(id="x", ptu_shares={"team-a": 2}).ptu_shares == {"team-a": 2}
|
||||
with pytest.raises(ValueError, match="ptu_count"):
|
||||
ModelInfo(id="x", ptu_count=100.5)
|
||||
|
||||
|
||||
def test_validate_helper_no_ptu_is_noop():
|
||||
_validate_ptu_model_info({"team_id": "t"})
|
||||
|
||||
|
|
@ -639,7 +657,7 @@ class TestAddNewModelPtuGate:
|
|||
monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR, raising=False)
|
||||
|
||||
@staticmethod
|
||||
def _patched_proxy(model_id: str):
|
||||
def _patched_proxy(model_id: str, prisma_client: MagicMock | None = None):
|
||||
"""Patch everything /model/new touches except the PTU gate, and hand back the DB writers."""
|
||||
db_row = LiteLLM_ProxyModelTable(
|
||||
model_id=model_id,
|
||||
|
|
@ -666,7 +684,7 @@ class TestAddNewModelPtuGate:
|
|||
proxy_server = "litellm.proxy.proxy_server"
|
||||
endpoints = "litellm.proxy.management_endpoints.model_management_endpoints"
|
||||
return (add_model_to_db, add_team_model_to_db), [
|
||||
patch(f"{proxy_server}.prisma_client", MagicMock()),
|
||||
patch(f"{proxy_server}.prisma_client", prisma_client if prisma_client is not None else MagicMock()),
|
||||
patch(f"{proxy_server}.store_model_in_db", True),
|
||||
patch(f"{proxy_server}.proxy_config", mock_proxy_config),
|
||||
patch(f"{proxy_server}.proxy_logging_obj", MagicMock()),
|
||||
|
|
@ -1284,3 +1302,144 @@ class TestPtuDeploymentsAreNotBilledPerToken:
|
|||
|
||||
assert "input_cost_per_token" in str(exc.value)
|
||||
add_team_model_to_db.assert_not_called()
|
||||
|
||||
|
||||
_SHARED_START = "2026-08-01T00:00:00Z"
|
||||
|
||||
|
||||
def test_validate_helper_accepts_shares_in_place_of_a_team_id():
|
||||
shared = {
|
||||
"ptu_count": 5,
|
||||
"cost_per_ptu_per_hour": 2.0,
|
||||
"ptu_effective_from": _SHARED_START,
|
||||
"ptu_shares": {"team-a": 3, "team-b": 2},
|
||||
}
|
||||
|
||||
_validate_ptu_model_info(shared)
|
||||
|
||||
assert ptu_terms(shared) is not None
|
||||
|
||||
|
||||
def test_validate_helper_refuses_shares_that_do_not_add_up_to_the_count():
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
_validate_ptu_model_info(
|
||||
{
|
||||
"ptu_count": 5,
|
||||
"cost_per_ptu_per_hour": 2.0,
|
||||
"ptu_effective_from": _SHARED_START,
|
||||
"ptu_shares": {"team-a": 3, "team-b": 1},
|
||||
}
|
||||
)
|
||||
assert exc.value.status_code == 400
|
||||
assert "4 of 5 allocated" in exc.value.detail
|
||||
|
||||
|
||||
def test_validate_helper_refuses_shares_declared_without_a_count_and_rate():
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
_validate_ptu_model_info({"ptu_shares": {"team-a": 3, "team-b": 2}, "ptu_effective_from": _SHARED_START})
|
||||
assert exc.value.status_code == 400
|
||||
assert exc.value.detail == "ptu_count and cost_per_ptu_per_hour are required when ptu_shares is set"
|
||||
|
||||
|
||||
class _TeamLookup:
|
||||
def __init__(self, existing: frozenset[str]) -> None:
|
||||
self.existing: Final = existing
|
||||
self.looked_up: tuple[str, ...] = ()
|
||||
self.batch_sizes: tuple[int, ...] = ()
|
||||
|
||||
async def find_unique(self, *, where: Mapping[str, object]) -> LiteLLM_TeamTable | None:
|
||||
raise AssertionError(f"one lookup per team is what the review asked to avoid: {where}")
|
||||
|
||||
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[LiteLLM_TeamTable]:
|
||||
team_filter: Final = where["team_id"]
|
||||
assert isinstance(team_filter, Mapping)
|
||||
requested: Final = tuple(str(team_id) for team_id in cast(Sequence[object], team_filter["in"]))
|
||||
self.looked_up = (*self.looked_up, *requested)
|
||||
self.batch_sizes = (*self.batch_sizes, len(requested))
|
||||
return tuple(LiteLLM_TeamTable(team_id=team_id) for team_id in requested if team_id in self.existing)
|
||||
|
||||
|
||||
def _shared_model_info(shares: Mapping[str, int]) -> Mapping[str, object]:
|
||||
return {
|
||||
"ptu_count": sum(shares.values()),
|
||||
"cost_per_ptu_per_hour": 2.0,
|
||||
"ptu_effective_from": _SHARED_START,
|
||||
"ptu_shares": dict(shares),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_share_team_check_refuses_a_team_that_does_not_exist():
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await _raise_if_ptu_share_teams_missing(
|
||||
_shared_model_info({"team-a": 3, "ghost-team": 2}), lambda: _TeamLookup(frozenset({"team-a"}))
|
||||
)
|
||||
assert exc.value.status_code == 400
|
||||
assert exc.value.detail == {"error": "Team id=ghost-team does not exist in db"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_share_team_check_accepts_shares_naming_existing_teams():
|
||||
lookup: Final = _TeamLookup(frozenset({"team-a", "team-b"}))
|
||||
await _raise_if_ptu_share_teams_missing(_shared_model_info({"team-a": 3, "team-b": 2}), lambda: lookup)
|
||||
assert lookup.looked_up == ("team-a", "team-b")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_share_team_check_splits_a_share_list_longer_than_one_in_list_chunk():
|
||||
team_ids: Final = tuple(f"team-{index}" for index in range(IN_LIST_CHUNK_SIZE + 1))
|
||||
lookup: Final = _TeamLookup(frozenset(team_ids))
|
||||
await _raise_if_ptu_share_teams_missing(_shared_model_info(dict.fromkeys(team_ids, 1)), lambda: lookup)
|
||||
assert lookup.looked_up == team_ids
|
||||
assert max(lookup.batch_sizes) <= IN_LIST_CHUNK_SIZE
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_share_team_check_leaves_a_team_id_holder_to_the_team_model_check():
|
||||
lookup: Final = _TeamLookup(frozenset())
|
||||
await _raise_if_ptu_share_teams_missing(
|
||||
{"team_id": "team-a", "ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "ptu_effective_from": _SHARED_START},
|
||||
lambda: lookup,
|
||||
)
|
||||
assert lookup.looked_up == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_new_refuses_shares_naming_a_team_that_does_not_exist(monkeypatch):
|
||||
monkeypatch.setenv(PTU_COST_ATTRIBUTION_ENV_VAR, "true")
|
||||
prisma_client = MagicMock()
|
||||
prisma_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[])
|
||||
(add_model_to_db, add_team_model_to_db), patches = TestAddNewModelPtuGate._patched_proxy(
|
||||
"ptu-shared-model", prisma_client=prisma_client
|
||||
)
|
||||
admin = UserAPIKeyAuth(user_id="test-admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
base = TestAddNewModelPtuGate._ptu_deployment("ptu-shared-model")
|
||||
shared = base.model_copy(
|
||||
update={"model_info": base.model_info.model_copy(update={"team_id": None, "ptu_shares": {"ghost-team": 15}})}
|
||||
)
|
||||
|
||||
with ExitStack() as stack:
|
||||
for active_patch in patches:
|
||||
stack.enter_context(active_patch)
|
||||
with pytest.raises(ProxyException, match="Team id=ghost-team does not exist in db") as exc:
|
||||
await add_new_model(model_params=shared, user_api_key_dict=admin)
|
||||
|
||||
assert exc.value.code == "400"
|
||||
prisma_client.db.litellm_teamtable.find_many.assert_awaited_once_with(where={"team_id": {"in": ["ghost-team"]}})
|
||||
add_model_to_db.assert_not_called()
|
||||
add_team_model_to_db.assert_not_called()
|
||||
|
||||
|
||||
def test_validate_helper_refuses_a_team_id_beside_shares():
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
_validate_ptu_model_info(
|
||||
{
|
||||
"team_id": "team-a",
|
||||
"ptu_count": 5,
|
||||
"cost_per_ptu_per_hour": 2.0,
|
||||
"ptu_effective_from": _SHARED_START,
|
||||
"ptu_shares": {"team-a": 5},
|
||||
}
|
||||
)
|
||||
assert exc.value.status_code == 400
|
||||
assert "cannot both be set" in exc.value.detail
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from pydantic import ValidationError
|
|||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.azure.ptu_capacity import AZURE_PTU_CAPACITY
|
||||
from litellm.proxy._types import (
|
||||
LiteLLM_BudgetTable,
|
||||
LiteLLM_BudgetTableFull,
|
||||
|
|
@ -16822,6 +16823,180 @@ def test_list_team_v2_answers_503_no_db_connection_when_the_callers_user_read_hi
|
|||
assert response.json() == _DB_OUTAGE_503_BODY
|
||||
|
||||
|
||||
# --- PTU-equivalent consumption on /team/daily/activity ------------------------------------
|
||||
|
||||
|
||||
_ONE_PTU_HOUR_OF_INPUT: Final = AZURE_PTU_CAPACITY["gpt-4.1"].normalized_tokens_per_ptu_hour
|
||||
|
||||
|
||||
def _ptu_activity_page():
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
BreakdownMetrics,
|
||||
DailySpendData,
|
||||
DailySpendMetadata,
|
||||
MetricWithMetadata,
|
||||
SpendAnalyticsPaginatedResponse,
|
||||
SpendMetrics,
|
||||
)
|
||||
|
||||
tokens: Final = _ONE_PTU_HOUR_OF_INPUT
|
||||
metrics = SpendMetrics(
|
||||
prompt_tokens=tokens, completion_tokens=0, total_tokens=tokens, api_requests=3, successful_requests=3
|
||||
)
|
||||
return SpendAnalyticsPaginatedResponse(
|
||||
results=[
|
||||
DailySpendData(
|
||||
date=datetime(2026, 9, 23).date(),
|
||||
metrics=metrics,
|
||||
breakdown=BreakdownMetrics(
|
||||
model_groups={"gpt-4.1-ptu": MetricWithMetadata(metrics=metrics, metadata={}, api_key_breakdown={})}
|
||||
),
|
||||
)
|
||||
],
|
||||
metadata=DailySpendMetadata(total_tokens=tokens, total_api_requests=3, total_successful_requests=3),
|
||||
)
|
||||
|
||||
|
||||
def _shared_ptu_router() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4.1-ptu",
|
||||
"litellm_params": {"model": "azure/gpt-4.1", "api_key": "sk-ptu", "api_base": "https://ptu.example"},
|
||||
"model_info": {
|
||||
"id": "shared-ptu",
|
||||
"base_model": "azure/gpt-4.1",
|
||||
"ptu_count": 50,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
"ptu_shares": {"team-a": 30, "team-b": 20},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("attribution_enabled", [True, False])
|
||||
async def test_team_daily_activity_reports_ptu_hours_only_while_attribution_is_on(
|
||||
mock_db_client, mock_admin_auth, monkeypatch, attribution_enabled
|
||||
):
|
||||
"""An hour of one PTU's input rate on gpt-4.1, in uncached input tokens, is one PTU-hour;
|
||||
the figure appears beside tokens only once the PTU flag is on, and the token totals are
|
||||
untouched either way."""
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_daily_activity
|
||||
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True" if attribution_enabled else "False")
|
||||
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[])
|
||||
page = _ptu_activity_page()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.management_endpoints.team_endpoints.get_daily_activity", AsyncMock(return_value=page)),
|
||||
patch("litellm.proxy.proxy_server.llm_router", _shared_ptu_router()),
|
||||
):
|
||||
result = await get_team_daily_activity(
|
||||
team_ids="team-a",
|
||||
start_date="2026-09-23",
|
||||
end_date="2026-09-24",
|
||||
model=None,
|
||||
api_key=None,
|
||||
page=1,
|
||||
page_size=10,
|
||||
exclude_team_ids=None,
|
||||
user_api_key_dict=mock_admin_auth,
|
||||
)
|
||||
|
||||
expected_ptu_hours = 1.0 if attribution_enabled else 0.0
|
||||
assert result.metadata.total_ptu_hours == expected_ptu_hours
|
||||
assert result.results[0].metrics.ptu_hours == expected_ptu_hours
|
||||
assert result.results[0].breakdown.model_groups["gpt-4.1-ptu"].metrics.ptu_hours == expected_ptu_hours
|
||||
assert result.metadata.total_tokens == _ONE_PTU_HOUR_OF_INPUT
|
||||
assert result.results[0].metrics.total_tokens == _ONE_PTU_HOUR_OF_INPUT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_daily_activity_sizes_a_team_scoped_deployment_by_its_public_name(
|
||||
mock_db_client, mock_admin_auth, monkeypatch
|
||||
):
|
||||
"""A deployment registered for one team is renamed to a unique internal name and keeps the
|
||||
name callers use in ``team_public_model_name``, which is the name the activity rows carry,
|
||||
so its sizing row is still found."""
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_daily_activity
|
||||
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[])
|
||||
page = _ptu_activity_page()
|
||||
team_scoped_router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4.1-ptu-3f9c1b",
|
||||
"litellm_params": {"model": "azure/gpt-4.1", "api_key": "sk-ptu", "api_base": "https://ptu.example"},
|
||||
"model_info": {
|
||||
"id": "team-a-ptu",
|
||||
"base_model": "azure/gpt-4.1",
|
||||
"team_id": "team-a",
|
||||
"team_public_model_name": "gpt-4.1-ptu",
|
||||
"ptu_count": 50,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.management_endpoints.team_endpoints.get_daily_activity", AsyncMock(return_value=page)),
|
||||
patch("litellm.proxy.proxy_server.llm_router", team_scoped_router),
|
||||
):
|
||||
result = await get_team_daily_activity(
|
||||
team_ids="team-a",
|
||||
start_date="2026-09-23",
|
||||
end_date="2026-09-24",
|
||||
model=None,
|
||||
api_key=None,
|
||||
page=1,
|
||||
page_size=10,
|
||||
exclude_team_ids=None,
|
||||
user_api_key_dict=mock_admin_auth,
|
||||
)
|
||||
|
||||
assert result.metadata.total_ptu_hours == 1.0
|
||||
assert result.results[0].breakdown.model_groups["gpt-4.1-ptu"].metrics.ptu_hours == 1.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_daily_activity_reports_no_ptu_hours_to_a_team_holding_no_share(
|
||||
mock_db_client, mock_admin_auth, monkeypatch
|
||||
):
|
||||
"""The deployment is split between team-a and team-b, so team-c's page alone converts none of
|
||||
its tokens on the group even though the group is sized."""
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_daily_activity
|
||||
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[])
|
||||
page = _ptu_activity_page()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.management_endpoints.team_endpoints.get_daily_activity", AsyncMock(return_value=page)),
|
||||
patch("litellm.proxy.proxy_server.llm_router", _shared_ptu_router()),
|
||||
):
|
||||
result = await get_team_daily_activity(
|
||||
team_ids="team-c",
|
||||
start_date="2026-09-23",
|
||||
end_date="2026-09-24",
|
||||
model=None,
|
||||
api_key=None,
|
||||
page=1,
|
||||
page_size=10,
|
||||
exclude_team_ids=None,
|
||||
user_api_key_dict=mock_admin_auth,
|
||||
)
|
||||
|
||||
assert result.metadata.total_ptu_hours == 0.0
|
||||
assert result.results[0].breakdown.model_groups["gpt-4.1-ptu"].metrics.ptu_hours == 0.0
|
||||
assert result.results[0].metrics.total_tokens == _ONE_PTU_HOUR_OF_INPUT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("start_date", "end_date"),
|
||||
(
|
||||
|
|
|
|||
|
|
@ -31,3 +31,15 @@ def test_reads_the_env_var_on_every_call(monkeypatch):
|
|||
|
||||
monkeypatch.setenv(PTU_COST_ATTRIBUTION_ENV_VAR, "true")
|
||||
assert is_ptu_cost_attribution_enabled() is True
|
||||
|
||||
|
||||
def test_reads_the_process_environment_and_never_a_secret_manager(monkeypatch):
|
||||
"""The flag is checked on every request and every rollup, so it never costs a round trip to
|
||||
a hosted secret manager even when one is configured for reads."""
|
||||
|
||||
def refuse(*args: object, **kwargs: object) -> object:
|
||||
raise AssertionError("secret manager consulted")
|
||||
|
||||
monkeypatch.setattr("litellm.secret_managers.main.get_secret", refuse)
|
||||
monkeypatch.setenv(PTU_COST_ATTRIBUTION_ENV_VAR, "true")
|
||||
assert is_ptu_cost_attribution_enabled() is True
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from litellm.proxy.spend_tracking.ptu_flat_cost_rollup import (
|
|||
PTUModel,
|
||||
_active_hours_on_day,
|
||||
_compute_daily_flat_cost,
|
||||
_parse_ptu_model,
|
||||
_parse_ptu_models,
|
||||
run_ptu_flat_cost_backfill,
|
||||
run_ptu_flat_cost_rollup,
|
||||
run_scheduled_ptu_rollup,
|
||||
|
|
@ -41,6 +41,14 @@ def _ptu_enabled(monkeypatch):
|
|||
_VALID_PTU = {"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"}
|
||||
|
||||
|
||||
def _parse_ptu_model(row):
|
||||
"""The single holding a team_id deployment parses to, else None; shared deployments
|
||||
parse to one holding per team and have their own tests below."""
|
||||
parsed = _parse_ptu_models(row)
|
||||
assert len(parsed) <= 1
|
||||
return parsed[0] if parsed else None
|
||||
|
||||
|
||||
def _model_row(model_id="m1", model_name="gpt-4o-mini-ptu", model_info=None, with_start=True):
|
||||
row = MagicMock()
|
||||
row.model_id = model_id
|
||||
|
|
@ -2082,3 +2090,54 @@ async def test_the_catch_up_pass_reaches_a_config_declared_deployment():
|
|||
assert len(charged) == 3, charged
|
||||
assert charged[-1] == yesterday
|
||||
assert all(row["ptu_flat_cost"] == pytest.approx(48.0) for row in table.rows.values())
|
||||
|
||||
|
||||
# --- a deployment split into per-team PTU shares -----------------------------------------
|
||||
|
||||
_SHARED_PTU = {"ptu_count": 50, "cost_per_ptu_per_hour": 1.0, "ptu_shares": {"team-b": 20, "team-a": 30}}
|
||||
|
||||
|
||||
def test_a_shared_deployment_parses_to_one_holding_per_team_carrying_its_share():
|
||||
parsed = _parse_ptu_models(_model_row(model_info=dict(_SHARED_PTU)))
|
||||
|
||||
assert [(m.team_id, m.ptu_count) for m in parsed] == [("team-a", 30), ("team-b", 20)]
|
||||
assert {m.model_id for m in parsed} == {"m1"}
|
||||
assert {m.model_name for m in parsed} == {"gpt-4o-mini-ptu"}
|
||||
assert {m.cost_per_ptu_per_hour for m in parsed} == {1.0}
|
||||
|
||||
|
||||
def test_a_shared_deployment_whose_shares_do_not_add_up_is_not_priced():
|
||||
assert _parse_ptu_models(_model_row(model_info={**_SHARED_PTU, "ptu_shares": {"team-a": 30}})) == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rollup_splits_a_shared_deployments_flat_cost_by_share():
|
||||
"""50 PTUs at $1/hour for a day is $1,200; team-a's 30 PTUs are $720 of it and team-b's
|
||||
20 are $480, each keyed on the same deployment id and public name."""
|
||||
prisma, table = _prisma_with_models([_model_row(model_info=dict(_SHARED_PTU))])
|
||||
|
||||
result = await run_ptu_flat_cost_rollup(prisma, target_date=DAY)
|
||||
|
||||
assert result.rows_written == 2
|
||||
assert result.models_processed == 1
|
||||
created = {
|
||||
call.kwargs["data"]["create"]["team_id"]: call.kwargs["data"]["create"] for call in table.upsert.await_args_list
|
||||
}
|
||||
assert created["team-a"]["ptu_flat_cost"] == pytest.approx(720.0)
|
||||
assert created["team-b"]["ptu_flat_cost"] == pytest.approx(480.0)
|
||||
assert sum(row["ptu_flat_cost"] for row in created.values()) == pytest.approx(50 * 1.0 * 24)
|
||||
assert {row["model"] for row in created.values()} == {"m1"}
|
||||
assert {row["model_group"] for row in created.values()} == {"gpt-4o-mini-ptu"}
|
||||
assert {row["api_key"] for row in created.values()} == {PTU_SENTINEL_API_KEY}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_shared_deployment_with_a_closed_window_is_reported_lapsed_once():
|
||||
closed = {**_SHARED_PTU, "ptu_effective_from": "2020-01-01T00:00:00Z", "ptu_effective_to": "2020-02-01T00:00:00Z"}
|
||||
prisma, _ = _prisma_with_models([_model_row(model_id="dep-shared", model_info=closed)])
|
||||
alert = AsyncMock()
|
||||
|
||||
result = await run_scheduled_ptu_rollup(prisma, target_date=DAY, alert=alert)
|
||||
|
||||
assert result.lapsed == ("gpt-4o-mini-ptu",)
|
||||
alert.assert_awaited_once()
|
||||
|
|
|
|||
267
tests/unit/router_utils/test_ptu_shares.py
Normal file
267
tests/unit/router_utils/test_ptu_shares.py
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
"""Tests for per-team PTU shares: who a shared deployment is served to and what a share is worth."""
|
||||
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.azure.ptu_capacity import AZURE_PTU_CAPACITY
|
||||
from litellm.router_utils.ptu_shares import (
|
||||
filter_ptu_shared_deployments,
|
||||
model_group_deployments,
|
||||
model_group_ptu_capacity,
|
||||
ptu_capacity_warning,
|
||||
PTUTeamCeiling,
|
||||
team_ptu_ceiling,
|
||||
team_servable_deployments,
|
||||
)
|
||||
|
||||
_GPT41: Final = AZURE_PTU_CAPACITY["gpt-4.1"]
|
||||
_GPT4O: Final = AZURE_PTU_CAPACITY["gpt-4o"]
|
||||
_GPT6SOL: Final = AZURE_PTU_CAPACITY["gpt-6-sol"]
|
||||
_SHARES: Final = {"team-a": 30, "team-b": 20}
|
||||
|
||||
|
||||
def _shared(
|
||||
model: str = "azure/gpt-4.1", shares: object = _SHARES, deployment_id: str = "shared", ptu_count: int | None = None
|
||||
) -> dict:
|
||||
return {
|
||||
"model_name": "gpt-4.1-ptu",
|
||||
"litellm_params": {"model": model},
|
||||
"model_info": {
|
||||
"id": deployment_id,
|
||||
"ptu_count": ptu_count or (sum(shares.values()) if isinstance(shares, dict) else 50),
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
"ptu_shares": shares,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _single_team(model: str = "azure/gpt-4.1") -> dict:
|
||||
return {
|
||||
"model_name": "gpt-4.1-ptu",
|
||||
"litellm_params": {"model": model},
|
||||
"model_info": {
|
||||
"id": "single",
|
||||
"team_id": "team-a",
|
||||
"ptu_count": 50,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
_OPEN: Final = {"model_name": "gpt-4.1-ptu", "litellm_params": {"model": "azure/gpt-4.1"}, "model_info": {"id": "open"}}
|
||||
|
||||
|
||||
def _unaliased_ceiling(deployments: list[dict], team_id: str, requested_model: str) -> PTUTeamCeiling | None:
|
||||
return team_ptu_ceiling(deployments, deployments, team_id, requested_model)
|
||||
|
||||
|
||||
def test_a_team_holding_a_share_keeps_the_shared_deployment():
|
||||
result: Final = filter_ptu_shared_deployments([_shared(), _OPEN], "team-a")
|
||||
assert [d["model_info"]["id"] for d in result.deployments] == ["shared", "open"]
|
||||
assert result.withheld is False
|
||||
|
||||
|
||||
def test_a_team_without_a_share_only_sees_the_unshared_deployments():
|
||||
result: Final = filter_ptu_shared_deployments([_shared(), _OPEN], "team-c")
|
||||
assert [d["model_info"]["id"] for d in result.deployments] == ["open"]
|
||||
assert result.withheld is True
|
||||
|
||||
|
||||
def test_a_caller_with_no_team_holds_no_share():
|
||||
for team_id in (None, ""):
|
||||
result = filter_ptu_shared_deployments([_shared()], team_id)
|
||||
assert result.deployments == ()
|
||||
assert result.withheld is True
|
||||
|
||||
|
||||
def test_a_single_team_deployment_and_a_malformed_share_map_are_not_filtered_here():
|
||||
"""A ``team_id`` deployment is scoped by the router's team filter, and an unusable
|
||||
``ptu_shares`` is refused at registration, so neither is withheld by the share filter."""
|
||||
result: Final = filter_ptu_shared_deployments([_single_team(), _shared(shares={"team-a": 0})], "team-z")
|
||||
assert [d["model_info"]["id"] for d in result.deployments] == ["single", "shared"]
|
||||
assert result.withheld is False
|
||||
|
||||
|
||||
_TERMLESS_SHARED: Final = {
|
||||
"model_name": "gpt-4.1-ptu",
|
||||
"litellm_params": {"model": "azure/gpt-4.1"},
|
||||
"model_info": {"id": "shared", "ptu_shares": _SHARES},
|
||||
}
|
||||
|
||||
|
||||
def test_a_declared_split_is_the_access_rule_even_without_pricing_terms():
|
||||
"""A map that reached the router without the count, rate and start registration now demands
|
||||
still names who the deployment is for, so it is served to those teams alone and sized from
|
||||
their shares; only the flat cost, which needs the terms, goes unattributed."""
|
||||
assert filter_ptu_shared_deployments([_TERMLESS_SHARED], "team-z").withheld is True
|
||||
assert filter_ptu_shared_deployments([_TERMLESS_SHARED], "team-a").withheld is False
|
||||
ceiling: Final = _unaliased_ceiling([_TERMLESS_SHARED], "team-a", "gpt-4.1-ptu")
|
||||
assert ceiling is not None
|
||||
assert ceiling.tpm_limit == 30 * _GPT41.input_tpm_per_ptu
|
||||
|
||||
|
||||
def test_a_team_is_served_from_its_shared_deployment_first_and_never_from_another_teams_reservation():
|
||||
"""team-b holds a share on the shared deployment and the open one serves everyone, while the
|
||||
deployment reserved for team-a alone is withheld from it the way the router withholds it."""
|
||||
servable: Final = team_servable_deployments([_single_team(), _OPEN, _shared()], "team-b")
|
||||
assert [d["model_info"]["id"] for d in servable] == ["shared", "open"]
|
||||
|
||||
|
||||
def test_the_owning_team_keeps_its_reservation_behind_its_share():
|
||||
servable: Final = team_servable_deployments([_single_team(), _OPEN, _shared()], "team-a")
|
||||
assert [d["model_info"]["id"] for d in servable] == ["shared", "single", "open"]
|
||||
|
||||
|
||||
def test_a_share_converts_to_the_models_input_tpm_per_ptu():
|
||||
ceiling: Final = _unaliased_ceiling([_shared()], "team-a", "gpt-4.1-ptu")
|
||||
assert ceiling == PTUTeamCeiling(
|
||||
model_group="gpt-4.1-ptu",
|
||||
tpm_limit=30 * _GPT41.input_tpm_per_ptu,
|
||||
output_to_input_ratio=_GPT41.output_to_input_ratio,
|
||||
cached_input_ratio=_GPT41.cached_input_ratio,
|
||||
)
|
||||
|
||||
|
||||
def test_shares_across_deployments_add_up_and_the_larger_output_ratio_wins():
|
||||
gpt4o: Final = _shared(model="azure/gpt-4o", shares={"team-a": 10}, deployment_id="shared-4o")
|
||||
ceiling: Final = _unaliased_ceiling([_shared(), gpt4o, _OPEN], "team-a", "gpt-4.1-ptu")
|
||||
assert ceiling is not None
|
||||
assert ceiling.tpm_limit == 30 * _GPT41.input_tpm_per_ptu + 10 * _GPT4O.input_tpm_per_ptu
|
||||
assert ceiling.output_to_input_ratio == max(_GPT41.output_to_input_ratio, _GPT4O.output_to_input_ratio)
|
||||
|
||||
|
||||
def test_the_larger_cached_input_ratio_wins_across_deployments():
|
||||
"""A team sharing two models is weighted by the one that charges more for cache reads,
|
||||
whichever order the deployments come in."""
|
||||
gpt6sol: Final = _shared(model="azure/gpt-6-sol", shares={"team-a": 10}, deployment_id="shared-6")
|
||||
ceiling: Final = _unaliased_ceiling([_shared(), gpt6sol], "team-a", "gpt-4.1-ptu")
|
||||
assert ceiling is not None
|
||||
assert _GPT41.cached_input_ratio < _GPT6SOL.cached_input_ratio
|
||||
assert ceiling.cached_input_ratio == _GPT6SOL.cached_input_ratio
|
||||
|
||||
|
||||
def test_a_group_is_served_by_name_or_by_a_team_scoped_deployments_public_name():
|
||||
"""A deployment registered for one team is renamed to a unique internal name and keeps
|
||||
the name callers use in ``team_public_model_name``."""
|
||||
team_scoped: Final = {
|
||||
"model_name": "gpt-4.1-ptu-3f9c1b",
|
||||
"litellm_params": {"model": "azure/gpt-4.1"},
|
||||
"model_info": {"id": "team-scoped", "team_id": "team-a", "team_public_model_name": "gpt-4.1-ptu"},
|
||||
}
|
||||
other: Final = {"model_name": "other", "litellm_params": {"model": "azure/gpt-4o"}, "model_info": {"id": "other"}}
|
||||
deployments: Final = [team_scoped, _OPEN, other]
|
||||
served: Final = model_group_deployments(deployments, "gpt-4.1-ptu")
|
||||
assert [d["model_info"]["id"] for d in served] == ["team-scoped", "open"]
|
||||
assert model_group_deployments(deployments, "gpt-4.1-ptu-3f9c1b") == (team_scoped,)
|
||||
assert model_group_deployments(deployments, "missing") == ()
|
||||
|
||||
|
||||
def test_no_share_or_no_sizing_row_sets_no_ceiling():
|
||||
assert _unaliased_ceiling([_shared()], "team-c", "gpt-4.1-ptu") is None
|
||||
assert _unaliased_ceiling([_shared(model="azure/unknown-deployment")], "team-a", "gpt-4.1-ptu") is None
|
||||
assert _unaliased_ceiling([_single_team(), _OPEN], "team-a", "gpt-4.1-ptu") is None
|
||||
|
||||
|
||||
def test_naming_a_shared_deployment_by_id_or_provider_model_draws_on_its_groups_ceiling():
|
||||
"""The router serves a deployment id or a provider model string when no group has that
|
||||
name, so those names share the group's ceiling instead of bypassing it."""
|
||||
payg: Final = {"model_name": "gpt-4.1-payg", "litellm_params": {"model": "azure/gpt-4.1"}, "model_info": {"id": "payg"}}
|
||||
deployments: Final = [payg, _shared(), _OPEN]
|
||||
by_group: Final = _unaliased_ceiling(deployments, "team-a", "gpt-4.1-ptu")
|
||||
assert by_group is not None
|
||||
assert by_group.model_group == "gpt-4.1-ptu"
|
||||
assert _unaliased_ceiling(deployments, "team-a", "shared") == by_group
|
||||
assert _unaliased_ceiling(deployments, "team-a", "azure/gpt-4.1") == by_group
|
||||
assert _unaliased_ceiling(deployments, "team-a", "payg") is None
|
||||
assert _unaliased_ceiling(deployments, "team-a", "missing") is None
|
||||
|
||||
|
||||
def test_a_group_name_wins_over_a_deployment_id_it_collides_with():
|
||||
"""The router routes a name that is both a group and a deployment id to the group."""
|
||||
colliding: Final = {
|
||||
"model_name": "shared",
|
||||
"litellm_params": {"model": "azure/gpt-4o"},
|
||||
"model_info": {"id": "colliding"},
|
||||
}
|
||||
assert _unaliased_ceiling([_shared(), colliding], "team-a", "shared") is None
|
||||
|
||||
|
||||
def test_a_team_scoped_deployment_named_by_id_draws_on_its_public_groups_ceiling():
|
||||
team_scoped: Final = {
|
||||
"model_name": "gpt-4.1-ptu-3f9c1b",
|
||||
"litellm_params": {"model": "azure/gpt-4.1"},
|
||||
"model_info": {**_shared()["model_info"], "id": "team-scoped", "team_public_model_name": "gpt-4.1-ptu"},
|
||||
}
|
||||
ceiling: Final = _unaliased_ceiling([team_scoped], "team-a", "team-scoped")
|
||||
assert ceiling is not None
|
||||
assert ceiling.model_group == "gpt-4.1-ptu"
|
||||
assert ceiling == _unaliased_ceiling([team_scoped], "team-a", "gpt-4.1-ptu")
|
||||
|
||||
|
||||
def test_alias_and_routing_group_copies_do_not_split_a_deployments_ceiling():
|
||||
"""The router lists alias and routing-group copies of a deployment under their own names
|
||||
ahead of its own rows, so every name still resolves to the deployment's group."""
|
||||
shared: Final = _shared()
|
||||
listed: Final = [{**shared, "model_name": "ptu-alias"}, {**shared, "model_name": "ptu-routing-group"}, shared]
|
||||
by_group: Final = team_ptu_ceiling(listed, [shared], "team-a", "gpt-4.1-ptu")
|
||||
assert by_group is not None
|
||||
assert by_group.model_group == "gpt-4.1-ptu"
|
||||
for name in ("shared", "azure/gpt-4.1", "ptu-routing-group"):
|
||||
assert team_ptu_ceiling(listed, [shared], "team-a", name) == by_group
|
||||
|
||||
|
||||
def test_a_provider_model_draws_on_the_group_where_the_team_holds_its_share():
|
||||
"""Two groups share deployments of one provider model among different teams, and the router
|
||||
serves each team only the one it holds a share of."""
|
||||
east: Final = _shared(shares={"team-a": 30}, deployment_id="east")
|
||||
west: Final = {**_shared(shares={"team-b": 20}, deployment_id="west"), "model_name": "gpt-4.1-ptu-west"}
|
||||
by_provider_model: Final = _unaliased_ceiling([east, west], "team-b", "azure/gpt-4.1")
|
||||
assert by_provider_model is not None
|
||||
assert by_provider_model.model_group == "gpt-4.1-ptu-west"
|
||||
assert by_provider_model == _unaliased_ceiling([east, west], "team-b", "gpt-4.1-ptu-west")
|
||||
assert _unaliased_ceiling([east, west], "team-a", "azure/gpt-4.1") == _unaliased_ceiling(
|
||||
[east, west], "team-a", "gpt-4.1-ptu"
|
||||
)
|
||||
|
||||
|
||||
def test_a_groups_capacity_comes_from_its_first_reserved_deployment_with_a_row():
|
||||
assert model_group_ptu_capacity([_OPEN, _single_team()]) is _GPT41
|
||||
assert model_group_ptu_capacity([_shared(model="azure/unknown"), _shared(model="azure/gpt-4o")]) is _GPT4O
|
||||
assert model_group_ptu_capacity([_OPEN]) is None
|
||||
assert model_group_ptu_capacity([]) is None
|
||||
|
||||
|
||||
def test_a_reserved_deployment_without_a_sizing_row_is_warned_about_by_name():
|
||||
warning: Final = ptu_capacity_warning("gpt-4.1-ptu", _shared(model="azure/my-ptu-deployment"))
|
||||
assert warning is not None
|
||||
assert "gpt-4.1-ptu" in warning
|
||||
assert "base_model" in warning
|
||||
|
||||
|
||||
def test_a_sized_reservation_and_an_unreserved_deployment_raise_no_warning():
|
||||
assert ptu_capacity_warning("gpt-4.1-ptu", _shared()) is None
|
||||
assert ptu_capacity_warning("gpt-4.1-ptu", _single_team()) is None
|
||||
unsized_open: Final = {**_OPEN, "litellm_params": {"model": "azure/my-ptu-deployment"}}
|
||||
assert ptu_capacity_warning("gpt-4.1-ptu", unsized_open) is None
|
||||
|
||||
|
||||
def test_an_unsized_single_team_azure_reservation_is_warned_about_its_ptu_hours_only():
|
||||
warning: Final = ptu_capacity_warning("gpt-4.1-ptu", _single_team(model="azure/my-ptu-deployment"))
|
||||
assert warning is not None
|
||||
assert "PTU hours" in warning
|
||||
assert "ceiling" not in warning
|
||||
|
||||
|
||||
def test_an_unsized_single_team_reservation_on_another_provider_is_not_warned_about():
|
||||
assert ptu_capacity_warning("claude-ptu", _single_team(model="anthropic/claude-sonnet-4-5")) is None
|
||||
|
||||
|
||||
def test_a_bare_model_name_counts_as_azure_through_custom_llm_provider():
|
||||
deployment: Final = {
|
||||
**_single_team(model="my-ptu-deployment"),
|
||||
"litellm_params": {"model": "my-ptu-deployment", "custom_llm_provider": "azure"},
|
||||
}
|
||||
warning: Final = ptu_capacity_warning("gpt-4.1-ptu", deployment)
|
||||
assert warning is not None
|
||||
assert "PTU hours" in warning
|
||||
|
|
@ -18773,6 +18773,88 @@ def test_bare_model_group_served_by_wildcard_deployment_has_provider_prefixed_co
|
|||
assert router._has_content_policy_fallback("claude-haiku-4-5", {}) is False
|
||||
|
||||
|
||||
def _shared_ptu_model_list() -> list:
|
||||
return [
|
||||
{
|
||||
"model_name": "gpt-4.1-ptu",
|
||||
"litellm_params": {"model": "gpt-4.1", "mock_response": "shared"},
|
||||
"model_info": {
|
||||
"id": "shared-deployment",
|
||||
"base_model": "azure/gpt-4.1",
|
||||
"ptu_count": 50,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
"ptu_effective_from": "2026-01-01T00:00:00Z",
|
||||
"ptu_shares": {"team-a": 30, "team-b": 20},
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4.1-ptu",
|
||||
"litellm_params": {"model": "gpt-4.1", "mock_response": "open"},
|
||||
"model_info": {"id": "open-deployment"},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_ptu_shares_hide_the_shared_deployment_from_teams_holding_no_share(monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
router = Router(model_list=_shared_ptu_model_list())
|
||||
for request_kwargs in ({"metadata": {"user_api_key_team_id": "team-c"}}, {"metadata": {}}, {}):
|
||||
_, deployments = router._common_checks_available_deployment(model="gpt-4.1-ptu", request_kwargs=request_kwargs)
|
||||
assert [d["model_info"]["id"] for d in deployments] == ["open-deployment"]
|
||||
for team_id in ("team-a", "team-b"):
|
||||
_, deployments = router._common_checks_available_deployment(
|
||||
model="gpt-4.1-ptu",
|
||||
request_kwargs={"metadata": {"user_api_key_team_id": team_id}},
|
||||
)
|
||||
assert {d["model_info"]["id"] for d in deployments} == {"shared-deployment", "open-deployment"}
|
||||
|
||||
|
||||
def test_ptu_shares_raise_when_only_shared_deployments_remain(monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
router = Router(model_list=_shared_ptu_model_list()[:1])
|
||||
with pytest.raises(litellm.BadRequestError, match="reserved for the teams holding a PTU share"):
|
||||
router._common_checks_available_deployment(
|
||||
model="gpt-4.1-ptu",
|
||||
request_kwargs={"metadata": {"user_api_key_team_id": "team-c"}},
|
||||
)
|
||||
_, deployments = router._common_checks_available_deployment(
|
||||
model="gpt-4.1-ptu",
|
||||
request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}},
|
||||
)
|
||||
assert [d["model_info"]["id"] for d in deployments] == ["shared-deployment"]
|
||||
|
||||
|
||||
def test_ptu_shares_hide_the_shared_deployment_even_while_the_feature_is_off(monkeypatch):
|
||||
"""The flag switches cost attribution on; a declared split is an access rule and holds
|
||||
without it, so a team never reaches a deployment reserved for others while the flag is
|
||||
off."""
|
||||
monkeypatch.delenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", raising=False)
|
||||
router = Router(model_list=_shared_ptu_model_list())
|
||||
_, deployments = router._common_checks_available_deployment(
|
||||
model="gpt-4.1-ptu",
|
||||
request_kwargs={"metadata": {"user_api_key_team_id": "team-c"}},
|
||||
)
|
||||
assert [d["model_info"]["id"] for d in deployments] == ["open-deployment"]
|
||||
|
||||
|
||||
def test_a_shared_ptu_deployment_whose_shares_do_not_add_up_is_refused_at_registration(monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True")
|
||||
model_list = _shared_ptu_model_list()[:1]
|
||||
model_list[0]["model_info"]["ptu_shares"] = {"team-a": 30}
|
||||
with pytest.raises(ValueError, match=r"gpt-4\.1-ptu.*30 of 50 allocated"):
|
||||
Router(model_list=model_list)
|
||||
|
||||
|
||||
def test_a_config_entry_declaring_shares_without_terms_is_refused_even_while_the_feature_is_off(monkeypatch):
|
||||
"""The split is enforced with the flag off, so its shape is checked with the flag off too: a
|
||||
malformed one is refused at boot instead of being honoured or ignored without a word."""
|
||||
monkeypatch.delenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", raising=False)
|
||||
model_list = _shared_ptu_model_list()[:1]
|
||||
model_list[0]["model_info"] = {"id": "shared-deployment", "ptu_shares": {"team-a": 30, "team-b": 20}}
|
||||
with pytest.raises(ValueError, match=r"ptu_count and cost_per_ptu_per_hour are required when ptu_shares is set"):
|
||||
Router(model_list=model_list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"verdict",
|
||||
|
|
|
|||
|
|
@ -2653,7 +2653,7 @@ def test_an_incomplete_reservation_is_refused_rather_than_served(dropped):
|
|||
@pytest.mark.parametrize(
|
||||
"dropped, expected",
|
||||
[
|
||||
("team_id", "team_id is required when PTU fields are set (one model maps to one team)"),
|
||||
("team_id", "team_id or ptu_shares is required when PTU fields are set"),
|
||||
("cost_per_ptu_per_hour", "ptu_count and cost_per_ptu_per_hour must be set together"),
|
||||
],
|
||||
ids=["no team_id", "count without rate"],
|
||||
|
|
|
|||
|
|
@ -8,7 +8,13 @@ import {
|
|||
getTopModels,
|
||||
type ProviderSpendRow,
|
||||
} from "./entityUsageAggregations";
|
||||
import { buildCostBreakdownTiles, buildSummaryTiles, hasFlatCost, type SummaryTile } from "./entityUsageSummary";
|
||||
import {
|
||||
buildCostBreakdownTiles,
|
||||
buildSummaryTiles,
|
||||
hasFlatCost,
|
||||
hasPtuHours,
|
||||
type SummaryTile,
|
||||
} from "./entityUsageSummary";
|
||||
import { MoneyCell } from "@/components/shared/table_cells";
|
||||
import { Card as ShadcnCard, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { hasCapability, type Capability } from "@/utils/capabilities";
|
||||
|
|
@ -299,6 +305,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
|
|||
|
||||
const capitalizedEntityLabel = entityType.charAt(0).toUpperCase() + entityType.slice(1);
|
||||
const showFlatCost = entityType === "team" && hasFlatCost(spendData.metadata);
|
||||
const showPtuHours = entityType === "team" && hasPtuHours(spendData.metadata);
|
||||
const userSpendTeamIds = useMemo(
|
||||
() =>
|
||||
selectedTags.length > 0
|
||||
|
|
@ -407,7 +414,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
|
|||
);
|
||||
|
||||
const breakdownTiles = showFlatCost && showCostBreakdown ? buildCostBreakdownTiles(spendData.metadata) : [];
|
||||
const summaryTiles = [...buildSummaryTiles(spendData.metadata, showFlatCost), ...breakdownTiles];
|
||||
const summaryTiles = [...buildSummaryTiles(spendData.metadata, showFlatCost, showPtuHours), ...breakdownTiles];
|
||||
|
||||
const modelViewTitle = modelViewType === "groups" ? "Top Public Model Names" : "Top Litellm Models";
|
||||
|
||||
|
|
@ -419,7 +426,9 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
|
|||
<ShadcnCard>
|
||||
<CardContent>
|
||||
<h3 className="text-lg font-medium text-foreground">{capitalizedEntityLabel} Spend Overview</h3>
|
||||
<div className="grid grid-cols-5 gap-4 mt-4">{summaryTiles.map(renderSummaryTile)}</div>
|
||||
<div className={`grid ${showPtuHours ? "grid-cols-6" : "grid-cols-5"} gap-4 mt-4`}>
|
||||
{summaryTiles.map(renderSummaryTile)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</ShadcnCard>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { buildCostBreakdownTiles, buildSummaryTiles, hasFlatCost } from "./entityUsageSummary";
|
||||
import { buildCostBreakdownTiles, buildSummaryTiles, hasFlatCost, hasPtuHours } from "./entityUsageSummary";
|
||||
|
||||
const metadata = {
|
||||
total_spend: 100,
|
||||
|
|
@ -10,6 +10,17 @@ const metadata = {
|
|||
total_tokens: 3456,
|
||||
};
|
||||
|
||||
describe("hasPtuHours", () => {
|
||||
it("is false when no PTU hours were consumed or the backend predates the field", () => {
|
||||
expect(hasPtuHours({ ...metadata, total_ptu_hours: 0 })).toBe(false);
|
||||
expect(hasPtuHours(metadata)).toBe(false);
|
||||
});
|
||||
|
||||
it("is true once a PTU model group has been used", () => {
|
||||
expect(hasPtuHours({ ...metadata, total_ptu_hours: 0.25 })).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("hasFlatCost", () => {
|
||||
it("is false when there is no flat cost to report", () => {
|
||||
expect(hasFlatCost({ ...metadata, total_flat_cost: 0 })).toBe(false);
|
||||
|
|
@ -28,6 +39,23 @@ describe("buildSummaryTiles", () => {
|
|||
expect(buildSummaryTiles(metadata, true)).toHaveLength(5);
|
||||
});
|
||||
|
||||
it("adds a sixth PTU Hours tile only when asked, after Total Tokens", () => {
|
||||
const withPtu = { ...metadata, total_ptu_hours: 1234.5678 };
|
||||
expect(buildSummaryTiles(withPtu, true)).toHaveLength(5);
|
||||
const tiles = buildSummaryTiles(withPtu, true, true);
|
||||
expect(tiles).toHaveLength(6);
|
||||
const last = tiles[tiles.length - 1];
|
||||
expect(last.title).toBe("PTU Hours");
|
||||
expect(last.value).toBe("1,234.57");
|
||||
expect(last.tooltip).toContain("PTU-hours");
|
||||
expect(tiles[tiles.length - 2].title).toBe("Total Tokens");
|
||||
});
|
||||
|
||||
it("treats missing PTU hours as zero when the tile is forced on", () => {
|
||||
const tiles = buildSummaryTiles(metadata, false, true);
|
||||
expect(tiles[tiles.length - 1].value).toBe("0.00");
|
||||
});
|
||||
|
||||
it("shows request-only spend under the original title when there is no flat cost", () => {
|
||||
const [first] = buildSummaryTiles(metadata, false);
|
||||
expect(first.title).toBe("Total Spend");
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ export interface SummaryTile {
|
|||
interface SpendSummaryMetadata {
|
||||
total_spend: number;
|
||||
total_flat_cost?: number;
|
||||
total_ptu_hours?: number;
|
||||
total_api_requests: number;
|
||||
total_successful_requests: number;
|
||||
total_failed_requests: number;
|
||||
|
|
@ -26,10 +27,28 @@ export const REQUEST_COST_TOOLTIP =
|
|||
export const FLAT_COST_TOOLTIP =
|
||||
"Reserved provisioned throughput, billed per hour whether or not requests are sent. Reported here only; it does not count toward team, key, user, or organization budgets.";
|
||||
|
||||
export const PTU_HOURS_TOOLTIP =
|
||||
"Provisioned throughput consumed, in PTU-hours: the tokens sent to PTU deployments converted through the Azure sizing table for each model. Compare it with the PTU-hours reserved to see how much of the reservation was used.";
|
||||
|
||||
export const hasFlatCost = (metadata: SpendSummaryMetadata): boolean => (metadata.total_flat_cost ?? 0) > 0;
|
||||
|
||||
export const buildSummaryTiles = (metadata: SpendSummaryMetadata, showFlatCost: boolean): SummaryTile[] => {
|
||||
export const hasPtuHours = (metadata: SpendSummaryMetadata): boolean => (metadata.total_ptu_hours ?? 0) > 0;
|
||||
|
||||
export const buildSummaryTiles = (
|
||||
metadata: SpendSummaryMetadata,
|
||||
showFlatCost: boolean,
|
||||
showPtuHours: boolean = false,
|
||||
): SummaryTile[] => {
|
||||
const flatCost = metadata.total_flat_cost ?? 0;
|
||||
const ptuHoursTile: SummaryTile[] = showPtuHours
|
||||
? [
|
||||
{
|
||||
title: "PTU Hours",
|
||||
value: formatNumberWithCommas(metadata.total_ptu_hours ?? 0, 2),
|
||||
tooltip: PTU_HOURS_TOOLTIP,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
return [
|
||||
showFlatCost
|
||||
? {
|
||||
|
|
@ -47,6 +66,7 @@ export const buildSummaryTiles = (metadata: SpendSummaryMetadata, showFlatCost:
|
|||
},
|
||||
{ title: "Failed Requests", value: metadata.total_failed_requests.toLocaleString(), className: "text-destructive" },
|
||||
{ title: "Total Tokens", value: metadata.total_tokens.toLocaleString() },
|
||||
...ptuHoursTile,
|
||||
];
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ export const EMPTY_DAILY_ACTIVITY_METADATA: DailyActivityMetadata = {
|
|||
total_pages: 1,
|
||||
total_spend: 0,
|
||||
total_flat_cost: 0,
|
||||
total_ptu_hours: 0,
|
||||
total_api_requests: 0,
|
||||
total_successful_requests: 0,
|
||||
total_failed_requests: 0,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
export interface SpendMetrics {
|
||||
spend: number;
|
||||
flat_cost?: number;
|
||||
ptu_hours?: number;
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
|
|
|
|||
14
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
14
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -31044,6 +31044,11 @@ export interface components {
|
|||
* @default 0
|
||||
*/
|
||||
total_prompt_tokens: number;
|
||||
/**
|
||||
* Total Ptu Hours
|
||||
* @default 0
|
||||
*/
|
||||
total_ptu_hours: number;
|
||||
/**
|
||||
* Total Response Time Ms
|
||||
* @default 0
|
||||
|
|
@ -45316,6 +45321,11 @@ export interface components {
|
|||
* @default 0
|
||||
*/
|
||||
prompt_tokens: number;
|
||||
/**
|
||||
* Ptu Hours
|
||||
* @default 0
|
||||
*/
|
||||
ptu_hours: number;
|
||||
/**
|
||||
* Spend
|
||||
* @default 0
|
||||
|
|
@ -49612,6 +49622,10 @@ export interface components {
|
|||
ptu_effective_from?: string | null;
|
||||
/** Ptu Effective To */
|
||||
ptu_effective_to?: string | null;
|
||||
/** Ptu Shares */
|
||||
ptu_shares?: {
|
||||
[key: string]: number;
|
||||
} | null;
|
||||
/** Team Id */
|
||||
team_id?: string | null;
|
||||
/** Team Public Model Name */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue