scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-11 03:38:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

fix(usage): measure throughput from timed tokens

Browse source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
atul naik 2026-10-03 20:16:53 +05:30
parent 5a9f1045ad
commit 45f3cf4d1b
22 changed files with 116 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm-proxy-extras/litellm_proxy_extras/migrations/20261003140000_add_timed_completion_tokens/migration.sql Normal file
View file

@ -0,0 +1,6 @@
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0;
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0;
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0;
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0;
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0;
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0;

10
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -4352,11 +4352,6 @@
},
"ProviderThroughputMetrics": {
"properties": {
"completion_tokens": {
"default": 0,
"title": "Completion Tokens",
"type": "integer"
},
"output_tokens_per_second": {
"anyOf": [
{
@ -4368,6 +4363,11 @@
],
"title": "Output Tokens Per Second"
},
"timed_completion_tokens": {
"default": 0,
"title": "Timed Completion Tokens",
"type": "integer"
},
"timed_requests": {
"default": 0,
"title": "Timed Requests",

1
litellm/proxy/_types.py
View file

@ -5703,6 +5703,7 @@ class BaseDailySpendTransaction(TypedDict):
failed_requests: int
total_response_time_ms: NotRequired[int] # writable-ok: the rollup queue accumulates into this key in place
timed_requests: NotRequired[int] # writable-ok: the rollup queue accumulates into this key in place
timed_completion_tokens: NotRequired[int] # writable-ok: the rollup queue accumulates into this key in place
class DailyTeamSpendTransaction(BaseDailySpendTransaction):

1
litellm/proxy/db/daily_spend_bulk_upsert.py
View file

@ -91,6 +91,7 @@ _COUNTER_COLUMNS: Final = (
"compression_saved_tokens",
"total_response_time_ms",
"timed_requests",
"timed_completion_tokens",
)
_SPEND_COLUMNS: Final = (
"spend",

1
litellm/proxy/db/db_spend_update_writer.py
View file

@ -2798,6 +2798,7 @@ class DBSpendUpdateWriter:
autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter,
total_response_time_ms=timed_duration_ms or 0,
timed_requests=0 if timed_duration_ms is None else 1,
timed_completion_tokens=0 if timed_duration_ms is None else cast(int, payload["completion_tokens"]),
)
return daily_transaction
except Exception as e:

4
litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
View file

@ -162,6 +162,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
payload.get("timed_requests", 0) or 0
) + daily_transaction.get("timed_requests", 0)
daily_transaction["timed_completion_tokens"] = (
payload.get("timed_completion_tokens", 0) or 0
) + daily_transaction.get("timed_completion_tokens", 0)
else:
aggregated_daily_spend_update_transactions[_key] = deepcopy(payload)
return aggregated_daily_spend_update_transactions

49
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -161,6 +161,9 @@ class DailySpendRecord(Protocol):
@property
def timed_requests(self) -> int: ...
@property
def timed_completion_tokens(self) -> int: ...
class _KeyMetadataDict(TypedDict, total=False):
key_alias: ReadOnly[str | None]
@ -238,15 +241,17 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) ->
def _provider_throughput(
completion_tokens: int,
timed_completion_tokens: int,
total_response_time_ms: int,
timed_requests: int,
) -> ProviderThroughputMetrics:
output_tokens_per_second: Final = (
completion_tokens * 1000 / total_response_time_ms if timed_requests > 0 and total_response_time_ms > 0 else None
timed_completion_tokens * 1000 / total_response_time_ms
if timed_completion_tokens > 0 and timed_requests > 0 and total_response_time_ms > 0
else None
)
return ProviderThroughputMetrics(
completion_tokens=completion_tokens,
timed_completion_tokens=timed_completion_tokens,
total_response_time_ms=total_response_time_ms,
timed_requests=timed_requests,
output_tokens_per_second=output_tokens_per_second,
@ -259,11 +264,14 @@ def _update_provider_throughput(
record: DailySpendRecord,
) -> None:
existing: Final = target.provider_breakdown.get(provider, ProviderThroughputMetrics())
target.provider_breakdown[provider] = _provider_throughput(
completion_tokens=existing.completion_tokens + (record.completion_tokens or 0),
total_response_time_ms=existing.total_response_time_ms + (record.total_response_time_ms or 0),
timed_requests=existing.timed_requests + (record.timed_requests or 0),
)
target.provider_breakdown = {
**target.provider_breakdown,
provider: _provider_throughput(
timed_completion_tokens=existing.timed_completion_tokens + (record.timed_completion_tokens or 0),
total_response_time_ms=existing.total_response_time_ms + (record.total_response_time_ms or 0),
timed_requests=existing.timed_requests + (record.timed_requests or 0),
),
}
def _is_user_agent_tag(tag: str | None) -> bool:
@ -827,7 +835,7 @@ def _aggregate_grouping_sets_records_sync(
target: dict[str, MetricWithMetadata],
parent_key: str,
provider: str,
metrics: SpendMetrics,
record: GroupingSetsRow,
) -> None:
parent: Final = target.get(parent_key)
if parent is None:
@ -836,18 +844,21 @@ def _aggregate_grouping_sets_records_sync(
metadata={},
provider_breakdown={
provider: _provider_throughput(
metrics.completion_tokens,
metrics.total_response_time_ms,
metrics.timed_requests,
record.timed_completion_tokens or 0,
record.total_response_time_ms or 0,
record.timed_requests or 0,
)
},
)
return
parent.provider_breakdown[provider] = _provider_throughput(
metrics.completion_tokens,
metrics.total_response_time_ms,
metrics.timed_requests,
)
parent.provider_breakdown = {
**parent.provider_breakdown,
provider: _provider_throughput(
record.timed_completion_tokens or 0,
record.total_response_time_ms or 0,
record.timed_requests or 0,
),
}
for record in records:
level = record.group_level
@ -882,7 +893,7 @@ def _aggregate_grouping_sets_records_sync(
breakdown.models,
record.model,
record.custom_llm_provider or "unknown",
metrics,
record,
)
elif level == _GROUP_DATE_MODEL_GROUP:
if record.model_group:
@ -901,7 +912,7 @@ def _aggregate_grouping_sets_records_sync(
breakdown.model_groups,
record.model_group,
record.custom_llm_provider or "unknown",
metrics,
record,
)
elif level == _GROUP_DATE_PROVIDER:
# Only PTU sentinel rows carry ptu_flat_cost and they have no provider, so at

6
litellm/proxy/schema.prisma
View file

@ -870,6 +870,7 @@ model LiteLLM_DailyUserSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@ -939,6 +940,7 @@ model LiteLLM_DailyOrganizationSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@ -977,6 +979,7 @@ model LiteLLM_DailyEndUserSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@ -1014,6 +1017,7 @@ model LiteLLM_DailyAgentSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@ -1051,6 +1055,7 @@ model LiteLLM_DailyTeamSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
ptu_flat_cost Float @default(0.0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@ -1091,6 +1096,7 @@ model LiteLLM_DailyTagSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt

3
litellm/repositories/daily_activity_sql.py
View file

@ -129,7 +129,8 @@ def _rollup_metric_select(table: DailyActivityTable) -> str:
SUM(successful_requests)::bigint AS successful_requests,
SUM(failed_requests)::bigint AS failed_requests,
SUM(total_response_time_ms)::bigint AS total_response_time_ms,
SUM(timed_requests)::bigint AS timed_requests"""
SUM(timed_requests)::bigint AS timed_requests,
SUM(timed_completion_tokens)::bigint AS timed_completion_tokens"""
def _validate_api_key_limit(api_key_limit: int) -> None:

2
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -57,7 +57,7 @@ class KeyMetricWithMetadata(MetricBase):
class ProviderThroughputMetrics(BaseModel):
completion_tokens: int = Field(default=0)
timed_completion_tokens: int = Field(default=0)
total_response_time_ms: int = Field(default=0)
timed_requests: int = Field(default=0)
output_tokens_per_second: float | None = Field(default=None)

2
litellm/types/repositories/daily_activity.py
View file

@ -124,6 +124,7 @@ class RollupMetricsRow:
failed_requests: int | None
total_response_time_ms: int | None
timed_requests: int | None
timed_completion_tokens: int | None
@dataclass(frozen=True, slots=True)
@ -184,6 +185,7 @@ class DailyActivityRow(Protocol):
failed_requests: int
total_response_time_ms: int
timed_requests: int
timed_completion_tokens: int
@dataclass(frozen=True, slots=True)

6
schema.prisma
View file

@ -870,6 +870,7 @@ model LiteLLM_DailyUserSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@ -939,6 +940,7 @@ model LiteLLM_DailyOrganizationSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@ -977,6 +979,7 @@ model LiteLLM_DailyEndUserSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@ -1014,6 +1017,7 @@ model LiteLLM_DailyAgentSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@ -1051,6 +1055,7 @@ model LiteLLM_DailyTeamSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
ptu_flat_cost Float @default(0.0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@ -1091,6 +1096,7 @@ model LiteLLM_DailyTagSpend {
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
timed_completion_tokens BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt

8
tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py
View file

@ -40,6 +40,7 @@ async def test_add_single_update(daily_spend_update_queue):
"spend": 10.0,
"prompt_tokens": 100,
"completion_tokens": 50,
"timed_completion_tokens": 50,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
@ -73,6 +74,7 @@ async def test_add_multiple_updates(daily_spend_update_queue):
"spend": 5.0,
"prompt_tokens": 200,
"completion_tokens": 30,
"timed_completion_tokens": 0,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
@ -181,6 +183,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key():
"spend": 10.0,
"prompt_tokens": 100,
"completion_tokens": 50,
"timed_completion_tokens": 50,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
@ -190,6 +193,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key():
"spend": 5.0,
"prompt_tokens": 200,
"completion_tokens": 30,
"timed_completion_tokens": 0,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
@ -199,6 +203,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key():
"spend": 15.0, # 10 + 5
"prompt_tokens": 300, # 100 + 200
"completion_tokens": 80, # 50 + 30
"timed_completion_tokens": 50,
"api_requests": 2, # 1 + 1
"successful_requests": 2, # 1 + 1
"failed_requests": 0, # 0 + 0
@ -235,6 +240,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions(
"spend": 10.0,
"prompt_tokens": 100,
"completion_tokens": 50,
"timed_completion_tokens": 50,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
@ -244,6 +250,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions(
"spend": 5.0,
"prompt_tokens": 200,
"completion_tokens": 30,
"timed_completion_tokens": 0,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
@ -253,6 +260,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions(
"spend": 15.0, # 10 + 5
"prompt_tokens": 300, # 100 + 200
"completion_tokens": 80, # 50 + 30
"timed_completion_tokens": 50,
"api_requests": 2, # 1 + 1
"successful_requests": 2, # 1 + 1
"failed_requests": 0, # 0 + 0

8
tests/unit/proxy/db/test_daily_spend_bulk_upsert.py
View file

@ -34,6 +34,7 @@ def tag_txn(**overrides):
"endpoint": "/chat/completions",
"prompt_tokens": 10,
"completion_tokens": 20,
"timed_completion_tokens": 20,
"spend": 0.25,
"api_requests": 1,
"successful_requests": 1,
@ -87,10 +88,10 @@ def test_one_statement_carries_every_row_in_the_batch():
assert sql.count("INSERT INTO") == 1
assert len(re.findall(r"ON CONFLICT", sql)) == 1
# 25 bound columns per row plus the inlined updated_at, so the row count is what
# 26 bound columns per row plus the inlined updated_at, so the row count is what
# separates one multi-row statement from a hundred single-row ones.
assert len(params) == 100 * 25
assert "$2500::text" in sql
assert len(params) == 100 * 26
assert "$2600::text" in sql
assert sql.count("(NOW() AT TIME ZONE 'UTC')") == 100 + 1
@ -115,6 +116,7 @@ def test_conflict_target_is_the_full_unique_constraint():
"failed_requests",
"total_response_time_ms",
"timed_requests",
"timed_completion_tokens",
],
)
def test_counters_increment_rather_than_overwrite(column):

2
tests/unit/proxy/db/test_db_spend_update_writer.py
View file

@ -3714,6 +3714,7 @@ async def test_daily_transaction_rolls_up_response_time_for_successful_requests(
assert transaction is not None
assert transaction["total_response_time_ms"] == request_duration_ms
assert transaction["timed_requests"] == 1
assert transaction["timed_completion_tokens"] == 5
@pytest.mark.asyncio
@ -3746,6 +3747,7 @@ async def test_daily_transaction_excludes_untimed_requests_from_response_time(
assert transaction is not None
assert transaction["total_response_time_ms"] == 0
assert transaction["timed_requests"] == 0
assert transaction["timed_completion_tokens"] == 0
def _deadlock_error():

26
tests/unit/proxy/management_endpoints/test_common_daily_activity.py
View file

@ -366,6 +366,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown():
"autorouter_savings_spend": 0.0,
"total_response_time_ms": 0,
"timed_requests": 0,
"timed_completion_tokens": 0,
"failed_requests": 0,
}
mock_rows = [
@ -950,6 +951,7 @@ def test_update_breakdown_metrics_includes_user_email():
autorouter_savings_spend=0,
total_response_time_ms=0,
timed_requests=0,
timed_completion_tokens=0,
total_tokens=2,
api_requests=1,
successful_requests=1,
@ -1031,6 +1033,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_1.autorouter_savings_spend = 0.0
mock_record_1.total_response_time_ms = 18_000
mock_record_1.timed_requests = 9
mock_record_1.timed_completion_tokens = 200
mock_record_1.api_requests = 10
mock_record_1.successful_requests = 9
mock_record_1.failed_requests = 1
@ -1057,6 +1060,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_2.autorouter_savings_spend = 0.0
mock_record_2.total_response_time_ms = 2_500
mock_record_2.timed_requests = 5
mock_record_2.timed_completion_tokens = 100
mock_record_2.api_requests = 5
mock_record_2.successful_requests = 5
mock_record_2.failed_requests = 0
@ -1129,6 +1133,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys():
"autorouter_savings_spend": 0.0,
"total_response_time_ms": 0,
"timed_requests": 0,
"timed_completion_tokens": 0,
"failed_requests": 0,
}
mock_rows = [
@ -1225,6 +1230,7 @@ async def test_aggregated_activity_flags_only_keys_that_key_info_can_still_resol
"autorouter_savings_spend": 0.0,
"total_response_time_ms": 0,
"timed_requests": 0,
"timed_completion_tokens": 0,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
@ -1295,6 +1301,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr
autorouter_savings_spend=0.0,
total_response_time_ms=0,
timed_requests=0,
timed_completion_tokens=0,
api_requests=1,
successful_requests=1,
failed_requests=0,
@ -1444,6 +1451,7 @@ async def test_get_daily_activity_aggregated_empty_result_set():
"autorouter_savings_spend": None,
"total_response_time_ms": None,
"timed_requests": None,
"timed_completion_tokens": None,
"api_requests": None,
"successful_requests": None,
"failed_requests": None,
@ -1492,6 +1500,7 @@ def _no_spend_record():
autorouter_savings_spend=None,
total_response_time_ms=None,
timed_requests=None,
timed_completion_tokens=None,
api_requests=None,
successful_requests=None,
failed_requests=None,
@ -1631,6 +1640,7 @@ def _spend_record(api_key, *, model="gpt-4o-mini-ptu", spend=0.0, ptu_flat_cost=
autorouter_savings_spend=0,
total_response_time_ms=0,
timed_requests=0,
timed_completion_tokens=0,
total_tokens=0,
api_requests=0,
successful_requests=0,
@ -1676,6 +1686,7 @@ def _grouping_row(
spend=0.0,
ptu_flat_cost=0.0,
completion_tokens=0,
timed_completion_tokens=0,
total_response_time_ms=0,
timed_requests=0,
):
@ -1693,6 +1704,7 @@ def _grouping_row(
ptu_flat_cost=ptu_flat_cost,
prompt_tokens=0,
completion_tokens=completion_tokens,
timed_completion_tokens=timed_completion_tokens,
cache_read_input_tokens=0,
cache_creation_input_tokens=0,
compression_saved_tokens=0,
@ -1809,7 +1821,8 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod
_GROUP_DATE_MODEL_PROVIDER,
model="gpt-4o",
custom_llm_provider="openai",
completion_tokens=900,
completion_tokens=1900,
timed_completion_tokens=900,
total_response_time_ms=3000,
timed_requests=3,
),
@ -1818,6 +1831,7 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod
model="gpt-4o",
custom_llm_provider="azure",
completion_tokens=400,
timed_completion_tokens=400,
total_response_time_ms=2000,
timed_requests=2,
),
@ -1826,6 +1840,7 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod
model_group="public-gpt-4o",
custom_llm_provider="openai",
completion_tokens=900,
timed_completion_tokens=900,
total_response_time_ms=3000,
timed_requests=3,
),
@ -1834,7 +1849,7 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod
day = _aggregate_grouping_sets_records_sync(records=records, api_key_metadata={})["results"][0]
assert day.breakdown.models["gpt-4o"].provider_breakdown["openai"].model_dump() == {
"completion_tokens": 900,
"timed_completion_tokens": 900,
"total_response_time_ms": 3000,
"timed_requests": 3,
"output_tokens_per_second": 300.0,
@ -1843,7 +1858,7 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod
assert day.breakdown.model_groups["public-gpt-4o"].provider_breakdown["openai"].output_tokens_per_second == 300.0
def test_grouping_sets_dispatcher_returns_no_throughput_without_positive_duration():
def test_grouping_sets_dispatcher_returns_no_throughput_for_legacy_rows_without_timed_tokens():
from litellm.proxy.management_endpoints.common_daily_activity import (
_GROUP_DATE_MODEL_PROVIDER,
_aggregate_grouping_sets_records_sync,
@ -1854,7 +1869,8 @@ def test_grouping_sets_dispatcher_returns_no_throughput_without_positive_duratio
_GROUP_DATE_MODEL_PROVIDER,
model="gpt-4o",
completion_tokens=900,
total_response_time_ms=0,
timed_completion_tokens=0,
total_response_time_ms=3000,
timed_requests=1,
)
]
@ -1931,6 +1947,7 @@ def test_update_breakdown_metrics_covers_mcp_endpoint_and_entity(ptu_cost_attrib
autorouter_savings_spend=0,
total_response_time_ms=2000,
timed_requests=2,
timed_completion_tokens=600,
total_tokens=0,
api_requests=0,
successful_requests=0,
@ -2288,6 +2305,7 @@ async def test_get_daily_activity_aggregated_with_entity_breakdown():
"autorouter_savings_spend": 0.0,
"total_response_time_ms": 0,
"timed_requests": 0,
"timed_completion_tokens": 0,
"failed_requests": 0,
"prompt_tokens": 0,
"completion_tokens": 0,

4
tests/unit/proxy/management_endpoints/test_daily_activity_routes.py
View file

@ -59,6 +59,7 @@ class _Activity:
failed_requests: int
total_response_time_ms: int
timed_requests: int
timed_completion_tokens: int
_ENTITY_CASES: Final[tuple[tuple[str, str, str], ...]] = (
@ -101,6 +102,7 @@ def _activity_for_entity(
failed_requests=0,
total_response_time_ms=100,
timed_requests=1,
timed_completion_tokens=5,
)
for api_key, date, model, spend, cache_read in key_rows
)
@ -157,6 +159,7 @@ def _seeded_activity() -> tuple[_Activity, ...]:
failed_requests=0,
total_response_time_ms=100,
timed_requests=1,
timed_completion_tokens=5,
)
for table in entity_ids
)
@ -180,6 +183,7 @@ def _metrics(rows: Sequence[_Activity]) -> Mapping[str, int | float]:
"failed_requests": sum(row.failed_requests for row in rows),
"total_response_time_ms": sum(row.total_response_time_ms for row in rows),
"timed_requests": sum(row.timed_requests for row in rows),
"timed_completion_tokens": sum(row.timed_completion_tokens for row in rows),
}

8
tests/unit/repositories/test_daily_activity_sql.py
View file

@ -237,19 +237,13 @@ def test_aggregate_query_sums_all_savings_drivers_and_response_time() -> None:
fields: Final = tuple(field for field in SpendMetrics.model_fields if field.endswith("_savings_spend")) + (
"total_response_time_ms",
"timed_requests",
"timed_completion_tokens",
)
assert fields
assert all(f"SUM({field})" in query.sql for field in fields)
def test_aggregated_query_groups_models_and_model_groups_by_provider() -> None:
query = build_aggregated_sql(_scope(), api_key_limit=constants.USAGE_TOP_API_KEYS_DEFAULT)
assert "(date, model, custom_llm_provider)" in query.sql
assert "(date, COALESCE(NULLIF(model_group, ''), model), custom_llm_provider)" in query.sql
def test_aggregated_query_binds_sentinel_and_api_key_limit_after_scope_values() -> None:
scope = _scope(entity_ids=None, api_keys=("key-1",))

4
ui/litellm-dashboard/src/components/UsagePage/keyActivityData.test.ts
View file

@ -58,7 +58,7 @@ const aggregatedResponse: DailyActivityAggregatedResponse = {
api_key_breakdown: { "key-hash": apiKeyActivity },
provider_breakdown: {
openai: {
completion_tokens: 900,
timed_completion_tokens: 900,
output_tokens_per_second: 300,
timed_requests: 3,
total_response_time_ms: 3000,
@ -115,7 +115,7 @@ describe("key activity data", () => {
]);
expect(toDailyData(aggregatedResponse)[0].breakdown.models["gpt-4o-mini"].provider_breakdown).toEqual({
openai: {
completion_tokens: 900,
timed_completion_tokens: 900,
output_tokens_per_second: 300,
timed_requests: 3,
total_response_time_ms: 3000,

2
ui/litellm-dashboard/src/components/UsagePage/types.ts
View file

@ -42,7 +42,7 @@ export interface MetricWithMetadata {
}
export interface ProviderThroughputMetrics {
completion_tokens: number;
timed_completion_tokens: number;
total_response_time_ms: number;
timed_requests: number;
output_tokens_per_second?: number | null;

2
ui/litellm-dashboard/src/components/activity_metrics.test.tsx
View file

@ -1416,7 +1416,7 @@ describe("processActivityData", () => {
api_key_breakdown: {},
provider_breakdown: {
openai: {
completion_tokens: 900,
timed_completion_tokens: 900,
total_response_time_ms: 3000,
timed_requests: 3,
output_tokens_per_second: 300,

10
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -41077,13 +41077,13 @@ export interface components {
};
/** ProviderThroughputMetrics */
ProviderThroughputMetrics: {
/**
* Completion Tokens
* @default 0
*/
completion_tokens: number;
/** Output Tokens Per Second */
output_tokens_per_second?: number | null;
/**
* Timed Completion Tokens
* @default 0
*/
timed_completion_tokens: number;
/**
* Timed Requests
* @default 0