diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261003140000_add_timed_completion_tokens/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261003140000_add_timed_completion_tokens/migration.sql new file mode 100644 index 00000000000..6d08b2092aa --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261003140000_add_timed_completion_tokens/migration.sql @@ -0,0 +1,6 @@ +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "timed_completion_tokens" BIGINT NOT NULL DEFAULT 0; diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 147174c73d0..06a696a6b6c 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -4352,11 +4352,6 @@ }, "ProviderThroughputMetrics": { "properties": { - "completion_tokens": { - "default": 0, - "title": "Completion Tokens", - "type": "integer" - }, "output_tokens_per_second": { "anyOf": [ { @@ -4368,6 +4363,11 @@ ], "title": "Output Tokens Per Second" }, + "timed_completion_tokens": { + "default": 0, + "title": "Timed Completion Tokens", + "type": "integer" + }, "timed_requests": { "default": 0, "title": "Timed Requests", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0abec51cc49..f1c43b3f1ca 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5703,6 +5703,7 @@ class BaseDailySpendTransaction(TypedDict): failed_requests: int total_response_time_ms: NotRequired[int] # writable-ok: the rollup queue accumulates into this key in place timed_requests: NotRequired[int] # writable-ok: the rollup queue accumulates into this key in place + timed_completion_tokens: NotRequired[int] # writable-ok: the rollup queue accumulates into this key in place class DailyTeamSpendTransaction(BaseDailySpendTransaction): diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index eb130a5196f..e4f8b62e6ae 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -91,6 +91,7 @@ _COUNTER_COLUMNS: Final = ( "compression_saved_tokens", "total_response_time_ms", "timed_requests", + "timed_completion_tokens", ) _SPEND_COLUMNS: Final = ( "spend", diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 26a21069c83..1897a6700ba 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -2798,6 +2798,7 @@ class DBSpendUpdateWriter: autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter, total_response_time_ms=timed_duration_ms or 0, timed_requests=0 if timed_duration_ms is None else 1, + timed_completion_tokens=0 if timed_duration_ms is None else cast(int, payload["completion_tokens"]), ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 288c85c3513..cc17bceba71 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -162,6 +162,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("timed_requests", 0) or 0 ) + daily_transaction.get("timed_requests", 0) + daily_transaction["timed_completion_tokens"] = ( + payload.get("timed_completion_tokens", 0) or 0 + ) + daily_transaction.get("timed_completion_tokens", 0) + else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) return aggregated_daily_spend_update_transactions diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 36ce7d0da5e..f663dcf120b 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -161,6 +161,9 @@ class DailySpendRecord(Protocol): @property def timed_requests(self) -> int: ... + @property + def timed_completion_tokens(self) -> int: ... + class _KeyMetadataDict(TypedDict, total=False): key_alias: ReadOnly[str | None] @@ -238,15 +241,17 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) -> def _provider_throughput( - completion_tokens: int, + timed_completion_tokens: int, total_response_time_ms: int, timed_requests: int, ) -> ProviderThroughputMetrics: output_tokens_per_second: Final = ( - completion_tokens * 1000 / total_response_time_ms if timed_requests > 0 and total_response_time_ms > 0 else None + timed_completion_tokens * 1000 / total_response_time_ms + if timed_completion_tokens > 0 and timed_requests > 0 and total_response_time_ms > 0 + else None ) return ProviderThroughputMetrics( - completion_tokens=completion_tokens, + timed_completion_tokens=timed_completion_tokens, total_response_time_ms=total_response_time_ms, timed_requests=timed_requests, output_tokens_per_second=output_tokens_per_second, @@ -259,11 +264,14 @@ def _update_provider_throughput( record: DailySpendRecord, ) -> None: existing: Final = target.provider_breakdown.get(provider, ProviderThroughputMetrics()) - target.provider_breakdown[provider] = _provider_throughput( - completion_tokens=existing.completion_tokens + (record.completion_tokens or 0), - total_response_time_ms=existing.total_response_time_ms + (record.total_response_time_ms or 0), - timed_requests=existing.timed_requests + (record.timed_requests or 0), - ) + target.provider_breakdown = { + **target.provider_breakdown, + provider: _provider_throughput( + timed_completion_tokens=existing.timed_completion_tokens + (record.timed_completion_tokens or 0), + total_response_time_ms=existing.total_response_time_ms + (record.total_response_time_ms or 0), + timed_requests=existing.timed_requests + (record.timed_requests or 0), + ), + } def _is_user_agent_tag(tag: str | None) -> bool: @@ -827,7 +835,7 @@ def _aggregate_grouping_sets_records_sync( target: dict[str, MetricWithMetadata], parent_key: str, provider: str, - metrics: SpendMetrics, + record: GroupingSetsRow, ) -> None: parent: Final = target.get(parent_key) if parent is None: @@ -836,18 +844,21 @@ def _aggregate_grouping_sets_records_sync( metadata={}, provider_breakdown={ provider: _provider_throughput( - metrics.completion_tokens, - metrics.total_response_time_ms, - metrics.timed_requests, + record.timed_completion_tokens or 0, + record.total_response_time_ms or 0, + record.timed_requests or 0, ) }, ) return - parent.provider_breakdown[provider] = _provider_throughput( - metrics.completion_tokens, - metrics.total_response_time_ms, - metrics.timed_requests, - ) + parent.provider_breakdown = { + **parent.provider_breakdown, + provider: _provider_throughput( + record.timed_completion_tokens or 0, + record.total_response_time_ms or 0, + record.timed_requests or 0, + ), + } for record in records: level = record.group_level @@ -882,7 +893,7 @@ def _aggregate_grouping_sets_records_sync( breakdown.models, record.model, record.custom_llm_provider or "unknown", - metrics, + record, ) elif level == _GROUP_DATE_MODEL_GROUP: if record.model_group: @@ -901,7 +912,7 @@ def _aggregate_grouping_sets_records_sync( breakdown.model_groups, record.model_group, record.custom_llm_provider or "unknown", - metrics, + record, ) elif level == _GROUP_DATE_PROVIDER: # Only PTU sentinel rows carry ptu_flat_cost and they have no provider, so at diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index cf76b764350..83b70ef6481 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -870,6 +870,7 @@ model LiteLLM_DailyUserSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -939,6 +940,7 @@ model LiteLLM_DailyOrganizationSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -977,6 +979,7 @@ model LiteLLM_DailyEndUserSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -1014,6 +1017,7 @@ model LiteLLM_DailyAgentSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -1051,6 +1055,7 @@ model LiteLLM_DailyTeamSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) ptu_flat_cost Float @default(0.0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -1091,6 +1096,7 @@ model LiteLLM_DailyTagSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 7495bb6abbf..945136a2e8b 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -129,7 +129,8 @@ def _rollup_metric_select(table: DailyActivityTable) -> str: SUM(successful_requests)::bigint AS successful_requests, SUM(failed_requests)::bigint AS failed_requests, SUM(total_response_time_ms)::bigint AS total_response_time_ms, - SUM(timed_requests)::bigint AS timed_requests""" + SUM(timed_requests)::bigint AS timed_requests, + SUM(timed_completion_tokens)::bigint AS timed_completion_tokens""" def _validate_api_key_limit(api_key_limit: int) -> None: diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 4fc27a03a99..ee0d599a720 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -57,7 +57,7 @@ class KeyMetricWithMetadata(MetricBase): class ProviderThroughputMetrics(BaseModel): - completion_tokens: int = Field(default=0) + timed_completion_tokens: int = Field(default=0) total_response_time_ms: int = Field(default=0) timed_requests: int = Field(default=0) output_tokens_per_second: float | None = Field(default=None) diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index df302234398..b4dae9cd9e1 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -124,6 +124,7 @@ class RollupMetricsRow: failed_requests: int | None total_response_time_ms: int | None timed_requests: int | None + timed_completion_tokens: int | None @dataclass(frozen=True, slots=True) @@ -184,6 +185,7 @@ class DailyActivityRow(Protocol): failed_requests: int total_response_time_ms: int timed_requests: int + timed_completion_tokens: int @dataclass(frozen=True, slots=True) diff --git a/schema.prisma b/schema.prisma index cf76b764350..83b70ef6481 100644 --- a/schema.prisma +++ b/schema.prisma @@ -870,6 +870,7 @@ model LiteLLM_DailyUserSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -939,6 +940,7 @@ model LiteLLM_DailyOrganizationSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -977,6 +979,7 @@ model LiteLLM_DailyEndUserSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -1014,6 +1017,7 @@ model LiteLLM_DailyAgentSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -1051,6 +1055,7 @@ model LiteLLM_DailyTeamSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) ptu_flat_cost Float @default(0.0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -1091,6 +1096,7 @@ model LiteLLM_DailyTagSpend { failed_requests BigInt @default(0) total_response_time_ms BigInt @default(0) timed_requests BigInt @default(0) + timed_completion_tokens BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index c17ba75db03..f12f6d076f5 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -40,6 +40,7 @@ async def test_add_single_update(daily_spend_update_queue): "spend": 10.0, "prompt_tokens": 100, "completion_tokens": 50, + "timed_completion_tokens": 50, "api_requests": 1, "successful_requests": 1, "failed_requests": 0, @@ -73,6 +74,7 @@ async def test_add_multiple_updates(daily_spend_update_queue): "spend": 5.0, "prompt_tokens": 200, "completion_tokens": 30, + "timed_completion_tokens": 0, "api_requests": 1, "successful_requests": 1, "failed_requests": 0, @@ -181,6 +183,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "spend": 10.0, "prompt_tokens": 100, "completion_tokens": 50, + "timed_completion_tokens": 50, "api_requests": 1, "successful_requests": 1, "failed_requests": 0, @@ -190,6 +193,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "spend": 5.0, "prompt_tokens": 200, "completion_tokens": 30, + "timed_completion_tokens": 0, "api_requests": 1, "successful_requests": 1, "failed_requests": 0, @@ -199,6 +203,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "spend": 15.0, # 10 + 5 "prompt_tokens": 300, # 100 + 200 "completion_tokens": 80, # 50 + 30 + "timed_completion_tokens": 50, "api_requests": 2, # 1 + 1 "successful_requests": 2, # 1 + 1 "failed_requests": 0, # 0 + 0 @@ -235,6 +240,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "spend": 10.0, "prompt_tokens": 100, "completion_tokens": 50, + "timed_completion_tokens": 50, "api_requests": 1, "successful_requests": 1, "failed_requests": 0, @@ -244,6 +250,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "spend": 5.0, "prompt_tokens": 200, "completion_tokens": 30, + "timed_completion_tokens": 0, "api_requests": 1, "successful_requests": 1, "failed_requests": 0, @@ -253,6 +260,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "spend": 15.0, # 10 + 5 "prompt_tokens": 300, # 100 + 200 "completion_tokens": 80, # 50 + 30 + "timed_completion_tokens": 50, "api_requests": 2, # 1 + 1 "successful_requests": 2, # 1 + 1 "failed_requests": 0, # 0 + 0 diff --git a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py index 7893fb82281..4941ae0f827 100644 --- a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py +++ b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py @@ -34,6 +34,7 @@ def tag_txn(**overrides): "endpoint": "/chat/completions", "prompt_tokens": 10, "completion_tokens": 20, + "timed_completion_tokens": 20, "spend": 0.25, "api_requests": 1, "successful_requests": 1, @@ -87,10 +88,10 @@ def test_one_statement_carries_every_row_in_the_batch(): assert sql.count("INSERT INTO") == 1 assert len(re.findall(r"ON CONFLICT", sql)) == 1 - # 25 bound columns per row plus the inlined updated_at, so the row count is what + # 26 bound columns per row plus the inlined updated_at, so the row count is what # separates one multi-row statement from a hundred single-row ones. - assert len(params) == 100 * 25 - assert "$2500::text" in sql + assert len(params) == 100 * 26 + assert "$2600::text" in sql assert sql.count("(NOW() AT TIME ZONE 'UTC')") == 100 + 1 @@ -115,6 +116,7 @@ def test_conflict_target_is_the_full_unique_constraint(): "failed_requests", "total_response_time_ms", "timed_requests", + "timed_completion_tokens", ], ) def test_counters_increment_rather_than_overwrite(column): diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 4de90d5d7f7..c70affce0df 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -3714,6 +3714,7 @@ async def test_daily_transaction_rolls_up_response_time_for_successful_requests( assert transaction is not None assert transaction["total_response_time_ms"] == request_duration_ms assert transaction["timed_requests"] == 1 + assert transaction["timed_completion_tokens"] == 5 @pytest.mark.asyncio @@ -3746,6 +3747,7 @@ async def test_daily_transaction_excludes_untimed_requests_from_response_time( assert transaction is not None assert transaction["total_response_time_ms"] == 0 assert transaction["timed_requests"] == 0 + assert transaction["timed_completion_tokens"] == 0 def _deadlock_error(): diff --git a/tests/unit/proxy/management_endpoints/test_common_daily_activity.py b/tests/unit/proxy/management_endpoints/test_common_daily_activity.py index c14e6e098d6..fcb7ce547dc 100644 --- a/tests/unit/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/unit/proxy/management_endpoints/test_common_daily_activity.py @@ -366,6 +366,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "autorouter_savings_spend": 0.0, "total_response_time_ms": 0, "timed_requests": 0, + "timed_completion_tokens": 0, "failed_requests": 0, } mock_rows = [ @@ -950,6 +951,7 @@ def test_update_breakdown_metrics_includes_user_email(): autorouter_savings_spend=0, total_response_time_ms=0, timed_requests=0, + timed_completion_tokens=0, total_tokens=2, api_requests=1, successful_requests=1, @@ -1031,6 +1033,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.autorouter_savings_spend = 0.0 mock_record_1.total_response_time_ms = 18_000 mock_record_1.timed_requests = 9 + mock_record_1.timed_completion_tokens = 200 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 mock_record_1.failed_requests = 1 @@ -1057,6 +1060,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.autorouter_savings_spend = 0.0 mock_record_2.total_response_time_ms = 2_500 mock_record_2.timed_requests = 5 + mock_record_2.timed_completion_tokens = 100 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 mock_record_2.failed_requests = 0 @@ -1129,6 +1133,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "autorouter_savings_spend": 0.0, "total_response_time_ms": 0, "timed_requests": 0, + "timed_completion_tokens": 0, "failed_requests": 0, } mock_rows = [ @@ -1225,6 +1230,7 @@ async def test_aggregated_activity_flags_only_keys_that_key_info_can_still_resol "autorouter_savings_spend": 0.0, "total_response_time_ms": 0, "timed_requests": 0, + "timed_completion_tokens": 0, "api_requests": 1, "successful_requests": 1, "failed_requests": 0, @@ -1295,6 +1301,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr autorouter_savings_spend=0.0, total_response_time_ms=0, timed_requests=0, + timed_completion_tokens=0, api_requests=1, successful_requests=1, failed_requests=0, @@ -1444,6 +1451,7 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "autorouter_savings_spend": None, "total_response_time_ms": None, "timed_requests": None, + "timed_completion_tokens": None, "api_requests": None, "successful_requests": None, "failed_requests": None, @@ -1492,6 +1500,7 @@ def _no_spend_record(): autorouter_savings_spend=None, total_response_time_ms=None, timed_requests=None, + timed_completion_tokens=None, api_requests=None, successful_requests=None, failed_requests=None, @@ -1631,6 +1640,7 @@ def _spend_record(api_key, *, model="gpt-4o-mini-ptu", spend=0.0, ptu_flat_cost= autorouter_savings_spend=0, total_response_time_ms=0, timed_requests=0, + timed_completion_tokens=0, total_tokens=0, api_requests=0, successful_requests=0, @@ -1676,6 +1686,7 @@ def _grouping_row( spend=0.0, ptu_flat_cost=0.0, completion_tokens=0, + timed_completion_tokens=0, total_response_time_ms=0, timed_requests=0, ): @@ -1693,6 +1704,7 @@ def _grouping_row( ptu_flat_cost=ptu_flat_cost, prompt_tokens=0, completion_tokens=completion_tokens, + timed_completion_tokens=timed_completion_tokens, cache_read_input_tokens=0, cache_creation_input_tokens=0, compression_saved_tokens=0, @@ -1809,7 +1821,8 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod _GROUP_DATE_MODEL_PROVIDER, model="gpt-4o", custom_llm_provider="openai", - completion_tokens=900, + completion_tokens=1900, + timed_completion_tokens=900, total_response_time_ms=3000, timed_requests=3, ), @@ -1818,6 +1831,7 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod model="gpt-4o", custom_llm_provider="azure", completion_tokens=400, + timed_completion_tokens=400, total_response_time_ms=2000, timed_requests=2, ), @@ -1826,6 +1840,7 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod model_group="public-gpt-4o", custom_llm_provider="openai", completion_tokens=900, + timed_completion_tokens=900, total_response_time_ms=3000, timed_requests=3, ), @@ -1834,7 +1849,7 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod day = _aggregate_grouping_sets_records_sync(records=records, api_key_metadata={})["results"][0] assert day.breakdown.models["gpt-4o"].provider_breakdown["openai"].model_dump() == { - "completion_tokens": 900, + "timed_completion_tokens": 900, "total_response_time_ms": 3000, "timed_requests": 3, "output_tokens_per_second": 300.0, @@ -1843,7 +1858,7 @@ def test_grouping_sets_dispatcher_returns_provider_throughput_for_models_and_mod assert day.breakdown.model_groups["public-gpt-4o"].provider_breakdown["openai"].output_tokens_per_second == 300.0 -def test_grouping_sets_dispatcher_returns_no_throughput_without_positive_duration(): +def test_grouping_sets_dispatcher_returns_no_throughput_for_legacy_rows_without_timed_tokens(): from litellm.proxy.management_endpoints.common_daily_activity import ( _GROUP_DATE_MODEL_PROVIDER, _aggregate_grouping_sets_records_sync, @@ -1854,7 +1869,8 @@ def test_grouping_sets_dispatcher_returns_no_throughput_without_positive_duratio _GROUP_DATE_MODEL_PROVIDER, model="gpt-4o", completion_tokens=900, - total_response_time_ms=0, + timed_completion_tokens=0, + total_response_time_ms=3000, timed_requests=1, ) ] @@ -1931,6 +1947,7 @@ def test_update_breakdown_metrics_covers_mcp_endpoint_and_entity(ptu_cost_attrib autorouter_savings_spend=0, total_response_time_ms=2000, timed_requests=2, + timed_completion_tokens=600, total_tokens=0, api_requests=0, successful_requests=0, @@ -2288,6 +2305,7 @@ async def test_get_daily_activity_aggregated_with_entity_breakdown(): "autorouter_savings_spend": 0.0, "total_response_time_ms": 0, "timed_requests": 0, + "timed_completion_tokens": 0, "failed_requests": 0, "prompt_tokens": 0, "completion_tokens": 0, diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index c3f4fdfef54..7b542d83c97 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -59,6 +59,7 @@ class _Activity: failed_requests: int total_response_time_ms: int timed_requests: int + timed_completion_tokens: int _ENTITY_CASES: Final[tuple[tuple[str, str, str], ...]] = ( @@ -101,6 +102,7 @@ def _activity_for_entity( failed_requests=0, total_response_time_ms=100, timed_requests=1, + timed_completion_tokens=5, ) for api_key, date, model, spend, cache_read in key_rows ) @@ -157,6 +159,7 @@ def _seeded_activity() -> tuple[_Activity, ...]: failed_requests=0, total_response_time_ms=100, timed_requests=1, + timed_completion_tokens=5, ) for table in entity_ids ) @@ -180,6 +183,7 @@ def _metrics(rows: Sequence[_Activity]) -> Mapping[str, int | float]: "failed_requests": sum(row.failed_requests for row in rows), "total_response_time_ms": sum(row.total_response_time_ms for row in rows), "timed_requests": sum(row.timed_requests for row in rows), + "timed_completion_tokens": sum(row.timed_completion_tokens for row in rows), } diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index d413615a0a1..761e24ac47c 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -237,19 +237,13 @@ def test_aggregate_query_sums_all_savings_drivers_and_response_time() -> None: fields: Final = tuple(field for field in SpendMetrics.model_fields if field.endswith("_savings_spend")) + ( "total_response_time_ms", "timed_requests", + "timed_completion_tokens", ) assert fields assert all(f"SUM({field})" in query.sql for field in fields) -def test_aggregated_query_groups_models_and_model_groups_by_provider() -> None: - query = build_aggregated_sql(_scope(), api_key_limit=constants.USAGE_TOP_API_KEYS_DEFAULT) - - assert "(date, model, custom_llm_provider)" in query.sql - assert "(date, COALESCE(NULLIF(model_group, ''), model), custom_llm_provider)" in query.sql - - def test_aggregated_query_binds_sentinel_and_api_key_limit_after_scope_values() -> None: scope = _scope(entity_ids=None, api_keys=("key-1",)) diff --git a/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.test.ts b/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.test.ts index 54145e3c9c6..33f6ba84038 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.test.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/keyActivityData.test.ts @@ -58,7 +58,7 @@ const aggregatedResponse: DailyActivityAggregatedResponse = { api_key_breakdown: { "key-hash": apiKeyActivity }, provider_breakdown: { openai: { - completion_tokens: 900, + timed_completion_tokens: 900, output_tokens_per_second: 300, timed_requests: 3, total_response_time_ms: 3000, @@ -115,7 +115,7 @@ describe("key activity data", () => { ]); expect(toDailyData(aggregatedResponse)[0].breakdown.models["gpt-4o-mini"].provider_breakdown).toEqual({ openai: { - completion_tokens: 900, + timed_completion_tokens: 900, output_tokens_per_second: 300, timed_requests: 3, total_response_time_ms: 3000, diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index 41a09874050..fa8fb6c3a4e 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -42,7 +42,7 @@ export interface MetricWithMetadata { } export interface ProviderThroughputMetrics { - completion_tokens: number; + timed_completion_tokens: number; total_response_time_ms: number; timed_requests: number; output_tokens_per_second?: number | null; diff --git a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx index f9fc2870aa3..869f679300c 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx @@ -1416,7 +1416,7 @@ describe("processActivityData", () => { api_key_breakdown: {}, provider_breakdown: { openai: { - completion_tokens: 900, + timed_completion_tokens: 900, total_response_time_ms: 3000, timed_requests: 3, output_tokens_per_second: 300, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f75f772e08c..1f437c6d38b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -41077,13 +41077,13 @@ export interface components { }; /** ProviderThroughputMetrics */ ProviderThroughputMetrics: { - /** - * Completion Tokens - * @default 0 - */ - completion_tokens: number; /** Output Tokens Per Second */ output_tokens_per_second?: number | null; + /** + * Timed Completion Tokens + * @default 0 + */ + timed_completion_tokens: number; /** * Timed Requests * @default 0