From 5b0fa890568d2fd154d202297200a4ff61085933 Mon Sep 17 00:00:00 2001 From: yassin Date: Tue, 15 Sep 2026 22:16:12 +0000 Subject: [PATCH 1/2] feat(ui): show average response time per model in usage model activity Roll request_duration_ms of successful, non-internal requests into the daily spend tables as total_response_time_ms plus timed_requests, expose both through the daily activity endpoints, and derive the average in the Usage -> Model Activity view of the Admin UI Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../migration.sql | 23 ++++ .../litellm_proxy_extras/schema.prisma | 12 ++ litellm/proxy/_lazy_openapi_snapshot.json | 20 +++ litellm/proxy/_types.py | 2 + litellm/proxy/db/daily_spend_bulk_upsert.py | 2 + litellm/proxy/db/db_spend_update_writer.py | 16 +++ .../daily_spend_update_queue.py | 8 ++ .../common_daily_activity.py | 24 +++- litellm/proxy/schema.prisma | 12 ++ .../common_daily_activity.py | 6 + schema.prisma | 12 ++ .../test_daily_spend_update_queue.py | 12 +- .../proxy/db/test_daily_spend_bulk_upsert.py | 17 ++- .../proxy/db/test_db_spend_update_writer.py | 70 ++++++++++ .../test_common_daily_activity.py | 79 +++++++++++ .../hooks/usePaginatedDailyActivity.test.ts | 2 + .../hooks/usePaginatedDailyActivity.ts | 4 + .../src/components/UsagePage/types.ts | 5 + .../UsagePage/utils/value_formatters.test.ts | 33 ++++- .../UsagePage/utils/value_formatters.tsx | 11 ++ .../src/components/activity_metrics.test.tsx | 127 ++++++++++++++++++ .../src/components/activity_metrics.tsx | 52 ++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 +++ 23 files changed, 557 insertions(+), 12 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_add_daily_response_time/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_add_daily_response_time/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_add_daily_response_time/migration.sql new file mode 100644 index 00000000000..79382ef9d63 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_add_daily_response_time/migration.sql @@ -0,0 +1,23 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 8072df5aa5b..754614d03ea 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -797,6 +797,8 @@ model LiteLLM_DailyUserSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -833,6 +835,8 @@ model LiteLLM_DailyOrganizationSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -869,6 +873,8 @@ model LiteLLM_DailyEndUserSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -904,6 +910,8 @@ model LiteLLM_DailyAgentSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -939,6 +947,8 @@ model LiteLLM_DailyTeamSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) ptu_flat_cost Float @default(0.0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -977,6 +987,8 @@ model LiteLLM_DailyTagSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index f3b579d22c7..fdb84a043e4 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -3125,6 +3125,11 @@ "title": "Total Prompt Tokens", "type": "integer" }, + "total_response_time_ms": { + "default": 0, + "title": "Total Response Time Ms", + "type": "integer" + }, "total_spend": { "default": 0.0, "title": "Total Spend", @@ -3135,6 +3140,11 @@ "title": "Total Successful Requests", "type": "integer" }, + "total_timed_requests": { + "default": 0, + "title": "Total Timed Requests", + "type": "integer" + }, "total_tokens": { "default": 0, "title": "Total Tokens", @@ -3643,6 +3653,16 @@ "title": "Successful Requests", "type": "integer" }, + "timed_requests": { + "default": 0, + "title": "Timed Requests", + "type": "integer" + }, + "total_response_time_ms": { + "default": 0, + "title": "Total Response Time Ms", + "type": "integer" + }, "total_tokens": { "default": 0, "title": "Total Tokens", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index afc2150934e..6b177460e30 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5214,6 +5214,8 @@ class BaseDailySpendTransaction(TypedDict): api_requests: int successful_requests: int failed_requests: int + total_response_time_ms: NotRequired[int] # writable-ok: the rollup queue accumulates into this key in place + timed_requests: NotRequired[int] # writable-ok: the rollup queue accumulates into this key in place class DailyTeamSpendTransaction(BaseDailySpendTransaction): diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index a143643577e..108b0e884ba 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -57,6 +57,8 @@ _COUNTER_COLUMNS: Final = ( "cache_read_input_tokens", "cache_creation_input_tokens", "compression_saved_tokens", + "total_response_time_ms", + "timed_requests", ) _SPEND_COLUMNS: Final = ( "spend", diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index eaa03c5d7f7..52e4a645d51 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -131,6 +131,19 @@ class _SpendTransactionManager(Protocol): async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... +def _timed_request_duration_ms( + payload: dict | SpendLogsPayload, + request_status: Literal["success", "failure"], + is_internal_call: bool, +) -> int | None: + if is_internal_call or request_status != "success": + return None + duration_ms: Final = payload.get("request_duration_ms") + if not isinstance(duration_ms, int) or duration_ms < 0: + return None + return duration_ms + + def _spend_update_tx(prisma_client: PrismaClient) -> _SpendTransactionManager: tx: Final[_SpendTransactionManager] = prisma_client.db.tx(timeout=timedelta(seconds=60)) return tx @@ -2191,6 +2204,7 @@ class DBSpendUpdateWriter: recorded_autorouter_savings=_metadata.get("autorouter_savings"), billed_at=payload.get("endTime"), ) + timed_duration_ms: Final = _timed_request_duration_ms(payload, request_status, is_internal_call) daily_transaction: Final = BaseDailySpendTransaction( date=date, @@ -2218,6 +2232,8 @@ class DBSpendUpdateWriter: prompt_caching_savings_spend=savings_spend.prompt_caching, gateway_injected_caching_savings_spend=savings_spend.gateway_injected_caching, autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter, + total_response_time_ms=timed_duration_ms or 0, + timed_requests=0 if timed_duration_ms is None else 1, ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 70a529900b2..c6381cd070b 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -142,6 +142,14 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("autorouter_savings_spend", 0) or 0 ) + daily_transaction.get("autorouter_savings_spend", 0) + daily_transaction["total_response_time_ms"] = ( + payload.get("total_response_time_ms", 0) or 0 + ) + daily_transaction.get("total_response_time_ms", 0) + + daily_transaction["timed_requests"] = ( + payload.get("timed_requests", 0) or 0 + ) + daily_transaction.get("timed_requests", 0) + else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) return aggregated_daily_spend_update_transactions diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 44ed0017e42..cac7a9b6d98 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -114,6 +114,12 @@ class DailySpendRecord(Protocol): @property def failed_requests(self) -> int: ... + @property + def total_response_time_ms(self) -> int: ... + + @property + def timed_requests(self) -> int: ... + class _KeyMetadataDict(TypedDict, total=False): key_alias: ReadOnly[str | None] @@ -162,6 +168,8 @@ class _GroupingSetsRow(SimpleNamespace): api_requests: int | None successful_requests: int | None failed_requests: int | None + total_response_time_ms: int | None + timed_requests: int | None class _EntityRollupRow(_GroupingSetsRow): @@ -217,6 +225,8 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) -> existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 existing_metrics.failed_requests += record.failed_requests or 0 + existing_metrics.total_response_time_ms += record.total_response_time_ms or 0 + existing_metrics.timed_requests += record.timed_requests or 0 return existing_metrics @@ -767,7 +777,9 @@ def _build_aggregated_sql_query( SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, - SUM(failed_requests)::bigint AS failed_requests + SUM(failed_requests)::bigint AS failed_requests, + SUM(total_response_time_ms)::bigint AS total_response_time_ms, + SUM(timed_requests)::bigint AS timed_requests FROM "{pg_table}" WHERE {where_clause} GROUP BY GROUPING SETS ( @@ -846,7 +858,9 @@ def _build_entity_rollup_sql_query( SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, - SUM(failed_requests)::bigint AS failed_requests + SUM(failed_requests)::bigint AS failed_requests, + SUM(total_response_time_ms)::bigint AS total_response_time_ms, + SUM(timed_requests)::bigint AS timed_requests FROM "{pg_table}" WHERE {where_clause} GROUP BY GROUPING SETS ( @@ -985,6 +999,8 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics: api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, failed_requests=record.failed_requests or 0, + total_response_time_ms=record.total_response_time_ms or 0, + timed_requests=record.timed_requests or 0, ) @@ -1246,6 +1262,8 @@ async def get_daily_activity( total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend, total_gateway_injected_caching_savings_spend=metadata_metrics.gateway_injected_caching_savings_spend, total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend, + total_response_time_ms=metadata_metrics.total_response_time_ms, + total_timed_requests=metadata_metrics.timed_requests, page=page, total_pages=-(-total_count // page_size), # Ceiling division has_more=(page * page_size) < total_count, @@ -1423,6 +1441,8 @@ async def get_daily_activity_aggregated( "totals" ].gateway_injected_caching_savings_spend, total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend, + total_response_time_ms=aggregated["totals"].total_response_time_ms, + total_timed_requests=aggregated["totals"].timed_requests, page=1, total_pages=1, has_more=False, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 8072df5aa5b..754614d03ea 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -797,6 +797,8 @@ model LiteLLM_DailyUserSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -833,6 +835,8 @@ model LiteLLM_DailyOrganizationSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -869,6 +873,8 @@ model LiteLLM_DailyEndUserSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -904,6 +910,8 @@ model LiteLLM_DailyAgentSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -939,6 +947,8 @@ model LiteLLM_DailyTeamSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) ptu_flat_cost Float @default(0.0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -977,6 +987,8 @@ model LiteLLM_DailyTagSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 090e5c42376..278af61a117 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -32,6 +32,8 @@ class SpendMetrics(BaseModel): successful_requests: int = Field(default=0) failed_requests: int = Field(default=0) api_requests: int = Field(default=0) + total_response_time_ms: int = Field(default=0) + timed_requests: int = Field(default=0) class MetricBase(BaseModel): @@ -93,6 +95,8 @@ class DailySpendMetadata(BaseModel): total_prompt_caching_savings_spend: float = Field(default=0.0) total_gateway_injected_caching_savings_spend: float = Field(default=0.0) total_autorouter_savings_spend: float = Field(default=0.0) + total_response_time_ms: int = Field(default=0) + total_timed_requests: int = Field(default=0) page: int = Field(default=1) total_pages: int = Field(default=1) has_more: bool = Field(default=False) @@ -125,6 +129,8 @@ class LiteLLM_DailyUserSpend(BaseModel): api_requests: int = 0 successful_requests: int = 0 failed_requests: int = 0 + total_response_time_ms: int = 0 + timed_requests: int = 0 class GroupedData(TypedDict): diff --git a/schema.prisma b/schema.prisma index 8072df5aa5b..754614d03ea 100644 --- a/schema.prisma +++ b/schema.prisma @@ -797,6 +797,8 @@ model LiteLLM_DailyUserSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -833,6 +835,8 @@ model LiteLLM_DailyOrganizationSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -869,6 +873,8 @@ model LiteLLM_DailyEndUserSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -904,6 +910,8 @@ model LiteLLM_DailyAgentSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@ -939,6 +947,8 @@ model LiteLLM_DailyTeamSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) ptu_flat_cost Float @default(0.0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -977,6 +987,8 @@ model LiteLLM_DailyTagSpend { api_requests BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index 681132105ad..c17ba75db03 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -209,6 +209,8 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "prompt_caching_savings_spend": 0, "gateway_injected_caching_savings_spend": 0, "autorouter_savings_spend": 0, + "total_response_time_ms": 0, + "timed_requests": 0, } updates = [{test_key: test_transaction1}, {test_key: test_transaction2}] @@ -261,6 +263,8 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "prompt_caching_savings_spend": 0, "gateway_injected_caching_savings_spend": 0, "autorouter_savings_spend": 0, + "total_response_time_ms": 0, + "timed_requests": 0, } # Add updates to queue @@ -550,7 +554,7 @@ async def test_every_optional_daily_metric_aggregates(daily_spend_update_queue): numeric_fields = [ name for name, annotation in BaseDailySpendTransaction.__annotations__.items() if _numeric(annotation) ] - assert "autorouter_savings_spend" in numeric_fields + assert {"autorouter_savings_spend", "total_response_time_ms", "timed_requests"} <= set(numeric_fields) increments = {field: index + 1 for index, field in enumerate(numeric_fields)} await daily_spend_update_queue.add_update({test_key: dict(increments)}) @@ -579,8 +583,12 @@ async def test_optional_metric_missing_from_an_older_payload_still_aggregates( } await daily_spend_update_queue.add_update({test_key: dict(base)}) - await daily_spend_update_queue.add_update({test_key: {**base, "autorouter_savings_spend": 0.25}}) + await daily_spend_update_queue.add_update( + {test_key: {**base, "autorouter_savings_spend": 0.25, "total_response_time_ms": 900, "timed_requests": 1}} + ) await daily_spend_update_queue.aggregate_queue_updates() updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25) + assert updates[0][test_key]["total_response_time_ms"] == 900 + assert updates[0][test_key]["timed_requests"] == 1 diff --git a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py index c1efb3e7220..510f77cecec 100644 --- a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py +++ b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py @@ -85,10 +85,10 @@ def test_one_statement_carries_every_row_in_the_batch(): assert sql.count("INSERT INTO") == 1 assert len(re.findall(r"ON CONFLICT", sql)) == 1 - # 23 bound columns per row plus the inlined updated_at, so the row count is what + # 25 bound columns per row plus the inlined updated_at, so the row count is what # separates one multi-row statement from a hundred single-row ones. - assert len(params) == 100 * 23 - assert "$2300::text" in sql + assert len(params) == 100 * 25 + assert "$2500::text" in sql assert sql.count("(NOW() AT TIME ZONE 'UTC')") == 100 + 1 @@ -104,7 +104,16 @@ def test_conflict_target_is_the_full_unique_constraint(): @pytest.mark.parametrize( "column", - ["prompt_tokens", "completion_tokens", "spend", "api_requests", "successful_requests", "failed_requests"], + [ + "prompt_tokens", + "completion_tokens", + "spend", + "api_requests", + "successful_requests", + "failed_requests", + "total_response_time_ms", + "timed_requests", + ], ) def test_counters_increment_rather_than_overwrite(column): """An overwrite would silently discard every earlier flush's spend for that row.""" diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 5e977712a1e..82b8b5c55c5 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -2748,6 +2748,76 @@ async def test_daily_transaction_internal_call_keeps_spend_but_not_request_count assert user_sent["successful_requests"] == 1 +def _response_time_payload(request_duration_ms: object, metadata: dict | None = None) -> dict: + return { + "request_id": "req-timed-1", + "user": "test-user", + "startTime": "2026-09-15T00:00:00", + "api_key": "test-key", + "model": "gpt-5.5", + "custom_llm_provider": "openai", + "model_group": "gpt-5.5", + "call_type": "acompletion", + "prompt_tokens": 10, + "completion_tokens": 5, + "spend": 0.01, + "request_duration_ms": request_duration_ms, + "metadata": json.dumps(metadata or {}), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_duration_ms", [1234, 0]) +async def test_daily_transaction_rolls_up_response_time_for_successful_requests(request_duration_ms: int): + """A successful user-sent request contributes its request_duration_ms to the daily + response-time sum and counts as one timed request, including a 0 ms duration.""" + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + transaction = await writer._common_add_spend_log_transaction_to_daily_transaction( + payload=_response_time_payload(request_duration_ms), + prisma_client=mock_prisma, + type="user", + ) + + assert transaction is not None + assert transaction["total_response_time_ms"] == request_duration_ms + assert transaction["timed_requests"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_status", "request_duration_ms", "metadata"), + [ + ("failure", 1234, {}), + ("success", None, {}), + ("success", -5, {}), + ("success", "1234", {}), + ("success", 1234, {"internal_call_origin": "shadow_eval_judge"}), + ], + ids=["failed", "missing", "negative", "non_int", "internal_call"], +) +async def test_daily_transaction_excludes_untimed_requests_from_response_time( + request_status: str, request_duration_ms: object, metadata: dict +): + """Failed, internal, and missing/invalid-duration requests never enter the response-time + average: both the duration sum and the timed_requests denominator stay at zero.""" + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value=request_status) + + transaction = await writer._common_add_spend_log_transaction_to_daily_transaction( + payload=_response_time_payload(request_duration_ms, metadata), + prisma_client=mock_prisma, + type="user", + ) + + assert transaction is not None + assert transaction["total_response_time_ms"] == 0 + assert transaction["timed_requests"] == 0 + + def _deadlock_error(): from prisma.errors import RawQueryError diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 71896a18f48..b6dd5d04131 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -157,6 +157,8 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "prompt_caching_savings_spend": 0.0, "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, + "total_response_time_ms": 0, + "timed_requests": 0, "failed_requests": 0, } mock_rows = [ @@ -647,6 +649,8 @@ def test_update_breakdown_metrics_includes_user_email(): prompt_caching_savings_spend=0, gateway_injected_caching_savings_spend=0, autorouter_savings_spend=0, + total_response_time_ms=0, + timed_requests=0, total_tokens=2, api_requests=1, successful_requests=1, @@ -722,6 +726,8 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.prompt_caching_savings_spend = 0.0 mock_record_1.gateway_injected_caching_savings_spend = 0.0 mock_record_1.autorouter_savings_spend = 0.0 + mock_record_1.total_response_time_ms = 18_000 + mock_record_1.timed_requests = 9 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 mock_record_1.failed_requests = 1 @@ -746,6 +752,8 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.prompt_caching_savings_spend = 0.0 mock_record_2.gateway_injected_caching_savings_spend = 0.0 mock_record_2.autorouter_savings_spend = 0.0 + mock_record_2.total_response_time_ms = 2_500 + mock_record_2.timed_requests = 5 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 mock_record_2.failed_requests = 0 @@ -778,6 +786,8 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): assert result.metadata.total_successful_requests == 14 # 9 + 5 assert result.metadata.total_failed_requests == 1 assert result.metadata.total_tokens == 1100 # (500+200) + (300+100) + assert result.metadata.total_response_time_ms == 20_500 + assert result.metadata.total_timed_requests == 14 # Verify breakdown still works assert len(result.results) == 1 @@ -786,6 +796,10 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): assert "staging" in daily.breakdown.entities assert daily.breakdown.entities["production"].metrics.spend == 25.0 assert daily.breakdown.entities["staging"].metrics.spend == 5.0 + assert daily.breakdown.models["gpt-4"].metrics.total_response_time_ms == 18_000 + assert daily.breakdown.models["gpt-4"].metrics.timed_requests == 9 + assert daily.breakdown.models["gpt-3.5-turbo"].metrics.total_response_time_ms == 2_500 + assert daily.breakdown.models["gpt-3.5-turbo"].metrics.timed_requests == 5 @pytest.mark.asyncio @@ -810,6 +824,8 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "prompt_caching_savings_spend": 0.0, "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, + "total_response_time_ms": 0, + "timed_requests": 0, "failed_requests": 0, } mock_rows = [ @@ -900,6 +916,8 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr prompt_caching_savings_spend=0.0, gateway_injected_caching_savings_spend=0.0, autorouter_savings_spend=0.0, + total_response_time_ms=0, + timed_requests=0, api_requests=1, successful_requests=1, failed_requests=0, @@ -1333,6 +1351,8 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "prompt_caching_savings_spend": None, "gateway_injected_caching_savings_spend": None, "autorouter_savings_spend": None, + "total_response_time_ms": None, + "timed_requests": None, "api_requests": None, "successful_requests": None, "failed_requests": None, @@ -1378,6 +1398,8 @@ def _no_spend_record(): prompt_caching_savings_spend=None, gateway_injected_caching_savings_spend=None, autorouter_savings_spend=None, + total_response_time_ms=None, + timed_requests=None, api_requests=None, successful_requests=None, failed_requests=None, @@ -1465,6 +1487,55 @@ class TestEverySavingsDriverSurvivesTheReadPath: ) +class TestResponseTimeSurvivesTheReadPath: + """The dashboard averages total_response_time_ms over timed_requests, so both halves + of the pair must be summed by the rollup query, accumulated across rows, carried by + a single-row conversion, and coalesced when a NULL aggregate comes back.""" + + _FIELDS = ("total_response_time_ms", "timed_requests") + + def test_both_halves_are_summed_by_the_rollup_query(self): + sql, _ = _build_aggregated_sql_query( + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id="user-1", + start_date="2026-09-01", + end_date="2026-09-30", + model=None, + api_key=None, + timezone_offset_minutes=None, + ) + for field in self._FIELDS: + assert f"SUM({field})" in sql, f"{field} is never summed, so the average reads as zero" + + def test_accumulating_rows_keeps_sum_and_count_paired(self): + first = _no_spend_record() + first.total_response_time_ms = 1500 + first.timed_requests = 2 + second = _no_spend_record() + second.total_response_time_ms = 500 + second.timed_requests = 1 + metrics = update_metrics(update_metrics(SpendMetrics(), first), second) + assert metrics.total_response_time_ms == 2000 + assert metrics.timed_requests == 3 + + def test_single_row_conversion_carries_both_halves(self): + record = _no_spend_record() + record.total_response_time_ms = 1234 + record.timed_requests = 4 + metrics = _record_to_spend_metrics(record) + assert metrics.total_response_time_ms == 1234 + assert metrics.timed_requests == 4 + + def test_null_aggregates_read_as_zero(self): + metrics = _record_to_spend_metrics(_no_spend_record()) + assert metrics.total_response_time_ms == 0 + assert metrics.timed_requests == 0 + accumulated = update_metrics(SpendMetrics(), _no_spend_record()) + assert accumulated.total_response_time_ms == 0 + assert accumulated.timed_requests == 0 + + @pytest.fixture def ptu_cost_attribution_enabled(monkeypatch): monkeypatch.setenv(PTU_COST_ATTRIBUTION_ENV_VAR, "true") @@ -1488,6 +1559,8 @@ def _spend_record(api_key, *, model="gpt-4o-mini-ptu", spend=0.0, ptu_flat_cost= prompt_caching_savings_spend=0, gateway_injected_caching_savings_spend=0, autorouter_savings_spend=0, + total_response_time_ms=0, + timed_requests=0, total_tokens=0, api_requests=0, successful_requests=0, @@ -1554,6 +1627,8 @@ def _grouping_row( prompt_caching_savings_spend=0.0, gateway_injected_caching_savings_spend=0.0, autorouter_savings_spend=0.0, + total_response_time_ms=0, + timed_requests=0, api_requests=0, successful_requests=0, failed_requests=0, @@ -1714,6 +1789,8 @@ def test_update_breakdown_metrics_covers_mcp_endpoint_and_entity(ptu_cost_attrib prompt_caching_savings_spend=0, gateway_injected_caching_savings_spend=0, autorouter_savings_spend=0, + total_response_time_ms=0, + timed_requests=0, total_tokens=0, api_requests=0, successful_requests=0, @@ -2118,6 +2195,8 @@ async def test_get_daily_activity_aggregated_with_entity_breakdown(): "prompt_caching_savings_spend": 0.0, "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, + "total_response_time_ms": 0, + "timed_requests": 0, "failed_requests": 0, "prompt_tokens": 0, "completion_tokens": 0, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts index 0537f469920..dd3a1238fb1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts @@ -43,6 +43,8 @@ describe("sumMetadata", () => { total_cache_read_input_tokens: 1, total_cache_creation_input_tokens: 1, total_flat_cost: 1, + total_response_time_ms: 1, + total_timed_requests: 1, }; const merged = sumMetadata(page, page); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts index e023feda2e3..a666ddf6256 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts @@ -30,6 +30,8 @@ const SUMMABLE_METADATA_KEYS = [ "total_cache_read_input_tokens", "total_cache_creation_input_tokens", "total_flat_cost", + "total_response_time_ms", + "total_timed_requests", ] as const; interface DailyActivityResponse { @@ -76,6 +78,8 @@ const EMPTY_DATA: DailyActivityResponse = { total_failed_requests: 0, total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, + total_response_time_ms: 0, + total_timed_requests: 0, total_pages: 1, has_more: false, page: 1, diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index fd4f1350020..e8bd3cb3a87 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -14,6 +14,8 @@ export interface SpendMetrics { prompt_caching_savings_spend?: number; gateway_injected_caching_savings_spend?: number; autorouter_savings_spend?: number; + total_response_time_ms?: number; + timed_requests?: number; } export type DailyData = { @@ -81,6 +83,8 @@ export interface ModelActivityData { prompt_tokens: number; completion_tokens: number; total_spend: number; + total_response_time_ms?: number; + total_timed_requests?: number; top_api_keys: TopApiKeyData[]; top_models: TopModelData[]; daily_data: { @@ -95,6 +99,7 @@ export interface ModelActivityData { failed_requests: number; cache_read_input_tokens: number; cache_creation_input_tokens: number; + avg_response_time_ms?: number | null; }; }[]; } diff --git a/ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.test.ts b/ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.test.ts index 3d2504e6652..09e2529301f 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.test.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.test.ts @@ -1,5 +1,36 @@ import { describe, expect, it } from "vitest"; -import { valueFormatter, valueFormatterSpend } from "./value_formatters"; +import { averageResponseTimeMs, formatResponseTime, valueFormatter, valueFormatterSpend } from "./value_formatters"; + +describe("averageResponseTimeMs", () => { + it("divides the summed duration by the number of timed requests", () => { + expect(averageResponseTimeMs(6000, 4)).toBe(1500); + expect(averageResponseTimeMs(0, 3)).toBe(0); + }); + + it("returns null instead of dividing by zero when nothing was timed", () => { + expect(averageResponseTimeMs(0, 0)).toBeNull(); + expect(averageResponseTimeMs(1200, 0)).toBeNull(); + }); +}); + +describe("formatResponseTime", () => { + it("shows sub-second durations in whole milliseconds", () => { + expect(formatResponseTime(0)).toBe("0ms"); + expect(formatResponseTime(412.6)).toBe("413ms"); + expect(formatResponseTime(999)).toBe("999ms"); + }); + + it("shows durations of a second or more in seconds with two decimals", () => { + expect(formatResponseTime(1000)).toBe("1.00s"); + expect(formatResponseTime(1500)).toBe("1.50s"); + expect(formatResponseTime(12345)).toBe("12.35s"); + }); + + it("shows a dash when there is no average to display", () => { + expect(formatResponseTime(null)).toBe("-"); + expect(formatResponseTime(undefined)).toBe("-"); + }); +}); describe("valueFormatter", () => { it("should format numbers >= 1,000,000 as millions with 2 decimal places", () => { diff --git a/ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.tsx b/ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.tsx index a1fb3ec8bb4..b1373698965 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/utils/value_formatters.tsx @@ -11,6 +11,17 @@ export function valueFormatter(number: number) { return number.toString(); } +export function averageResponseTimeMs(totalResponseTimeMs: number, timedRequests: number): number | null { + if (timedRequests <= 0) return null; + return totalResponseTimeMs / timedRequests; +} + +export function formatResponseTime(ms: number | null | undefined) { + if (ms == null) return "-"; + if (ms < 1000) return `${Math.round(ms)}ms`; + return `${(ms / 1000).toFixed(2)}s`; +} + export function valueFormatterSpend(number: number) { if (number === 0) return "$0"; if (number >= 1_000_000_000) { diff --git a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx index 914fe1872b6..2b3e74d6fe6 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx @@ -1424,6 +1424,133 @@ describe("processActivityData", () => { expect(result).toEqual({}); }); + + it("sums response time per model and derives a per-day average over timed requests", () => { + const dayWithModel = (date: string, metrics: Partial & Record) => + createMockDailyData(date, EMPTY_SPEND_METRICS, { + ...EMPTY_BREAKDOWN, + models: { + "gpt-5.5": { metrics: { ...EMPTY_SPEND_METRICS, ...metrics }, metadata: {}, api_key_breakdown: {} }, + }, + }); + const fourTimedRequests = { + api_requests: 4, + successful_requests: 4, + total_response_time_ms: 6000, + timed_requests: 4, + }; + const oneTimedOneFailed = { + api_requests: 2, + successful_requests: 1, + failed_requests: 1, + total_response_time_ms: 500, + timed_requests: 1, + }; + const onlyFailures = { api_requests: 1, successful_requests: 0, failed_requests: 1 }; + const activity: { results: DailyData[] } = { + results: [ + dayWithModel("2025-01-02", fourTimedRequests), + dayWithModel("2025-01-01", oneTimedOneFailed), + dayWithModel("2025-01-03", onlyFailures), + ], + }; + + const result = processActivityData(activity, "models"); + + expect(result["gpt-5.5"].total_response_time_ms).toBe(6500); + expect(result["gpt-5.5"].total_timed_requests).toBe(5); + expect(result["gpt-5.5"].daily_data.map((day) => day.metrics.avg_response_time_ms)).toEqual([500, 1500, null]); + }); + + it("treats rollups written before response time existed as zero timed requests", () => { + const activity: { results: DailyData[] } = { + results: [ + createMockDailyData("2025-01-01", EMPTY_SPEND_METRICS, { + ...EMPTY_BREAKDOWN, + models: { + "gpt-5.5": { + metrics: { ...EMPTY_SPEND_METRICS, api_requests: 3, successful_requests: 3 }, + metadata: {}, + api_key_breakdown: {}, + }, + }, + }), + ], + }; + + const result = processActivityData(activity, "models"); + + expect(result["gpt-5.5"].total_response_time_ms).toBe(0); + expect(result["gpt-5.5"].total_timed_requests).toBe(0); + expect(result["gpt-5.5"].daily_data[0].metrics.avg_response_time_ms).toBeNull(); + }); +}); + +describe("ActivityMetrics response time", () => { + const timedModel = createMockModelActivityData("GPT-5.5", { + total_response_time_ms: 6000, + total_timed_requests: 4, + daily_data: [ + { + date: "2025-01-01", + metrics: { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + api_requests: 3, + spend: 1, + successful_requests: 3, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + avg_response_time_ms: 2000, + }, + }, + { + date: "2025-01-02", + metrics: { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + api_requests: 1, + spend: 1, + successful_requests: 1, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + avg_response_time_ms: 1000, + }, + }, + ], + }); + + it("shows the model's average response time in the summary card and the collapsed header", () => { + render(); + + expect(screen.getByText("Avg Response Time")).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "1.50s" })).toBeInTheDocument(); + expect(screen.getByText("over 4 timed successful requests")).toBeInTheDocument(); + expect(screen.getByText("1.50s avg response")).toBeInTheDocument(); + }); + + it("renders the per-day response time chart with duration-formatted axis ticks", () => { + render(); + + expect(screen.getByText("Avg Response Time per day")).toBeInTheDocument(); + expect(screen.getByText("Avg Response Time Ms")).toBeInTheDocument(); + expect(screen.getAllByText(/^\d+(\.\d+)?(ms|s)$/).length).toBeGreaterThan(1); + }); + + it("shows a dash and no response time chart when the model has no timed requests", () => { + render(); + + expect(screen.getByText("Avg Response Time")).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "-" })).toBeInTheDocument(); + expect(screen.getByText("over 0 timed successful requests")).toBeInTheDocument(); + expect(screen.queryByText(/avg response$/)).not.toBeInTheDocument(); + expect(screen.queryByText("Avg Response Time per day")).not.toBeInTheDocument(); + expect(screen.queryByText("Avg Response Time Ms")).not.toBeInTheDocument(); + }); }); describe("formatKeyLabel", () => { diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index 7c40a91be29..b1df2b598e2 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -1,4 +1,4 @@ -import { AreaChart, BarChart, CustomLegend, CustomTooltip } from "@/components/shared/charts"; +import { AreaChart, BarChart, CustomLegend, CustomTooltip, LineChart } from "@/components/shared/charts"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { resolveTeamAliasFromTeamID } from "@/utils/teamUtils"; import { Card, CardContent } from "@/components/ui/card"; @@ -9,13 +9,16 @@ import { Team } from "./key_team_helpers/key_list"; import KeyModelUsageView from "./UsagePage/components/KeyModelUsageView"; import { keyActivityLabel } from "./UsagePage/keyActivityLabel"; import { DailyData, KeyMetricWithMetadata, ModelActivityData, TopApiKeyData, TopModelData } from "./UsagePage/types"; -import { valueFormatter } from "./UsagePage/utils/value_formatters"; +import { averageResponseTimeMs, formatResponseTime, valueFormatter } from "./UsagePage/utils/value_formatters"; interface ActivityMetricsProps { modelMetrics: Record; hidePromptCachingMetrics?: boolean; } +const modelAverageResponseTimeMs = (metrics: ModelActivityData): number | null => + averageResponseTimeMs(metrics.total_response_time_ms ?? 0, metrics.total_timed_requests ?? 0); + const ModelSection = ({ modelName, metrics, @@ -28,7 +31,7 @@ const ModelSection = ({ return (
{/* Summary Cards */} -
+

Total Requests

@@ -62,6 +65,17 @@ const ModelSection = ({

+ + +

Avg Response Time

+

+ {formatResponseTime(modelAverageResponseTimeMs(metrics))} +

+

+ over {(metrics.total_timed_requests ?? 0).toLocaleString()} timed successful requests +

+ +
{metrics.top_api_keys && metrics.top_api_keys.length > 0 && ( @@ -154,6 +168,27 @@ const ModelSection = ({ + {(metrics.total_timed_requests ?? 0) > 0 && ( + + +
+

Avg Response Time per day

+ +
+ + + + )} +
@@ -416,6 +451,9 @@ export const ActivityMetrics: React.FC = ({ modelMetrics,
${formatNumberWithCommas(modelMetrics[modelName].total_spend, 2)} {modelMetrics[modelName].total_requests.toLocaleString()} requests + {modelAverageResponseTimeMs(modelMetrics[modelName]) != null && ( + {formatResponseTime(modelAverageResponseTimeMs(modelMetrics[modelName]))} avg response + )}
} @@ -471,11 +509,15 @@ export const processActivityData = ( total_spend: 0, total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, + total_response_time_ms: 0, + total_timed_requests: 0, top_api_keys: [], top_models: [], daily_data: [], }; } + const dayResponseTimeMs = modelData.metrics.total_response_time_ms || 0; + const dayTimedRequests = modelData.metrics.timed_requests || 0; // Update totals modelMetrics[model].total_requests += modelData.metrics.api_requests; modelMetrics[model].prompt_tokens += modelData.metrics.prompt_tokens; @@ -486,6 +528,9 @@ export const processActivityData = ( modelMetrics[model].total_failed_requests += modelData.metrics.failed_requests; modelMetrics[model].total_cache_read_input_tokens += modelData.metrics.cache_read_input_tokens || 0; modelMetrics[model].total_cache_creation_input_tokens += modelData.metrics.cache_creation_input_tokens || 0; + modelMetrics[model].total_response_time_ms = + (modelMetrics[model].total_response_time_ms ?? 0) + dayResponseTimeMs; + modelMetrics[model].total_timed_requests = (modelMetrics[model].total_timed_requests ?? 0) + dayTimedRequests; // Add daily data modelMetrics[model].daily_data.push({ @@ -500,6 +545,7 @@ export const processActivityData = ( failed_requests: modelData.metrics.failed_requests, cache_read_input_tokens: modelData.metrics.cache_read_input_tokens || 0, cache_creation_input_tokens: modelData.metrics.cache_creation_input_tokens || 0, + avg_response_time_ms: averageResponseTimeMs(dayResponseTimeMs, dayTimedRequests), }, }); }); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4c16a8613eb..3b4498c2a1d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27505,6 +27505,11 @@ export interface components { * @default 0 */ total_prompt_tokens: number; + /** + * Total Response Time Ms + * @default 0 + */ + total_response_time_ms: number; /** * Total Spend * @default 0 @@ -27515,6 +27520,11 @@ export interface components { * @default 0 */ total_successful_requests: number; + /** + * Total Timed Requests + * @default 0 + */ + total_timed_requests: number; /** * Total Tokens * @default 0 @@ -37045,6 +37055,16 @@ export interface components { * @default 0 */ successful_requests: number; + /** + * Timed Requests + * @default 0 + */ + timed_requests: number; + /** + * Total Response Time Ms + * @default 0 + */ + total_response_time_ms: number; /** * Total Tokens * @default 0 From be082013f7cef4c167a92fee621f0fe8e20e733f Mon Sep 17 00:00:00 2001 From: yassin Date: Tue, 15 Sep 2026 22:38:55 +0000 Subject: [PATCH 2/2] fix(ui): label the response time chart tooltip with the series name Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/components/activity_metrics.test.tsx | 14 ++++++++++++- .../src/components/activity_metrics.tsx | 21 ++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx index 2b3e74d6fe6..1bd7655b5e3 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.test.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.test.tsx @@ -1,7 +1,8 @@ import { fireEvent, render, screen } from "@testing-library/react"; import React from "react"; import { beforeAll, describe, expect, it, vi } from "vitest"; -import { ActivityMetrics, formatKeyLabel, processActivityData } from "./activity_metrics"; +import { ActivityMetrics, formatKeyLabel, processActivityData, ResponseTimeTooltip } from "./activity_metrics"; +import type { ChartTooltipProps } from "@/components/shared/charts"; import { Team } from "./key_team_helpers/key_list"; import { DailyData, KeyMetricWithMetadata, ModelActivityData } from "./UsagePage/types"; @@ -1541,6 +1542,17 @@ describe("ActivityMetrics response time", () => { expect(screen.getAllByText(/^\d+(\.\d+)?(ms|s)$/).length).toBeGreaterThan(1); }); + it("labels the chart tooltip with the readable series name and a formatted duration", () => { + const payload = [ + { dataKey: "metrics.avg_response_time_ms", value: 1500, color: "#f59e0b", payload: timedModel.daily_data[0] }, + ] as NonNullable; + render(); + + expect(screen.getByText("Avg Response Time Ms")).toBeInTheDocument(); + expect(screen.getByText("1.50s")).toBeInTheDocument(); + expect(screen.queryByText("metrics.avg_response_time_ms")).not.toBeInTheDocument(); + }); + it("shows a dash and no response time chart when the model has no timed requests", () => { render(); diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index b1df2b598e2..95315f8ec27 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -1,4 +1,13 @@ -import { AreaChart, BarChart, CustomLegend, CustomTooltip, LineChart } from "@/components/shared/charts"; +import { + AreaChart, + BarChart, + type ChartTooltipProps, + CustomLegend, + CustomTooltip, + formatCategoryName, + LineChart, + ValueTooltip, +} from "@/components/shared/charts"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { resolveTeamAliasFromTeamID } from "@/utils/teamUtils"; import { Card, CardContent } from "@/components/ui/card"; @@ -19,6 +28,15 @@ interface ActivityMetricsProps { const modelAverageResponseTimeMs = (metrics: ModelActivityData): number | null => averageResponseTimeMs(metrics.total_response_time_ms ?? 0, metrics.total_timed_requests ?? 0); +export const ResponseTimeTooltip = ({ active, payload, label }: ChartTooltipProps) => ( + ({ ...item, name: formatCategoryName(String(item.dataKey ?? "")) }))} + label={label} + valueFormatter={formatResponseTime} + /> +); + const ModelSection = ({ modelName, metrics, @@ -182,6 +200,7 @@ const ModelSection = ({ categories={["metrics.avg_response_time_ms"]} colors={["amber"]} valueFormatter={formatResponseTime} + customTooltip={ResponseTimeTooltip} connectNulls={true} showLegend={false} />