scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 74

feat(proxy): carry response time metrics through LiteLLM_DailyGlobalSpend

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-16 01:58:37 +00:00
parent 225fc53f08
commit 0601d2bb03
8 changed files with 29 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_add_daily_global_spend/migration.sql
View file

@ -20,6 +20,8 @@ CREATE TABLE IF NOT EXISTS "LiteLLM_DailyGlobalSpend" (
"api_requests" BIGINT NOT NULL DEFAULT 0,
"successful_requests" BIGINT NOT NULL DEFAULT 0,
"failed_requests" BIGINT NOT NULL DEFAULT 0,
"total_response_time_ms" BIGINT NOT NULL DEFAULT 0,
"timed_requests" BIGINT NOT NULL DEFAULT 0,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,

2
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -837,6 +837,8 @@ model LiteLLM_DailyGlobalSpend {
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt

2
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -767,6 +767,8 @@ _KEY_FREE_SOURCE_COLUMNS: Final = (
"api_requests",
"successful_requests",
"failed_requests",
"total_response_time_ms",
"timed_requests",
)

2
litellm/proxy/schema.prisma
View file

@ -837,6 +837,8 @@ model LiteLLM_DailyGlobalSpend {
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt

2
litellm/proxy/spend_tracking/daily_global_spend_rollup.py
View file

@ -43,6 +43,8 @@ _METRIC_COLUMNS: Final = (
"api_requests",
"successful_requests",
"failed_requests",
"total_response_time_ms",
"timed_requests",
"compression_savings_spend",
"prompt_caching_savings_spend",
"gateway_injected_caching_savings_spend",

2
schema.prisma
View file

@ -837,6 +837,8 @@ model LiteLLM_DailyGlobalSpend {
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
total_response_time_ms BigInt @default(0)
timed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt

6
tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
View file

@ -1771,6 +1771,10 @@ async def test_get_daily_activity_aggregated_serves_closed_days_from_the_global_
]
_seed_daily_user_spend(_aggregated_postgresql, rows)
with _aggregated_postgresql.cursor() as cur:
cur.execute(
'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms = prompt_tokens * 25, '
"timed_requests = api_requests"
)
cur.execute(_GLOBAL_SPEND_MIGRATION.read_text()) # pyright: ignore[reportArgumentType] # DDL literal
cur.execute(
re.sub(r"\$(\d+)", r"%(p\1)s", RECONCILE_DAY_SQL), # pyright: ignore[reportArgumentType] # $N -> psycopg
@ -1805,6 +1809,8 @@ async def test_get_daily_activity_aggregated_serves_closed_days_from_the_global_
assert global_sql[0].count('FROM "LiteLLM_DailyUserSpend"') == 3
assert from_global.model_dump() == from_per_key.model_dump()
assert from_global.metadata.total_spend == pytest.approx(2 * sum(float(i + 1) for i in range(n_keys)))
assert from_global.metadata.total_response_time_ms == 2 * n_keys * 10 * 25
assert from_global.metadata.total_timed_requests == 2 * n_keys
assert {day.date.isoformat() for day in from_global.results} == {"2026-06-01", "2026-06-02"}
assert len(from_global.results[0].breakdown.api_keys) == USAGE_TOP_API_KEYS_LIMIT
assert set(from_global.results[0].breakdown.model_groups) == {"gpt-5", "claude"}

13
tests/test_litellm/proxy/spend_tracking/test_daily_global_spend_rollup.py
View file

@ -336,6 +336,8 @@ _DAILY_USER_SPEND_DDL: Final = """
api_requests BIGINT DEFAULT 0,
successful_requests BIGINT DEFAULT 0,
failed_requests BIGINT DEFAULT 0,
total_response_time_ms BIGINT DEFAULT 0,
timed_requests BIGINT DEFAULT 0,
created_at TIMESTAMP DEFAULT now(),
updated_at TIMESTAMP,
UNIQUE (user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint)
@ -345,12 +347,14 @@ _DAILY_USER_SPEND_DDL: Final = """
_PER_KEY_SUMS_SQL: Final = """
SELECT COALESCE(model, '') AS model, COALESCE(model_group, '') AS model_group,
COALESCE(custom_llm_provider, '') AS custom_llm_provider,
SUM(spend) AS spend, SUM(prompt_tokens) AS prompt_tokens, SUM(api_requests) AS api_requests
SUM(spend) AS spend, SUM(prompt_tokens) AS prompt_tokens, SUM(api_requests) AS api_requests,
SUM(total_response_time_ms) AS total_response_time_ms, SUM(timed_requests) AS timed_requests
FROM "LiteLLM_DailyUserSpend" WHERE date = %s
GROUP BY 1, 2, 3 ORDER BY 1, 2, 3
"""
_GLOBAL_ROWS_SQL: Final = """
SELECT model, model_group, custom_llm_provider, spend, prompt_tokens, api_requests
SELECT model, model_group, custom_llm_provider, spend, prompt_tokens, api_requests,
total_response_time_ms, timed_requests
FROM "LiteLLM_DailyGlobalSpend" WHERE date = %s ORDER BY 1, 2, 3
"""
@ -380,6 +384,8 @@ def _user_txn(**overrides):
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
"total_response_time_ms": 800,
"timed_requests": 1,
**overrides,
}
@ -393,6 +399,8 @@ def _normalized(rows: list[dict[str, object]]) -> list[tuple[object, ...]]:
float(r["spend"]),
int(r["prompt_tokens"]),
int(r["api_requests"]),
int(r["total_response_time_ms"]),
int(r["timed_requests"]),
) # pyright: ignore[reportArgumentType] # dict_row values are untyped
for r in rows
]
@ -436,5 +444,6 @@ def test_reconcile_day_sql_makes_the_global_day_equal_the_per_key_sums(_rollup_p
assert _normalized(global_rows) == _normalized(per_key)
assert sum(float(r["spend"]) for r in global_rows) == pytest.approx(15.0) # pyright: ignore[reportArgumentType] # dict_row values are untyped
assert sum(int(r["total_response_time_ms"]) for r in global_rows) == 1600 # pyright: ignore[reportArgumentType] # dict_row values are untyped
assert [(r["model"], r["model_group"]) for r in global_rows] == [("gpt-5", ""), ("gpt-5", "gpt-5")]
assert untouched == []