From 1da3bdc5cc55d8e4181c54d15676c969a4d1e446 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 8 Aug 2026 09:40:13 -0700 Subject: [PATCH] fix: default shadow eval cost_actual to 0 so judge spend accumulates The column was declared Float? with no default, so every row started NULL. The verdict writer increments it, and NULL + x is NULL in Postgres, meaning judge spend never accumulated and the UI always showed no spend. Makes the column non-null with a default of 0 across all three schema copies and the migration, and tightens the response model to a plain float. --- .../migration.sql | 2 +- .../litellm_proxy_extras/schema.prisma | 2 +- litellm/proxy/schema.prisma | 2 +- .../auto_router_endpoints.py | 2 +- schema.prisma | 2 +- .../integrations/test_shadow_eval_logger.py | 32 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 +- 7 files changed, 39 insertions(+), 6 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260807000000_add_shadow_eval_job/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260807000000_add_shadow_eval_job/migration.sql index e9a08a28eaa..d793cba185c 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260807000000_add_shadow_eval_job/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260807000000_add_shadow_eval_job/migration.sql @@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalJob" ( "failed_count" INTEGER NOT NULL DEFAULT 0, "result_json" JSONB, "cost_estimate" DOUBLE PRECISION, - "cost_actual" DOUBLE PRECISION, + "cost_actual" DOUBLE PRECISION NOT NULL DEFAULT 0, "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "created_by" TEXT, "completed_at" TIMESTAMP(3), diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 401ff58ac06..91e0becc0c3 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1469,7 +1469,7 @@ model LiteLLM_ShadowEvalJob { result_json Json? cost_estimate Float? // estimated cost of running the judge - cost_actual Float? // actual cost (judge calls * price) + cost_actual Float @default(0) // actual cost (judge calls * price) created_at DateTime @default(now()) created_by String? // user_id of the key that created this job diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 401ff58ac06..91e0becc0c3 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1469,7 +1469,7 @@ model LiteLLM_ShadowEvalJob { result_json Json? cost_estimate Float? // estimated cost of running the judge - cost_actual Float? // actual cost (judge calls * price) + cost_actual Float @default(0) // actual cost (judge calls * price) created_at DateTime @default(now()) created_by String? // user_id of the key that created this job diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index bdd536f31fe..0646ea3b3b2 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -221,6 +221,6 @@ class GetShadowEvalJobResponse(BaseModel): default=None, description="Present once at least one verdict has been recorded" ) cost_estimate: float | None = None - cost_actual: float | None = Field(default=None, description="Running total of judge-call spend for this job") + cost_actual: float = Field(default=0.0, description="Running total of judge-call spend for this job") created_at: str completed_at: str | None = None diff --git a/schema.prisma b/schema.prisma index 401ff58ac06..91e0becc0c3 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1469,7 +1469,7 @@ model LiteLLM_ShadowEvalJob { result_json Json? cost_estimate Float? // estimated cost of running the judge - cost_actual Float? // actual cost (judge calls * price) + cost_actual Float @default(0) // actual cost (judge calls * price) created_at DateTime @default(now()) created_by String? // user_id of the key that created this job diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index c2389ccbaeb..845f696f9e7 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -261,6 +261,38 @@ class TestStoppedJobCannotBeReactivated: assert call_kwargs["data"]["status"] == "running" +@pytest.mark.asyncio +class TestVerdictWriteAccumulatesCost: + async def test_cost_actual_uses_increment_not_a_raw_set(self): + """Regression: cost_actual must accumulate across verdicts, not overwrite. + + The DB column defaults to 0 (not NULL) precisely so this increment lands + on a real number instead of NULL + x = NULL. If this update ever + regresses to a flat `"cost_actual": judge_cost` assignment, only the + last verdict's cost would survive instead of the running total. + """ + prisma = MagicMock() + prisma.db.litellm_shadowevalverdict.create = AsyncMock() + prisma.db.litellm_shadowevaljob.update_many = AsyncMock() + + logger = ShadowEvalLogger(router_provider=lambda: MagicMock(), prisma_provider=lambda: prisma) + logger._call_router_shadow = AsyncMock(return_value=("shadow text", "shadow-model", "SIMPLE", 10)) + logger._call_judge = AsyncMock(return_value=("real", 0.9, "clearer", 0.05)) + + job = {"id": "j1", "router_name": "r", "judge_model": "m", "shadow_percentage": 100.0, "status": "running"} + await logger._run_shadow_eval( + job=job, + request_id="req-1", + messages=[{"role": "user", "content": "hi"}], + response_obj={"choices": [{"message": {"content": "real text"}}]}, + real_model="gpt-4o", + model_parameters={}, + ) + + _, call_kwargs = prisma.db.litellm_shadowevaljob.update_many.call_args + assert call_kwargs["data"]["cost_actual"] == {"increment": 0.05} + + class TestExtractResponseText: def test_dict_response(self): resp = {"choices": [{"message": {"content": "hello"}}]} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2e80a44187c..63eb71c7790 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25339,8 +25339,9 @@ export interface components { /** * Cost Actual * @description Running total of judge-call spend for this job + * @default 0 */ - cost_actual?: number | null; + cost_actual: number; /** Cost Estimate */ cost_estimate?: number | null; /** Created At */