fix: default shadow eval cost_actual to 0 so judge spend accumulates

The column was declared Float? with no default, so every row started NULL.
The verdict writer increments it, and NULL + x is NULL in Postgres, meaning
judge spend never accumulated and the UI always showed no spend.

Makes the column non-null with a default of 0 across all three schema copies
and the migration, and tightens the response model to a plain float.
This commit is contained in:
Abhimanyu Kapur 2026-08-08 09:40:13 -07:00
parent aba922ff40
commit 1da3bdc5cc
7 changed files with 39 additions and 6 deletions

View file

@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalJob" (
"failed_count" INTEGER NOT NULL DEFAULT 0,
"result_json" JSONB,
"cost_estimate" DOUBLE PRECISION,
"cost_actual" DOUBLE PRECISION,
"cost_actual" DOUBLE PRECISION NOT NULL DEFAULT 0,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"completed_at" TIMESTAMP(3),

View file

@ -1469,7 +1469,7 @@ model LiteLLM_ShadowEvalJob {
result_json Json?
cost_estimate Float? // estimated cost of running the judge
cost_actual Float? // actual cost (judge calls * price)
cost_actual Float @default(0) // actual cost (judge calls * price)
created_at DateTime @default(now())
created_by String? // user_id of the key that created this job

View file

@ -1469,7 +1469,7 @@ model LiteLLM_ShadowEvalJob {
result_json Json?
cost_estimate Float? // estimated cost of running the judge
cost_actual Float? // actual cost (judge calls * price)
cost_actual Float @default(0) // actual cost (judge calls * price)
created_at DateTime @default(now())
created_by String? // user_id of the key that created this job

View file

@ -221,6 +221,6 @@ class GetShadowEvalJobResponse(BaseModel):
default=None, description="Present once at least one verdict has been recorded"
)
cost_estimate: float | None = None
cost_actual: float | None = Field(default=None, description="Running total of judge-call spend for this job")
cost_actual: float = Field(default=0.0, description="Running total of judge-call spend for this job")
created_at: str
completed_at: str | None = None

View file

@ -1469,7 +1469,7 @@ model LiteLLM_ShadowEvalJob {
result_json Json?
cost_estimate Float? // estimated cost of running the judge
cost_actual Float? // actual cost (judge calls * price)
cost_actual Float @default(0) // actual cost (judge calls * price)
created_at DateTime @default(now())
created_by String? // user_id of the key that created this job

View file

@ -261,6 +261,38 @@ class TestStoppedJobCannotBeReactivated:
assert call_kwargs["data"]["status"] == "running"
@pytest.mark.asyncio
class TestVerdictWriteAccumulatesCost:
async def test_cost_actual_uses_increment_not_a_raw_set(self):
"""Regression: cost_actual must accumulate across verdicts, not overwrite.
The DB column defaults to 0 (not NULL) precisely so this increment lands
on a real number instead of NULL + x = NULL. If this update ever
regresses to a flat `"cost_actual": judge_cost` assignment, only the
last verdict's cost would survive instead of the running total.
"""
prisma = MagicMock()
prisma.db.litellm_shadowevalverdict.create = AsyncMock()
prisma.db.litellm_shadowevaljob.update_many = AsyncMock()
logger = ShadowEvalLogger(router_provider=lambda: MagicMock(), prisma_provider=lambda: prisma)
logger._call_router_shadow = AsyncMock(return_value=("shadow text", "shadow-model", "SIMPLE", 10))
logger._call_judge = AsyncMock(return_value=("real", 0.9, "clearer", 0.05))
job = {"id": "j1", "router_name": "r", "judge_model": "m", "shadow_percentage": 100.0, "status": "running"}
await logger._run_shadow_eval(
job=job,
request_id="req-1",
messages=[{"role": "user", "content": "hi"}],
response_obj={"choices": [{"message": {"content": "real text"}}]},
real_model="gpt-4o",
model_parameters={},
)
_, call_kwargs = prisma.db.litellm_shadowevaljob.update_many.call_args
assert call_kwargs["data"]["cost_actual"] == {"increment": 0.05}
class TestExtractResponseText:
def test_dict_response(self):
resp = {"choices": [{"message": {"content": "hello"}}]}

View file

@ -25339,8 +25339,9 @@ export interface components {
/**
* Cost Actual
* @description Running total of judge-call spend for this job
* @default 0
*/
cost_actual?: number | null;
cost_actual: number;
/** Cost Estimate */
cost_estimate?: number | null;
/** Created At */