mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
fix: default shadow eval cost_actual to 0 so judge spend accumulates
The column was declared Float? with no default, so every row started NULL. The verdict writer increments it, and NULL + x is NULL in Postgres, meaning judge spend never accumulated and the UI always showed no spend. Makes the column non-null with a default of 0 across all three schema copies and the migration, and tightens the response model to a plain float.
This commit is contained in:
parent
aba922ff40
commit
1da3bdc5cc
7 changed files with 39 additions and 6 deletions
|
|
@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalJob" (
|
|||
"failed_count" INTEGER NOT NULL DEFAULT 0,
|
||||
"result_json" JSONB,
|
||||
"cost_estimate" DOUBLE PRECISION,
|
||||
"cost_actual" DOUBLE PRECISION,
|
||||
"cost_actual" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"created_by" TEXT,
|
||||
"completed_at" TIMESTAMP(3),
|
||||
|
|
|
|||
|
|
@ -1469,7 +1469,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
result_json Json?
|
||||
|
||||
cost_estimate Float? // estimated cost of running the judge
|
||||
cost_actual Float? // actual cost (judge calls * price)
|
||||
cost_actual Float @default(0) // actual cost (judge calls * price)
|
||||
|
||||
created_at DateTime @default(now())
|
||||
created_by String? // user_id of the key that created this job
|
||||
|
|
|
|||
|
|
@ -1469,7 +1469,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
result_json Json?
|
||||
|
||||
cost_estimate Float? // estimated cost of running the judge
|
||||
cost_actual Float? // actual cost (judge calls * price)
|
||||
cost_actual Float @default(0) // actual cost (judge calls * price)
|
||||
|
||||
created_at DateTime @default(now())
|
||||
created_by String? // user_id of the key that created this job
|
||||
|
|
|
|||
|
|
@ -221,6 +221,6 @@ class GetShadowEvalJobResponse(BaseModel):
|
|||
default=None, description="Present once at least one verdict has been recorded"
|
||||
)
|
||||
cost_estimate: float | None = None
|
||||
cost_actual: float | None = Field(default=None, description="Running total of judge-call spend for this job")
|
||||
cost_actual: float = Field(default=0.0, description="Running total of judge-call spend for this job")
|
||||
created_at: str
|
||||
completed_at: str | None = None
|
||||
|
|
|
|||
|
|
@ -1469,7 +1469,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
result_json Json?
|
||||
|
||||
cost_estimate Float? // estimated cost of running the judge
|
||||
cost_actual Float? // actual cost (judge calls * price)
|
||||
cost_actual Float @default(0) // actual cost (judge calls * price)
|
||||
|
||||
created_at DateTime @default(now())
|
||||
created_by String? // user_id of the key that created this job
|
||||
|
|
|
|||
|
|
@ -261,6 +261,38 @@ class TestStoppedJobCannotBeReactivated:
|
|||
assert call_kwargs["data"]["status"] == "running"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestVerdictWriteAccumulatesCost:
|
||||
async def test_cost_actual_uses_increment_not_a_raw_set(self):
|
||||
"""Regression: cost_actual must accumulate across verdicts, not overwrite.
|
||||
|
||||
The DB column defaults to 0 (not NULL) precisely so this increment lands
|
||||
on a real number instead of NULL + x = NULL. If this update ever
|
||||
regresses to a flat `"cost_actual": judge_cost` assignment, only the
|
||||
last verdict's cost would survive instead of the running total.
|
||||
"""
|
||||
prisma = MagicMock()
|
||||
prisma.db.litellm_shadowevalverdict.create = AsyncMock()
|
||||
prisma.db.litellm_shadowevaljob.update_many = AsyncMock()
|
||||
|
||||
logger = ShadowEvalLogger(router_provider=lambda: MagicMock(), prisma_provider=lambda: prisma)
|
||||
logger._call_router_shadow = AsyncMock(return_value=("shadow text", "shadow-model", "SIMPLE", 10))
|
||||
logger._call_judge = AsyncMock(return_value=("real", 0.9, "clearer", 0.05))
|
||||
|
||||
job = {"id": "j1", "router_name": "r", "judge_model": "m", "shadow_percentage": 100.0, "status": "running"}
|
||||
await logger._run_shadow_eval(
|
||||
job=job,
|
||||
request_id="req-1",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
response_obj={"choices": [{"message": {"content": "real text"}}]},
|
||||
real_model="gpt-4o",
|
||||
model_parameters={},
|
||||
)
|
||||
|
||||
_, call_kwargs = prisma.db.litellm_shadowevaljob.update_many.call_args
|
||||
assert call_kwargs["data"]["cost_actual"] == {"increment": 0.05}
|
||||
|
||||
|
||||
class TestExtractResponseText:
|
||||
def test_dict_response(self):
|
||||
resp = {"choices": [{"message": {"content": "hello"}}]}
|
||||
|
|
|
|||
3
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
3
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -25339,8 +25339,9 @@ export interface components {
|
|||
/**
|
||||
* Cost Actual
|
||||
* @description Running total of judge-call spend for this job
|
||||
* @default 0
|
||||
*/
|
||||
cost_actual?: number | null;
|
||||
cost_actual: number;
|
||||
/** Cost Estimate */
|
||||
cost_estimate?: number | null;
|
||||
/** Created At */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue