mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
fix(ui): correct cache-miss rate math in provider cache cost formula
Address review feedback on the input cost formula rows. The cache-miss input cost subtracted only the cache-read portion, so when a request also had cache-write (creation) cost the miss rate was inflated; subtract both the cache-read and cache-write portions. Guard the derived rate against a negative cache-miss cost, which inconsistent backend costs could produce, so the formula hides instead of showing a negative rate. Show the per-token input formula only when cache_read_cost is present, since otherwise input_cost blends the miss and hit rates and cannot be split accurately, and render it directly under the Input Cost line so it attaches to the right row rather than floating below the whole section.
This commit is contained in:
Dhruv
parent
e3b09214b7
commit
2c8a6e8a05
4 changed files with 128 additions and 43 deletions
|
|
@ -218,7 +218,9 @@ describe("CostBreakdownViewer", () => {
|
|||
});
|
||||
|
||||
describe("provider prompt-cache formula breakdown (issue #32045)", () => {
|
||||
// Numbers taken directly from the issue.
|
||||
// The issue's numbers. cache_read_cost is present so the per-token miss/hit
|
||||
// rates are derivable (input_cost - cache_read_cost gives the full-rate miss
|
||||
// cost). This is the case where the formula is accurate and is shown.
|
||||
const issueBreakdown: CostBreakdown = {
|
||||
input_cost: 0.06476666,
|
||||
cache_read_cost: 0.0645696,
|
||||
|
|
@ -226,7 +228,15 @@ describe("CostBreakdownViewer", () => {
|
|||
original_cost: 0.06486395,
|
||||
};
|
||||
|
||||
it("splits the input cost into cache-miss and cache-hit token formulas", async () => {
|
||||
// JSX interpolation splits the formula across child text nodes, so match on the
|
||||
// element whose full textContent matches while none of its children already do
|
||||
// (i.e. the innermost matching element), avoiding ambiguous multi-node matches.
|
||||
const hasText = (needle: RegExp) => (_: string, node: Element | null) => {
|
||||
if (node?.textContent == null || !needle.test(node.textContent)) return false;
|
||||
return !Array.from(node.children).some((child) => child.textContent != null && needle.test(child.textContent));
|
||||
};
|
||||
|
||||
it("splits the input cost into cache-miss and cache-hit token formulas with correct rates", async () => {
|
||||
renderWithProviders(
|
||||
<CostBreakdownViewer
|
||||
costBreakdown={issueBreakdown}
|
||||
|
|
@ -234,15 +244,16 @@ describe("CostBreakdownViewer", () => {
|
|||
promptTokens={430798}
|
||||
completionTokens={47}
|
||||
providerCacheReadTokens={430464}
|
||||
cacheReadTokens={430464}
|
||||
/>,
|
||||
);
|
||||
|
||||
await expandCostBreakdown();
|
||||
|
||||
// Assert the full formula incl. the derived per-token rate, trimmed of trailing
|
||||
// zeros and float noise, so 5.9e-7 shows as $0.00000059 (not $0.00000059000).
|
||||
expect(screen.getByText(/334 cache miss tokens \* \$0\.00000059/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/430,464 cache hit tokens \* \$0\.00000015/)).toBeInTheDocument();
|
||||
// miss cost = 0.06476666 - 0.0645696 = 0.00019706; /334 = 5.9e-7 -> $0.00000059
|
||||
// hit rate = 0.0645696 / 430464 = 1.5e-7 -> $0.00000015
|
||||
expect(screen.getByText(hasText(/334 cache miss tokens \* \$0\.00000059/))).toBeInTheDocument();
|
||||
expect(screen.getByText(hasText(/430,464 cache hit tokens \* \$0\.00000015/))).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows the output cost and original LLM cost formulas", async () => {
|
||||
|
|
@ -253,13 +264,67 @@ describe("CostBreakdownViewer", () => {
|
|||
promptTokens={430798}
|
||||
completionTokens={47}
|
||||
providerCacheReadTokens={430464}
|
||||
cacheReadTokens={430464}
|
||||
/>,
|
||||
);
|
||||
|
||||
await expandCostBreakdown();
|
||||
|
||||
expect(screen.getByText(/47 completion tokens \* \$0\.00000207/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/\$0\.06476666 input cost \+ \$0\.00009729 output cost/)).toBeInTheDocument();
|
||||
expect(screen.getByText(hasText(/47 completion tokens \* \$0\.00000207/))).toBeInTheDocument();
|
||||
expect(screen.getByText(hasText(/\$0\.06476666 input cost \+ \$0\.00009729 output cost/))).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("subtracts BOTH cache-read and cache-write cost from the cache-miss rate", async () => {
|
||||
// 400 miss tokens. input_cost 0.001 = miss + read(0.0006) + write(0.0002).
|
||||
// miss cost = 0.0002; /400 = 5e-7 -> $0.0000005. Omitting cache_creation_cost
|
||||
// (Greptile #2) would give 0.0004/400 = 1e-6, an inflated rate.
|
||||
renderWithProviders(
|
||||
<CostBreakdownViewer
|
||||
costBreakdown={{
|
||||
input_cost: 0.001,
|
||||
cache_read_cost: 0.0006,
|
||||
cache_creation_cost: 0.0002,
|
||||
output_cost: 0.0001,
|
||||
original_cost: 0.0011,
|
||||
}}
|
||||
totalSpend={0.0011}
|
||||
promptTokens={1000}
|
||||
completionTokens={10}
|
||||
providerCacheReadTokens={600}
|
||||
cacheReadTokens={600}
|
||||
/>,
|
||||
);
|
||||
|
||||
await expandCostBreakdown();
|
||||
|
||||
expect(screen.getByText(hasText(/400 cache miss tokens \* \$0\.0000005/))).toBeInTheDocument();
|
||||
// Guard against the inflated $0.000001 rate that omitting cache-write produces.
|
||||
expect(screen.queryByText(hasText(/400 cache miss tokens \* \$0\.000001/))).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides the input formula (no negative rate) when read+write cost exceeds input cost", async () => {
|
||||
// Inconsistent backend costs: read+write > input_cost -> negative miss cost.
|
||||
// deriveRate rejects it (Greptile #3), so the miss/hit formula is not shown.
|
||||
renderWithProviders(
|
||||
<CostBreakdownViewer
|
||||
costBreakdown={{
|
||||
input_cost: 0.001,
|
||||
cache_read_cost: 0.0009,
|
||||
cache_creation_cost: 0.0003,
|
||||
output_cost: 0.0001,
|
||||
original_cost: 0.0011,
|
||||
}}
|
||||
totalSpend={0.0011}
|
||||
promptTokens={1000}
|
||||
completionTokens={10}
|
||||
providerCacheReadTokens={600}
|
||||
cacheReadTokens={600}
|
||||
/>,
|
||||
);
|
||||
|
||||
await expandCostBreakdown();
|
||||
|
||||
expect(screen.queryByText(hasText(/cache miss tokens \*/))).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("does NOT show the formula when there is no provider prompt cache hit", async () => {
|
||||
|
|
@ -275,8 +340,8 @@ describe("CostBreakdownViewer", () => {
|
|||
|
||||
await expandCostBreakdown();
|
||||
|
||||
expect(screen.queryByText(/cache miss tokens \*/)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/cache hit tokens \*/)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(hasText(/cache miss tokens \*/))).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(hasText(/cache hit tokens \*/))).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("does NOT show the formula for a LiteLLM response-cache hit (costs are $0)", async () => {
|
||||
|
|
@ -287,6 +352,7 @@ describe("CostBreakdownViewer", () => {
|
|||
promptTokens={430798}
|
||||
completionTokens={47}
|
||||
providerCacheReadTokens={430464}
|
||||
cacheReadTokens={430464}
|
||||
cacheHit="true"
|
||||
/>,
|
||||
);
|
||||
|
|
@ -294,7 +360,7 @@ describe("CostBreakdownViewer", () => {
|
|||
await expandCostBreakdown();
|
||||
|
||||
// Response-cache hit forces $0 and must not print a per-token cache formula.
|
||||
expect(screen.queryByText(/cache miss tokens \*/)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(hasText(/cache miss tokens \*/))).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -101,16 +101,39 @@ export const CostBreakdownViewer: React.FC<CostBreakdownViewerProps> = ({
|
|||
const originalCost = isCached ? 0 : costBreakdown?.original_cost;
|
||||
const totalCost = isCached ? 0 : costBreakdown?.total_cost ?? totalSpend;
|
||||
|
||||
// Provider prompt-cache formula breakdown (issue #32045). Only shown when the
|
||||
// provider reported cache-hit prompt tokens, and never when the LiteLLM response
|
||||
// cache served the request (isCached) since costs are $0 there.
|
||||
// Provider prompt-cache formula breakdown (issue #32045). Never shown when the
|
||||
// LiteLLM response cache served the request (isCached) since costs are $0 there.
|
||||
const tokenSplit = splitPromptTokens(promptTokens ?? 0, providerCacheReadTokens ?? 0);
|
||||
const showProviderCacheFormula = !isCached && tokenSplit.cacheHitTokens > 0;
|
||||
const hasItemizedCacheCost =
|
||||
costBreakdown?.cache_read_cost !== undefined || costBreakdown?.cache_creation_cost !== undefined;
|
||||
const cacheReadCost = costBreakdown?.cache_read_cost;
|
||||
const cacheMissInputCost = inputCost !== undefined ? inputCost - (cacheReadCost ?? 0) : undefined;
|
||||
// Full-rate (cache-miss) input cost is the input cost minus the discounted
|
||||
// cache-read and premium cache-write portions. deriveRate rejects a negative
|
||||
// result, so inconsistent backend costs hide the row instead of showing a
|
||||
// negative rate.
|
||||
const cacheMissInputCost =
|
||||
inputCost !== undefined ? inputCost - (cacheReadCost ?? 0) - (costBreakdown?.cache_creation_cost ?? 0) : undefined;
|
||||
const cacheMissRate = deriveRate(cacheMissInputCost, tokenSplit.cacheMissTokens);
|
||||
const cacheReadRate = deriveRate(cacheReadCost, tokenSplit.cacheHitTokens);
|
||||
const outputRate = deriveRate(outputCost, completionTokens);
|
||||
// The per-token input formula is only accurate when cache_read_cost is known,
|
||||
// since otherwise input_cost blends the miss and hit rates and cannot be split.
|
||||
// Render it directly under the Input Cost line (inside the itemized branch) so
|
||||
// it attaches to the right row rather than floating below the whole section.
|
||||
const showInputCacheFormula =
|
||||
!isCached && tokenSplit.cacheHitTokens > 0 && cacheMissRate !== undefined && cacheReadRate !== undefined;
|
||||
const showProviderCacheFormula = !isCached && tokenSplit.cacheHitTokens > 0;
|
||||
|
||||
const inputCacheFormula = showInputCacheFormula ? (
|
||||
<div className="pl-1 text-xs text-gray-500 font-mono space-y-0.5">
|
||||
<div>
|
||||
= {formatNumberWithCommas(tokenSplit.cacheMissTokens)} cache miss tokens * {formatRate(cacheMissRate)}
|
||||
</div>
|
||||
<div>
|
||||
+ {formatNumberWithCommas(tokenSplit.cacheHitTokens)} cache hit tokens * {formatRate(cacheReadRate)}
|
||||
</div>
|
||||
</div>
|
||||
) : null;
|
||||
|
||||
return (
|
||||
<div className="bg-white rounded-lg shadow-sm w-full max-w-full overflow-hidden mb-6">
|
||||
|
|
@ -136,9 +159,7 @@ export const CostBreakdownViewer: React.FC<CostBreakdownViewerProps> = ({
|
|||
{/* Step 1: Base Token Costs */}
|
||||
<div className="space-y-2 max-w-2xl">
|
||||
{(() => {
|
||||
const hasCacheBreakdown =
|
||||
costBreakdown?.cache_read_cost !== undefined || costBreakdown?.cache_creation_cost !== undefined;
|
||||
if (hasCacheBreakdown) {
|
||||
if (hasItemizedCacheCost) {
|
||||
// Separate line items: Input / Cache Read / Cache Write
|
||||
const rawCost = isCached
|
||||
? 0
|
||||
|
|
@ -158,6 +179,7 @@ export const CostBreakdownViewer: React.FC<CostBreakdownViewerProps> = ({
|
|||
)}
|
||||
</span>
|
||||
</div>
|
||||
{inputCacheFormula}
|
||||
{(costBreakdown?.cache_read_cost ?? 0) > 0 && (
|
||||
<div className="flex text-sm">
|
||||
<span className="text-gray-600 font-medium w-1/3">Cache Read Cost:</span>
|
||||
|
|
@ -188,31 +210,22 @@ export const CostBreakdownViewer: React.FC<CostBreakdownViewerProps> = ({
|
|||
);
|
||||
}
|
||||
return (
|
||||
<div className="flex text-sm">
|
||||
<span className="text-gray-600 font-medium w-1/3">Input Cost:</span>
|
||||
<span className="text-gray-900">
|
||||
{formatCost(inputCost)}
|
||||
{promptTokens !== undefined && (
|
||||
<span className="text-gray-500 font-normal ml-1">
|
||||
({promptTokens.toLocaleString()} prompt tokens)
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
<>
|
||||
<div className="flex text-sm">
|
||||
<span className="text-gray-600 font-medium w-1/3">Input Cost:</span>
|
||||
<span className="text-gray-900">
|
||||
{formatCost(inputCost)}
|
||||
{promptTokens !== undefined && (
|
||||
<span className="text-gray-500 font-normal ml-1">
|
||||
({promptTokens.toLocaleString()} prompt tokens)
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
{inputCacheFormula}
|
||||
</>
|
||||
);
|
||||
})()}
|
||||
{showProviderCacheFormula && (
|
||||
<div className="pl-1 text-xs text-gray-500 font-mono space-y-0.5">
|
||||
<div>
|
||||
= {formatNumberWithCommas(tokenSplit.cacheMissTokens)} cache miss tokens *{" "}
|
||||
{formatRate(cacheMissRate)}
|
||||
</div>
|
||||
<div>
|
||||
+ {formatNumberWithCommas(tokenSplit.cacheHitTokens)} cache hit tokens *{" "}
|
||||
{formatRate(cacheReadRate)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex text-sm">
|
||||
<span className="text-gray-600 font-medium w-1/3">Output Cost:</span>
|
||||
<span className="text-gray-900">
|
||||
|
|
|
|||
|
|
@ -99,4 +99,10 @@ describe("deriveRate", () => {
|
|||
it("returns undefined when cost is missing", () => {
|
||||
expect(deriveRate(undefined, 100)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns undefined for a negative cost so no negative rate is shown", () => {
|
||||
// Guards against inconsistent backend costs where read+write cost exceeds
|
||||
// input cost, which would otherwise render a negative per-token rate.
|
||||
expect(deriveRate(-0.0001, 100)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ export function splitPromptTokens(promptTokens: number, cacheReadTokens: number)
|
|||
* the formula rather than print `Infinity` or `NaN`.
|
||||
*/
|
||||
export function deriveRate(cost: number | undefined, tokens: number | undefined): number | undefined {
|
||||
if (cost === undefined || cost === null) return undefined;
|
||||
if (cost === undefined || cost === null || cost < 0) return undefined;
|
||||
if (tokens === undefined || tokens === null || tokens <= 0) return undefined;
|
||||
const rate = cost / tokens;
|
||||
return Number.isFinite(rate) ? rate : undefined;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue