mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(rust): replay executed-python cost fixtures across seven surfaces
generate_python_fixtures.py runs the real Python calculators (generic_cost_per_token, get_billed_token_rates, get_batch_cost_rates, _is_off_peak, the responses and Anthropic usage transforms, and calculate_prompt_caching_savings) over synthetic multi-feature inputs and records their outputs; python_fixture_replay.rs replays all 84 rows. Rates and timestamps are ours, so nothing pins vendor prices. Expose batch_cost_rates_from_model_info so the batch rows can select rates straight from model metadata the way Python's public helper does. Verified the replay fails when the batch threshold predicate is mutated
This commit is contained in:
parent
b401bc9be2
commit
8b2be1b566
5 changed files with 3007 additions and 2 deletions
|
|
@ -147,7 +147,7 @@ The Rust module tree does not mirror Python's overall cost module tree. The Pyth
|
|||
| `litellm.cost_calculator.default_video_cost_calculator` | `litellm_cost::non_token::calculate_video` | `test_video_generation.py::test_video_generation_cost_*` | `python_cost_calculator.rs::default_video_cost_calculator_*` |
|
||||
| `litellm.llms.openai.cost_calculation.cost_router` | `litellm_cost::openai_cost::cost_router` | No mapped Python test | `python_openai_cost.rs::openai_cost_router_selects_transcription_duration_path` |
|
||||
| `litellm.llms.openai.cost_calculation._video_resolution_to_cost_field_suffix`, `_video_output_cost_per_second`, `video_generation_cost` with supplied metadata | Matching functions in `litellm_cost::openai_cost` | `test_video_generation.py::test_video_generation_cost_*` | `python_openai_cost.rs::video_*` |
|
||||
| `litellm.litellm_core_utils.llm_cost_calc.utils.get_batch_cost_rates` | `litellm_cost::batch::get_batch_cost_rates` | `test_cost_calculator.py::test_batch_cost_calculator_*` | `python_cost_calculator.rs::get_batch_cost_rates_*` |
|
||||
| `litellm.litellm_core_utils.llm_cost_calc.utils.get_batch_cost_rates` | `litellm_cost::batch::get_batch_cost_rates`, `litellm_cost::batch::batch_cost_rates_from_model_info` | `test_cost_calculator.py::test_batch_cost_calculator_*` | `python_cost_calculator.rs::get_batch_cost_rates_*` |
|
||||
| `litellm.cost_calculator.batch_cost_calculator` | `litellm_cost::batch::batch_cost_from_model_info`, `ModelInfoCatalog::cost_per_token_for_call`, `litellm_cost::batch::batch_cost_calculator` | `test_cost_calculator.py::test_batch_cost_calculator_*`; `test_llm_cost_calc_utils.py::test_get_batch_cost_rates_crosses_each_components_own_tier`, `test_get_batch_cost_rates_has_no_cache_write_rate_without_a_cache_write_batch_key` | `python_call_dispatch.rs::batch_call_*`, `python_cost_calculator.rs::batch_cost_calculator_*`, `batch_model_info_crosses_input_and_output_thresholds_independently`, `batch_model_info_bills_cache_writes_as_input_without_a_batch_cache_rate` |
|
||||
| `litellm.cost_calculator.cost_per_token` cache normalization | `litellm_cost::custom_pricing::normalize_cache_usage` | `test_cost_calculator.py::test_custom_pricing_*` | `python_custom_pricing.rs::normalize_cache_usage_*` |
|
||||
| `litellm.cost_calculator._cost_per_token_custom_pricing_helper` | `litellm_cost::custom_pricing::cost_per_token_custom_pricing_helper` | `test_cost_calculator.py::test_custom_pricing_*` | `python_custom_pricing.rs::cost_per_token_custom_pricing_helper_*` |
|
||||
|
|
@ -307,4 +307,4 @@ The caller states whether `prompt_tokens` includes cache tokens. Threshold selec
|
|||
|
||||
The model-info off-peak path supports UTC windows, wrapping windows, and weekday rules with an IANA timezone. The lower-level compiled pricing plan still supports one non-wrapping UTC daily window. The caller supplies the provider-specific threshold policy. Negative or non-finite rates, ambiguous rules, inconsistent cache counts, incomplete write splits, invalid plan windows and overflow return errors. Callers must decline unsupported inputs before native execution if their public contract accepts those shapes
|
||||
|
||||
This crate accepts a caller-supplied catalog and only supports part of Python's model selection and response normalization. It does not fetch provider prices or cover every provider-stated cost shape. The non-token functions accept already selected prices and do not yet match every Python fallback or warning. This crate does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage
|
||||
This crate accepts a caller-supplied catalog and only supports part of Python's model selection and response normalization. It does not fetch provider prices or cover every provider-stated cost shape. The non-token functions accept already selected prices and do not yet match every Python fallback or warning. This crate does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage. A second differential fixture set lives in `tests/python_fixtures.json`, generated by `tests/generate_python_fixtures.py` by executing the real Python calculators (`generic_cost_per_token`, `get_billed_token_rates`, `get_batch_cost_rates`, `_is_off_peak`, the responses and Anthropic usage transforms, and `calculate_prompt_caching_savings`) over synthetic multi-feature inputs; `tests/python_fixture_replay.rs` replays all rows and fails on any drift
|
||||
|
|
|
|||
|
|
@ -262,6 +262,34 @@ fn selected_model_rate(
|
|||
})
|
||||
}
|
||||
|
||||
pub fn batch_cost_rates_from_model_info(
|
||||
model_info: &Value,
|
||||
prompt_tokens: u64,
|
||||
inclusive: bool,
|
||||
) -> BatchCostRates {
|
||||
let policy = if inclusive {
|
||||
ThresholdPolicy::Inclusive
|
||||
} else {
|
||||
ThresholdPolicy::Exclusive
|
||||
};
|
||||
BatchCostRates {
|
||||
input: selected_model_rate(model_info, "input_cost_per_token", prompt_tokens, policy),
|
||||
output: selected_model_rate(model_info, "output_cost_per_token", prompt_tokens, policy),
|
||||
cache_read: selected_model_rate(
|
||||
model_info,
|
||||
"cache_read_input_token_cost",
|
||||
prompt_tokens,
|
||||
policy,
|
||||
),
|
||||
cache_creation: selected_model_rate(
|
||||
model_info,
|
||||
"cache_creation_input_token_cost",
|
||||
prompt_tokens,
|
||||
policy,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn batch_cost_from_model_info(
|
||||
model_info: &Value,
|
||||
usage: &ChatUsage,
|
||||
|
|
|
|||
835
litellm-rust/crates/cost/tests/generate_python_fixtures.py
Normal file
835
litellm-rust/crates/cost/tests/generate_python_fixtures.py
Normal file
|
|
@ -0,0 +1,835 @@
|
|||
"""Generates differential fixtures by executing the real Python cost calculators.
|
||||
|
||||
The inputs are ours (synthetic rates and realistic usage shapes); only the
|
||||
outputs are Python's. Run from litellm-rust with:
|
||||
|
||||
PYTHONPATH=.. ../.venv/bin/python crates/cost/tests/generate_python_fixtures.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
_is_off_peak,
|
||||
calculate_prompt_caching_savings,
|
||||
generic_cost_per_token,
|
||||
get_batch_cost_rates,
|
||||
get_billed_token_rates,
|
||||
)
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.utils import register_model
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
UTC = timezone.utc
|
||||
|
||||
|
||||
def details(
|
||||
cached: int | None = None,
|
||||
cache_write: int | None = None,
|
||||
audio_in: int | None = None,
|
||||
audio_out: int | None = None,
|
||||
image: int | None = None,
|
||||
text: int | None = None,
|
||||
reasoning: int | None = None,
|
||||
) -> dict[str, int]:
|
||||
prompt: dict[str, int] = {}
|
||||
completion: dict[str, int] = {}
|
||||
if cached is not None:
|
||||
prompt["cached_tokens"] = cached
|
||||
if cache_write is not None:
|
||||
prompt["cache_write_tokens"] = cache_write
|
||||
if audio_in is not None:
|
||||
prompt["audio_tokens"] = audio_in
|
||||
if audio_out is not None:
|
||||
completion["audio_tokens"] = audio_out
|
||||
if image is not None:
|
||||
completion["image_tokens"] = image
|
||||
if text is not None:
|
||||
completion["text_tokens"] = text
|
||||
if reasoning is not None:
|
||||
completion["reasoning_tokens"] = reasoning
|
||||
return {"prompt": prompt, "completion": completion}
|
||||
|
||||
|
||||
def usage(prompt: int, completion: int, shape: dict[str, int] | None = None) -> dict[str, Any]:
|
||||
shape = shape or {}
|
||||
return {
|
||||
"prompt_tokens": prompt,
|
||||
"completion_tokens": completion,
|
||||
"prompt_tokens_details": shape.get("prompt") or None,
|
||||
"completion_tokens_details": shape.get("completion") or None,
|
||||
}
|
||||
|
||||
|
||||
def make_usage(payload: dict[str, Any]) -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=payload["prompt_tokens"],
|
||||
completion_tokens=payload["completion_tokens"],
|
||||
prompt_tokens_details=payload.get("prompt_tokens_details") or {},
|
||||
completion_tokens_details=payload.get("completion_tokens_details") or {},
|
||||
)
|
||||
|
||||
|
||||
GENERIC_CASES: tuple[tuple[str, dict[str, Any], dict[str, Any], dict[str, Any]], ...] = (
|
||||
(
|
||||
"audio_mixed_modalities",
|
||||
usage(1000, 500, details(audio_in=200, audio_out=150, text=350, reasoning=0)),
|
||||
{
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
"input_cost_per_audio_token": 1e-5,
|
||||
"output_cost_per_audio_token": 2e-5,
|
||||
"output_cost_per_reasoning_token": 8e-6,
|
||||
},
|
||||
{},
|
||||
),
|
||||
(
|
||||
"anthropic_cache_ttl_split",
|
||||
usage(
|
||||
10_000,
|
||||
900,
|
||||
details(
|
||||
cached=8_000,
|
||||
text=800,
|
||||
reasoning=100,
|
||||
),
|
||||
),
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"cache_read_input_token_cost": 3e-7,
|
||||
"output_cost_per_reasoning_token": 6e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
},
|
||||
{},
|
||||
),
|
||||
(
|
||||
"cache_write_plus_inclusive_threshold",
|
||||
usage(130_000, 2_000, details(cached=40_000, cache_write=5_000)),
|
||||
{
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 10e-6,
|
||||
"cache_creation_input_token_cost": 5e-6,
|
||||
"input_cost_per_token_above_128k_tokens": 4e-6,
|
||||
"output_cost_per_token_above_128k_tokens": 20e-6,
|
||||
"litellm_provider": "xai",
|
||||
},
|
||||
{},
|
||||
),
|
||||
(
|
||||
"exclusive_threshold_at_boundary",
|
||||
usage(128_000, 500, details(cached=1_000)),
|
||||
{
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 10e-6,
|
||||
"input_cost_per_token_above_128k_tokens": 4e-6,
|
||||
"output_cost_per_token_above_128k_tokens": 20e-6,
|
||||
"litellm_provider": "openai",
|
||||
},
|
||||
{},
|
||||
),
|
||||
(
|
||||
"service_tier_priority_rates",
|
||||
usage(5_000, 1_000, details(reasoning=400)),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_priority": 3e-6,
|
||||
"output_cost_per_token_priority": 6e-6,
|
||||
"output_cost_per_reasoning_token_priority": 9e-6,
|
||||
},
|
||||
{"service_tier": "priority"},
|
||||
),
|
||||
(
|
||||
"service_tier_flex_uses_flex_rates",
|
||||
usage(5_000, 1_000),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_flex": 5e-7,
|
||||
"output_cost_per_token_flex": 1e-6,
|
||||
},
|
||||
{"service_tier": "flex"},
|
||||
),
|
||||
(
|
||||
"service_tier_unpriced_falls_back_to_base",
|
||||
usage(5_000, 1_000),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
},
|
||||
{"service_tier": "flex"},
|
||||
),
|
||||
(
|
||||
"data_residency_uplift",
|
||||
usage(1_000, 200),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"regional_processing_uplift_multiplier_eu": 1.5,
|
||||
},
|
||||
{"data_residency": "eu"},
|
||||
),
|
||||
(
|
||||
"image_output_tokens",
|
||||
usage(500, 700, details(image=600, text=100)),
|
||||
{
|
||||
"input_cost_per_token": 5e-7,
|
||||
"output_cost_per_token": 1e-6,
|
||||
"output_cost_per_image_token": 5e-6,
|
||||
},
|
||||
{},
|
||||
),
|
||||
(
|
||||
"free_cache_read_and_write",
|
||||
usage(1_000, 200, details(cached=500, cache_write=300)),
|
||||
{
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
},
|
||||
{},
|
||||
),
|
||||
(
|
||||
"reasoning_default_follows_output",
|
||||
usage(1_000, 600, details(reasoning=250, text=350)),
|
||||
{"input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6},
|
||||
{},
|
||||
),
|
||||
(
|
||||
"off_peak_window_active",
|
||||
usage(1_000, 200, details(reasoning=50)),
|
||||
{
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
"off_peak_pricing": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"output_cost_per_reasoning_token": 1e-6,
|
||||
"hours_utc": "22:00-06:00",
|
||||
},
|
||||
},
|
||||
{"current_time": datetime(2026, 3, 12, 2, 30, tzinfo=UTC)},
|
||||
),
|
||||
(
|
||||
"off_peak_window_inactive",
|
||||
usage(1_000, 200, details(reasoning=50)),
|
||||
{
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
"off_peak_pricing": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"hours_utc": "22:00-06:00",
|
||||
},
|
||||
},
|
||||
{"current_time": datetime(2026, 3, 12, 14, 0, tzinfo=UTC)},
|
||||
),
|
||||
(
|
||||
"off_peak_weekday_calendar",
|
||||
usage(1_000, 200),
|
||||
{
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
"off_peak_pricing": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"hours_utc": "00:00-00:00",
|
||||
"weekdays": ["sat", "sun"],
|
||||
"weekday_timezone": "America/New_York",
|
||||
},
|
||||
},
|
||||
{"current_time": datetime(2026, 3, 14, 12, 0, tzinfo=UTC)},
|
||||
),
|
||||
(
|
||||
"off_peak_weekday_blocked_on_weekday",
|
||||
usage(1_000, 200),
|
||||
{
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
"off_peak_pricing": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"hours_utc": "00:00-00:00",
|
||||
"weekdays": ["sat", "sun"],
|
||||
"weekday_timezone": "America/New_York",
|
||||
},
|
||||
},
|
||||
{"current_time": datetime(2026, 3, 12, 12, 0, tzinfo=UTC)},
|
||||
),
|
||||
(
|
||||
"tiered_base_costs_low_tier",
|
||||
usage(500, 100, details(reasoning=40)),
|
||||
{
|
||||
"tiered_pricing": [
|
||||
{"range": [0, 16_000], "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "output_cost_per_reasoning_token": 3e-6},
|
||||
{"range": [16_000, 128_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6, "output_cost_per_reasoning_token": 6e-6},
|
||||
]
|
||||
},
|
||||
{"custom_llm_provider": "dashscope"},
|
||||
),
|
||||
(
|
||||
"tiered_base_costs_high_tier",
|
||||
usage(50_000, 100, details(reasoning=40)),
|
||||
{
|
||||
"tiered_pricing": [
|
||||
{"range": [0, 16_000], "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "output_cost_per_reasoning_token": 3e-6},
|
||||
{"range": [16_000, 128_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6, "output_cost_per_reasoning_token": 6e-6},
|
||||
]
|
||||
},
|
||||
{"custom_llm_provider": "dashscope"},
|
||||
),
|
||||
(
|
||||
"tier_boundary_stays_in_lower_tier",
|
||||
usage(16_000, 100, details(reasoning=40)),
|
||||
{
|
||||
"tiered_pricing": [
|
||||
{"range": [0, 16_000], "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "output_cost_per_reasoning_token": 3e-6},
|
||||
{"range": [16_000, 128_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6, "output_cost_per_reasoning_token": 6e-6},
|
||||
]
|
||||
},
|
||||
{"custom_llm_provider": "dashscope"},
|
||||
),
|
||||
(
|
||||
"tier_cache_rates_fall_back_to_tier_input",
|
||||
usage(20_000, 100, details(cached=5_000, cache_write=1_000)),
|
||||
{
|
||||
"input_cost_per_token": 5e-6,
|
||||
"output_cost_per_token": 10e-6,
|
||||
"cache_read_input_token_cost": 1e-6,
|
||||
"tiered_pricing": [
|
||||
{"range": [0, 16_000], "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6},
|
||||
{"range": [16_000, 128_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6},
|
||||
],
|
||||
},
|
||||
{"custom_llm_provider": "dashscope"},
|
||||
),
|
||||
(
|
||||
"cache_read_falls_back_to_input_rate",
|
||||
usage(1_000, 200, details(cached=600)),
|
||||
{"input_cost_per_token": 3e-6, "output_cost_per_token": 9e-6, "litellm_provider": "openai"},
|
||||
{},
|
||||
),
|
||||
(
|
||||
"fireworks_cache_read_default_applies",
|
||||
usage(1_000, 200, details(cached=600)),
|
||||
{"input_cost_per_token": 2e-7, "output_cost_per_token": 6e-7, "litellm_provider": "fireworks_ai"},
|
||||
{"custom_llm_provider": "fireworks_ai"},
|
||||
),
|
||||
(
|
||||
"fireworks_cache_read_default_off_peak_untouched",
|
||||
usage(1_000, 200, details(cached=600)),
|
||||
{
|
||||
"input_cost_per_token": 2e-7,
|
||||
"output_cost_per_token": 6e-7,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"off_peak_pricing": {"input_cost_per_token": 1e-7, "hours_utc": "00:00-00:00"},
|
||||
},
|
||||
{"custom_llm_provider": "fireworks_ai", "current_time": datetime(2026, 3, 12, 2, 30, tzinfo=UTC)},
|
||||
),
|
||||
(
|
||||
"zero_rated_everything",
|
||||
usage(10_000, 5_000, details(cached=5_000, reasoning=1_000)),
|
||||
{"input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
|
||||
{},
|
||||
),
|
||||
(
|
||||
"vertex_regional_endpoint_uplift",
|
||||
usage(1_000, 200),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"regional_endpoint_uplift_multiplier": 1.10,
|
||||
},
|
||||
{"vertex_location": "us-east1", "custom_llm_provider": "vertex_ai"},
|
||||
),
|
||||
(
|
||||
"vertex_global_location_no_uplift",
|
||||
usage(1_000, 200),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"regional_endpoint_uplift_multiplier": 1.10,
|
||||
},
|
||||
{"vertex_location": "global", "custom_llm_provider": "vertex_ai"},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def generic_surface() -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for name, payload, model_info, extra in GENERIC_CASES:
|
||||
call = {
|
||||
"usage": payload,
|
||||
"model_info": model_info,
|
||||
"custom_llm_provider": extra.get("custom_llm_provider", "openai"),
|
||||
"service_tier": extra.get("service_tier"),
|
||||
"data_residency": extra.get("data_residency"),
|
||||
"vertex_location": extra.get("vertex_location"),
|
||||
"current_time": extra["current_time"].isoformat() if "current_time" in extra else None,
|
||||
}
|
||||
result = generic_cost_per_token(
|
||||
model="fixture-model",
|
||||
usage=make_usage(payload),
|
||||
custom_llm_provider=call["custom_llm_provider"],
|
||||
service_tier=call["service_tier"],
|
||||
data_residency=call["data_residency"],
|
||||
model_info=model_info,
|
||||
vertex_location=call["vertex_location"],
|
||||
current_time=extra.get("current_time"),
|
||||
)
|
||||
rows.append({"id": name, **call, "expected_input_cost": result[0], "expected_output_cost": result[1]})
|
||||
return rows
|
||||
|
||||
|
||||
BATCH_CASES: tuple[tuple[str, dict[str, Any], int, int, str], ...] = (
|
||||
(
|
||||
"single_batch_threshold_crossed",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"input_cost_per_token_batches": 1.5e-6,
|
||||
"output_cost_per_token_batches": 7.5e-6,
|
||||
"input_cost_per_token_above_100k_tokens_batches": 7.5e-7,
|
||||
"output_cost_per_token_above_100k_tokens_batches": 3.75e-6,
|
||||
},
|
||||
150_000,
|
||||
20_000,
|
||||
"anthropic",
|
||||
),
|
||||
(
|
||||
"batch_threshold_not_crossed",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"input_cost_per_token_batches": 1.5e-6,
|
||||
"output_cost_per_token_batches": 7.5e-6,
|
||||
"input_cost_per_token_above_100k_tokens_batches": 7.5e-7,
|
||||
"output_cost_per_token_above_100k_tokens_batches": 3.75e-6,
|
||||
},
|
||||
99_999,
|
||||
20_000,
|
||||
"anthropic",
|
||||
),
|
||||
(
|
||||
"inclusive_batch_provider_at_boundary",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"input_cost_per_token_batches": 1.5e-6,
|
||||
"output_cost_per_token_batches": 7.5e-6,
|
||||
"input_cost_per_token_above_100k_tokens_batches": 7.5e-7,
|
||||
"output_cost_per_token_above_100k_tokens_batches": 3.75e-6,
|
||||
},
|
||||
100_000,
|
||||
20_000,
|
||||
"vertex_ai",
|
||||
),
|
||||
(
|
||||
"batch_cache_rates_use_batch_fractions",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"cache_read_input_token_cost": 3e-7,
|
||||
"cache_creation_input_token_cost": 3.75e-6,
|
||||
"input_cost_per_token_batches": 1.5e-6,
|
||||
"output_cost_per_token_batches": 7.5e-6,
|
||||
"cache_read_input_token_cost_batches": 1.5e-7,
|
||||
"cache_creation_input_token_cost_batches": 1.875e-6,
|
||||
},
|
||||
50_000,
|
||||
10_000,
|
||||
"anthropic",
|
||||
),
|
||||
(
|
||||
"batch_cache_rates_absent_stay_none",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"cache_read_input_token_cost": 3e-7,
|
||||
"input_cost_per_token_batches": 1.5e-6,
|
||||
"output_cost_per_token_batches": 7.5e-6,
|
||||
},
|
||||
50_000,
|
||||
10_000,
|
||||
"anthropic",
|
||||
),
|
||||
(
|
||||
"no_batch_rates_falls_back_to_base",
|
||||
{"input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6},
|
||||
50_000,
|
||||
10_000,
|
||||
"anthropic",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def batch_surface() -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for name, model_info, prompt, completion, provider in BATCH_CASES:
|
||||
payload = usage(prompt, completion, details(cached=2_000, cache_write=1_000))
|
||||
rates = get_batch_cost_rates(model_info, make_usage(payload), provider)
|
||||
rows.append(
|
||||
{
|
||||
"id": name,
|
||||
"usage": payload,
|
||||
"model_info": model_info,
|
||||
"custom_llm_provider": provider,
|
||||
"expected_input": rates.input,
|
||||
"expected_output": rates.output,
|
||||
"expected_cache_read": rates.cache_read,
|
||||
"expected_cache_creation": rates.cache_creation,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
OFF_PEAK_TIMES = (
|
||||
"2026-03-12T02:30:00+00:00",
|
||||
"2026-03-12T14:00:00+00:00",
|
||||
"2026-03-14T12:00:00+00:00",
|
||||
"2026-03-14T20:00:00+00:00",
|
||||
"2026-07-04T01:00:00+00:00",
|
||||
)
|
||||
OFF_PEAK_BLOCKS: tuple[tuple[str, dict[str, Any]], ...] = (
|
||||
("simple_window", {"hours_utc": "22:00-06:00"}),
|
||||
("multi_window", {"hours_utc": ["00:30-02:00", "22:00-23:30"]}),
|
||||
("weekend_only", {"hours_utc": "00:00-00:00", "weekdays": ["sat", "sun"]}),
|
||||
("weekday_calendar_est", {"hours_utc": "00:00-00:00", "weekdays": ["mon", "wed"], "weekday_timezone": "America/New_York"}),
|
||||
("window_rules_with_weekdays", {"windows": [{"hours_utc": "00:00-06:00", "weekdays": ["sat", "sun"]}]}),
|
||||
("window_rule_wrong_weekday", {"windows": [{"hours_utc": "00:00-06:00", "weekdays": ["mon"]}]}),
|
||||
("malformed_window_ignored", {"hours_utc": "22-06"}),
|
||||
)
|
||||
|
||||
|
||||
def off_peak_surface() -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for block_name, block in OFF_PEAK_BLOCKS:
|
||||
for moment in OFF_PEAK_TIMES:
|
||||
at = datetime.fromisoformat(moment)
|
||||
rows.append(
|
||||
{
|
||||
"id": f"{block_name}_{moment[11:16]}",
|
||||
"off_peak": block,
|
||||
"current_time": moment,
|
||||
"expected_is_off_peak": _is_off_peak(block, at),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
RESPONSES_USAGE_CASES: tuple[tuple[str, dict[str, Any]], ...] = (
|
||||
(
|
||||
"cached_details_with_web_search",
|
||||
{
|
||||
"input_tokens": 1_200,
|
||||
"output_tokens": 300,
|
||||
"input_token_details": {"cached_tokens": 800, "web_search_requests": 2},
|
||||
"output_token_details": {"reasoning_tokens": 100},
|
||||
},
|
||||
),
|
||||
(
|
||||
"realtime_plural_details",
|
||||
{
|
||||
"input_tokens": 900,
|
||||
"output_tokens": 250,
|
||||
"input_token_details": {"text_tokens": 700, "cached_tokens": 200, "audio_tokens": 0},
|
||||
"output_token_details": {"text_tokens": 150, "audio_tokens": 100},
|
||||
},
|
||||
),
|
||||
(
|
||||
"missing_totals_derived",
|
||||
{"input_tokens": 410, "output_tokens": 90},
|
||||
),
|
||||
(
|
||||
"already_chat_shaped_passthrough",
|
||||
{"prompt_tokens": 500, "completion_tokens": 100, "total_tokens": 600},
|
||||
),
|
||||
(
|
||||
"nested_cached_details",
|
||||
{
|
||||
"input_tokens": 1_000,
|
||||
"output_tokens": 200,
|
||||
"input_token_details": {"text_tokens": 400, "cached_tokens": 600},
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def responses_usage_surface() -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for name, raw in RESPONSES_USAGE_CASES:
|
||||
transformed = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw)
|
||||
rows.append({"id": name, "raw": raw, "expected": transformed.model_dump(exclude_none=True)})
|
||||
return rows
|
||||
|
||||
|
||||
CACHING_SAVINGS_CASES: tuple[tuple[str, dict[str, Any], dict[str, Any], str | None], ...] = (
|
||||
(
|
||||
"anthropic_read_discount_write_premium",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"cache_read_input_token_cost": 3e-7,
|
||||
"cache_creation_input_token_cost": 3.75e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
},
|
||||
usage(10_000, 500, details(cached=8_000, cache_write=1_500)),
|
||||
None,
|
||||
),
|
||||
(
|
||||
"openai_cache_savings",
|
||||
{
|
||||
"input_cost_per_token": 1.25e-6,
|
||||
"output_cost_per_token": 1e-5,
|
||||
"cache_read_input_token_cost": 6.25e-8,
|
||||
"cache_creation_input_token_cost": 1.25e-6,
|
||||
"litellm_provider": "openai",
|
||||
},
|
||||
usage(50_000, 2_000, details(cached=45_000, cache_write=3_000)),
|
||||
None,
|
||||
),
|
||||
(
|
||||
"unpriced_cache_no_savings",
|
||||
{"input_cost_per_token": 2e-6, "output_cost_per_token": 8e-6, "litellm_provider": "openai"},
|
||||
usage(10_000, 500, details(cached=8_000)),
|
||||
None,
|
||||
),
|
||||
(
|
||||
"priority_tier_discount",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"cache_read_input_token_cost": 3e-7,
|
||||
"cache_creation_input_token_cost": 3.75e-6,
|
||||
"input_cost_per_token_priority": 6e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
},
|
||||
usage(10_000, 500, details(cached=8_000, cache_write=1_500)),
|
||||
"priority",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def caching_savings_surface() -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for name, model_info, payload, service_tier in CACHING_SAVINGS_CASES:
|
||||
savings = calculate_prompt_caching_savings(
|
||||
model_info=model_info,
|
||||
usage=make_usage(payload),
|
||||
custom_llm_provider=model_info.get("litellm_provider"),
|
||||
service_tier=service_tier,
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"id": name,
|
||||
"model_info": model_info,
|
||||
"usage": payload,
|
||||
"custom_llm_provider": model_info.get("litellm_provider"),
|
||||
"service_tier": service_tier,
|
||||
"expected_savings": savings,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
ANTHROPIC_USAGE_CASES: tuple[tuple[str, dict[str, Any], str | None, bool], ...] = (
|
||||
(
|
||||
"plain_input_output",
|
||||
{"input_tokens": 1_000, "output_tokens": 300},
|
||||
None,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"cache_creation_and_read",
|
||||
{
|
||||
"input_tokens": 5_000,
|
||||
"output_tokens": 200,
|
||||
"cache_creation_input_tokens": 2_000,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 1_500, "ephemeral_1h_input_tokens": 500},
|
||||
"cache_read_input_tokens": 3_000,
|
||||
},
|
||||
None,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"iterations_with_thinking_tokens",
|
||||
{
|
||||
"input_tokens": 600,
|
||||
"output_tokens": 300,
|
||||
"iterations": [
|
||||
{"input_tokens": 100, "output_tokens": 150, "output_tokens_details": {"thinking_tokens": 90}},
|
||||
{"input_tokens": 200, "output_tokens": 150, "output_tokens_details": {"thinking_tokens": 60}},
|
||||
],
|
||||
},
|
||||
None,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"iterations_usage",
|
||||
{
|
||||
"input_tokens": 600,
|
||||
"output_tokens": 250,
|
||||
"server_tool_use": {"web_search_requests": 3},
|
||||
},
|
||||
None,
|
||||
False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def anthropic_usage_surface() -> list[dict[str, Any]]:
|
||||
config = AnthropicConfig()
|
||||
rows: list[dict[str, Any]] = []
|
||||
for name, raw, reasoning, has_thinking in ANTHROPIC_USAGE_CASES:
|
||||
transformed = config.calculate_usage(raw, reasoning)
|
||||
rows.append(
|
||||
{
|
||||
"id": name,
|
||||
"raw": raw,
|
||||
"reasoning_content": reasoning,
|
||||
"response_has_thinking_block": has_thinking,
|
||||
"expected": transformed.model_dump(exclude_none=True),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
BILLED_RATES_CASES: tuple[tuple[str, dict[str, Any], dict[str, Any], dict[str, Any]], ...] = (
|
||||
(
|
||||
"geo_multiplier_applies",
|
||||
usage(1_000, 200),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"provider_specific_entry": {"us": 1.1},
|
||||
},
|
||||
{"usage_extra": {"inference_geo": "us"}, "custom_llm_provider": "anthropic"},
|
||||
),
|
||||
(
|
||||
"geo_global_is_neutral",
|
||||
usage(1_000, 200),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"provider_specific_entry": {"us": 1.1},
|
||||
},
|
||||
{"usage_extra": {"inference_geo": "global"}, "custom_llm_provider": "anthropic"},
|
||||
),
|
||||
(
|
||||
"all_multipliers_compose",
|
||||
usage(1_000, 200),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 1e-7,
|
||||
"provider_specific_entry": {"us": 1.1},
|
||||
"regional_processing_uplift_multiplier_eu": 1.5,
|
||||
"regional_endpoint_uplift_multiplier": 1.10,
|
||||
},
|
||||
{
|
||||
"usage_extra": {"inference_geo": "us"},
|
||||
"data_residency": "eu",
|
||||
"vertex_location": "us-east1",
|
||||
"custom_llm_provider": "anthropic",
|
||||
},
|
||||
),
|
||||
(
|
||||
"service_tier_priority_rates",
|
||||
usage(1_000, 200),
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_priority": 3e-6,
|
||||
"output_cost_per_token_priority": 6e-6,
|
||||
},
|
||||
{"service_tier": "priority"},
|
||||
),
|
||||
(
|
||||
"off_peak_rates_and_reasoning",
|
||||
usage(1_000, 200, details(reasoning=50)),
|
||||
{
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
"cache_read_input_token_cost": 2e-7,
|
||||
"cache_creation_input_token_cost": 5e-6,
|
||||
"off_peak_pricing": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"output_cost_per_reasoning_token": 1e-6,
|
||||
"hours_utc": "22:00-06:00",
|
||||
},
|
||||
},
|
||||
{"current_time": datetime(2026, 3, 12, 2, 30, tzinfo=UTC)},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def billed_rates_surface() -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for name, payload, model_info, extra in BILLED_RATES_CASES:
|
||||
usage_obj = make_usage(payload)
|
||||
for key, value in extra.get("usage_extra", {}).items():
|
||||
setattr(usage_obj, key, value)
|
||||
model_name = f"fixture-billed/{name.replace('_', '-')}"
|
||||
register_model({model_name: model_info})
|
||||
rates = get_billed_token_rates(
|
||||
model=model_name,
|
||||
usage=usage_obj,
|
||||
custom_llm_provider=extra.get("custom_llm_provider", "openai"),
|
||||
service_tier=extra.get("service_tier"),
|
||||
data_residency=extra.get("data_residency"),
|
||||
vertex_location=extra.get("vertex_location"),
|
||||
current_time=extra.get("current_time"),
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"id": name,
|
||||
"model": model_name,
|
||||
"usage": payload,
|
||||
"usage_extra": extra.get("usage_extra", {}),
|
||||
"model_info": model_info,
|
||||
"custom_llm_provider": extra.get("custom_llm_provider", "openai"),
|
||||
"service_tier": extra.get("service_tier"),
|
||||
"data_residency": extra.get("data_residency"),
|
||||
"vertex_location": extra.get("vertex_location"),
|
||||
"current_time": extra["current_time"].isoformat() if "current_time" in extra else None,
|
||||
"expected": {
|
||||
"input_cost_per_token": rates.input_cost_per_token,
|
||||
"output_cost_per_token": rates.output_cost_per_token,
|
||||
"cache_read_input_token_cost": rates.cache_read_input_token_cost,
|
||||
"cache_read_input_audio_token_cost": rates.cache_read_input_audio_token_cost,
|
||||
"cache_creation_input_token_cost": rates.cache_creation_input_token_cost,
|
||||
"cache_creation_input_token_cost_above_1hr": rates.cache_creation_input_token_cost_above_1hr,
|
||||
"output_cost_per_reasoning_token": rates.output_cost_per_reasoning_token,
|
||||
},
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> None:
|
||||
revision = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True).strip()
|
||||
fixture = {
|
||||
"python_commit": revision,
|
||||
"generic": generic_surface(),
|
||||
"batch": batch_surface(),
|
||||
"billed_rates": billed_rates_surface(),
|
||||
"off_peak": off_peak_surface(),
|
||||
"responses_usage": responses_usage_surface(),
|
||||
"caching_savings": caching_savings_surface(),
|
||||
"anthropic_usage": anthropic_usage_surface(),
|
||||
}
|
||||
target = Path(__file__).parent / "python_fixtures.json"
|
||||
target.write_text(json.dumps(fixture, indent=1) + "\n")
|
||||
counts = {key: len(value) for key, value in fixture.items() if isinstance(value, list)}
|
||||
print(json.dumps(counts))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
364
litellm-rust/crates/cost/tests/python_fixture_replay.rs
Normal file
364
litellm-rust/crates/cost/tests/python_fixture_replay.rs
Normal file
|
|
@ -0,0 +1,364 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::Rate;
|
||||
use litellm_cost::anthropic_usage::transform_anthropic_usage_to_chat_usage;
|
||||
use litellm_cost::base_rate_selection::uses_inclusive_token_thresholds;
|
||||
use litellm_cost::batch::batch_cost_rates_from_model_info;
|
||||
use litellm_cost::billed_token_rates::{BilledRatesRequest, get_billed_token_rates};
|
||||
use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
||||
use litellm_cost::off_peak::is_off_peak;
|
||||
use litellm_cost::prompt_caching_savings::{
|
||||
PromptCachingSavingsRequest, calculate_prompt_caching_savings,
|
||||
};
|
||||
use litellm_cost::provider_cache::apply_provider_cache_read_default;
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn fixtures() -> Value {
|
||||
serde_json::from_str(include_str!("python_fixtures.json")).unwrap()
|
||||
}
|
||||
|
||||
fn instant(iso: &str) -> Timestamp {
|
||||
iso.parse().unwrap()
|
||||
}
|
||||
|
||||
fn opt_str<'a>(value: &'a Value, key: &str) -> Option<&'a str> {
|
||||
value.get(key).and_then(Value::as_str)
|
||||
}
|
||||
|
||||
fn chat_usage(payload: &Value) -> litellm_cost::responses_usage::ChatUsage {
|
||||
get_usage_object(&json!({"usage": payload}))
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn assert_close(actual: f64, expected: f64, id: &str) {
|
||||
assert!(
|
||||
(actual - expected).abs() <= expected.abs() * 1e-12,
|
||||
"{id}: rust={actual:?} python={expected:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generic_token_costs_match_executed_python_fixtures() {
|
||||
for row in fixtures().get("generic").unwrap().as_array().unwrap() {
|
||||
let id = row.get("id").and_then(Value::as_str).unwrap();
|
||||
let usage = chat_usage(row.get("usage").unwrap());
|
||||
let provider = opt_str(row, "custom_llm_provider");
|
||||
let model_info =
|
||||
apply_provider_cache_read_default(row.get("model_info").unwrap(), provider);
|
||||
let at = opt_str(row, "current_time")
|
||||
.map_or_else(|| "2026-03-12T00:00:00Z".parse().unwrap(), instant);
|
||||
let (input_cost, output_cost) = calculate_generic_cost_from_model_info_with_region(
|
||||
&usage,
|
||||
&model_info,
|
||||
opt_str(row, "service_tier"),
|
||||
uses_inclusive_token_thresholds(provider),
|
||||
opt_str(row, "data_residency"),
|
||||
opt_str(row, "vertex_location"),
|
||||
at,
|
||||
);
|
||||
assert_close(
|
||||
input_cost,
|
||||
row.get("expected_input_cost").unwrap().as_f64().unwrap(),
|
||||
id,
|
||||
);
|
||||
assert_close(
|
||||
output_cost,
|
||||
row.get("expected_output_cost").unwrap().as_f64().unwrap(),
|
||||
id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billed_token_rates_match_executed_python_fixtures() {
|
||||
for row in fixtures().get("billed_rates").unwrap().as_array().unwrap() {
|
||||
let id = row.get("id").and_then(Value::as_str).unwrap();
|
||||
let mut payload = row.get("usage").unwrap().clone();
|
||||
if let Some(extra) = row.get("usage_extra").and_then(Value::as_object) {
|
||||
for (key, value) in extra {
|
||||
payload
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
let usage = chat_usage(&payload);
|
||||
let provider = opt_str(row, "custom_llm_provider");
|
||||
let at = opt_str(row, "current_time")
|
||||
.map_or_else(|| "2026-03-12T00:00:00Z".parse().unwrap(), instant);
|
||||
let rates = get_billed_token_rates(BilledRatesRequest {
|
||||
model_info: Some(row.get("model_info").unwrap()),
|
||||
usage: &usage,
|
||||
provider,
|
||||
service_tier: opt_str(row, "service_tier"),
|
||||
data_residency: opt_str(row, "data_residency"),
|
||||
vertex_location: opt_str(row, "vertex_location"),
|
||||
at,
|
||||
custom_cost_per_token: None,
|
||||
})
|
||||
.unwrap_or_else(|| panic!("{id}: rates unresolved"));
|
||||
let expected = row.get("expected").unwrap();
|
||||
for field in [
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_read_input_audio_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
"cache_creation_input_token_cost_above_1hr",
|
||||
"output_cost_per_reasoning_token",
|
||||
] {
|
||||
let want = expected.get(field).unwrap().as_f64().unwrap();
|
||||
let got = match field {
|
||||
"input_cost_per_token" => rates.input_cost_per_token,
|
||||
"output_cost_per_token" => rates.output_cost_per_token,
|
||||
"cache_read_input_token_cost" => rates.cache_read_input_token_cost,
|
||||
"cache_read_input_audio_token_cost" => rates.cache_read_input_audio_token_cost,
|
||||
"cache_creation_input_token_cost" => rates.cache_creation_input_token_cost,
|
||||
"cache_creation_input_token_cost_above_1hr" => {
|
||||
rates.cache_creation_input_token_cost_above_1hr
|
||||
}
|
||||
_ => rates.output_cost_per_reasoning_token,
|
||||
};
|
||||
assert_close(got, want, &format!("{id}/{field}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_cost_rates_match_executed_python_fixtures() {
|
||||
for row in fixtures().get("batch").unwrap().as_array().unwrap() {
|
||||
let id = row.get("id").and_then(Value::as_str).unwrap();
|
||||
let model_info = row.get("model_info").unwrap();
|
||||
let provider = opt_str(row, "custom_llm_provider").unwrap();
|
||||
let prompt_tokens = row
|
||||
.get("usage")
|
||||
.unwrap()
|
||||
.get("prompt_tokens")
|
||||
.unwrap()
|
||||
.as_u64()
|
||||
.unwrap();
|
||||
let rates = batch_cost_rates_from_model_info(
|
||||
model_info,
|
||||
prompt_tokens,
|
||||
uses_inclusive_token_thresholds(Some(provider)),
|
||||
);
|
||||
let as_option = |rate: Rate| match rate {
|
||||
Rate::Value(value) => Some(value),
|
||||
Rate::Missing | Rate::Null => None,
|
||||
};
|
||||
let option_equals = |actual: Option<f64>, key: &str| match (actual, row.get(key)) {
|
||||
(Some(got), Some(want)) => {
|
||||
assert_close(got, want.as_f64().unwrap(), &format!("{id}/{key}"))
|
||||
}
|
||||
(None, None) | (None, Some(Value::Null)) => {}
|
||||
(got, want) => panic!("{id}/{key}: rust={got:?} python={want:?}"),
|
||||
};
|
||||
option_equals(as_option(rates.input), "expected_input");
|
||||
option_equals(as_option(rates.output), "expected_output");
|
||||
option_equals(as_option(rates.cache_read), "expected_cache_read");
|
||||
option_equals(as_option(rates.cache_creation), "expected_cache_creation");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn off_peak_decisions_match_executed_python_fixtures() {
|
||||
for row in fixtures().get("off_peak").unwrap().as_array().unwrap() {
|
||||
let id = row.get("id").and_then(Value::as_str).unwrap();
|
||||
let block = row.get("off_peak").unwrap();
|
||||
let at = instant(row.get("current_time").and_then(Value::as_str).unwrap());
|
||||
let expected = row.get("expected_is_off_peak").unwrap().as_bool().unwrap();
|
||||
assert_eq!(is_off_peak(block, at), expected, "{id}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_usage_transforms_match_executed_python_fixtures() {
|
||||
for row in fixtures()
|
||||
.get("responses_usage")
|
||||
.unwrap()
|
||||
.as_array()
|
||||
.unwrap()
|
||||
{
|
||||
let id = row.get("id").and_then(Value::as_str).unwrap();
|
||||
let usage = get_usage_object(&json!({"usage": row.get("raw").unwrap()}))
|
||||
.unwrap()
|
||||
.unwrap_or_else(|| panic!("{id}: usage unparsed"));
|
||||
assert_usage_matches(&usage, row.get("expected").unwrap(), id);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anthropic_usage_transforms_match_executed_python_fixtures() {
|
||||
for row in fixtures()
|
||||
.get("anthropic_usage")
|
||||
.unwrap()
|
||||
.as_array()
|
||||
.unwrap()
|
||||
{
|
||||
let id = row.get("id").and_then(Value::as_str).unwrap();
|
||||
let usage = transform_anthropic_usage_to_chat_usage(
|
||||
row.get("raw").unwrap(),
|
||||
None,
|
||||
row.get("response_has_thinking_block")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false),
|
||||
)
|
||||
.unwrap();
|
||||
assert_usage_matches(&usage, row.get("expected").unwrap(), id);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_usage_matches(
|
||||
usage: &litellm_cost::responses_usage::ChatUsage,
|
||||
expected: &Value,
|
||||
id: &str,
|
||||
) {
|
||||
let count = |key: &str| expected.get(key).and_then(Value::as_u64).unwrap_or(0);
|
||||
assert_eq!(usage.prompt_tokens, count("prompt_tokens"), "{id}/prompt");
|
||||
assert_eq!(
|
||||
usage.completion_tokens,
|
||||
count("completion_tokens"),
|
||||
"{id}/completion"
|
||||
);
|
||||
assert_eq!(usage.total_tokens, count("total_tokens"), "{id}/total");
|
||||
let details = usage.prompt_tokens_details.clone().unwrap_or_default();
|
||||
let expected_details = expected.get("prompt_tokens_details");
|
||||
let detail = |key: &str| {
|
||||
expected_details
|
||||
.and_then(|d| d.get(key))
|
||||
.and_then(Value::as_u64)
|
||||
};
|
||||
assert_eq!(
|
||||
details.cached_tokens,
|
||||
count_of(expected_details, "cached_tokens"),
|
||||
"{id}/cached"
|
||||
);
|
||||
assert_eq!(
|
||||
details.web_search_requests,
|
||||
detail("web_search_requests"),
|
||||
"{id}/web_search_requests"
|
||||
);
|
||||
assert_eq!(
|
||||
details.cache_write_tokens,
|
||||
detail("cache_write_tokens"),
|
||||
"{id}/cache_write"
|
||||
);
|
||||
if let Some(want) = detail("cache_creation_tokens") {
|
||||
assert_eq!(
|
||||
details.cache_creation_tokens,
|
||||
Some(want),
|
||||
"{id}/cache_creation"
|
||||
);
|
||||
}
|
||||
if let (Some(want), Some(got)) = (
|
||||
expected_details
|
||||
.and_then(|d| d.get("cache_creation_token_details"))
|
||||
.and_then(Value::as_object),
|
||||
details.cache_creation_token_details.as_ref().map(|v| {
|
||||
json!({
|
||||
"ephemeral_5m_input_tokens": v.ephemeral_5m_input_tokens,
|
||||
"ephemeral_1h_input_tokens": v.ephemeral_1h_input_tokens,
|
||||
})
|
||||
}),
|
||||
) {
|
||||
assert_eq!(
|
||||
got.get("ephemeral_5m_input_tokens").and_then(Value::as_u64),
|
||||
want.get("ephemeral_5m_input_tokens")
|
||||
.and_then(Value::as_u64),
|
||||
"{id}/ttl_5m"
|
||||
);
|
||||
assert_eq!(
|
||||
got.get("ephemeral_1h_input_tokens").and_then(Value::as_u64),
|
||||
want.get("ephemeral_1h_input_tokens")
|
||||
.and_then(Value::as_u64),
|
||||
"{id}/ttl_1h"
|
||||
);
|
||||
}
|
||||
let completion = usage.completion_tokens_details.clone();
|
||||
let expected_completion = expected.get("completion_tokens_details");
|
||||
assert_eq!(
|
||||
completion.as_ref().and_then(|d| d.reasoning_tokens),
|
||||
expected_completion
|
||||
.and_then(|d| d.get("reasoning_tokens"))
|
||||
.and_then(Value::as_u64),
|
||||
"{id}/reasoning"
|
||||
);
|
||||
assert_eq!(
|
||||
completion.as_ref().and_then(|d| d.text_tokens),
|
||||
expected_completion
|
||||
.and_then(|d| d.get("text_tokens"))
|
||||
.and_then(Value::as_u64),
|
||||
"{id}/text"
|
||||
);
|
||||
if let Some(server_tool_use) = expected.get("server_tool_use").and_then(Value::as_object) {
|
||||
for (key, value) in server_tool_use {
|
||||
assert_eq!(
|
||||
usage.extra.get("server_tool_use").and_then(|v| v.get(key)),
|
||||
Some(value),
|
||||
"{id}/server_tool_use/{key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn count_of(details: Option<&Value>, key: &str) -> u64 {
|
||||
details
|
||||
.and_then(|d| d.get(key))
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_caching_savings_match_executed_python_fixtures() {
|
||||
for row in fixtures()
|
||||
.get("caching_savings")
|
||||
.unwrap()
|
||||
.as_array()
|
||||
.unwrap()
|
||||
{
|
||||
let id = row.get("id").and_then(Value::as_str).unwrap();
|
||||
let usage = chat_usage(row.get("usage").unwrap());
|
||||
let savings = calculate_prompt_caching_savings(PromptCachingSavingsRequest {
|
||||
model_info: row.get("model_info").unwrap(),
|
||||
usage: &usage,
|
||||
provider: opt_str(row, "custom_llm_provider"),
|
||||
service_tier: opt_str(row, "service_tier"),
|
||||
data_residency: opt_str(row, "data_residency"),
|
||||
vertex_location: opt_str(row, "vertex_location"),
|
||||
at: "2026-03-12T00:00:00Z".parse().unwrap(),
|
||||
});
|
||||
assert_close(
|
||||
savings,
|
||||
row.get("expected_savings").unwrap().as_f64().unwrap(),
|
||||
id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixture_ids_are_unique_per_surface() {
|
||||
let mut seen: HashMap<String, usize> = HashMap::new();
|
||||
let root = fixtures();
|
||||
for surface in [
|
||||
"generic",
|
||||
"batch",
|
||||
"billed_rates",
|
||||
"off_peak",
|
||||
"responses_usage",
|
||||
"caching_savings",
|
||||
"anthropic_usage",
|
||||
] {
|
||||
for row in root.get(surface).unwrap().as_array().unwrap() {
|
||||
let id = row.get("id").and_then(Value::as_str).unwrap().to_owned();
|
||||
*seen.entry(format!("{surface}/{id}")).or_default() += 1;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
seen.values().all(|count| *count == 1),
|
||||
"duplicate fixture ids: {:?}",
|
||||
seen.iter().filter(|(_, c)| **c > 1).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
1778
litellm-rust/crates/cost/tests/python_fixtures.json
Normal file
1778
litellm-rust/crates/cost/tests/python_fixtures.json
Normal file
File diff suppressed because it is too large
Load diff
Loading…
Add table
Reference in a new issue