diff --git a/litellm-rust/crates/cost/README.md b/litellm-rust/crates/cost/README.md index b4fd81528ee..21839643ecf 100644 --- a/litellm-rust/crates/cost/README.md +++ b/litellm-rust/crates/cost/README.md @@ -147,7 +147,7 @@ The Rust module tree does not mirror Python's overall cost module tree. The Pyth | `litellm.cost_calculator.default_video_cost_calculator` | `litellm_cost::non_token::calculate_video` | `test_video_generation.py::test_video_generation_cost_*` | `python_cost_calculator.rs::default_video_cost_calculator_*` | | `litellm.llms.openai.cost_calculation.cost_router` | `litellm_cost::openai_cost::cost_router` | No mapped Python test | `python_openai_cost.rs::openai_cost_router_selects_transcription_duration_path` | | `litellm.llms.openai.cost_calculation._video_resolution_to_cost_field_suffix`, `_video_output_cost_per_second`, `video_generation_cost` with supplied metadata | Matching functions in `litellm_cost::openai_cost` | `test_video_generation.py::test_video_generation_cost_*` | `python_openai_cost.rs::video_*` | -| `litellm.litellm_core_utils.llm_cost_calc.utils.get_batch_cost_rates` | `litellm_cost::batch::get_batch_cost_rates` | `test_cost_calculator.py::test_batch_cost_calculator_*` | `python_cost_calculator.rs::get_batch_cost_rates_*` | +| `litellm.litellm_core_utils.llm_cost_calc.utils.get_batch_cost_rates` | `litellm_cost::batch::get_batch_cost_rates`, `litellm_cost::batch::batch_cost_rates_from_model_info` | `test_cost_calculator.py::test_batch_cost_calculator_*` | `python_cost_calculator.rs::get_batch_cost_rates_*` | | `litellm.cost_calculator.batch_cost_calculator` | `litellm_cost::batch::batch_cost_from_model_info`, `ModelInfoCatalog::cost_per_token_for_call`, `litellm_cost::batch::batch_cost_calculator` | `test_cost_calculator.py::test_batch_cost_calculator_*`; `test_llm_cost_calc_utils.py::test_get_batch_cost_rates_crosses_each_components_own_tier`, `test_get_batch_cost_rates_has_no_cache_write_rate_without_a_cache_write_batch_key` | `python_call_dispatch.rs::batch_call_*`, `python_cost_calculator.rs::batch_cost_calculator_*`, `batch_model_info_crosses_input_and_output_thresholds_independently`, `batch_model_info_bills_cache_writes_as_input_without_a_batch_cache_rate` | | `litellm.cost_calculator.cost_per_token` cache normalization | `litellm_cost::custom_pricing::normalize_cache_usage` | `test_cost_calculator.py::test_custom_pricing_*` | `python_custom_pricing.rs::normalize_cache_usage_*` | | `litellm.cost_calculator._cost_per_token_custom_pricing_helper` | `litellm_cost::custom_pricing::cost_per_token_custom_pricing_helper` | `test_cost_calculator.py::test_custom_pricing_*` | `python_custom_pricing.rs::cost_per_token_custom_pricing_helper_*` | @@ -307,4 +307,4 @@ The caller states whether `prompt_tokens` includes cache tokens. Threshold selec The model-info off-peak path supports UTC windows, wrapping windows, and weekday rules with an IANA timezone. The lower-level compiled pricing plan still supports one non-wrapping UTC daily window. The caller supplies the provider-specific threshold policy. Negative or non-finite rates, ambiguous rules, inconsistent cache counts, incomplete write splits, invalid plan windows and overflow return errors. Callers must decline unsupported inputs before native execution if their public contract accepts those shapes -This crate accepts a caller-supplied catalog and only supports part of Python's model selection and response normalization. It does not fetch provider prices or cover every provider-stated cost shape. The non-token functions accept already selected prices and do not yet match every Python fallback or warning. This crate does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage +This crate accepts a caller-supplied catalog and only supports part of Python's model selection and response normalization. It does not fetch provider prices or cover every provider-stated cost shape. The non-token functions accept already selected prices and do not yet match every Python fallback or warning. This crate does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage. A second differential fixture set lives in `tests/python_fixtures.json`, generated by `tests/generate_python_fixtures.py` by executing the real Python calculators (`generic_cost_per_token`, `get_billed_token_rates`, `get_batch_cost_rates`, `_is_off_peak`, the responses and Anthropic usage transforms, and `calculate_prompt_caching_savings`) over synthetic multi-feature inputs; `tests/python_fixture_replay.rs` replays all rows and fails on any drift diff --git a/litellm-rust/crates/cost/src/batch.rs b/litellm-rust/crates/cost/src/batch.rs index 1bb68f2da38..487947f7e86 100644 --- a/litellm-rust/crates/cost/src/batch.rs +++ b/litellm-rust/crates/cost/src/batch.rs @@ -262,6 +262,34 @@ fn selected_model_rate( }) } +pub fn batch_cost_rates_from_model_info( + model_info: &Value, + prompt_tokens: u64, + inclusive: bool, +) -> BatchCostRates { + let policy = if inclusive { + ThresholdPolicy::Inclusive + } else { + ThresholdPolicy::Exclusive + }; + BatchCostRates { + input: selected_model_rate(model_info, "input_cost_per_token", prompt_tokens, policy), + output: selected_model_rate(model_info, "output_cost_per_token", prompt_tokens, policy), + cache_read: selected_model_rate( + model_info, + "cache_read_input_token_cost", + prompt_tokens, + policy, + ), + cache_creation: selected_model_rate( + model_info, + "cache_creation_input_token_cost", + prompt_tokens, + policy, + ), + } +} + pub fn batch_cost_from_model_info( model_info: &Value, usage: &ChatUsage, diff --git a/litellm-rust/crates/cost/tests/generate_python_fixtures.py b/litellm-rust/crates/cost/tests/generate_python_fixtures.py new file mode 100644 index 00000000000..124c21c1cc6 --- /dev/null +++ b/litellm-rust/crates/cost/tests/generate_python_fixtures.py @@ -0,0 +1,835 @@ +"""Generates differential fixtures by executing the real Python cost calculators. + +The inputs are ours (synthetic rates and realistic usage shapes); only the +outputs are Python's. Run from litellm-rust with: + + PYTHONPATH=.. ../.venv/bin/python crates/cost/tests/generate_python_fixtures.py +""" + +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + _is_off_peak, + calculate_prompt_caching_savings, + generic_cost_per_token, + get_batch_cost_rates, + get_billed_token_rates, +) +from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.utils import register_model +from litellm.types.utils import Usage + +UTC = timezone.utc + + +def details( + cached: int | None = None, + cache_write: int | None = None, + audio_in: int | None = None, + audio_out: int | None = None, + image: int | None = None, + text: int | None = None, + reasoning: int | None = None, +) -> dict[str, int]: + prompt: dict[str, int] = {} + completion: dict[str, int] = {} + if cached is not None: + prompt["cached_tokens"] = cached + if cache_write is not None: + prompt["cache_write_tokens"] = cache_write + if audio_in is not None: + prompt["audio_tokens"] = audio_in + if audio_out is not None: + completion["audio_tokens"] = audio_out + if image is not None: + completion["image_tokens"] = image + if text is not None: + completion["text_tokens"] = text + if reasoning is not None: + completion["reasoning_tokens"] = reasoning + return {"prompt": prompt, "completion": completion} + + +def usage(prompt: int, completion: int, shape: dict[str, int] | None = None) -> dict[str, Any]: + shape = shape or {} + return { + "prompt_tokens": prompt, + "completion_tokens": completion, + "prompt_tokens_details": shape.get("prompt") or None, + "completion_tokens_details": shape.get("completion") or None, + } + + +def make_usage(payload: dict[str, Any]) -> Usage: + return Usage( + prompt_tokens=payload["prompt_tokens"], + completion_tokens=payload["completion_tokens"], + prompt_tokens_details=payload.get("prompt_tokens_details") or {}, + completion_tokens_details=payload.get("completion_tokens_details") or {}, + ) + + +GENERIC_CASES: tuple[tuple[str, dict[str, Any], dict[str, Any], dict[str, Any]], ...] = ( + ( + "audio_mixed_modalities", + usage(1000, 500, details(audio_in=200, audio_out=150, text=350, reasoning=0)), + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 4e-6, + "input_cost_per_audio_token": 1e-5, + "output_cost_per_audio_token": 2e-5, + "output_cost_per_reasoning_token": 8e-6, + }, + {}, + ), + ( + "anthropic_cache_ttl_split", + usage( + 10_000, + 900, + details( + cached=8_000, + text=800, + reasoning=100, + ), + ), + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "output_cost_per_reasoning_token": 6e-6, + "litellm_provider": "anthropic", + }, + {}, + ), + ( + "cache_write_plus_inclusive_threshold", + usage(130_000, 2_000, details(cached=40_000, cache_write=5_000)), + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 10e-6, + "cache_creation_input_token_cost": 5e-6, + "input_cost_per_token_above_128k_tokens": 4e-6, + "output_cost_per_token_above_128k_tokens": 20e-6, + "litellm_provider": "xai", + }, + {}, + ), + ( + "exclusive_threshold_at_boundary", + usage(128_000, 500, details(cached=1_000)), + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 10e-6, + "input_cost_per_token_above_128k_tokens": 4e-6, + "output_cost_per_token_above_128k_tokens": 20e-6, + "litellm_provider": "openai", + }, + {}, + ), + ( + "service_tier_priority_rates", + usage(5_000, 1_000, details(reasoning=400)), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_priority": 3e-6, + "output_cost_per_token_priority": 6e-6, + "output_cost_per_reasoning_token_priority": 9e-6, + }, + {"service_tier": "priority"}, + ), + ( + "service_tier_flex_uses_flex_rates", + usage(5_000, 1_000), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_flex": 5e-7, + "output_cost_per_token_flex": 1e-6, + }, + {"service_tier": "flex"}, + ), + ( + "service_tier_unpriced_falls_back_to_base", + usage(5_000, 1_000), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + }, + {"service_tier": "flex"}, + ), + ( + "data_residency_uplift", + usage(1_000, 200), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "regional_processing_uplift_multiplier_eu": 1.5, + }, + {"data_residency": "eu"}, + ), + ( + "image_output_tokens", + usage(500, 700, details(image=600, text=100)), + { + "input_cost_per_token": 5e-7, + "output_cost_per_token": 1e-6, + "output_cost_per_image_token": 5e-6, + }, + {}, + ), + ( + "free_cache_read_and_write", + usage(1_000, 200, details(cached=500, cache_write=300)), + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 4e-6, + "cache_read_input_token_cost": 0.0, + "cache_creation_input_token_cost": 0.0, + }, + {}, + ), + ( + "reasoning_default_follows_output", + usage(1_000, 600, details(reasoning=250, text=350)), + {"input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6}, + {}, + ), + ( + "off_peak_window_active", + usage(1_000, 200, details(reasoning=50)), + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 4e-6, + "off_peak_pricing": { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_reasoning_token": 1e-6, + "hours_utc": "22:00-06:00", + }, + }, + {"current_time": datetime(2026, 3, 12, 2, 30, tzinfo=UTC)}, + ), + ( + "off_peak_window_inactive", + usage(1_000, 200, details(reasoning=50)), + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 4e-6, + "off_peak_pricing": { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "hours_utc": "22:00-06:00", + }, + }, + {"current_time": datetime(2026, 3, 12, 14, 0, tzinfo=UTC)}, + ), + ( + "off_peak_weekday_calendar", + usage(1_000, 200), + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 4e-6, + "off_peak_pricing": { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "hours_utc": "00:00-00:00", + "weekdays": ["sat", "sun"], + "weekday_timezone": "America/New_York", + }, + }, + {"current_time": datetime(2026, 3, 14, 12, 0, tzinfo=UTC)}, + ), + ( + "off_peak_weekday_blocked_on_weekday", + usage(1_000, 200), + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 4e-6, + "off_peak_pricing": { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "hours_utc": "00:00-00:00", + "weekdays": ["sat", "sun"], + "weekday_timezone": "America/New_York", + }, + }, + {"current_time": datetime(2026, 3, 12, 12, 0, tzinfo=UTC)}, + ), + ( + "tiered_base_costs_low_tier", + usage(500, 100, details(reasoning=40)), + { + "tiered_pricing": [ + {"range": [0, 16_000], "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "output_cost_per_reasoning_token": 3e-6}, + {"range": [16_000, 128_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6, "output_cost_per_reasoning_token": 6e-6}, + ] + }, + {"custom_llm_provider": "dashscope"}, + ), + ( + "tiered_base_costs_high_tier", + usage(50_000, 100, details(reasoning=40)), + { + "tiered_pricing": [ + {"range": [0, 16_000], "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "output_cost_per_reasoning_token": 3e-6}, + {"range": [16_000, 128_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6, "output_cost_per_reasoning_token": 6e-6}, + ] + }, + {"custom_llm_provider": "dashscope"}, + ), + ( + "tier_boundary_stays_in_lower_tier", + usage(16_000, 100, details(reasoning=40)), + { + "tiered_pricing": [ + {"range": [0, 16_000], "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "output_cost_per_reasoning_token": 3e-6}, + {"range": [16_000, 128_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6, "output_cost_per_reasoning_token": 6e-6}, + ] + }, + {"custom_llm_provider": "dashscope"}, + ), + ( + "tier_cache_rates_fall_back_to_tier_input", + usage(20_000, 100, details(cached=5_000, cache_write=1_000)), + { + "input_cost_per_token": 5e-6, + "output_cost_per_token": 10e-6, + "cache_read_input_token_cost": 1e-6, + "tiered_pricing": [ + {"range": [0, 16_000], "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + {"range": [16_000, 128_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6}, + ], + }, + {"custom_llm_provider": "dashscope"}, + ), + ( + "cache_read_falls_back_to_input_rate", + usage(1_000, 200, details(cached=600)), + {"input_cost_per_token": 3e-6, "output_cost_per_token": 9e-6, "litellm_provider": "openai"}, + {}, + ), + ( + "fireworks_cache_read_default_applies", + usage(1_000, 200, details(cached=600)), + {"input_cost_per_token": 2e-7, "output_cost_per_token": 6e-7, "litellm_provider": "fireworks_ai"}, + {"custom_llm_provider": "fireworks_ai"}, + ), + ( + "fireworks_cache_read_default_off_peak_untouched", + usage(1_000, 200, details(cached=600)), + { + "input_cost_per_token": 2e-7, + "output_cost_per_token": 6e-7, + "litellm_provider": "fireworks_ai", + "off_peak_pricing": {"input_cost_per_token": 1e-7, "hours_utc": "00:00-00:00"}, + }, + {"custom_llm_provider": "fireworks_ai", "current_time": datetime(2026, 3, 12, 2, 30, tzinfo=UTC)}, + ), + ( + "zero_rated_everything", + usage(10_000, 5_000, details(cached=5_000, reasoning=1_000)), + {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + {}, + ), + ( + "vertex_regional_endpoint_uplift", + usage(1_000, 200), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "regional_endpoint_uplift_multiplier": 1.10, + }, + {"vertex_location": "us-east1", "custom_llm_provider": "vertex_ai"}, + ), + ( + "vertex_global_location_no_uplift", + usage(1_000, 200), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "regional_endpoint_uplift_multiplier": 1.10, + }, + {"vertex_location": "global", "custom_llm_provider": "vertex_ai"}, + ), +) + + +def generic_surface() -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for name, payload, model_info, extra in GENERIC_CASES: + call = { + "usage": payload, + "model_info": model_info, + "custom_llm_provider": extra.get("custom_llm_provider", "openai"), + "service_tier": extra.get("service_tier"), + "data_residency": extra.get("data_residency"), + "vertex_location": extra.get("vertex_location"), + "current_time": extra["current_time"].isoformat() if "current_time" in extra else None, + } + result = generic_cost_per_token( + model="fixture-model", + usage=make_usage(payload), + custom_llm_provider=call["custom_llm_provider"], + service_tier=call["service_tier"], + data_residency=call["data_residency"], + model_info=model_info, + vertex_location=call["vertex_location"], + current_time=extra.get("current_time"), + ) + rows.append({"id": name, **call, "expected_input_cost": result[0], "expected_output_cost": result[1]}) + return rows + + +BATCH_CASES: tuple[tuple[str, dict[str, Any], int, int, str], ...] = ( + ( + "single_batch_threshold_crossed", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "input_cost_per_token_batches": 1.5e-6, + "output_cost_per_token_batches": 7.5e-6, + "input_cost_per_token_above_100k_tokens_batches": 7.5e-7, + "output_cost_per_token_above_100k_tokens_batches": 3.75e-6, + }, + 150_000, + 20_000, + "anthropic", + ), + ( + "batch_threshold_not_crossed", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "input_cost_per_token_batches": 1.5e-6, + "output_cost_per_token_batches": 7.5e-6, + "input_cost_per_token_above_100k_tokens_batches": 7.5e-7, + "output_cost_per_token_above_100k_tokens_batches": 3.75e-6, + }, + 99_999, + 20_000, + "anthropic", + ), + ( + "inclusive_batch_provider_at_boundary", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "input_cost_per_token_batches": 1.5e-6, + "output_cost_per_token_batches": 7.5e-6, + "input_cost_per_token_above_100k_tokens_batches": 7.5e-7, + "output_cost_per_token_above_100k_tokens_batches": 3.75e-6, + }, + 100_000, + 20_000, + "vertex_ai", + ), + ( + "batch_cache_rates_use_batch_fractions", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "cache_creation_input_token_cost": 3.75e-6, + "input_cost_per_token_batches": 1.5e-6, + "output_cost_per_token_batches": 7.5e-6, + "cache_read_input_token_cost_batches": 1.5e-7, + "cache_creation_input_token_cost_batches": 1.875e-6, + }, + 50_000, + 10_000, + "anthropic", + ), + ( + "batch_cache_rates_absent_stay_none", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "input_cost_per_token_batches": 1.5e-6, + "output_cost_per_token_batches": 7.5e-6, + }, + 50_000, + 10_000, + "anthropic", + ), + ( + "no_batch_rates_falls_back_to_base", + {"input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6}, + 50_000, + 10_000, + "anthropic", + ), +) + + +def batch_surface() -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for name, model_info, prompt, completion, provider in BATCH_CASES: + payload = usage(prompt, completion, details(cached=2_000, cache_write=1_000)) + rates = get_batch_cost_rates(model_info, make_usage(payload), provider) + rows.append( + { + "id": name, + "usage": payload, + "model_info": model_info, + "custom_llm_provider": provider, + "expected_input": rates.input, + "expected_output": rates.output, + "expected_cache_read": rates.cache_read, + "expected_cache_creation": rates.cache_creation, + } + ) + return rows + + +OFF_PEAK_TIMES = ( + "2026-03-12T02:30:00+00:00", + "2026-03-12T14:00:00+00:00", + "2026-03-14T12:00:00+00:00", + "2026-03-14T20:00:00+00:00", + "2026-07-04T01:00:00+00:00", +) +OFF_PEAK_BLOCKS: tuple[tuple[str, dict[str, Any]], ...] = ( + ("simple_window", {"hours_utc": "22:00-06:00"}), + ("multi_window", {"hours_utc": ["00:30-02:00", "22:00-23:30"]}), + ("weekend_only", {"hours_utc": "00:00-00:00", "weekdays": ["sat", "sun"]}), + ("weekday_calendar_est", {"hours_utc": "00:00-00:00", "weekdays": ["mon", "wed"], "weekday_timezone": "America/New_York"}), + ("window_rules_with_weekdays", {"windows": [{"hours_utc": "00:00-06:00", "weekdays": ["sat", "sun"]}]}), + ("window_rule_wrong_weekday", {"windows": [{"hours_utc": "00:00-06:00", "weekdays": ["mon"]}]}), + ("malformed_window_ignored", {"hours_utc": "22-06"}), +) + + +def off_peak_surface() -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for block_name, block in OFF_PEAK_BLOCKS: + for moment in OFF_PEAK_TIMES: + at = datetime.fromisoformat(moment) + rows.append( + { + "id": f"{block_name}_{moment[11:16]}", + "off_peak": block, + "current_time": moment, + "expected_is_off_peak": _is_off_peak(block, at), + } + ) + return rows + + +RESPONSES_USAGE_CASES: tuple[tuple[str, dict[str, Any]], ...] = ( + ( + "cached_details_with_web_search", + { + "input_tokens": 1_200, + "output_tokens": 300, + "input_token_details": {"cached_tokens": 800, "web_search_requests": 2}, + "output_token_details": {"reasoning_tokens": 100}, + }, + ), + ( + "realtime_plural_details", + { + "input_tokens": 900, + "output_tokens": 250, + "input_token_details": {"text_tokens": 700, "cached_tokens": 200, "audio_tokens": 0}, + "output_token_details": {"text_tokens": 150, "audio_tokens": 100}, + }, + ), + ( + "missing_totals_derived", + {"input_tokens": 410, "output_tokens": 90}, + ), + ( + "already_chat_shaped_passthrough", + {"prompt_tokens": 500, "completion_tokens": 100, "total_tokens": 600}, + ), + ( + "nested_cached_details", + { + "input_tokens": 1_000, + "output_tokens": 200, + "input_token_details": {"text_tokens": 400, "cached_tokens": 600}, + }, + ), +) + + +def responses_usage_surface() -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for name, raw in RESPONSES_USAGE_CASES: + transformed = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw) + rows.append({"id": name, "raw": raw, "expected": transformed.model_dump(exclude_none=True)}) + return rows + + +CACHING_SAVINGS_CASES: tuple[tuple[str, dict[str, Any], dict[str, Any], str | None], ...] = ( + ( + "anthropic_read_discount_write_premium", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "cache_creation_input_token_cost": 3.75e-6, + "litellm_provider": "anthropic", + }, + usage(10_000, 500, details(cached=8_000, cache_write=1_500)), + None, + ), + ( + "openai_cache_savings", + { + "input_cost_per_token": 1.25e-6, + "output_cost_per_token": 1e-5, + "cache_read_input_token_cost": 6.25e-8, + "cache_creation_input_token_cost": 1.25e-6, + "litellm_provider": "openai", + }, + usage(50_000, 2_000, details(cached=45_000, cache_write=3_000)), + None, + ), + ( + "unpriced_cache_no_savings", + {"input_cost_per_token": 2e-6, "output_cost_per_token": 8e-6, "litellm_provider": "openai"}, + usage(10_000, 500, details(cached=8_000)), + None, + ), + ( + "priority_tier_discount", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 3e-7, + "cache_creation_input_token_cost": 3.75e-6, + "input_cost_per_token_priority": 6e-6, + "litellm_provider": "anthropic", + }, + usage(10_000, 500, details(cached=8_000, cache_write=1_500)), + "priority", + ), +) + + +def caching_savings_surface() -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for name, model_info, payload, service_tier in CACHING_SAVINGS_CASES: + savings = calculate_prompt_caching_savings( + model_info=model_info, + usage=make_usage(payload), + custom_llm_provider=model_info.get("litellm_provider"), + service_tier=service_tier, + ) + rows.append( + { + "id": name, + "model_info": model_info, + "usage": payload, + "custom_llm_provider": model_info.get("litellm_provider"), + "service_tier": service_tier, + "expected_savings": savings, + } + ) + return rows + + +ANTHROPIC_USAGE_CASES: tuple[tuple[str, dict[str, Any], str | None, bool], ...] = ( + ( + "plain_input_output", + {"input_tokens": 1_000, "output_tokens": 300}, + None, + False, + ), + ( + "cache_creation_and_read", + { + "input_tokens": 5_000, + "output_tokens": 200, + "cache_creation_input_tokens": 2_000, + "cache_creation": {"ephemeral_5m_input_tokens": 1_500, "ephemeral_1h_input_tokens": 500}, + "cache_read_input_tokens": 3_000, + }, + None, + False, + ), + ( + "iterations_with_thinking_tokens", + { + "input_tokens": 600, + "output_tokens": 300, + "iterations": [ + {"input_tokens": 100, "output_tokens": 150, "output_tokens_details": {"thinking_tokens": 90}}, + {"input_tokens": 200, "output_tokens": 150, "output_tokens_details": {"thinking_tokens": 60}}, + ], + }, + None, + False, + ), + ( + "iterations_usage", + { + "input_tokens": 600, + "output_tokens": 250, + "server_tool_use": {"web_search_requests": 3}, + }, + None, + False, + ), +) + + +def anthropic_usage_surface() -> list[dict[str, Any]]: + config = AnthropicConfig() + rows: list[dict[str, Any]] = [] + for name, raw, reasoning, has_thinking in ANTHROPIC_USAGE_CASES: + transformed = config.calculate_usage(raw, reasoning) + rows.append( + { + "id": name, + "raw": raw, + "reasoning_content": reasoning, + "response_has_thinking_block": has_thinking, + "expected": transformed.model_dump(exclude_none=True), + } + ) + return rows + + +BILLED_RATES_CASES: tuple[tuple[str, dict[str, Any], dict[str, Any], dict[str, Any]], ...] = ( + ( + "geo_multiplier_applies", + usage(1_000, 200), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "provider_specific_entry": {"us": 1.1}, + }, + {"usage_extra": {"inference_geo": "us"}, "custom_llm_provider": "anthropic"}, + ), + ( + "geo_global_is_neutral", + usage(1_000, 200), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "provider_specific_entry": {"us": 1.1}, + }, + {"usage_extra": {"inference_geo": "global"}, "custom_llm_provider": "anthropic"}, + ), + ( + "all_multipliers_compose", + usage(1_000, 200), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 1e-7, + "provider_specific_entry": {"us": 1.1}, + "regional_processing_uplift_multiplier_eu": 1.5, + "regional_endpoint_uplift_multiplier": 1.10, + }, + { + "usage_extra": {"inference_geo": "us"}, + "data_residency": "eu", + "vertex_location": "us-east1", + "custom_llm_provider": "anthropic", + }, + ), + ( + "service_tier_priority_rates", + usage(1_000, 200), + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_priority": 3e-6, + "output_cost_per_token_priority": 6e-6, + }, + {"service_tier": "priority"}, + ), + ( + "off_peak_rates_and_reasoning", + usage(1_000, 200, details(reasoning=50)), + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 4e-6, + "cache_read_input_token_cost": 2e-7, + "cache_creation_input_token_cost": 5e-6, + "off_peak_pricing": { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_reasoning_token": 1e-6, + "hours_utc": "22:00-06:00", + }, + }, + {"current_time": datetime(2026, 3, 12, 2, 30, tzinfo=UTC)}, + ), +) + + +def billed_rates_surface() -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for name, payload, model_info, extra in BILLED_RATES_CASES: + usage_obj = make_usage(payload) + for key, value in extra.get("usage_extra", {}).items(): + setattr(usage_obj, key, value) + model_name = f"fixture-billed/{name.replace('_', '-')}" + register_model({model_name: model_info}) + rates = get_billed_token_rates( + model=model_name, + usage=usage_obj, + custom_llm_provider=extra.get("custom_llm_provider", "openai"), + service_tier=extra.get("service_tier"), + data_residency=extra.get("data_residency"), + vertex_location=extra.get("vertex_location"), + current_time=extra.get("current_time"), + ) + rows.append( + { + "id": name, + "model": model_name, + "usage": payload, + "usage_extra": extra.get("usage_extra", {}), + "model_info": model_info, + "custom_llm_provider": extra.get("custom_llm_provider", "openai"), + "service_tier": extra.get("service_tier"), + "data_residency": extra.get("data_residency"), + "vertex_location": extra.get("vertex_location"), + "current_time": extra["current_time"].isoformat() if "current_time" in extra else None, + "expected": { + "input_cost_per_token": rates.input_cost_per_token, + "output_cost_per_token": rates.output_cost_per_token, + "cache_read_input_token_cost": rates.cache_read_input_token_cost, + "cache_read_input_audio_token_cost": rates.cache_read_input_audio_token_cost, + "cache_creation_input_token_cost": rates.cache_creation_input_token_cost, + "cache_creation_input_token_cost_above_1hr": rates.cache_creation_input_token_cost_above_1hr, + "output_cost_per_reasoning_token": rates.output_cost_per_reasoning_token, + }, + } + ) + return rows + + +def main() -> None: + revision = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True).strip() + fixture = { + "python_commit": revision, + "generic": generic_surface(), + "batch": batch_surface(), + "billed_rates": billed_rates_surface(), + "off_peak": off_peak_surface(), + "responses_usage": responses_usage_surface(), + "caching_savings": caching_savings_surface(), + "anthropic_usage": anthropic_usage_surface(), + } + target = Path(__file__).parent / "python_fixtures.json" + target.write_text(json.dumps(fixture, indent=1) + "\n") + counts = {key: len(value) for key, value in fixture.items() if isinstance(value, list)} + print(json.dumps(counts)) + + +if __name__ == "__main__": + main() diff --git a/litellm-rust/crates/cost/tests/python_fixture_replay.rs b/litellm-rust/crates/cost/tests/python_fixture_replay.rs new file mode 100644 index 00000000000..9eb535d71af --- /dev/null +++ b/litellm-rust/crates/cost/tests/python_fixture_replay.rs @@ -0,0 +1,364 @@ +use std::collections::HashMap; + +use jiff::Timestamp; +use litellm_cost::Rate; +use litellm_cost::anthropic_usage::transform_anthropic_usage_to_chat_usage; +use litellm_cost::base_rate_selection::uses_inclusive_token_thresholds; +use litellm_cost::batch::batch_cost_rates_from_model_info; +use litellm_cost::billed_token_rates::{BilledRatesRequest, get_billed_token_rates}; +use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region; +use litellm_cost::off_peak::is_off_peak; +use litellm_cost::prompt_caching_savings::{ + PromptCachingSavingsRequest, calculate_prompt_caching_savings, +}; +use litellm_cost::provider_cache::apply_provider_cache_read_default; +use litellm_cost::usage_dispatch::get_usage_object; +use serde_json::{Value, json}; + +fn fixtures() -> Value { + serde_json::from_str(include_str!("python_fixtures.json")).unwrap() +} + +fn instant(iso: &str) -> Timestamp { + iso.parse().unwrap() +} + +fn opt_str<'a>(value: &'a Value, key: &str) -> Option<&'a str> { + value.get(key).and_then(Value::as_str) +} + +fn chat_usage(payload: &Value) -> litellm_cost::responses_usage::ChatUsage { + get_usage_object(&json!({"usage": payload})) + .unwrap() + .unwrap() +} + +fn assert_close(actual: f64, expected: f64, id: &str) { + assert!( + (actual - expected).abs() <= expected.abs() * 1e-12, + "{id}: rust={actual:?} python={expected:?}" + ); +} + +#[test] +fn generic_token_costs_match_executed_python_fixtures() { + for row in fixtures().get("generic").unwrap().as_array().unwrap() { + let id = row.get("id").and_then(Value::as_str).unwrap(); + let usage = chat_usage(row.get("usage").unwrap()); + let provider = opt_str(row, "custom_llm_provider"); + let model_info = + apply_provider_cache_read_default(row.get("model_info").unwrap(), provider); + let at = opt_str(row, "current_time") + .map_or_else(|| "2026-03-12T00:00:00Z".parse().unwrap(), instant); + let (input_cost, output_cost) = calculate_generic_cost_from_model_info_with_region( + &usage, + &model_info, + opt_str(row, "service_tier"), + uses_inclusive_token_thresholds(provider), + opt_str(row, "data_residency"), + opt_str(row, "vertex_location"), + at, + ); + assert_close( + input_cost, + row.get("expected_input_cost").unwrap().as_f64().unwrap(), + id, + ); + assert_close( + output_cost, + row.get("expected_output_cost").unwrap().as_f64().unwrap(), + id, + ); + } +} + +#[test] +fn billed_token_rates_match_executed_python_fixtures() { + for row in fixtures().get("billed_rates").unwrap().as_array().unwrap() { + let id = row.get("id").and_then(Value::as_str).unwrap(); + let mut payload = row.get("usage").unwrap().clone(); + if let Some(extra) = row.get("usage_extra").and_then(Value::as_object) { + for (key, value) in extra { + payload + .as_object_mut() + .unwrap() + .insert(key.clone(), value.clone()); + } + } + let usage = chat_usage(&payload); + let provider = opt_str(row, "custom_llm_provider"); + let at = opt_str(row, "current_time") + .map_or_else(|| "2026-03-12T00:00:00Z".parse().unwrap(), instant); + let rates = get_billed_token_rates(BilledRatesRequest { + model_info: Some(row.get("model_info").unwrap()), + usage: &usage, + provider, + service_tier: opt_str(row, "service_tier"), + data_residency: opt_str(row, "data_residency"), + vertex_location: opt_str(row, "vertex_location"), + at, + custom_cost_per_token: None, + }) + .unwrap_or_else(|| panic!("{id}: rates unresolved")); + let expected = row.get("expected").unwrap(); + for field in [ + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_read_input_audio_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "output_cost_per_reasoning_token", + ] { + let want = expected.get(field).unwrap().as_f64().unwrap(); + let got = match field { + "input_cost_per_token" => rates.input_cost_per_token, + "output_cost_per_token" => rates.output_cost_per_token, + "cache_read_input_token_cost" => rates.cache_read_input_token_cost, + "cache_read_input_audio_token_cost" => rates.cache_read_input_audio_token_cost, + "cache_creation_input_token_cost" => rates.cache_creation_input_token_cost, + "cache_creation_input_token_cost_above_1hr" => { + rates.cache_creation_input_token_cost_above_1hr + } + _ => rates.output_cost_per_reasoning_token, + }; + assert_close(got, want, &format!("{id}/{field}")); + } + } +} + +#[test] +fn batch_cost_rates_match_executed_python_fixtures() { + for row in fixtures().get("batch").unwrap().as_array().unwrap() { + let id = row.get("id").and_then(Value::as_str).unwrap(); + let model_info = row.get("model_info").unwrap(); + let provider = opt_str(row, "custom_llm_provider").unwrap(); + let prompt_tokens = row + .get("usage") + .unwrap() + .get("prompt_tokens") + .unwrap() + .as_u64() + .unwrap(); + let rates = batch_cost_rates_from_model_info( + model_info, + prompt_tokens, + uses_inclusive_token_thresholds(Some(provider)), + ); + let as_option = |rate: Rate| match rate { + Rate::Value(value) => Some(value), + Rate::Missing | Rate::Null => None, + }; + let option_equals = |actual: Option, key: &str| match (actual, row.get(key)) { + (Some(got), Some(want)) => { + assert_close(got, want.as_f64().unwrap(), &format!("{id}/{key}")) + } + (None, None) | (None, Some(Value::Null)) => {} + (got, want) => panic!("{id}/{key}: rust={got:?} python={want:?}"), + }; + option_equals(as_option(rates.input), "expected_input"); + option_equals(as_option(rates.output), "expected_output"); + option_equals(as_option(rates.cache_read), "expected_cache_read"); + option_equals(as_option(rates.cache_creation), "expected_cache_creation"); + } +} + +#[test] +fn off_peak_decisions_match_executed_python_fixtures() { + for row in fixtures().get("off_peak").unwrap().as_array().unwrap() { + let id = row.get("id").and_then(Value::as_str).unwrap(); + let block = row.get("off_peak").unwrap(); + let at = instant(row.get("current_time").and_then(Value::as_str).unwrap()); + let expected = row.get("expected_is_off_peak").unwrap().as_bool().unwrap(); + assert_eq!(is_off_peak(block, at), expected, "{id}"); + } +} + +#[test] +fn responses_usage_transforms_match_executed_python_fixtures() { + for row in fixtures() + .get("responses_usage") + .unwrap() + .as_array() + .unwrap() + { + let id = row.get("id").and_then(Value::as_str).unwrap(); + let usage = get_usage_object(&json!({"usage": row.get("raw").unwrap()})) + .unwrap() + .unwrap_or_else(|| panic!("{id}: usage unparsed")); + assert_usage_matches(&usage, row.get("expected").unwrap(), id); + } +} + +#[test] +fn anthropic_usage_transforms_match_executed_python_fixtures() { + for row in fixtures() + .get("anthropic_usage") + .unwrap() + .as_array() + .unwrap() + { + let id = row.get("id").and_then(Value::as_str).unwrap(); + let usage = transform_anthropic_usage_to_chat_usage( + row.get("raw").unwrap(), + None, + row.get("response_has_thinking_block") + .and_then(Value::as_bool) + .unwrap_or(false), + ) + .unwrap(); + assert_usage_matches(&usage, row.get("expected").unwrap(), id); + } +} + +fn assert_usage_matches( + usage: &litellm_cost::responses_usage::ChatUsage, + expected: &Value, + id: &str, +) { + let count = |key: &str| expected.get(key).and_then(Value::as_u64).unwrap_or(0); + assert_eq!(usage.prompt_tokens, count("prompt_tokens"), "{id}/prompt"); + assert_eq!( + usage.completion_tokens, + count("completion_tokens"), + "{id}/completion" + ); + assert_eq!(usage.total_tokens, count("total_tokens"), "{id}/total"); + let details = usage.prompt_tokens_details.clone().unwrap_or_default(); + let expected_details = expected.get("prompt_tokens_details"); + let detail = |key: &str| { + expected_details + .and_then(|d| d.get(key)) + .and_then(Value::as_u64) + }; + assert_eq!( + details.cached_tokens, + count_of(expected_details, "cached_tokens"), + "{id}/cached" + ); + assert_eq!( + details.web_search_requests, + detail("web_search_requests"), + "{id}/web_search_requests" + ); + assert_eq!( + details.cache_write_tokens, + detail("cache_write_tokens"), + "{id}/cache_write" + ); + if let Some(want) = detail("cache_creation_tokens") { + assert_eq!( + details.cache_creation_tokens, + Some(want), + "{id}/cache_creation" + ); + } + if let (Some(want), Some(got)) = ( + expected_details + .and_then(|d| d.get("cache_creation_token_details")) + .and_then(Value::as_object), + details.cache_creation_token_details.as_ref().map(|v| { + json!({ + "ephemeral_5m_input_tokens": v.ephemeral_5m_input_tokens, + "ephemeral_1h_input_tokens": v.ephemeral_1h_input_tokens, + }) + }), + ) { + assert_eq!( + got.get("ephemeral_5m_input_tokens").and_then(Value::as_u64), + want.get("ephemeral_5m_input_tokens") + .and_then(Value::as_u64), + "{id}/ttl_5m" + ); + assert_eq!( + got.get("ephemeral_1h_input_tokens").and_then(Value::as_u64), + want.get("ephemeral_1h_input_tokens") + .and_then(Value::as_u64), + "{id}/ttl_1h" + ); + } + let completion = usage.completion_tokens_details.clone(); + let expected_completion = expected.get("completion_tokens_details"); + assert_eq!( + completion.as_ref().and_then(|d| d.reasoning_tokens), + expected_completion + .and_then(|d| d.get("reasoning_tokens")) + .and_then(Value::as_u64), + "{id}/reasoning" + ); + assert_eq!( + completion.as_ref().and_then(|d| d.text_tokens), + expected_completion + .and_then(|d| d.get("text_tokens")) + .and_then(Value::as_u64), + "{id}/text" + ); + if let Some(server_tool_use) = expected.get("server_tool_use").and_then(Value::as_object) { + for (key, value) in server_tool_use { + assert_eq!( + usage.extra.get("server_tool_use").and_then(|v| v.get(key)), + Some(value), + "{id}/server_tool_use/{key}" + ); + } + } +} + +fn count_of(details: Option<&Value>, key: &str) -> u64 { + details + .and_then(|d| d.get(key)) + .and_then(Value::as_u64) + .unwrap_or(0) +} + +#[test] +fn prompt_caching_savings_match_executed_python_fixtures() { + for row in fixtures() + .get("caching_savings") + .unwrap() + .as_array() + .unwrap() + { + let id = row.get("id").and_then(Value::as_str).unwrap(); + let usage = chat_usage(row.get("usage").unwrap()); + let savings = calculate_prompt_caching_savings(PromptCachingSavingsRequest { + model_info: row.get("model_info").unwrap(), + usage: &usage, + provider: opt_str(row, "custom_llm_provider"), + service_tier: opt_str(row, "service_tier"), + data_residency: opt_str(row, "data_residency"), + vertex_location: opt_str(row, "vertex_location"), + at: "2026-03-12T00:00:00Z".parse().unwrap(), + }); + assert_close( + savings, + row.get("expected_savings").unwrap().as_f64().unwrap(), + id, + ); + } +} + +#[test] +fn fixture_ids_are_unique_per_surface() { + let mut seen: HashMap = HashMap::new(); + let root = fixtures(); + for surface in [ + "generic", + "batch", + "billed_rates", + "off_peak", + "responses_usage", + "caching_savings", + "anthropic_usage", + ] { + for row in root.get(surface).unwrap().as_array().unwrap() { + let id = row.get("id").and_then(Value::as_str).unwrap().to_owned(); + *seen.entry(format!("{surface}/{id}")).or_default() += 1; + } + } + assert!( + seen.values().all(|count| *count == 1), + "duplicate fixture ids: {:?}", + seen.iter().filter(|(_, c)| **c > 1).collect::>() + ); +} diff --git a/litellm-rust/crates/cost/tests/python_fixtures.json b/litellm-rust/crates/cost/tests/python_fixtures.json new file mode 100644 index 00000000000..4268cf4219c --- /dev/null +++ b/litellm-rust/crates/cost/tests/python_fixtures.json @@ -0,0 +1,1778 @@ +{ + "python_commit": "b401bc9be2509e46fc0126087db2542b129ad56c", + "generic": [ + { + "id": "audio_mixed_modalities", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 500, + "prompt_tokens_details": { + "audio_tokens": 200 + }, + "completion_tokens_details": { + "audio_tokens": 150, + "text_tokens": 350, + "reasoning_tokens": 0 + } + }, + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "input_cost_per_audio_token": 1e-05, + "output_cost_per_audio_token": 2e-05, + "output_cost_per_reasoning_token": 8e-06 + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.0036, + "expected_output_cost": 0.0044 + }, + { + "id": "anthropic_cache_ttl_split", + "usage": { + "prompt_tokens": 10000, + "completion_tokens": 900, + "prompt_tokens_details": { + "cached_tokens": 8000 + }, + "completion_tokens_details": { + "text_tokens": 800, + "reasoning_tokens": 100 + } + }, + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "output_cost_per_reasoning_token": 6e-06, + "litellm_provider": "anthropic" + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.0084, + "expected_output_cost": 0.0126 + }, + { + "id": "cache_write_plus_inclusive_threshold", + "usage": { + "prompt_tokens": 130000, + "completion_tokens": 2000, + "prompt_tokens_details": { + "cached_tokens": 40000, + "cache_write_tokens": 5000 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_creation_input_token_cost": 5e-06, + "input_cost_per_token_above_128k_tokens": 4e-06, + "output_cost_per_token_above_128k_tokens": 2e-05, + "litellm_provider": "xai" + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.525, + "expected_output_cost": 0.04 + }, + { + "id": "exclusive_threshold_at_boundary", + "usage": { + "prompt_tokens": 128000, + "completion_tokens": 500, + "prompt_tokens_details": { + "cached_tokens": 1000 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "input_cost_per_token_above_128k_tokens": 4e-06, + "output_cost_per_token_above_128k_tokens": 2e-05, + "litellm_provider": "openai" + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.256, + "expected_output_cost": 0.005 + }, + { + "id": "service_tier_priority_rates", + "usage": { + "prompt_tokens": 5000, + "completion_tokens": 1000, + "prompt_tokens_details": null, + "completion_tokens_details": { + "reasoning_tokens": 400 + } + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "input_cost_per_token_priority": 3e-06, + "output_cost_per_token_priority": 6e-06, + "output_cost_per_reasoning_token_priority": 9e-06 + }, + "custom_llm_provider": "openai", + "service_tier": "priority", + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.015000000000000001, + "expected_output_cost": 0.0072 + }, + { + "id": "service_tier_flex_uses_flex_rates", + "usage": { + "prompt_tokens": 5000, + "completion_tokens": 1000, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "input_cost_per_token_flex": 5e-07, + "output_cost_per_token_flex": 1e-06 + }, + "custom_llm_provider": "openai", + "service_tier": "flex", + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.0025, + "expected_output_cost": 0.001 + }, + { + "id": "service_tier_unpriced_falls_back_to_base", + "usage": { + "prompt_tokens": 5000, + "completion_tokens": 1000, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06 + }, + "custom_llm_provider": "openai", + "service_tier": "flex", + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.005, + "expected_output_cost": 0.002 + }, + { + "id": "data_residency_uplift", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "regional_processing_uplift_multiplier_eu": 1.5 + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": "eu", + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.0015, + "expected_output_cost": 0.0006 + }, + { + "id": "image_output_tokens", + "usage": { + "prompt_tokens": 500, + "completion_tokens": 700, + "prompt_tokens_details": null, + "completion_tokens_details": { + "image_tokens": 600, + "text_tokens": 100 + } + }, + "model_info": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 1e-06, + "output_cost_per_image_token": 5e-06 + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.00025, + "expected_output_cost": 0.0031 + }, + { + "id": "free_cache_read_and_write", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": { + "cached_tokens": 500, + "cache_write_tokens": 300 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 0.0, + "cache_creation_input_token_cost": 0.0 + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.00039999999999999996, + "expected_output_cost": 0.0007999999999999999 + }, + { + "id": "reasoning_default_follows_output", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 600, + "prompt_tokens_details": null, + "completion_tokens_details": { + "text_tokens": 350, + "reasoning_tokens": 250 + } + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06 + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.001, + "expected_output_cost": 0.0024000000000000002 + }, + { + "id": "off_peak_window_active", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": { + "reasoning_tokens": 50 + } + }, + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "off_peak_pricing": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "output_cost_per_reasoning_token": 1e-06, + "hours_utc": "22:00-06:00" + } + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": "2026-03-12T02:30:00+00:00", + "expected_input_cost": 0.001, + "expected_output_cost": 0.00034999999999999994 + }, + { + "id": "off_peak_window_inactive", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": { + "reasoning_tokens": 50 + } + }, + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "off_peak_pricing": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "hours_utc": "22:00-06:00" + } + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": "2026-03-12T14:00:00+00:00", + "expected_input_cost": 0.002, + "expected_output_cost": 0.0007999999999999999 + }, + { + "id": "off_peak_weekday_calendar", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "off_peak_pricing": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "hours_utc": "00:00-00:00", + "weekdays": [ + "sat", + "sun" + ], + "weekday_timezone": "America/New_York" + } + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": "2026-03-14T12:00:00+00:00", + "expected_input_cost": 0.001, + "expected_output_cost": 0.00039999999999999996 + }, + { + "id": "off_peak_weekday_blocked_on_weekday", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "off_peak_pricing": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "hours_utc": "00:00-00:00", + "weekdays": [ + "sat", + "sun" + ], + "weekday_timezone": "America/New_York" + } + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": "2026-03-12T12:00:00+00:00", + "expected_input_cost": 0.001, + "expected_output_cost": 0.00039999999999999996 + }, + { + "id": "tiered_base_costs_low_tier", + "usage": { + "prompt_tokens": 500, + "completion_tokens": 100, + "prompt_tokens_details": null, + "completion_tokens_details": { + "reasoning_tokens": 40 + } + }, + "model_info": { + "tiered_pricing": [ + { + "range": [ + 0, + 16000 + ], + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "output_cost_per_reasoning_token": 3e-06 + }, + { + "range": [ + 16000, + 128000 + ], + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "output_cost_per_reasoning_token": 6e-06 + } + ] + }, + "custom_llm_provider": "dashscope", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.0005, + "expected_output_cost": 0.00023999999999999998 + }, + { + "id": "tiered_base_costs_high_tier", + "usage": { + "prompt_tokens": 50000, + "completion_tokens": 100, + "prompt_tokens_details": null, + "completion_tokens_details": { + "reasoning_tokens": 40 + } + }, + "model_info": { + "tiered_pricing": [ + { + "range": [ + 0, + 16000 + ], + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "output_cost_per_reasoning_token": 3e-06 + }, + { + "range": [ + 16000, + 128000 + ], + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "output_cost_per_reasoning_token": 6e-06 + } + ] + }, + "custom_llm_provider": "dashscope", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.09999999999999999, + "expected_output_cost": 0.00047999999999999996 + }, + { + "id": "tier_boundary_stays_in_lower_tier", + "usage": { + "prompt_tokens": 16000, + "completion_tokens": 100, + "prompt_tokens_details": null, + "completion_tokens_details": { + "reasoning_tokens": 40 + } + }, + "model_info": { + "tiered_pricing": [ + { + "range": [ + 0, + 16000 + ], + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "output_cost_per_reasoning_token": 3e-06 + }, + { + "range": [ + 16000, + 128000 + ], + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "output_cost_per_reasoning_token": 6e-06 + } + ] + }, + "custom_llm_provider": "dashscope", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.016, + "expected_output_cost": 0.00023999999999999998 + }, + { + "id": "tier_cache_rates_fall_back_to_tier_input", + "usage": { + "prompt_tokens": 20000, + "completion_tokens": 100, + "prompt_tokens_details": { + "cached_tokens": 5000, + "cache_write_tokens": 1000 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1e-06, + "tiered_pricing": [ + { + "range": [ + 0, + 16000 + ], + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06 + }, + { + "range": [ + 16000, + 128000 + ], + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06 + } + ] + }, + "custom_llm_provider": "dashscope", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.04, + "expected_output_cost": 0.00039999999999999996 + }, + { + "id": "cache_read_falls_back_to_input_rate", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": { + "cached_tokens": 600 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 9e-06, + "litellm_provider": "openai" + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.003, + "expected_output_cost": 0.0018 + }, + { + "id": "fireworks_cache_read_default_applies", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": { + "cached_tokens": 600 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai" + }, + "custom_llm_provider": "fireworks_ai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.00014, + "expected_output_cost": 0.00011999999999999999 + }, + { + "id": "fireworks_cache_read_default_off_peak_untouched", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": { + "cached_tokens": 600 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "off_peak_pricing": { + "input_cost_per_token": 1e-07, + "hours_utc": "00:00-00:00" + } + }, + "custom_llm_provider": "fireworks_ai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": "2026-03-12T02:30:00+00:00", + "expected_input_cost": 7e-05, + "expected_output_cost": 0.00011999999999999999 + }, + { + "id": "zero_rated_everything", + "usage": { + "prompt_tokens": 10000, + "completion_tokens": 5000, + "prompt_tokens_details": { + "cached_tokens": 5000 + }, + "completion_tokens_details": { + "reasoning_tokens": 1000 + } + }, + "model_info": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0 + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected_input_cost": 0.0, + "expected_output_cost": 0.0 + }, + { + "id": "vertex_regional_endpoint_uplift", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "regional_endpoint_uplift_multiplier": 1.1 + }, + "custom_llm_provider": "vertex_ai", + "service_tier": null, + "data_residency": null, + "vertex_location": "us-east1", + "current_time": null, + "expected_input_cost": 0.0011, + "expected_output_cost": 0.00044 + }, + { + "id": "vertex_global_location_no_uplift", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "regional_endpoint_uplift_multiplier": 1.1 + }, + "custom_llm_provider": "vertex_ai", + "service_tier": null, + "data_residency": null, + "vertex_location": "global", + "current_time": null, + "expected_input_cost": 0.001, + "expected_output_cost": 0.00039999999999999996 + } + ], + "batch": [ + { + "id": "single_batch_threshold_crossed", + "usage": { + "prompt_tokens": 150000, + "completion_tokens": 20000, + "prompt_tokens_details": { + "cached_tokens": 2000, + "cache_write_tokens": 1000 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "input_cost_per_token_batches": 1.5e-06, + "output_cost_per_token_batches": 7.5e-06, + "input_cost_per_token_above_100k_tokens_batches": 7.5e-07, + "output_cost_per_token_above_100k_tokens_batches": 3.75e-06 + }, + "custom_llm_provider": "anthropic", + "expected_input": 7.5e-07, + "expected_output": 3.75e-06, + "expected_cache_read": null, + "expected_cache_creation": null + }, + { + "id": "batch_threshold_not_crossed", + "usage": { + "prompt_tokens": 99999, + "completion_tokens": 20000, + "prompt_tokens_details": { + "cached_tokens": 2000, + "cache_write_tokens": 1000 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "input_cost_per_token_batches": 1.5e-06, + "output_cost_per_token_batches": 7.5e-06, + "input_cost_per_token_above_100k_tokens_batches": 7.5e-07, + "output_cost_per_token_above_100k_tokens_batches": 3.75e-06 + }, + "custom_llm_provider": "anthropic", + "expected_input": 1.5e-06, + "expected_output": 7.5e-06, + "expected_cache_read": null, + "expected_cache_creation": null + }, + { + "id": "inclusive_batch_provider_at_boundary", + "usage": { + "prompt_tokens": 100000, + "completion_tokens": 20000, + "prompt_tokens_details": { + "cached_tokens": 2000, + "cache_write_tokens": 1000 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "input_cost_per_token_batches": 1.5e-06, + "output_cost_per_token_batches": 7.5e-06, + "input_cost_per_token_above_100k_tokens_batches": 7.5e-07, + "output_cost_per_token_above_100k_tokens_batches": 3.75e-06 + }, + "custom_llm_provider": "vertex_ai", + "expected_input": 1.5e-06, + "expected_output": 7.5e-06, + "expected_cache_read": null, + "expected_cache_creation": null + }, + { + "id": "batch_cache_rates_use_batch_fractions", + "usage": { + "prompt_tokens": 50000, + "completion_tokens": 10000, + "prompt_tokens_details": { + "cached_tokens": 2000, + "cache_write_tokens": 1000 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost": 3.75e-06, + "input_cost_per_token_batches": 1.5e-06, + "output_cost_per_token_batches": 7.5e-06, + "cache_read_input_token_cost_batches": 1.5e-07, + "cache_creation_input_token_cost_batches": 1.875e-06 + }, + "custom_llm_provider": "anthropic", + "expected_input": 1.5e-06, + "expected_output": 7.5e-06, + "expected_cache_read": 1.5e-07, + "expected_cache_creation": 1.875e-06 + }, + { + "id": "batch_cache_rates_absent_stay_none", + "usage": { + "prompt_tokens": 50000, + "completion_tokens": 10000, + "prompt_tokens_details": { + "cached_tokens": 2000, + "cache_write_tokens": 1000 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token_batches": 1.5e-06, + "output_cost_per_token_batches": 7.5e-06 + }, + "custom_llm_provider": "anthropic", + "expected_input": 1.5e-06, + "expected_output": 7.5e-06, + "expected_cache_read": null, + "expected_cache_creation": null + }, + { + "id": "no_batch_rates_falls_back_to_base", + "usage": { + "prompt_tokens": 50000, + "completion_tokens": 10000, + "prompt_tokens_details": { + "cached_tokens": 2000, + "cache_write_tokens": 1000 + }, + "completion_tokens_details": null + }, + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05 + }, + "custom_llm_provider": "anthropic", + "expected_input": null, + "expected_output": null, + "expected_cache_read": null, + "expected_cache_creation": null + } + ], + "billed_rates": [ + { + "id": "geo_multiplier_applies", + "model": "fixture-billed/geo-multiplier-applies", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "usage_extra": { + "inference_geo": "us" + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "provider_specific_entry": { + "us": 1.1 + } + }, + "custom_llm_provider": "anthropic", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected": { + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 2.2e-06, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_audio_token_cost": 1.1e-06, + "cache_creation_input_token_cost": 1.1e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-06, + "output_cost_per_reasoning_token": 2.2e-06 + } + }, + { + "id": "geo_global_is_neutral", + "model": "fixture-billed/geo-global-is-neutral", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "usage_extra": { + "inference_geo": "global" + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "provider_specific_entry": { + "us": 1.1 + } + }, + "custom_llm_provider": "anthropic", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_audio_token_cost": 1e-06, + "cache_creation_input_token_cost": 1e-06, + "cache_creation_input_token_cost_above_1hr": 1e-06, + "output_cost_per_reasoning_token": 2e-06 + } + }, + { + "id": "all_multipliers_compose", + "model": "fixture-billed/all-multipliers-compose", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "usage_extra": { + "inference_geo": "us" + }, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1e-07, + "provider_specific_entry": { + "us": 1.1 + }, + "regional_processing_uplift_multiplier_eu": 1.5, + "regional_endpoint_uplift_multiplier": 1.1 + }, + "custom_llm_provider": "anthropic", + "service_tier": null, + "data_residency": "eu", + "vertex_location": "us-east1", + "current_time": null, + "expected": { + "input_cost_per_token": 1.8150000000000004e-06, + "output_cost_per_token": 3.630000000000001e-06, + "cache_read_input_token_cost": 1.8150000000000003e-07, + "cache_read_input_audio_token_cost": 1.8150000000000003e-07, + "cache_creation_input_token_cost": 1.8150000000000004e-06, + "cache_creation_input_token_cost_above_1hr": 1.8150000000000004e-06, + "output_cost_per_reasoning_token": 3.630000000000001e-06 + } + }, + { + "id": "service_tier_priority_rates", + "model": "fixture-billed/service-tier-priority-rates", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": null + }, + "usage_extra": {}, + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "input_cost_per_token_priority": 3e-06, + "output_cost_per_token_priority": 6e-06 + }, + "custom_llm_provider": "openai", + "service_tier": "priority", + "data_residency": null, + "vertex_location": null, + "current_time": null, + "expected": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 3e-06, + "cache_read_input_audio_token_cost": 3e-06, + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 3e-06, + "output_cost_per_reasoning_token": 6e-06 + } + }, + { + "id": "off_peak_rates_and_reasoning", + "model": "fixture-billed/off-peak-rates-and-reasoning", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "prompt_tokens_details": null, + "completion_tokens_details": { + "reasoning_tokens": 50 + } + }, + "usage_extra": {}, + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2e-07, + "cache_creation_input_token_cost": 5e-06, + "off_peak_pricing": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "output_cost_per_reasoning_token": 1e-06, + "hours_utc": "22:00-06:00" + } + }, + "custom_llm_provider": "openai", + "service_tier": null, + "data_residency": null, + "vertex_location": null, + "current_time": "2026-03-12T02:30:00+00:00", + "expected": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_audio_token_cost": 2e-07, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 5e-06, + "output_cost_per_reasoning_token": 1e-06 + } + } + ], + "off_peak": [ + { + "id": "simple_window_02:30", + "off_peak": { + "hours_utc": "22:00-06:00" + }, + "current_time": "2026-03-12T02:30:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "simple_window_14:00", + "off_peak": { + "hours_utc": "22:00-06:00" + }, + "current_time": "2026-03-12T14:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "simple_window_12:00", + "off_peak": { + "hours_utc": "22:00-06:00" + }, + "current_time": "2026-03-14T12:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "simple_window_20:00", + "off_peak": { + "hours_utc": "22:00-06:00" + }, + "current_time": "2026-03-14T20:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "simple_window_01:00", + "off_peak": { + "hours_utc": "22:00-06:00" + }, + "current_time": "2026-07-04T01:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "multi_window_02:30", + "off_peak": { + "hours_utc": [ + "00:30-02:00", + "22:00-23:30" + ] + }, + "current_time": "2026-03-12T02:30:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "multi_window_14:00", + "off_peak": { + "hours_utc": [ + "00:30-02:00", + "22:00-23:30" + ] + }, + "current_time": "2026-03-12T14:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "multi_window_12:00", + "off_peak": { + "hours_utc": [ + "00:30-02:00", + "22:00-23:30" + ] + }, + "current_time": "2026-03-14T12:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "multi_window_20:00", + "off_peak": { + "hours_utc": [ + "00:30-02:00", + "22:00-23:30" + ] + }, + "current_time": "2026-03-14T20:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "multi_window_01:00", + "off_peak": { + "hours_utc": [ + "00:30-02:00", + "22:00-23:30" + ] + }, + "current_time": "2026-07-04T01:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekend_only_02:30", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "sat", + "sun" + ] + }, + "current_time": "2026-03-12T02:30:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekend_only_14:00", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "sat", + "sun" + ] + }, + "current_time": "2026-03-12T14:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekend_only_12:00", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "sat", + "sun" + ] + }, + "current_time": "2026-03-14T12:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekend_only_20:00", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "sat", + "sun" + ] + }, + "current_time": "2026-03-14T20:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekend_only_01:00", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "sat", + "sun" + ] + }, + "current_time": "2026-07-04T01:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekday_calendar_est_02:30", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "mon", + "wed" + ], + "weekday_timezone": "America/New_York" + }, + "current_time": "2026-03-12T02:30:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekday_calendar_est_14:00", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "mon", + "wed" + ], + "weekday_timezone": "America/New_York" + }, + "current_time": "2026-03-12T14:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekday_calendar_est_12:00", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "mon", + "wed" + ], + "weekday_timezone": "America/New_York" + }, + "current_time": "2026-03-14T12:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekday_calendar_est_20:00", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "mon", + "wed" + ], + "weekday_timezone": "America/New_York" + }, + "current_time": "2026-03-14T20:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "weekday_calendar_est_01:00", + "off_peak": { + "hours_utc": "00:00-00:00", + "weekdays": [ + "mon", + "wed" + ], + "weekday_timezone": "America/New_York" + }, + "current_time": "2026-07-04T01:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "window_rules_with_weekdays_02:30", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "sat", + "sun" + ] + } + ] + }, + "current_time": "2026-03-12T02:30:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "window_rules_with_weekdays_14:00", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "sat", + "sun" + ] + } + ] + }, + "current_time": "2026-03-12T14:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "window_rules_with_weekdays_12:00", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "sat", + "sun" + ] + } + ] + }, + "current_time": "2026-03-14T12:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "window_rules_with_weekdays_20:00", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "sat", + "sun" + ] + } + ] + }, + "current_time": "2026-03-14T20:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "window_rules_with_weekdays_01:00", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "sat", + "sun" + ] + } + ] + }, + "current_time": "2026-07-04T01:00:00+00:00", + "expected_is_off_peak": true + }, + { + "id": "window_rule_wrong_weekday_02:30", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "mon" + ] + } + ] + }, + "current_time": "2026-03-12T02:30:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "window_rule_wrong_weekday_14:00", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "mon" + ] + } + ] + }, + "current_time": "2026-03-12T14:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "window_rule_wrong_weekday_12:00", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "mon" + ] + } + ] + }, + "current_time": "2026-03-14T12:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "window_rule_wrong_weekday_20:00", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "mon" + ] + } + ] + }, + "current_time": "2026-03-14T20:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "window_rule_wrong_weekday_01:00", + "off_peak": { + "windows": [ + { + "hours_utc": "00:00-06:00", + "weekdays": [ + "mon" + ] + } + ] + }, + "current_time": "2026-07-04T01:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "malformed_window_ignored_02:30", + "off_peak": { + "hours_utc": "22-06" + }, + "current_time": "2026-03-12T02:30:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "malformed_window_ignored_14:00", + "off_peak": { + "hours_utc": "22-06" + }, + "current_time": "2026-03-12T14:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "malformed_window_ignored_12:00", + "off_peak": { + "hours_utc": "22-06" + }, + "current_time": "2026-03-14T12:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "malformed_window_ignored_20:00", + "off_peak": { + "hours_utc": "22-06" + }, + "current_time": "2026-03-14T20:00:00+00:00", + "expected_is_off_peak": false + }, + { + "id": "malformed_window_ignored_01:00", + "off_peak": { + "hours_utc": "22-06" + }, + "current_time": "2026-07-04T01:00:00+00:00", + "expected_is_off_peak": false + } + ], + "responses_usage": [ + { + "id": "cached_details_with_web_search", + "raw": { + "input_tokens": 1200, + "output_tokens": 300, + "input_token_details": { + "cached_tokens": 800, + "web_search_requests": 2 + }, + "output_token_details": { + "reasoning_tokens": 100 + } + }, + "expected": { + "completion_tokens": 300, + "prompt_tokens": 1200, + "total_tokens": 1500, + "completion_tokens_details": { + "reasoning_tokens": 100 + }, + "prompt_tokens_details": { + "cached_tokens": 800, + "web_search_requests": 2 + } + } + }, + { + "id": "realtime_plural_details", + "raw": { + "input_tokens": 900, + "output_tokens": 250, + "input_token_details": { + "text_tokens": 700, + "cached_tokens": 200, + "audio_tokens": 0 + }, + "output_token_details": { + "text_tokens": 150, + "audio_tokens": 100 + } + }, + "expected": { + "completion_tokens": 250, + "prompt_tokens": 900, + "total_tokens": 1150, + "completion_tokens_details": { + "audio_tokens": 100, + "text_tokens": 150 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 200, + "text_tokens": 700 + } + } + }, + { + "id": "missing_totals_derived", + "raw": { + "input_tokens": 410, + "output_tokens": 90 + }, + "expected": { + "completion_tokens": 90, + "prompt_tokens": 410, + "total_tokens": 500 + } + }, + { + "id": "already_chat_shaped_passthrough", + "raw": { + "prompt_tokens": 500, + "completion_tokens": 100, + "total_tokens": 600 + }, + "expected": { + "completion_tokens": 100, + "prompt_tokens": 500, + "total_tokens": 600 + } + }, + { + "id": "nested_cached_details", + "raw": { + "input_tokens": 1000, + "output_tokens": 200, + "input_token_details": { + "text_tokens": 400, + "cached_tokens": 600 + } + }, + "expected": { + "completion_tokens": 200, + "prompt_tokens": 1000, + "total_tokens": 1200, + "prompt_tokens_details": { + "cached_tokens": 600, + "text_tokens": 400 + } + } + } + ], + "caching_savings": [ + { + "id": "anthropic_read_discount_write_premium", + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost": 3.75e-06, + "litellm_provider": "anthropic" + }, + "usage": { + "prompt_tokens": 10000, + "completion_tokens": 500, + "prompt_tokens_details": { + "cached_tokens": 8000, + "cache_write_tokens": 1500 + }, + "completion_tokens_details": null + }, + "custom_llm_provider": "anthropic", + "service_tier": null, + "expected_savings": 0.020475 + }, + { + "id": "openai_cache_savings", + "model_info": { + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 6.25e-08, + "cache_creation_input_token_cost": 1.25e-06, + "litellm_provider": "openai" + }, + "usage": { + "prompt_tokens": 50000, + "completion_tokens": 2000, + "prompt_tokens_details": { + "cached_tokens": 45000, + "cache_write_tokens": 3000 + }, + "completion_tokens_details": null + }, + "custom_llm_provider": "openai", + "service_tier": null, + "expected_savings": 0.053437500000000006 + }, + { + "id": "unpriced_cache_no_savings", + "model_info": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "litellm_provider": "openai" + }, + "usage": { + "prompt_tokens": 10000, + "completion_tokens": 500, + "prompt_tokens_details": { + "cached_tokens": 8000 + }, + "completion_tokens_details": null + }, + "custom_llm_provider": "openai", + "service_tier": null, + "expected_savings": 0.0 + }, + { + "id": "priority_tier_discount", + "model_info": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost": 3.75e-06, + "input_cost_per_token_priority": 6e-06, + "litellm_provider": "anthropic" + }, + "usage": { + "prompt_tokens": 10000, + "completion_tokens": 500, + "prompt_tokens_details": { + "cached_tokens": 8000, + "cache_write_tokens": 1500 + }, + "completion_tokens_details": null + }, + "custom_llm_provider": "anthropic", + "service_tier": "priority", + "expected_savings": 0.048975000000000005 + } + ], + "anthropic_usage": [ + { + "id": "plain_input_output", + "raw": { + "input_tokens": 1000, + "output_tokens": 300 + }, + "reasoning_content": null, + "response_has_thinking_block": false, + "expected": { + "completion_tokens": 300, + "prompt_tokens": 1000, + "total_tokens": 1300, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 300 + }, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 1000, + "cache_write_tokens": 0, + "cache_creation_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + }, + { + "id": "cache_creation_and_read", + "raw": { + "input_tokens": 5000, + "output_tokens": 200, + "cache_creation_input_tokens": 2000, + "cache_creation": { + "ephemeral_5m_input_tokens": 1500, + "ephemeral_1h_input_tokens": 500 + }, + "cache_read_input_tokens": 3000 + }, + "reasoning_content": null, + "response_has_thinking_block": false, + "expected": { + "completion_tokens": 200, + "prompt_tokens": 10000, + "total_tokens": 10200, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 200 + }, + "prompt_tokens_details": { + "cached_tokens": 3000, + "text_tokens": 5000, + "cache_write_tokens": 2000, + "cache_creation_tokens": 2000, + "cache_creation_token_details": { + "ephemeral_5m_input_tokens": 1500, + "ephemeral_1h_input_tokens": 500 + } + }, + "cache_creation_input_tokens": 2000, + "cache_read_input_tokens": 3000 + } + }, + { + "id": "iterations_with_thinking_tokens", + "raw": { + "input_tokens": 600, + "output_tokens": 300, + "iterations": [ + { + "input_tokens": 100, + "output_tokens": 150, + "output_tokens_details": { + "thinking_tokens": 90 + } + }, + { + "input_tokens": 200, + "output_tokens": 150, + "output_tokens_details": { + "thinking_tokens": 60 + } + } + ] + }, + "reasoning_content": null, + "response_has_thinking_block": false, + "expected": { + "completion_tokens": 300, + "prompt_tokens": 300, + "total_tokens": 600, + "completion_tokens_details": { + "reasoning_tokens": 150, + "text_tokens": 150 + }, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 300, + "cache_write_tokens": 0, + "cache_creation_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "iterations": [ + { + "input_tokens": 100, + "output_tokens": 150, + "output_tokens_details": { + "thinking_tokens": 90 + } + }, + { + "input_tokens": 200, + "output_tokens": 150, + "output_tokens_details": { + "thinking_tokens": 60 + } + } + ] + } + }, + { + "id": "iterations_usage", + "raw": { + "input_tokens": 600, + "output_tokens": 250, + "server_tool_use": { + "web_search_requests": 3 + } + }, + "reasoning_content": null, + "response_has_thinking_block": false, + "expected": { + "completion_tokens": 250, + "prompt_tokens": 600, + "total_tokens": 850, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 250 + }, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 600, + "cache_write_tokens": 0, + "cache_creation_tokens": 0 + }, + "server_tool_use": { + "web_search_requests": 3 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + } + ] +}