mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(cost): match Python service-tier suffixes and enforce mirrors markers
get_cost_per_unit now delegates to the wire cost_per_unit policy pinned against executed Python, so _auto-suffixed keys fall back to their base key and _standard no longer does, matching _SERVICE_TIER_SUFFIXES; threshold detection excludes the same suffix set Python derives from its ServiceTier enum plus _batches. The two private tier_key copies in generic_input and generic_output collapse onto base_rate_selection's shared one. scripts/check_cost_test_mirrors.py requires every tests/python_*.rs file to carry a // mirrors: marker naming a real Python test under tests/ (or the fixture generator for replay files), runs in the rust CI job, and all 57 files now carry verified markers
This commit is contained in:
parent
6908fe19e6
commit
cbd6e82a82
64 changed files with 244 additions and 53 deletions
4
.github/workflows/test-rust.yml
vendored
4
.github/workflows/test-rust.yml
vendored
|
|
@ -113,6 +113,10 @@ jobs:
|
|||
working-directory: .
|
||||
run: |
|
||||
uv sync --frozen --no-install-project
|
||||
|
||||
- name: Cost test mirrors markers
|
||||
working-directory: .
|
||||
run: python3 scripts/check_cost_test_mirrors.py
|
||||
echo "PYTHONPATH=$PWD/.venv/lib/$(ls .venv/lib)/site-packages" >> "$GITHUB_ENV"
|
||||
|
||||
- run: rustup toolchain install --no-self-update
|
||||
|
|
|
|||
|
|
@ -7,6 +7,15 @@ use crate::off_peak::{
|
|||
};
|
||||
use crate::tiered_pricing::{select_tier_for_input, tier_rate};
|
||||
|
||||
pub const NON_STANDARD_THRESHOLD_SUFFIXES: [&str; 6] = [
|
||||
"_ultrafast",
|
||||
"_priority",
|
||||
"_auto",
|
||||
"_flex",
|
||||
"_fast",
|
||||
"_batches",
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct TokenBaseRates {
|
||||
pub input: f64,
|
||||
|
|
@ -102,16 +111,9 @@ fn crossed_threshold(
|
|||
.iter()
|
||||
.filter(|(key, value)| {
|
||||
key.starts_with("input_cost_per_token_above_")
|
||||
&& ![
|
||||
"_ultrafast",
|
||||
"_priority",
|
||||
"_standard",
|
||||
"_flex",
|
||||
"_fast",
|
||||
"_batches",
|
||||
]
|
||||
.iter()
|
||||
.any(|suffix| key.ends_with(suffix))
|
||||
&& !NON_STANDARD_THRESHOLD_SUFFIXES
|
||||
.iter()
|
||||
.any(|suffix| key.ends_with(suffix))
|
||||
&& !value.is_null()
|
||||
})
|
||||
.filter_map(|(key, _)| threshold_number(key).map(|threshold| (key.as_str(), threshold)))
|
||||
|
|
|
|||
|
|
@ -1,35 +1,17 @@
|
|||
use serde_json::Value;
|
||||
|
||||
use crate::base_rate_selection::tier_key;
|
||||
use crate::generic_usage::ParsedPromptDetails;
|
||||
use crate::responses_usage::CacheCreationTokenDetails;
|
||||
|
||||
fn number(value: &Value) -> Option<f64> {
|
||||
match value {
|
||||
Value::Number(value) => value.as_f64(),
|
||||
Value::String(value) => value.trim().parse().ok(),
|
||||
Value::Bool(value) => Some(u8::from(*value) as f64),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_cost_per_unit(
|
||||
model_info: &Value,
|
||||
cost_key: &str,
|
||||
default_value: Option<f64>,
|
||||
) -> Option<f64> {
|
||||
let direct = model_info.get(cost_key);
|
||||
if let Some(parsed) = direct.and_then(number) {
|
||||
return Some(parsed);
|
||||
}
|
||||
if direct.is_none_or(Value::is_null) {
|
||||
for suffix in ["_ultrafast", "_priority", "_standard", "_flex", "_fast"] {
|
||||
if cost_key.contains(suffix) {
|
||||
let base = cost_key.replace(suffix, "");
|
||||
return model_info.get(&base).and_then(number).or(default_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
default_value
|
||||
model_info.as_object().map_or(default_value, |fields| {
|
||||
crate::wire::cost_per_unit(fields, cost_key, default_value)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn calculate_cost_component(model_info: &Value, cost_key: &str, usage_value: f64) -> f64 {
|
||||
|
|
@ -63,15 +45,6 @@ pub struct InputBaseRates {
|
|||
pub cache_creation_above_1hr: f64,
|
||||
}
|
||||
|
||||
fn tier_key(base: &str, service_tier: Option<&str>) -> String {
|
||||
match service_tier.map(str::to_ascii_lowercase).as_deref() {
|
||||
Some("flex") => format!("{base}_flex"),
|
||||
Some("priority" | "fast") => format!("{base}_priority"),
|
||||
Some("ultrafast") => format!("{base}_ultrafast"),
|
||||
_ => base.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calculate_input_cost(
|
||||
details: &ParsedPromptDetails,
|
||||
model_info: &Value,
|
||||
|
|
|
|||
|
|
@ -1,18 +1,10 @@
|
|||
use serde_json::Value;
|
||||
|
||||
use crate::base_rate_selection::tier_key;
|
||||
use crate::generic_input::get_cost_per_unit;
|
||||
use crate::generic_usage::parse_completion_tokens_details;
|
||||
use crate::responses_usage::ChatUsage;
|
||||
|
||||
fn tier_key(base: &str, service_tier: Option<&str>) -> String {
|
||||
match service_tier.map(str::to_ascii_lowercase).as_deref() {
|
||||
Some("flex") => format!("{base}_flex"),
|
||||
Some("priority" | "fast") => format!("{base}_priority"),
|
||||
Some("ultrafast") => format!("{base}_ultrafast"),
|
||||
_ => base.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resolve_reasoning_token_cost(
|
||||
model_info: &Value,
|
||||
service_tier: Option<&str>,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: unit/a2a_protocol/test_cost_calculator.py::test_asend_message_uses_input_output_cost_per_token
|
||||
// mirrors: test_litellm/proxy/_experimental/mcp_server/test_mcp_cost_calculator.py::TestMCPCostCalculator::test_calculate_mcp_tool_call_cost_with_tool_specific_cost
|
||||
|
||||
use litellm_cost::a2a_cost::calculate_a2a_cost;
|
||||
use litellm_cost::mcp_cost::calculate_mcp_tool_call_cost;
|
||||
use rstest::rstest;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_anthropic_geo_and_fast_multipliers_compose
|
||||
// mirrors: test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py::test_get_cost_for_anthropic_web_search_with_dict_server_tool_use
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py::test_calculate_usage_aggregates_cache_creation_split_across_iterations
|
||||
|
||||
use litellm_cost::anthropic_usage::{
|
||||
is_anthropic_usage_object, transform_anthropic_usage_to_chat_usage,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py::TestAzureModelRouterFlatCost
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py::TestAzureMAIImageGeneration::test_mai_image_cost_calculator_token_based
|
||||
// mirrors: test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py::test_flux2_flex_cost_prefers_deployment_input_cost_per_pixel
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
// mirrors: test_litellm/interactions/test_background_cost_polling.py::test_poll_intervals_double_up_to_the_cap_and_stay_inside_the_timeout
|
||||
|
||||
use litellm_cost::background_cost_polling::{
|
||||
is_pollable_background_interaction, missing_usage_is_expected, poll_intervals,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_get_token_base_cost_picks_highest_crossed_tier
|
||||
|
||||
use litellm_cost::base_rate_selection::{
|
||||
TokenBaseRates, get_tiered_reasoning_rate, get_token_base_cost_without_off_peak,
|
||||
uses_inclusive_token_thresholds,
|
||||
|
|
@ -97,6 +99,35 @@ fn get_token_base_cost_without_off_peak_uses_tier_specific_threshold_keys() {
|
|||
assert_eq!(rates.cache_read, 6e-6);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn threshold_detection_excludes_auto_and_keeps_plain_thresholds() {
|
||||
let model_info = json!({
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
"input_cost_per_token_above_128k_tokens": 5e-6,
|
||||
"input_cost_per_token_above_128k_tokens_auto": 9e-6,
|
||||
"output_cost_per_token_above_128k_tokens": 7e-6
|
||||
});
|
||||
let rates = get_token_base_cost_without_off_peak(&model_info, 128_001, Some("auto"), false);
|
||||
assert_eq!(
|
||||
rates.input, 5e-6,
|
||||
"_auto-suffixed threshold keys are non-standard; the plain threshold rate applies"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn non_standard_threshold_suffixes_match_python_service_tier_suffixes() {
|
||||
let mut python_suffixes: Vec<String> = litellm_cost::pricing::ServiceTier::SUFFIXES
|
||||
.into_iter()
|
||||
.map(|tier| format!("_{}", tier.as_str()))
|
||||
.collect();
|
||||
python_suffixes.push("_batches".to_string());
|
||||
assert_eq!(
|
||||
litellm_cost::base_rate_selection::NON_STANDARD_THRESHOLD_SUFFIXES.to_vec(),
|
||||
python_suffixes
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn get_token_base_cost_without_off_peak_uses_one_tier_for_all_token_rates() {
|
||||
let model_info = json!({
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: image_gen_tests/test_bedrock_image_gen_unit_tests.py::test_cost_calculator_stability1
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_per_query_priced_rerank_deployment_completion_cost_is_nonzero
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::catalog::{CatalogCallError, CostCall, ModelCostRequest, ModelInfoCatalog};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_cost_discount_not_applied_to_other_providers
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_completion_cost_extracts_service_tier_from_response
|
||||
// mirrors: test_litellm/llms/gemini/test_cost_calculator.py::test_map_traffic_type_to_service_tier
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_cost_calculator_with_response_cost_in_additional_headers
|
||||
// mirrors: local_testing/test_completion_cost.py
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::catalog::ModelInfoCatalog;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_token_type_cost_breakdown_reconciles_with_generic_total
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_cost::billed_token_rates::{BilledTokenRates, TokenTypeCostBreakdown};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_default_image_cost_calculator
|
||||
// mirrors: integration/pricing/test_ocr_page_pricing.py
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::batch::{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_custom_pricing_anthropic_style_cache_tokens_not_double_counted
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::catalog::{CatalogCustomError, CostCall, ModelCostRequest, ModelInfoCatalog};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/dashscope/test_dashscope_cost_calculator.py::TestDashscopeCostCalculator
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/fal_ai/test_cost_calculator.py
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: crates/cost/tests/generate_python_fixtures.py
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/gemini/test_cost_calculator.py
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_cost::gemini_cost::{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_generic_cost_per_token_overlapping_cached_and_image_tokens
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::generic_cost::{
|
||||
ResolvedTokenRates, billable_prompt_details, calculate_generic_cost_from_model_info,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_calculate_cache_writing_cost
|
||||
|
||||
use litellm_cost::generic_input::{
|
||||
InputBaseRates, calculate_cache_writing_cost, calculate_cost_component, calculate_input_cost,
|
||||
get_cost_per_unit,
|
||||
|
|
@ -24,6 +26,8 @@ fn rates() -> InputBaseRates {
|
|||
#[case(json!({"input_cost_per_audio_token": 0.0}), "input_cost_per_audio_token", Some(1e-6), Some(0.0))]
|
||||
#[case(json!({"input_cost_per_audio_token": "3e-6"}), "input_cost_per_audio_token_priority", None, Some(3e-6))]
|
||||
#[case(json!({"input_cost_per_audio_token": "bad"}), "input_cost_per_audio_token", Some(1e-6), Some(1e-6))]
|
||||
#[case(json!({"input_cost_per_audio_token": 3e-6}), "input_cost_per_audio_token_auto", None, Some(3e-6))]
|
||||
#[case(json!({"input_cost_per_audio_token": 3e-6}), "input_cost_per_audio_token_standard", None, None)]
|
||||
fn get_cost_per_unit_parses_string_rates_and_falls_back_from_tier(
|
||||
#[case] model_info: serde_json::Value,
|
||||
#[case] key: &str,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output
|
||||
|
||||
use litellm_cost::generic_output::{calculate_output_cost, resolve_reasoning_token_cost};
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use rstest::rstest;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: local_testing/test_completion_cost.py::test_gemini_completion_cost
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_missing_cache_read_rate_resolves_to_input_rate
|
||||
|
||||
use litellm_cost::generic_usage::{
|
||||
get_billable_input_tokens, parse_completion_tokens_details, parse_prompt_tokens_details,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: unit/llms/groq/test_groq_cost_calculator.py::test_bills_searches_and_opens_together
|
||||
|
||||
use litellm_cost::groq_cost::cost_per_web_search_request;
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use rstest::rstest;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py::test_bedrock_guardrail_cost_prices_each_counter
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_cost::guardrail_cost::{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_route_image_generation_cost_honors_deployment_model_info
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/gemini/test_cost_calculator.py
|
||||
// mirrors: test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py::test_vertex_image_generation_cost_adds_web_search_grounding
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py::test_detects_interactions_usage_object
|
||||
|
||||
use litellm_cost::interactions_usage::{
|
||||
is_interactions_usage_object, transform_interactions_usage_object,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_duplicate_openai_prefix_matches_model_cost
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_select_model_name_keeps_base_model_free_of_region
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::catalog::ModelInfoCatalog;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_is_within_off_peak_window_same_day
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::base_rate_selection::get_token_base_cost;
|
||||
use litellm_cost::off_peak::{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_video_generation.py::TestVideoGeneration
|
||||
// mirrors: unit/llms/openai/transcriptions/test_transcription_duration_hidden.py::TestCostCalculatorReadsDurationFromHiddenParams
|
||||
|
||||
use litellm_cost::non_token::Error;
|
||||
use litellm_cost::openai_cost::{
|
||||
cost_router, video_generation_cost, video_output_cost_per_second,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_gpt_image_cost_calculator.py
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_per_second_pricing
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/perplexity/test_perplexity_cost_calculator.py::TestPerplexityCostCalculator
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
// mirrors: crates/cost/tests/generate_python_fixtures.py (pinning + model_info_projection surfaces)
|
||||
|
||||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: crates/cost/tests/generate_python_fixtures.py
|
||||
// (pinning + model_info_projection surfaces)
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use litellm_cost::pricing::{Metric, ServiceTier, tokenize};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py::test_fireworks_cache_read_rates_match_breakdown_and_caching_savings
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py::test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/openai/test_cost_calculation.py::test_shipped_per_second_models_bill_a_non_zero_cost
|
||||
// mirrors: test_litellm/llms/databricks/test_databricks_cost_calculator.py
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/test_ptu_pricing.py::test_a_complete_reservation_is_accepted
|
||||
|
||||
use litellm_cost::ptu_pricing::{
|
||||
MAX_COST_PER_PTU_PER_HOUR, MAX_PTU_COUNT, PTU_ZEROED_PRICING_FIELDS, SEARCH_CONTEXT_SIZES,
|
||||
azure_spillover, declares_ptu, is_spilled_over_ptu_request, ptu_config_error,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_realtime_stream_combines_text_and_audio_token_details
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_data_residency_applies_uplift
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
||||
use litellm_cost::regional_uplift::{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/responses/test_responses_utils.py::TestResponseAPILoggingUtils
|
||||
|
||||
use litellm_cost::responses_usage::{
|
||||
UsageError, is_response_api_usage, text_tokens_without_nested_reasoning,
|
||||
transform_response_api_usage_to_chat_usage,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/rag/test_main.py::test_aquery_billed_cost_includes_priced_vector_store_search
|
||||
|
||||
use litellm_cost::retrieval_cost::{rerank_cost, vector_store_search_cost};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py::TestAgentCoreSearchEdgeCases::test_search_cost_lookup_is_mapped
|
||||
// mirrors: unit/llms/parallel_ai/test_parallel_ai_search.py::test_explicit_mode_wins_over_processor
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::catalog::ModelInfoCatalog;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_generic_cost_per_token_tiered_pricing_is_all_or_nothing
|
||||
|
||||
use litellm_cost::tiered_pricing::{select_tier_for_input, tier_rate};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: local_testing/test_completion_cost.py::test_together_ai_qwen_completion_cost
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py::test_web_search_cost_low
|
||||
|
||||
use litellm_cost::tool_call_cost_tracking::{
|
||||
DefaultToolRates, ResponseKind, chat_completion_response_includes_annotations,
|
||||
count_web_search_calls, extract_file_search_params, extract_token_counts,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py::test_get_cost_for_built_in_tools_web_search
|
||||
|
||||
use litellm_cost::tool_call_cost_tracking::{DefaultToolRates, ResponseKind};
|
||||
use litellm_cost::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py
|
||||
|
||||
use litellm_cost::transcription_usage::{
|
||||
is_transcription_usage_object, transform_transcription_usage_object,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py
|
||||
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_vertex_uplift_composes_with_above_128k_pricing
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: crates/cost/tests/generate_python_fixtures.py (wire_usage + wire_response surfaces)
|
||||
// mirrors: crates/cost/tests/generate_python_fixtures.py
|
||||
// (wire_usage + wire_response surfaces)
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/llms/xai/test_xai_cost_calculator.py::TestXAICostCalculator
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py::test_missing_pricing_key_names_every_rate_the_usage_needs
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::catalog::ModelInfoCatalog;
|
||||
|
|
|
|||
63
scripts/check_cost_test_mirrors.py
Normal file
63
scripts/check_cost_test_mirrors.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Requires every tests/python_*.rs file in the cost crate to name the Python
|
||||
test it mirrors with a `// mirrors: <target>` line in its header.
|
||||
|
||||
The target names a real file under litellm/tests/ (or the fixture generator
|
||||
when the file pins executed-Python output rather than a specific test), and
|
||||
may qualify a test with `::` segments. Exit 1 with one line per offender.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
COST_TESTS = REPO_ROOT / "litellm-rust" / "crates" / "cost" / "tests"
|
||||
PYTHON_TESTS = REPO_ROOT / "tests"
|
||||
GENERATOR = "crates/cost/tests/generate_python_fixtures.py"
|
||||
MIRROR_RE = re.compile(r"^//\s*mirrors:\s*(\S.*)$", re.MULTILINE)
|
||||
TARGET_RE = re.compile(r"^([\w./-]+\.py)(?:::[\w.-]+)*$")
|
||||
|
||||
|
||||
def header(source: str) -> str:
|
||||
lines = source.splitlines()
|
||||
body_start = next(
|
||||
(index for index, line in enumerate(lines) if line.strip().startswith(("mod ", "use ", "fn ", "#["))),
|
||||
len(lines),
|
||||
)
|
||||
return "\n".join(lines[:body_start])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
rust_files = sorted(COST_TESTS.glob("python_*.rs"))
|
||||
if not rust_files:
|
||||
print("no tests/python_*.rs files found; checker is misconfigured", file=sys.stderr)
|
||||
return 1
|
||||
offenders: list[str] = []
|
||||
for rust_file in rust_files:
|
||||
source = rust_file.read_text()
|
||||
markers = MIRROR_RE.findall(header(source))
|
||||
if not markers:
|
||||
offenders.append(f"{rust_file.name}: missing `// mirrors:` marker")
|
||||
continue
|
||||
for target in markers:
|
||||
match = TARGET_RE.match(target)
|
||||
if not match:
|
||||
offenders.append(f"{rust_file.name}: malformed mirrors target {target!r}")
|
||||
continue
|
||||
path = match.group(1)
|
||||
if path == GENERATOR:
|
||||
continue
|
||||
if not (PYTHON_TESTS / path.removeprefix("litellm/tests/")).is_file():
|
||||
offenders.append(
|
||||
f"{rust_file.name}: mirrors target {path!r} is not a file under litellm/tests/"
|
||||
)
|
||||
for line in offenders:
|
||||
print(line)
|
||||
return 1 if offenders else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Reference in a new issue