From cbd6e82a8226bcd10c28405290ebda5cf7779bbc Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 23 Sep 2026 16:39:31 -0700 Subject: [PATCH] fix(cost): match Python service-tier suffixes and enforce mirrors markers get_cost_per_unit now delegates to the wire cost_per_unit policy pinned against executed Python, so _auto-suffixed keys fall back to their base key and _standard no longer does, matching _SERVICE_TIER_SUFFIXES; threshold detection excludes the same suffix set Python derives from its ServiceTier enum plus _batches. The two private tier_key copies in generic_input and generic_output collapse onto base_rate_selection's shared one. scripts/check_cost_test_mirrors.py requires every tests/python_*.rs file to carry a // mirrors: marker naming a real Python test under tests/ (or the fixture generator for replay files), runs in the rust CI job, and all 57 files now carry verified markers --- .github/workflows/test-rust.yml | 4 ++ .../crates/cost/src/base_rate_selection.rs | 22 ++++--- litellm-rust/crates/cost/src/generic_input.rs | 35 ++--------- .../crates/cost/src/generic_output.rs | 10 +-- .../crates/cost/tests/python_a2a_mcp_cost.rs | 3 + .../cost/tests/python_anthropic_cost.rs | 3 + .../cost/tests/python_anthropic_usage.rs | 2 + .../crates/cost/tests/python_azure_ai_cost.rs | 2 + .../cost/tests/python_azure_ai_image_cost.rs | 3 + .../tests/python_background_cost_polling.rs | 2 + .../cost/tests/python_base_rate_selection.rs | 31 +++++++++ .../cost/tests/python_bedrock_image_cost.rs | 2 + .../cost/tests/python_billed_token_rates.rs | 2 + .../crates/cost/tests/python_call_dispatch.rs | 2 + .../cost/tests/python_completion_cost.rs | 2 + .../cost/tests/python_completion_input.rs | 3 + .../cost/tests/python_completion_response.rs | 3 + .../cost/tests/python_cost_breakdown.rs | 2 + .../cost/tests/python_cost_calculator.rs | 3 + .../cost/tests/python_custom_pricing.rs | 2 + .../cost/tests/python_dashscope_cost.rs | 2 + .../cost/tests/python_fal_ai_image_cost.rs | 2 + .../cost/tests/python_fireworks_cost.rs | 2 + .../cost/tests/python_fixture_replay.rs | 2 + .../crates/cost/tests/python_gemini_cost.rs | 2 + .../crates/cost/tests/python_generic_cost.rs | 2 + .../crates/cost/tests/python_generic_input.rs | 4 ++ .../cost/tests/python_generic_output.rs | 2 + .../tests/python_generic_provider_routes.rs | 2 + .../crates/cost/tests/python_generic_usage.rs | 2 + .../crates/cost/tests/python_groq_cost.rs | 2 + .../cost/tests/python_guardrail_cost.rs | 2 + .../cost/tests/python_image_cost_router.rs | 2 + .../cost/tests/python_image_response_cost.rs | 3 + .../cost/tests/python_interactions_usage.rs | 2 + .../crates/cost/tests/python_model_lookup.rs | 2 + .../cost/tests/python_model_selection.rs | 2 + .../crates/cost/tests/python_off_peak.rs | 2 + .../crates/cost/tests/python_openai_cost.rs | 3 + .../cost/tests/python_openai_image_cost.rs | 2 + .../crates/cost/tests/python_per_second.rs | 2 + .../cost/tests/python_perplexity_cost.rs | 2 + .../cost/tests/python_pricing_pinning.rs | 5 +- .../tests/python_prompt_caching_savings.rs | 2 + .../cost/tests/python_provider_cache.rs | 2 + .../cost/tests/python_provider_routes.rs | 3 + .../crates/cost/tests/python_ptu_pricing.rs | 2 + .../crates/cost/tests/python_realtime_cost.rs | 2 + .../cost/tests/python_regional_uplift.rs | 2 + .../cost/tests/python_responses_usage.rs | 2 + .../cost/tests/python_retrieval_cost.rs | 2 + .../crates/cost/tests/python_search_cost.rs | 3 + .../crates/cost/tests/python_speech_cost.rs | 2 + .../cost/tests/python_tiered_pricing.rs | 2 + .../crates/cost/tests/python_together_cost.rs | 2 + .../tests/python_tool_call_cost_tracking.rs | 2 + .../cost/tests/python_tool_cost_dispatch.rs | 2 + .../cost/tests/python_transcription_usage.rs | 2 + .../cost/tests/python_usage_dispatch.rs | 2 + .../crates/cost/tests/python_vertex_cost.rs | 2 + .../crates/cost/tests/python_wire_contract.rs | 3 +- .../crates/cost/tests/python_xai_cost.rs | 2 + .../cost/tests/python_zero_cost_diagnostic.rs | 2 + scripts/check_cost_test_mirrors.py | 63 +++++++++++++++++++ 64 files changed, 244 insertions(+), 53 deletions(-) create mode 100644 scripts/check_cost_test_mirrors.py diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 64e16b512e1..5d80e420e54 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -113,6 +113,10 @@ jobs: working-directory: . run: | uv sync --frozen --no-install-project + + - name: Cost test mirrors markers + working-directory: . + run: python3 scripts/check_cost_test_mirrors.py echo "PYTHONPATH=$PWD/.venv/lib/$(ls .venv/lib)/site-packages" >> "$GITHUB_ENV" - run: rustup toolchain install --no-self-update diff --git a/litellm-rust/crates/cost/src/base_rate_selection.rs b/litellm-rust/crates/cost/src/base_rate_selection.rs index df024fabf0f..53f6be66b29 100644 --- a/litellm-rust/crates/cost/src/base_rate_selection.rs +++ b/litellm-rust/crates/cost/src/base_rate_selection.rs @@ -7,6 +7,15 @@ use crate::off_peak::{ }; use crate::tiered_pricing::{select_tier_for_input, tier_rate}; +pub const NON_STANDARD_THRESHOLD_SUFFIXES: [&str; 6] = [ + "_ultrafast", + "_priority", + "_auto", + "_flex", + "_fast", + "_batches", +]; + #[derive(Clone, Copy, Debug, PartialEq)] pub struct TokenBaseRates { pub input: f64, @@ -102,16 +111,9 @@ fn crossed_threshold( .iter() .filter(|(key, value)| { key.starts_with("input_cost_per_token_above_") - && ![ - "_ultrafast", - "_priority", - "_standard", - "_flex", - "_fast", - "_batches", - ] - .iter() - .any(|suffix| key.ends_with(suffix)) + && !NON_STANDARD_THRESHOLD_SUFFIXES + .iter() + .any(|suffix| key.ends_with(suffix)) && !value.is_null() }) .filter_map(|(key, _)| threshold_number(key).map(|threshold| (key.as_str(), threshold))) diff --git a/litellm-rust/crates/cost/src/generic_input.rs b/litellm-rust/crates/cost/src/generic_input.rs index b5aeae854b4..09878ec5023 100644 --- a/litellm-rust/crates/cost/src/generic_input.rs +++ b/litellm-rust/crates/cost/src/generic_input.rs @@ -1,35 +1,17 @@ use serde_json::Value; +use crate::base_rate_selection::tier_key; use crate::generic_usage::ParsedPromptDetails; use crate::responses_usage::CacheCreationTokenDetails; -fn number(value: &Value) -> Option { - match value { - Value::Number(value) => value.as_f64(), - Value::String(value) => value.trim().parse().ok(), - Value::Bool(value) => Some(u8::from(*value) as f64), - _ => None, - } -} - pub fn get_cost_per_unit( model_info: &Value, cost_key: &str, default_value: Option, ) -> Option { - let direct = model_info.get(cost_key); - if let Some(parsed) = direct.and_then(number) { - return Some(parsed); - } - if direct.is_none_or(Value::is_null) { - for suffix in ["_ultrafast", "_priority", "_standard", "_flex", "_fast"] { - if cost_key.contains(suffix) { - let base = cost_key.replace(suffix, ""); - return model_info.get(&base).and_then(number).or(default_value); - } - } - } - default_value + model_info.as_object().map_or(default_value, |fields| { + crate::wire::cost_per_unit(fields, cost_key, default_value) + }) } pub fn calculate_cost_component(model_info: &Value, cost_key: &str, usage_value: f64) -> f64 { @@ -63,15 +45,6 @@ pub struct InputBaseRates { pub cache_creation_above_1hr: f64, } -fn tier_key(base: &str, service_tier: Option<&str>) -> String { - match service_tier.map(str::to_ascii_lowercase).as_deref() { - Some("flex") => format!("{base}_flex"), - Some("priority" | "fast") => format!("{base}_priority"), - Some("ultrafast") => format!("{base}_ultrafast"), - _ => base.to_owned(), - } -} - pub fn calculate_input_cost( details: &ParsedPromptDetails, model_info: &Value, diff --git a/litellm-rust/crates/cost/src/generic_output.rs b/litellm-rust/crates/cost/src/generic_output.rs index f04e743900a..b93c7b5fb57 100644 --- a/litellm-rust/crates/cost/src/generic_output.rs +++ b/litellm-rust/crates/cost/src/generic_output.rs @@ -1,18 +1,10 @@ use serde_json::Value; +use crate::base_rate_selection::tier_key; use crate::generic_input::get_cost_per_unit; use crate::generic_usage::parse_completion_tokens_details; use crate::responses_usage::ChatUsage; -fn tier_key(base: &str, service_tier: Option<&str>) -> String { - match service_tier.map(str::to_ascii_lowercase).as_deref() { - Some("flex") => format!("{base}_flex"), - Some("priority" | "fast") => format!("{base}_priority"), - Some("ultrafast") => format!("{base}_ultrafast"), - _ => base.to_owned(), - } -} - pub fn resolve_reasoning_token_cost( model_info: &Value, service_tier: Option<&str>, diff --git a/litellm-rust/crates/cost/tests/python_a2a_mcp_cost.rs b/litellm-rust/crates/cost/tests/python_a2a_mcp_cost.rs index 6113529422a..2380ae65a85 100644 --- a/litellm-rust/crates/cost/tests/python_a2a_mcp_cost.rs +++ b/litellm-rust/crates/cost/tests/python_a2a_mcp_cost.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: unit/a2a_protocol/test_cost_calculator.py::test_asend_message_uses_input_output_cost_per_token +// mirrors: test_litellm/proxy/_experimental/mcp_server/test_mcp_cost_calculator.py::TestMCPCostCalculator::test_calculate_mcp_tool_call_cost_with_tool_specific_cost + use litellm_cost::a2a_cost::calculate_a2a_cost; use litellm_cost::mcp_cost::calculate_mcp_tool_call_cost; use rstest::rstest; diff --git a/litellm-rust/crates/cost/tests/python_anthropic_cost.rs b/litellm-rust/crates/cost/tests/python_anthropic_cost.rs index f6199e8163d..23a7eebb81f 100644 --- a/litellm-rust/crates/cost/tests/python_anthropic_cost.rs +++ b/litellm-rust/crates/cost/tests/python_anthropic_cost.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_anthropic_geo_and_fast_multipliers_compose +// mirrors: test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py::test_get_cost_for_anthropic_web_search_with_dict_server_tool_use + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_anthropic_usage.rs b/litellm-rust/crates/cost/tests/python_anthropic_usage.rs index 0011fe4c65b..ec8ba029093 100644 --- a/litellm-rust/crates/cost/tests/python_anthropic_usage.rs +++ b/litellm-rust/crates/cost/tests/python_anthropic_usage.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py::test_calculate_usage_aggregates_cache_creation_split_across_iterations + use litellm_cost::anthropic_usage::{ is_anthropic_usage_object, transform_anthropic_usage_to_chat_usage, }; diff --git a/litellm-rust/crates/cost/tests/python_azure_ai_cost.rs b/litellm-rust/crates/cost/tests/python_azure_ai_cost.rs index bd015804c31..f1f40eef506 100644 --- a/litellm-rust/crates/cost/tests/python_azure_ai_cost.rs +++ b/litellm-rust/crates/cost/tests/python_azure_ai_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py::TestAzureModelRouterFlatCost + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_azure_ai_image_cost.rs b/litellm-rust/crates/cost/tests/python_azure_ai_image_cost.rs index d85714eaae7..d4e5bd860d6 100644 --- a/litellm-rust/crates/cost/tests/python_azure_ai_image_cost.rs +++ b/litellm-rust/crates/cost/tests/python_azure_ai_image_cost.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py::TestAzureMAIImageGeneration::test_mai_image_cost_calculator_token_based +// mirrors: test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py::test_flux2_flex_cost_prefers_deployment_input_cost_per_pixel + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_background_cost_polling.rs b/litellm-rust/crates/cost/tests/python_background_cost_polling.rs index c2aa4f06a0d..827ea7033b5 100644 --- a/litellm-rust/crates/cost/tests/python_background_cost_polling.rs +++ b/litellm-rust/crates/cost/tests/python_background_cost_polling.rs @@ -1,3 +1,5 @@ +// mirrors: test_litellm/interactions/test_background_cost_polling.py::test_poll_intervals_double_up_to_the_cap_and_stay_inside_the_timeout + use litellm_cost::background_cost_polling::{ is_pollable_background_interaction, missing_usage_is_expected, poll_intervals, }; diff --git a/litellm-rust/crates/cost/tests/python_base_rate_selection.rs b/litellm-rust/crates/cost/tests/python_base_rate_selection.rs index adc8695d038..07ea046de87 100644 --- a/litellm-rust/crates/cost/tests/python_base_rate_selection.rs +++ b/litellm-rust/crates/cost/tests/python_base_rate_selection.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_get_token_base_cost_picks_highest_crossed_tier + use litellm_cost::base_rate_selection::{ TokenBaseRates, get_tiered_reasoning_rate, get_token_base_cost_without_off_peak, uses_inclusive_token_thresholds, @@ -97,6 +99,35 @@ fn get_token_base_cost_without_off_peak_uses_tier_specific_threshold_keys() { assert_eq!(rates.cache_read, 6e-6); } +#[rstest] +fn threshold_detection_excludes_auto_and_keeps_plain_thresholds() { + let model_info = json!({ + "input_cost_per_token": 2e-6, + "output_cost_per_token": 4e-6, + "input_cost_per_token_above_128k_tokens": 5e-6, + "input_cost_per_token_above_128k_tokens_auto": 9e-6, + "output_cost_per_token_above_128k_tokens": 7e-6 + }); + let rates = get_token_base_cost_without_off_peak(&model_info, 128_001, Some("auto"), false); + assert_eq!( + rates.input, 5e-6, + "_auto-suffixed threshold keys are non-standard; the plain threshold rate applies" + ); +} + +#[rstest] +fn non_standard_threshold_suffixes_match_python_service_tier_suffixes() { + let mut python_suffixes: Vec = litellm_cost::pricing::ServiceTier::SUFFIXES + .into_iter() + .map(|tier| format!("_{}", tier.as_str())) + .collect(); + python_suffixes.push("_batches".to_string()); + assert_eq!( + litellm_cost::base_rate_selection::NON_STANDARD_THRESHOLD_SUFFIXES.to_vec(), + python_suffixes + ); +} + #[rstest] fn get_token_base_cost_without_off_peak_uses_one_tier_for_all_token_rates() { let model_info = json!({ diff --git a/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs b/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs index a0e5d2ba872..341fd3bcf7d 100644 --- a/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs +++ b/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: image_gen_tests/test_bedrock_image_gen_unit_tests.py::test_cost_calculator_stability1 + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_billed_token_rates.rs b/litellm-rust/crates/cost/tests/python_billed_token_rates.rs index 9a4259cbc55..55c28eda516 100644 --- a/litellm-rust/crates/cost/tests/python_billed_token_rates.rs +++ b/litellm-rust/crates/cost/tests/python_billed_token_rates.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_call_dispatch.rs b/litellm-rust/crates/cost/tests/python_call_dispatch.rs index e010583ff95..0ca011add31 100644 --- a/litellm-rust/crates/cost/tests/python_call_dispatch.rs +++ b/litellm-rust/crates/cost/tests/python_call_dispatch.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_per_query_priced_rerank_deployment_completion_cost_is_nonzero + use std::collections::HashMap; use litellm_cost::catalog::{CatalogCallError, CostCall, ModelCostRequest, ModelInfoCatalog}; diff --git a/litellm-rust/crates/cost/tests/python_completion_cost.rs b/litellm-rust/crates/cost/tests/python_completion_cost.rs index d207999ed43..b77e455ff70 100644 --- a/litellm-rust/crates/cost/tests/python_completion_cost.rs +++ b/litellm-rust/crates/cost/tests/python_completion_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_cost_discount_not_applied_to_other_providers + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_completion_input.rs b/litellm-rust/crates/cost/tests/python_completion_input.rs index 202b00ebddb..99457cee434 100644 --- a/litellm-rust/crates/cost/tests/python_completion_input.rs +++ b/litellm-rust/crates/cost/tests/python_completion_input.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_completion_cost_extracts_service_tier_from_response +// mirrors: test_litellm/llms/gemini/test_cost_calculator.py::test_map_traffic_type_to_service_tier + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_completion_response.rs b/litellm-rust/crates/cost/tests/python_completion_response.rs index aa0ad9a388c..d6f7ed5842e 100644 --- a/litellm-rust/crates/cost/tests/python_completion_response.rs +++ b/litellm-rust/crates/cost/tests/python_completion_response.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_cost_calculator_with_response_cost_in_additional_headers +// mirrors: local_testing/test_completion_cost.py + use std::collections::HashMap; use litellm_cost::catalog::ModelInfoCatalog; diff --git a/litellm-rust/crates/cost/tests/python_cost_breakdown.rs b/litellm-rust/crates/cost/tests/python_cost_breakdown.rs index c7c12a76b1c..649b9027d47 100644 --- a/litellm-rust/crates/cost/tests/python_cost_breakdown.rs +++ b/litellm-rust/crates/cost/tests/python_cost_breakdown.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_token_type_cost_breakdown_reconciles_with_generic_total + use std::collections::BTreeMap; use litellm_cost::billed_token_rates::{BilledTokenRates, TokenTypeCostBreakdown}; diff --git a/litellm-rust/crates/cost/tests/python_cost_calculator.rs b/litellm-rust/crates/cost/tests/python_cost_calculator.rs index fe65c60844a..eac63c27a7a 100644 --- a/litellm-rust/crates/cost/tests/python_cost_calculator.rs +++ b/litellm-rust/crates/cost/tests/python_cost_calculator.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_default_image_cost_calculator +// mirrors: integration/pricing/test_ocr_page_pricing.py + use std::collections::HashMap; use litellm_cost::batch::{ diff --git a/litellm-rust/crates/cost/tests/python_custom_pricing.rs b/litellm-rust/crates/cost/tests/python_custom_pricing.rs index 4b6db67b941..f1884ed645f 100644 --- a/litellm-rust/crates/cost/tests/python_custom_pricing.rs +++ b/litellm-rust/crates/cost/tests/python_custom_pricing.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_custom_pricing_anthropic_style_cache_tokens_not_double_counted + use std::collections::HashMap; use litellm_cost::catalog::{CatalogCustomError, CostCall, ModelCostRequest, ModelInfoCatalog}; diff --git a/litellm-rust/crates/cost/tests/python_dashscope_cost.rs b/litellm-rust/crates/cost/tests/python_dashscope_cost.rs index df3e3182f92..1b8ccf9e3ee 100644 --- a/litellm-rust/crates/cost/tests/python_dashscope_cost.rs +++ b/litellm-rust/crates/cost/tests/python_dashscope_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/dashscope/test_dashscope_cost_calculator.py::TestDashscopeCostCalculator + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs b/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs index 23fcdb9e3c2..694be457ff0 100644 --- a/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs +++ b/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/fal_ai/test_cost_calculator.py + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_fireworks_cost.rs b/litellm-rust/crates/cost/tests/python_fireworks_cost.rs index be911604b41..8002727419b 100644 --- a/litellm-rust/crates/cost/tests/python_fireworks_cost.rs +++ b/litellm-rust/crates/cost/tests/python_fireworks_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_fixture_replay.rs b/litellm-rust/crates/cost/tests/python_fixture_replay.rs index 562c4088c9e..2a797f22bf4 100644 --- a/litellm-rust/crates/cost/tests/python_fixture_replay.rs +++ b/litellm-rust/crates/cost/tests/python_fixture_replay.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: crates/cost/tests/generate_python_fixtures.py + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_gemini_cost.rs b/litellm-rust/crates/cost/tests/python_gemini_cost.rs index ad25eb2b99c..a817d004acd 100644 --- a/litellm-rust/crates/cost/tests/python_gemini_cost.rs +++ b/litellm-rust/crates/cost/tests/python_gemini_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/gemini/test_cost_calculator.py + use std::collections::BTreeMap; use litellm_cost::gemini_cost::{ diff --git a/litellm-rust/crates/cost/tests/python_generic_cost.rs b/litellm-rust/crates/cost/tests/python_generic_cost.rs index 2541b853a2f..eead3224345 100644 --- a/litellm-rust/crates/cost/tests/python_generic_cost.rs +++ b/litellm-rust/crates/cost/tests/python_generic_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_generic_cost_per_token_overlapping_cached_and_image_tokens + use jiff::Timestamp; use litellm_cost::generic_cost::{ ResolvedTokenRates, billable_prompt_details, calculate_generic_cost_from_model_info, diff --git a/litellm-rust/crates/cost/tests/python_generic_input.rs b/litellm-rust/crates/cost/tests/python_generic_input.rs index f55430be6dd..3a612b4596c 100644 --- a/litellm-rust/crates/cost/tests/python_generic_input.rs +++ b/litellm-rust/crates/cost/tests/python_generic_input.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_calculate_cache_writing_cost + use litellm_cost::generic_input::{ InputBaseRates, calculate_cache_writing_cost, calculate_cost_component, calculate_input_cost, get_cost_per_unit, @@ -24,6 +26,8 @@ fn rates() -> InputBaseRates { #[case(json!({"input_cost_per_audio_token": 0.0}), "input_cost_per_audio_token", Some(1e-6), Some(0.0))] #[case(json!({"input_cost_per_audio_token": "3e-6"}), "input_cost_per_audio_token_priority", None, Some(3e-6))] #[case(json!({"input_cost_per_audio_token": "bad"}), "input_cost_per_audio_token", Some(1e-6), Some(1e-6))] +#[case(json!({"input_cost_per_audio_token": 3e-6}), "input_cost_per_audio_token_auto", None, Some(3e-6))] +#[case(json!({"input_cost_per_audio_token": 3e-6}), "input_cost_per_audio_token_standard", None, None)] fn get_cost_per_unit_parses_string_rates_and_falls_back_from_tier( #[case] model_info: serde_json::Value, #[case] key: &str, diff --git a/litellm-rust/crates/cost/tests/python_generic_output.rs b/litellm-rust/crates/cost/tests/python_generic_output.rs index 1b9562deb73..dc2be8beb2e 100644 --- a/litellm-rust/crates/cost/tests/python_generic_output.rs +++ b/litellm-rust/crates/cost/tests/python_generic_output.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output + use litellm_cost::generic_output::{calculate_output_cost, resolve_reasoning_token_cost}; use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; diff --git a/litellm-rust/crates/cost/tests/python_generic_provider_routes.rs b/litellm-rust/crates/cost/tests/python_generic_provider_routes.rs index e83996f30ec..a1b6ebf5486 100644 --- a/litellm-rust/crates/cost/tests/python_generic_provider_routes.rs +++ b/litellm-rust/crates/cost/tests/python_generic_provider_routes.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: local_testing/test_completion_cost.py::test_gemini_completion_cost + use std::collections::HashMap; use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; diff --git a/litellm-rust/crates/cost/tests/python_generic_usage.rs b/litellm-rust/crates/cost/tests/python_generic_usage.rs index 055a74b217d..b285278d432 100644 --- a/litellm-rust/crates/cost/tests/python_generic_usage.rs +++ b/litellm-rust/crates/cost/tests/python_generic_usage.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_missing_cache_read_rate_resolves_to_input_rate + use litellm_cost::generic_usage::{ get_billable_input_tokens, parse_completion_tokens_details, parse_prompt_tokens_details, }; diff --git a/litellm-rust/crates/cost/tests/python_groq_cost.rs b/litellm-rust/crates/cost/tests/python_groq_cost.rs index bd5ffdc66aa..fbe3a272604 100644 --- a/litellm-rust/crates/cost/tests/python_groq_cost.rs +++ b/litellm-rust/crates/cost/tests/python_groq_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: unit/llms/groq/test_groq_cost_calculator.py::test_bills_searches_and_opens_together + use litellm_cost::groq_cost::cost_per_web_search_request; use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; diff --git a/litellm-rust/crates/cost/tests/python_guardrail_cost.rs b/litellm-rust/crates/cost/tests/python_guardrail_cost.rs index e3fb36c4453..be761617839 100644 --- a/litellm-rust/crates/cost/tests/python_guardrail_cost.rs +++ b/litellm-rust/crates/cost/tests/python_guardrail_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py::test_bedrock_guardrail_cost_prices_each_counter + use std::collections::BTreeMap; use litellm_cost::guardrail_cost::{ diff --git a/litellm-rust/crates/cost/tests/python_image_cost_router.rs b/litellm-rust/crates/cost/tests/python_image_cost_router.rs index 65895c294fc..28d51c990f3 100644 --- a/litellm-rust/crates/cost/tests/python_image_cost_router.rs +++ b/litellm-rust/crates/cost/tests/python_image_cost_router.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_route_image_generation_cost_honors_deployment_model_info + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_image_response_cost.rs b/litellm-rust/crates/cost/tests/python_image_response_cost.rs index 008f919e72c..8366d6c700d 100644 --- a/litellm-rust/crates/cost/tests/python_image_response_cost.rs +++ b/litellm-rust/crates/cost/tests/python_image_response_cost.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/gemini/test_cost_calculator.py +// mirrors: test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py::test_vertex_image_generation_cost_adds_web_search_grounding + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_interactions_usage.rs b/litellm-rust/crates/cost/tests/python_interactions_usage.rs index d1113504f49..3bfb1c94ecc 100644 --- a/litellm-rust/crates/cost/tests/python_interactions_usage.rs +++ b/litellm-rust/crates/cost/tests/python_interactions_usage.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py::test_detects_interactions_usage_object + use litellm_cost::interactions_usage::{ is_interactions_usage_object, transform_interactions_usage_object, }; diff --git a/litellm-rust/crates/cost/tests/python_model_lookup.rs b/litellm-rust/crates/cost/tests/python_model_lookup.rs index 4c257bc4526..1fd389cdcb7 100644 --- a/litellm-rust/crates/cost/tests/python_model_lookup.rs +++ b/litellm-rust/crates/cost/tests/python_model_lookup.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_duplicate_openai_prefix_matches_model_cost + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_model_selection.rs b/litellm-rust/crates/cost/tests/python_model_selection.rs index 8a7ebb976e6..a1bc8334972 100644 --- a/litellm-rust/crates/cost/tests/python_model_selection.rs +++ b/litellm-rust/crates/cost/tests/python_model_selection.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_select_model_name_keeps_base_model_free_of_region + use std::collections::HashMap; use litellm_cost::catalog::ModelInfoCatalog; diff --git a/litellm-rust/crates/cost/tests/python_off_peak.rs b/litellm-rust/crates/cost/tests/python_off_peak.rs index 797b3052f7f..2e6ca52a942 100644 --- a/litellm-rust/crates/cost/tests/python_off_peak.rs +++ b/litellm-rust/crates/cost/tests/python_off_peak.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_is_within_off_peak_window_same_day + use jiff::Timestamp; use litellm_cost::base_rate_selection::get_token_base_cost; use litellm_cost::off_peak::{ diff --git a/litellm-rust/crates/cost/tests/python_openai_cost.rs b/litellm-rust/crates/cost/tests/python_openai_cost.rs index f3754e4a1e3..02c8e3b8a43 100644 --- a/litellm-rust/crates/cost/tests/python_openai_cost.rs +++ b/litellm-rust/crates/cost/tests/python_openai_cost.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_video_generation.py::TestVideoGeneration +// mirrors: unit/llms/openai/transcriptions/test_transcription_duration_hidden.py::TestCostCalculatorReadsDurationFromHiddenParams + use litellm_cost::non_token::Error; use litellm_cost::openai_cost::{ cost_router, video_generation_cost, video_output_cost_per_second, diff --git a/litellm-rust/crates/cost/tests/python_openai_image_cost.rs b/litellm-rust/crates/cost/tests/python_openai_image_cost.rs index 0f9e61ff747..45ece5913b5 100644 --- a/litellm-rust/crates/cost/tests/python_openai_image_cost.rs +++ b/litellm-rust/crates/cost/tests/python_openai_image_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_gpt_image_cost_calculator.py + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_per_second.rs b/litellm-rust/crates/cost/tests/python_per_second.rs index 9c74623c559..abf96be2711 100644 --- a/litellm-rust/crates/cost/tests/python_per_second.rs +++ b/litellm-rust/crates/cost/tests/python_per_second.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_per_second_pricing + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_perplexity_cost.rs b/litellm-rust/crates/cost/tests/python_perplexity_cost.rs index 0494646873a..a3383c0741e 100644 --- a/litellm-rust/crates/cost/tests/python_perplexity_cost.rs +++ b/litellm-rust/crates/cost/tests/python_perplexity_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/perplexity/test_perplexity_cost_calculator.py::TestPerplexityCostCalculator + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_pricing_pinning.rs b/litellm-rust/crates/cost/tests/python_pricing_pinning.rs index 60ed09a54cc..dba518faf9a 100644 --- a/litellm-rust/crates/cost/tests/python_pricing_pinning.rs +++ b/litellm-rust/crates/cost/tests/python_pricing_pinning.rs @@ -1,7 +1,8 @@ -// mirrors: crates/cost/tests/generate_python_fixtures.py (pinning + model_info_projection surfaces) - #![allow(clippy::disallowed_types)] +// mirrors: crates/cost/tests/generate_python_fixtures.py +// (pinning + model_info_projection surfaces) + use serde_json::Value; use litellm_cost::pricing::{Metric, ServiceTier, tokenize}; diff --git a/litellm-rust/crates/cost/tests/python_prompt_caching_savings.rs b/litellm-rust/crates/cost/tests/python_prompt_caching_savings.rs index 4029789006a..6c6d2a09177 100644 --- a/litellm-rust/crates/cost/tests/python_prompt_caching_savings.rs +++ b/litellm-rust/crates/cost/tests/python_prompt_caching_savings.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py::test_fireworks_cache_read_rates_match_breakdown_and_caching_savings + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_provider_cache.rs b/litellm-rust/crates/cost/tests/python_provider_cache.rs index b6a8c9387e4..f972f5584d0 100644 --- a/litellm-rust/crates/cost/tests/python_provider_cache.rs +++ b/litellm-rust/crates/cost/tests/python_provider_cache.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py::test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_provider_routes.rs b/litellm-rust/crates/cost/tests/python_provider_routes.rs index e39f68fefce..f7dda087ff0 100644 --- a/litellm-rust/crates/cost/tests/python_provider_routes.rs +++ b/litellm-rust/crates/cost/tests/python_provider_routes.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/openai/test_cost_calculation.py::test_shipped_per_second_models_bill_a_non_zero_cost +// mirrors: test_litellm/llms/databricks/test_databricks_cost_calculator.py + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_ptu_pricing.rs b/litellm-rust/crates/cost/tests/python_ptu_pricing.rs index cfd794e99e6..fd496ab6939 100644 --- a/litellm-rust/crates/cost/tests/python_ptu_pricing.rs +++ b/litellm-rust/crates/cost/tests/python_ptu_pricing.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/test_ptu_pricing.py::test_a_complete_reservation_is_accepted + use litellm_cost::ptu_pricing::{ MAX_COST_PER_PTU_PER_HOUR, MAX_PTU_COUNT, PTU_ZEROED_PRICING_FIELDS, SEARCH_CONTEXT_SIZES, azure_spillover, declares_ptu, is_spilled_over_ptu_request, ptu_config_error, diff --git a/litellm-rust/crates/cost/tests/python_realtime_cost.rs b/litellm-rust/crates/cost/tests/python_realtime_cost.rs index 62c914c0ec8..422fb360f40 100644 --- a/litellm-rust/crates/cost/tests/python_realtime_cost.rs +++ b/litellm-rust/crates/cost/tests/python_realtime_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_realtime_stream_combines_text_and_audio_token_details + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_regional_uplift.rs b/litellm-rust/crates/cost/tests/python_regional_uplift.rs index 538e2b34e4e..722975e8e0b 100644 --- a/litellm-rust/crates/cost/tests/python_regional_uplift.rs +++ b/litellm-rust/crates/cost/tests/python_regional_uplift.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_data_residency_applies_uplift + use jiff::Timestamp; use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region; use litellm_cost::regional_uplift::{ diff --git a/litellm-rust/crates/cost/tests/python_responses_usage.rs b/litellm-rust/crates/cost/tests/python_responses_usage.rs index a54339d628f..24848d218d9 100644 --- a/litellm-rust/crates/cost/tests/python_responses_usage.rs +++ b/litellm-rust/crates/cost/tests/python_responses_usage.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/responses/test_responses_utils.py::TestResponseAPILoggingUtils + use litellm_cost::responses_usage::{ UsageError, is_response_api_usage, text_tokens_without_nested_reasoning, transform_response_api_usage_to_chat_usage, diff --git a/litellm-rust/crates/cost/tests/python_retrieval_cost.rs b/litellm-rust/crates/cost/tests/python_retrieval_cost.rs index 26b77ae53ce..a7142ca7f16 100644 --- a/litellm-rust/crates/cost/tests/python_retrieval_cost.rs +++ b/litellm-rust/crates/cost/tests/python_retrieval_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/rag/test_main.py::test_aquery_billed_cost_includes_priced_vector_store_search + use litellm_cost::retrieval_cost::{rerank_cost, vector_store_search_cost}; use rstest::rstest; use serde_json::json; diff --git a/litellm-rust/crates/cost/tests/python_search_cost.rs b/litellm-rust/crates/cost/tests/python_search_cost.rs index 1390ad2164e..cef859ca828 100644 --- a/litellm-rust/crates/cost/tests/python_search_cost.rs +++ b/litellm-rust/crates/cost/tests/python_search_cost.rs @@ -1,5 +1,8 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py::TestAgentCoreSearchEdgeCases::test_search_cost_lookup_is_mapped +// mirrors: unit/llms/parallel_ai/test_parallel_ai_search.py::test_explicit_mode_wins_over_processor + use std::collections::HashMap; use litellm_cost::catalog::ModelInfoCatalog; diff --git a/litellm-rust/crates/cost/tests/python_speech_cost.rs b/litellm-rust/crates/cost/tests/python_speech_cost.rs index 5b20dcd0dfd..4e4312bdc74 100644 --- a/litellm-rust/crates/cost/tests/python_speech_cost.rs +++ b/litellm-rust/crates/cost/tests/python_speech_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_tiered_pricing.rs b/litellm-rust/crates/cost/tests/python_tiered_pricing.rs index dfe4e2e2c48..1e9fbbb967d 100644 --- a/litellm-rust/crates/cost/tests/python_tiered_pricing.rs +++ b/litellm-rust/crates/cost/tests/python_tiered_pricing.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py::test_generic_cost_per_token_tiered_pricing_is_all_or_nothing + use litellm_cost::tiered_pricing::{select_tier_for_input, tier_rate}; use rstest::rstest; use serde_json::{Value, json}; diff --git a/litellm-rust/crates/cost/tests/python_together_cost.rs b/litellm-rust/crates/cost/tests/python_together_cost.rs index ff10a88cbf8..8da16a050de 100644 --- a/litellm-rust/crates/cost/tests/python_together_cost.rs +++ b/litellm-rust/crates/cost/tests/python_together_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: local_testing/test_completion_cost.py::test_together_ai_qwen_completion_cost + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_tool_call_cost_tracking.rs b/litellm-rust/crates/cost/tests/python_tool_call_cost_tracking.rs index 05e0598f0cf..1ddea1ed125 100644 --- a/litellm-rust/crates/cost/tests/python_tool_call_cost_tracking.rs +++ b/litellm-rust/crates/cost/tests/python_tool_call_cost_tracking.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py::test_web_search_cost_low + use litellm_cost::tool_call_cost_tracking::{ DefaultToolRates, ResponseKind, chat_completion_response_includes_annotations, count_web_search_calls, extract_file_search_params, extract_token_counts, diff --git a/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs b/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs index a062adfbe20..21813869ce7 100644 --- a/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs +++ b/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py::test_get_cost_for_built_in_tools_web_search + use litellm_cost::tool_call_cost_tracking::{DefaultToolRates, ResponseKind}; use litellm_cost::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools}; use litellm_cost::usage_dispatch::get_usage_object; diff --git a/litellm-rust/crates/cost/tests/python_transcription_usage.rs b/litellm-rust/crates/cost/tests/python_transcription_usage.rs index 0e0a2fa1e6d..e0136ed4881 100644 --- a/litellm-rust/crates/cost/tests/python_transcription_usage.rs +++ b/litellm-rust/crates/cost/tests/python_transcription_usage.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py + use litellm_cost::transcription_usage::{ is_transcription_usage_object, transform_transcription_usage_object, }; diff --git a/litellm-rust/crates/cost/tests/python_usage_dispatch.rs b/litellm-rust/crates/cost/tests/python_usage_dispatch.rs index 8e6d0da6b20..7a258e49409 100644 --- a/litellm-rust/crates/cost/tests/python_usage_dispatch.rs +++ b/litellm-rust/crates/cost/tests/python_usage_dispatch.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_usage_object_transformation.py + use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; use serde_json::{Value, json}; diff --git a/litellm-rust/crates/cost/tests/python_vertex_cost.rs b/litellm-rust/crates/cost/tests/python_vertex_cost.rs index 61590c780d4..fa9e72869ba 100644 --- a/litellm-rust/crates/cost/tests/python_vertex_cost.rs +++ b/litellm-rust/crates/cost/tests/python_vertex_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/test_cost_calculator.py::test_vertex_uplift_composes_with_above_128k_pricing + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_wire_contract.rs b/litellm-rust/crates/cost/tests/python_wire_contract.rs index 2bbfda34b73..88039693f36 100644 --- a/litellm-rust/crates/cost/tests/python_wire_contract.rs +++ b/litellm-rust/crates/cost/tests/python_wire_contract.rs @@ -1,6 +1,7 @@ #![allow(clippy::disallowed_types)] -// mirrors: crates/cost/tests/generate_python_fixtures.py (wire_usage + wire_response surfaces) +// mirrors: crates/cost/tests/generate_python_fixtures.py +// (wire_usage + wire_response surfaces) use serde_json::Value; diff --git a/litellm-rust/crates/cost/tests/python_xai_cost.rs b/litellm-rust/crates/cost/tests/python_xai_cost.rs index 8d82b89de89..62b6d940bee 100644 --- a/litellm-rust/crates/cost/tests/python_xai_cost.rs +++ b/litellm-rust/crates/cost/tests/python_xai_cost.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/llms/xai/test_xai_cost_calculator.py::TestXAICostCalculator + use std::collections::HashMap; use jiff::Timestamp; diff --git a/litellm-rust/crates/cost/tests/python_zero_cost_diagnostic.rs b/litellm-rust/crates/cost/tests/python_zero_cost_diagnostic.rs index 3b1c61366de..c68e16e2bb4 100644 --- a/litellm-rust/crates/cost/tests/python_zero_cost_diagnostic.rs +++ b/litellm-rust/crates/cost/tests/python_zero_cost_diagnostic.rs @@ -1,5 +1,7 @@ #![allow(clippy::disallowed_types)] +// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py::test_missing_pricing_key_names_every_rate_the_usage_needs + use std::collections::HashMap; use litellm_cost::catalog::ModelInfoCatalog; diff --git a/scripts/check_cost_test_mirrors.py b/scripts/check_cost_test_mirrors.py new file mode 100644 index 00000000000..5c701040e5d --- /dev/null +++ b/scripts/check_cost_test_mirrors.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Requires every tests/python_*.rs file in the cost crate to name the Python +test it mirrors with a `// mirrors: ` line in its header. + +The target names a real file under litellm/tests/ (or the fixture generator +when the file pins executed-Python output rather than a specific test), and +may qualify a test with `::` segments. Exit 1 with one line per offender. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +COST_TESTS = REPO_ROOT / "litellm-rust" / "crates" / "cost" / "tests" +PYTHON_TESTS = REPO_ROOT / "tests" +GENERATOR = "crates/cost/tests/generate_python_fixtures.py" +MIRROR_RE = re.compile(r"^//\s*mirrors:\s*(\S.*)$", re.MULTILINE) +TARGET_RE = re.compile(r"^([\w./-]+\.py)(?:::[\w.-]+)*$") + + +def header(source: str) -> str: + lines = source.splitlines() + body_start = next( + (index for index, line in enumerate(lines) if line.strip().startswith(("mod ", "use ", "fn ", "#["))), + len(lines), + ) + return "\n".join(lines[:body_start]) + + +def main() -> int: + rust_files = sorted(COST_TESTS.glob("python_*.rs")) + if not rust_files: + print("no tests/python_*.rs files found; checker is misconfigured", file=sys.stderr) + return 1 + offenders: list[str] = [] + for rust_file in rust_files: + source = rust_file.read_text() + markers = MIRROR_RE.findall(header(source)) + if not markers: + offenders.append(f"{rust_file.name}: missing `// mirrors:` marker") + continue + for target in markers: + match = TARGET_RE.match(target) + if not match: + offenders.append(f"{rust_file.name}: malformed mirrors target {target!r}") + continue + path = match.group(1) + if path == GENERATOR: + continue + if not (PYTHON_TESTS / path.removeprefix("litellm/tests/")).is_file(): + offenders.append( + f"{rust_file.name}: mirrors target {path!r} is not a file under litellm/tests/" + ) + for line in offenders: + print(line) + return 1 if offenders else 0 + + +if __name__ == "__main__": + raise SystemExit(main())