From 583dfa97c1f8d7d13f02da195e52bc8a1481cd6c Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 23 Sep 2026 23:16:44 -0700 Subject: [PATCH] test(cost): pin every provider model list and rule edge case against Python Adds Python-recorded cases for each model list get_llm_provider routes through, the finetune and guardrail exclusions, legacy rules that inherit capabilities through extends, rules with a malformed fill_missing_for_providers, and the Stability 1 empty size default. Drops the bedrock pricing-only filter: every key it excludes starts with bedrock/, which the provider-prefix branch resolves first. Co-Authored-By: Claude Opus 5.5 --- .../crates/cost/src/get_llm_provider.rs | 11 +- .../cost/tests/python_bedrock_image_cost.rs | 1 + .../crates/cost/tests/python_model_info.rs | 145 ++++++++++++++++++ 3 files changed, 147 insertions(+), 10 deletions(-) diff --git a/litellm-rust/crates/cost/src/get_llm_provider.rs b/litellm-rust/crates/cost/src/get_llm_provider.rs index 20658a58c52..34671f543b7 100644 --- a/litellm-rust/crates/cost/src/get_llm_provider.rs +++ b/litellm-rust/crates/cost/src/get_llm_provider.rs @@ -2,7 +2,6 @@ use std::collections::{HashMap, HashSet}; use std::str::FromStr; use std::sync::LazyLock; -use regex::Regex; use serde_json::Value; use crate::fallback_generalizations::FallbackGeneralizations; @@ -45,10 +44,6 @@ static JSON_PROVIDERS: LazyLock> = LazyLock::new(|| { .collect() }); -static BEDROCK_PRICING_ONLY: LazyLock = LazyLock::new(|| { - Regex::new(r"^bedrock/[a-zA-Z0-9_-]+/.+$").expect("valid bedrock pricing pattern") -}); - #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum ModelList { OpenAiChatCompletion, @@ -85,10 +80,6 @@ fn is_openai_finetune_model(key: &str) -> bool { key.starts_with("ft:") && key.matches(':').count() <= 1 } -fn is_bedrock_pricing_only_model(key: &str) -> bool { - BEDROCK_PRICING_ONLY.is_match(key) -} - fn vertex_member(key: &str) -> String { key.replace("vertex_ai/", "") } @@ -122,7 +113,7 @@ fn model_list_membership(key: &str, info: &Value) -> Option<(ModelList, String)> "nlp_cloud" => ModelList::NlpCloud, "aleph_alpha" => ModelList::AlephAlpha, "bedrock" if mode == Some("guardrail") => return None, - "bedrock" if !is_bedrock_pricing_only_model(key) => ModelList::Bedrock, + "bedrock" => ModelList::Bedrock, "bedrock_converse" => ModelList::BedrockConverse, "watsonx" => ModelList::Watsonx, "gradient_ai" => ModelList::GradientAi, diff --git a/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs b/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs index 8e6d30daa13..12874e9020b 100644 --- a/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs +++ b/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs @@ -76,6 +76,7 @@ fn bedrock_image_catalog() -> ModelInfoCatalog { #[case::stability1_max_tier("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": 60}), Ok(0.16))] #[case::stability1_missing_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({}), Ok(0.08))] #[case::stability1_default_size("stability.sd-zz-v1", None, json!({"steps": true}), Ok(0.08))] +#[case::stability1_empty_size_uses_the_default("stability.sd-zz-v1", Some(""), json!({}), Ok(0.08))] #[case::stability1_null_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": null}), Err(CostError::InvalidSteps))] #[case::stability1_string_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": "60"}), Err(CostError::InvalidSteps))] #[case::stability1_keeps_the_routing_prefix_in_the_key( diff --git a/litellm-rust/crates/cost/tests/python_model_info.rs b/litellm-rust/crates/cost/tests/python_model_info.rs index 4cddd308f4c..dd4f629bcdd 100644 --- a/litellm-rust/crates/cost/tests/python_model_info.rs +++ b/litellm-rust/crates/cost/tests/python_model_info.rs @@ -44,6 +44,11 @@ fn catalog() -> ModelInfoCatalog { provider("fireworks_ai"), ), ("gpt-4.1".to_owned(), provider("openai")), + ("amazon.zz-model".to_owned(), provider("bedrock")), + ( + "zz-guardrail".to_owned(), + json!({"litellm_provider": "bedrock", "mode": "guardrail"}), + ), ("Mixed-Case-ZZ".to_owned(), provider("together_ai")), ( "fallback_generalizations".to_owned(), @@ -51,11 +56,29 @@ fn catalog() -> ModelInfoCatalog { {"name": "zz-route", "pattern": "^zzroute-", "model_info": {"litellm_provider": "anthropic"}}, {"name": "zz-cap", "pattern": "zzcap-\\d+", "model_info": {"mode": "chat", "max_tokens": 10}}, {"name": "zz-cap2", "pattern": "zzcap-9", "model_info": {"max_tokens": 20}}, + { + "name": "zz-bad", + "pattern": "^zzbad-", + "model_info": {"mode": "chat"}, + "fill_missing_for_providers": "openai" + }, + { + "name": "zz-good", + "pattern": "^zzgood-", + "model_info": {"mode": "chat", "max_tokens": 5}, + "fill_missing_for_providers": ["openai"] + }, { "name": "zz-legacy", "pattern": "^legacy-zz", "extends": "zz-cap", "model_info": {"litellm_provider": "xai", "max_tokens": 30} + }, + { + "name": "zz-legacy-inherits", + "pattern": "^legacy-yy", + "extends": "zz-cap", + "model_info": {"litellm_provider": "xai"} } ]}), ), @@ -130,6 +153,9 @@ type Resolved = Result<(&'static str, Option<&'static str>, Option), CostEr #[case::capability_backfills_the_provider("zzcap-9", Some("openai"), Ok(("openai/zzcap-9", Some("openai"), Some(20))))] #[case::routing_rule_alone_has_no_capabilities("zzroute-x", None, Err(CostError::ModelNotFound))] #[case::legacy_rule_routes_and_extends("legacy-zz-1", None, Ok(("legacy-zz-1", Some("xai"), Some(30))))] +#[case::legacy_rule_inherits_parent_capabilities("legacy-yy-1", None, Ok(("legacy-yy-1", Some("xai"), Some(10))))] +#[case::malformed_fill_missing_skips_the_rule("zzbad-1", None, Err(CostError::ModelNotFound))] +#[case::listed_fill_missing_keeps_the_rule("zzgood-1", None, Ok(("zzgood-1", None, Some(5))))] #[case::unmapped("nothing-matches", None, Err(CostError::ModelNotFound))] fn get_model_info_resolves_keys_like_python( #[case] model: &str, @@ -201,6 +227,9 @@ fn capability_rules_lose_to_an_exact_key_owned_by_another_provider() { #[case::json_registry_provider("gmi/any", Some("gmi"))] #[case::static_cohere_embedding("embed-v4.0", Some("cohere"))] #[case::static_openai_image("dall-e-3", Some("openai"))] +#[case::finetune_pricing_key_is_not_a_chat_model("ft:gpt-yy", None)] +#[case::bedrock_family("amazon.zz-model", Some("bedrock"))] +#[case::bedrock_guardrail_is_not_a_model("zz-guardrail", None)] #[case::unroutable("GPT-ZZ", None)] fn get_llm_provider_infers_like_python(#[case] model: &str, #[case] expected: Option<&str>) { assert_eq!( @@ -260,3 +289,119 @@ fn unmapped_models_price_through_capability_rules( expected ); } + +fn provider_list_catalog() -> ModelInfoCatalog { + let providers = [ + "openai", + "text-completion-openai", + "cohere", + "cohere_chat", + "mistral", + "anthropic", + "empower", + "openrouter", + "vertex_ai-text-models", + "vertex_ai-code-text-models", + "vertex_ai-language-models", + "vertex_ai-vision-models", + "vertex_ai-chat-models", + "vertex_ai-code-chat-models", + "vertex_ai-embedding-models", + "vertex_ai-llama_models", + "vertex_ai-mistral_models", + "vertex_ai-ai21_models", + "vertex_ai-image-models", + "vertex_ai-video-models", + "nlp_cloud", + "aleph_alpha", + "bedrock", + "bedrock_converse", + "watsonx", + "gradient_ai", + "xai", + ]; + let entries = providers + .into_iter() + .map(|provider| { + ( + format!("zz-{provider}"), + json!({"litellm_provider": provider}), + ) + }) + .chain([ + ( + "vertex_ai/zz-vimg".to_owned(), + json!({"litellm_provider": "vertex_ai-image-models"}), + ), + ( + "zz-ai21-chat".to_owned(), + json!({"litellm_provider": "ai21", "mode": "chat"}), + ), + ( + "zz-ai21-text".to_owned(), + json!({"litellm_provider": "ai21", "mode": "completion"}), + ), + ( + "claude-2".to_owned(), + json!({"litellm_provider": "anthropic"}), + ), + ]) + .collect(); + ModelInfoCatalog::new(entries) +} + +// expected values recorded from get_llm_provider after add_known_models() folds this catalog into the model lists +#[rstest] +#[case::claude_2("claude-2", Some("anthropic_text"))] +#[case::vertex_ai_zz_vimg("vertex_ai/zz-vimg", Some("vertex_ai"))] +#[case::zz_ai21_chat("zz-ai21-chat", Some("ai21_chat"))] +#[case::zz_ai21_text("zz-ai21-text", Some("ai21_chat"))] +#[case::zz_aleph_alpha("zz-aleph_alpha", Some("aleph_alpha"))] +#[case::zz_anthropic("zz-anthropic", Some("anthropic"))] +#[case::zz_bedrock("zz-bedrock", Some("bedrock"))] +#[case::zz_bedrock_converse("zz-bedrock_converse", Some("bedrock"))] +#[case::zz_cohere("zz-cohere", Some("cohere"))] +#[case::zz_cohere_chat("zz-cohere_chat", Some("cohere_chat"))] +#[case::zz_empower("zz-empower", Some("empower"))] +#[case::zz_gradient_ai("zz-gradient_ai", Some("gradient_ai"))] +#[case::zz_mistral("zz-mistral", None)] +#[case::zz_nlp_cloud("zz-nlp_cloud", Some("nlp_cloud"))] +#[case::zz_openai("zz-openai", Some("openai"))] +#[case::zz_openrouter("zz-openrouter", Some("openrouter"))] +#[case::zz_text_completion_openai("zz-text-completion-openai", Some("text-completion-openai"))] +#[case::zz_vertex_ai_ai21_models("zz-vertex_ai-ai21_models", None)] +#[case::zz_vertex_ai_chat_models("zz-vertex_ai-chat-models", Some("vertex_ai"))] +#[case::zz_vertex_ai_code_chat_models("zz-vertex_ai-code-chat-models", Some("vertex_ai"))] +#[case::zz_vertex_ai_code_text_models("zz-vertex_ai-code-text-models", Some("vertex_ai"))] +#[case::zz_vertex_ai_embedding_models("zz-vertex_ai-embedding-models", Some("vertex_ai"))] +#[case::zz_vertex_ai_image_models("zz-vertex_ai-image-models", Some("vertex_ai"))] +#[case::zz_vertex_ai_language_models("zz-vertex_ai-language-models", Some("vertex_ai"))] +#[case::zz_vertex_ai_llama_models("zz-vertex_ai-llama_models", None)] +#[case::zz_vertex_ai_mistral_models("zz-vertex_ai-mistral_models", None)] +#[case::zz_vertex_ai_text_models("zz-vertex_ai-text-models", Some("vertex_ai"))] +#[case::zz_vertex_ai_video_models("zz-vertex_ai-video-models", Some("vertex_ai"))] +#[case::zz_vertex_ai_vision_models("zz-vertex_ai-vision-models", Some("vertex_ai"))] +#[case::zz_watsonx("zz-watsonx", Some("watsonx"))] +#[case::zz_xai("zz-xai", None)] +#[case::zz_vimg("zz-vimg", Some("vertex_ai"))] +#[case::bytez_x("bytez/x", Some("bytez"))] +#[case::amazon_nova_x("amazon_nova-x", Some("amazon_nova"))] +#[case::sap_x("sap/x", Some("sap"))] +#[case::gpt_image_9("gpt-image-9", Some("openai"))] +#[case::sora_2("sora-2", Some("openai"))] +#[case::ft_gpt_4o_o_1("ft:gpt-4o:o::1", Some("openai"))] +#[case::ft_gpt_3_5_turbo_o_1("ft:gpt-3.5-turbo:o::1", Some("openai"))] +#[case::replicate_64_char_version_id(&*format!("owner/m:{}b", "a".repeat(63)), Some("replicate"))] +#[case::long_id_that_is_not_a_replicate_version(&*format!("owner/m:{}", "a".repeat(70)), None)] +fn get_llm_provider_routes_every_model_list_like_python( + #[case] model: &str, + #[case] expected: Option<&str>, +) { + assert_eq!( + provider_list_catalog() + .get_llm_provider(model) + .map(|resolved| resolved.custom_llm_provider) + .as_deref(), + expected + ); +}