mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(cost): pin every provider model list and rule edge case against Python
Adds Python-recorded cases for each model list get_llm_provider routes through, the finetune and guardrail exclusions, legacy rules that inherit capabilities through extends, rules with a malformed fill_missing_for_providers, and the Stability 1 empty size default. Drops the bedrock pricing-only filter: every key it excludes starts with bedrock/, which the provider-prefix branch resolves first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
edbf88daf2
commit
583dfa97c1
3 changed files with 147 additions and 10 deletions
|
|
@ -2,7 +2,6 @@ use std::collections::{HashMap, HashSet};
|
|||
use std::str::FromStr;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::fallback_generalizations::FallbackGeneralizations;
|
||||
|
|
@ -45,10 +44,6 @@ static JSON_PROVIDERS: LazyLock<HashSet<String>> = LazyLock::new(|| {
|
|||
.collect()
|
||||
});
|
||||
|
||||
static BEDROCK_PRICING_ONLY: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"^bedrock/[a-zA-Z0-9_-]+/.+$").expect("valid bedrock pricing pattern")
|
||||
});
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub enum ModelList {
|
||||
OpenAiChatCompletion,
|
||||
|
|
@ -85,10 +80,6 @@ fn is_openai_finetune_model(key: &str) -> bool {
|
|||
key.starts_with("ft:") && key.matches(':').count() <= 1
|
||||
}
|
||||
|
||||
fn is_bedrock_pricing_only_model(key: &str) -> bool {
|
||||
BEDROCK_PRICING_ONLY.is_match(key)
|
||||
}
|
||||
|
||||
fn vertex_member(key: &str) -> String {
|
||||
key.replace("vertex_ai/", "")
|
||||
}
|
||||
|
|
@ -122,7 +113,7 @@ fn model_list_membership(key: &str, info: &Value) -> Option<(ModelList, String)>
|
|||
"nlp_cloud" => ModelList::NlpCloud,
|
||||
"aleph_alpha" => ModelList::AlephAlpha,
|
||||
"bedrock" if mode == Some("guardrail") => return None,
|
||||
"bedrock" if !is_bedrock_pricing_only_model(key) => ModelList::Bedrock,
|
||||
"bedrock" => ModelList::Bedrock,
|
||||
"bedrock_converse" => ModelList::BedrockConverse,
|
||||
"watsonx" => ModelList::Watsonx,
|
||||
"gradient_ai" => ModelList::GradientAi,
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ fn bedrock_image_catalog() -> ModelInfoCatalog {
|
|||
#[case::stability1_max_tier("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": 60}), Ok(0.16))]
|
||||
#[case::stability1_missing_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({}), Ok(0.08))]
|
||||
#[case::stability1_default_size("stability.sd-zz-v1", None, json!({"steps": true}), Ok(0.08))]
|
||||
#[case::stability1_empty_size_uses_the_default("stability.sd-zz-v1", Some(""), json!({}), Ok(0.08))]
|
||||
#[case::stability1_null_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": null}), Err(CostError::InvalidSteps))]
|
||||
#[case::stability1_string_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": "60"}), Err(CostError::InvalidSteps))]
|
||||
#[case::stability1_keeps_the_routing_prefix_in_the_key(
|
||||
|
|
|
|||
|
|
@ -44,6 +44,11 @@ fn catalog() -> ModelInfoCatalog {
|
|||
provider("fireworks_ai"),
|
||||
),
|
||||
("gpt-4.1".to_owned(), provider("openai")),
|
||||
("amazon.zz-model".to_owned(), provider("bedrock")),
|
||||
(
|
||||
"zz-guardrail".to_owned(),
|
||||
json!({"litellm_provider": "bedrock", "mode": "guardrail"}),
|
||||
),
|
||||
("Mixed-Case-ZZ".to_owned(), provider("together_ai")),
|
||||
(
|
||||
"fallback_generalizations".to_owned(),
|
||||
|
|
@ -51,11 +56,29 @@ fn catalog() -> ModelInfoCatalog {
|
|||
{"name": "zz-route", "pattern": "^zzroute-", "model_info": {"litellm_provider": "anthropic"}},
|
||||
{"name": "zz-cap", "pattern": "zzcap-\\d+", "model_info": {"mode": "chat", "max_tokens": 10}},
|
||||
{"name": "zz-cap2", "pattern": "zzcap-9", "model_info": {"max_tokens": 20}},
|
||||
{
|
||||
"name": "zz-bad",
|
||||
"pattern": "^zzbad-",
|
||||
"model_info": {"mode": "chat"},
|
||||
"fill_missing_for_providers": "openai"
|
||||
},
|
||||
{
|
||||
"name": "zz-good",
|
||||
"pattern": "^zzgood-",
|
||||
"model_info": {"mode": "chat", "max_tokens": 5},
|
||||
"fill_missing_for_providers": ["openai"]
|
||||
},
|
||||
{
|
||||
"name": "zz-legacy",
|
||||
"pattern": "^legacy-zz",
|
||||
"extends": "zz-cap",
|
||||
"model_info": {"litellm_provider": "xai", "max_tokens": 30}
|
||||
},
|
||||
{
|
||||
"name": "zz-legacy-inherits",
|
||||
"pattern": "^legacy-yy",
|
||||
"extends": "zz-cap",
|
||||
"model_info": {"litellm_provider": "xai"}
|
||||
}
|
||||
]}),
|
||||
),
|
||||
|
|
@ -130,6 +153,9 @@ type Resolved = Result<(&'static str, Option<&'static str>, Option<u64>), CostEr
|
|||
#[case::capability_backfills_the_provider("zzcap-9", Some("openai"), Ok(("openai/zzcap-9", Some("openai"), Some(20))))]
|
||||
#[case::routing_rule_alone_has_no_capabilities("zzroute-x", None, Err(CostError::ModelNotFound))]
|
||||
#[case::legacy_rule_routes_and_extends("legacy-zz-1", None, Ok(("legacy-zz-1", Some("xai"), Some(30))))]
|
||||
#[case::legacy_rule_inherits_parent_capabilities("legacy-yy-1", None, Ok(("legacy-yy-1", Some("xai"), Some(10))))]
|
||||
#[case::malformed_fill_missing_skips_the_rule("zzbad-1", None, Err(CostError::ModelNotFound))]
|
||||
#[case::listed_fill_missing_keeps_the_rule("zzgood-1", None, Ok(("zzgood-1", None, Some(5))))]
|
||||
#[case::unmapped("nothing-matches", None, Err(CostError::ModelNotFound))]
|
||||
fn get_model_info_resolves_keys_like_python(
|
||||
#[case] model: &str,
|
||||
|
|
@ -201,6 +227,9 @@ fn capability_rules_lose_to_an_exact_key_owned_by_another_provider() {
|
|||
#[case::json_registry_provider("gmi/any", Some("gmi"))]
|
||||
#[case::static_cohere_embedding("embed-v4.0", Some("cohere"))]
|
||||
#[case::static_openai_image("dall-e-3", Some("openai"))]
|
||||
#[case::finetune_pricing_key_is_not_a_chat_model("ft:gpt-yy", None)]
|
||||
#[case::bedrock_family("amazon.zz-model", Some("bedrock"))]
|
||||
#[case::bedrock_guardrail_is_not_a_model("zz-guardrail", None)]
|
||||
#[case::unroutable("GPT-ZZ", None)]
|
||||
fn get_llm_provider_infers_like_python(#[case] model: &str, #[case] expected: Option<&str>) {
|
||||
assert_eq!(
|
||||
|
|
@ -260,3 +289,119 @@ fn unmapped_models_price_through_capability_rules(
|
|||
expected
|
||||
);
|
||||
}
|
||||
|
||||
fn provider_list_catalog() -> ModelInfoCatalog {
|
||||
let providers = [
|
||||
"openai",
|
||||
"text-completion-openai",
|
||||
"cohere",
|
||||
"cohere_chat",
|
||||
"mistral",
|
||||
"anthropic",
|
||||
"empower",
|
||||
"openrouter",
|
||||
"vertex_ai-text-models",
|
||||
"vertex_ai-code-text-models",
|
||||
"vertex_ai-language-models",
|
||||
"vertex_ai-vision-models",
|
||||
"vertex_ai-chat-models",
|
||||
"vertex_ai-code-chat-models",
|
||||
"vertex_ai-embedding-models",
|
||||
"vertex_ai-llama_models",
|
||||
"vertex_ai-mistral_models",
|
||||
"vertex_ai-ai21_models",
|
||||
"vertex_ai-image-models",
|
||||
"vertex_ai-video-models",
|
||||
"nlp_cloud",
|
||||
"aleph_alpha",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"watsonx",
|
||||
"gradient_ai",
|
||||
"xai",
|
||||
];
|
||||
let entries = providers
|
||||
.into_iter()
|
||||
.map(|provider| {
|
||||
(
|
||||
format!("zz-{provider}"),
|
||||
json!({"litellm_provider": provider}),
|
||||
)
|
||||
})
|
||||
.chain([
|
||||
(
|
||||
"vertex_ai/zz-vimg".to_owned(),
|
||||
json!({"litellm_provider": "vertex_ai-image-models"}),
|
||||
),
|
||||
(
|
||||
"zz-ai21-chat".to_owned(),
|
||||
json!({"litellm_provider": "ai21", "mode": "chat"}),
|
||||
),
|
||||
(
|
||||
"zz-ai21-text".to_owned(),
|
||||
json!({"litellm_provider": "ai21", "mode": "completion"}),
|
||||
),
|
||||
(
|
||||
"claude-2".to_owned(),
|
||||
json!({"litellm_provider": "anthropic"}),
|
||||
),
|
||||
])
|
||||
.collect();
|
||||
ModelInfoCatalog::new(entries)
|
||||
}
|
||||
|
||||
// expected values recorded from get_llm_provider after add_known_models() folds this catalog into the model lists
|
||||
#[rstest]
|
||||
#[case::claude_2("claude-2", Some("anthropic_text"))]
|
||||
#[case::vertex_ai_zz_vimg("vertex_ai/zz-vimg", Some("vertex_ai"))]
|
||||
#[case::zz_ai21_chat("zz-ai21-chat", Some("ai21_chat"))]
|
||||
#[case::zz_ai21_text("zz-ai21-text", Some("ai21_chat"))]
|
||||
#[case::zz_aleph_alpha("zz-aleph_alpha", Some("aleph_alpha"))]
|
||||
#[case::zz_anthropic("zz-anthropic", Some("anthropic"))]
|
||||
#[case::zz_bedrock("zz-bedrock", Some("bedrock"))]
|
||||
#[case::zz_bedrock_converse("zz-bedrock_converse", Some("bedrock"))]
|
||||
#[case::zz_cohere("zz-cohere", Some("cohere"))]
|
||||
#[case::zz_cohere_chat("zz-cohere_chat", Some("cohere_chat"))]
|
||||
#[case::zz_empower("zz-empower", Some("empower"))]
|
||||
#[case::zz_gradient_ai("zz-gradient_ai", Some("gradient_ai"))]
|
||||
#[case::zz_mistral("zz-mistral", None)]
|
||||
#[case::zz_nlp_cloud("zz-nlp_cloud", Some("nlp_cloud"))]
|
||||
#[case::zz_openai("zz-openai", Some("openai"))]
|
||||
#[case::zz_openrouter("zz-openrouter", Some("openrouter"))]
|
||||
#[case::zz_text_completion_openai("zz-text-completion-openai", Some("text-completion-openai"))]
|
||||
#[case::zz_vertex_ai_ai21_models("zz-vertex_ai-ai21_models", None)]
|
||||
#[case::zz_vertex_ai_chat_models("zz-vertex_ai-chat-models", Some("vertex_ai"))]
|
||||
#[case::zz_vertex_ai_code_chat_models("zz-vertex_ai-code-chat-models", Some("vertex_ai"))]
|
||||
#[case::zz_vertex_ai_code_text_models("zz-vertex_ai-code-text-models", Some("vertex_ai"))]
|
||||
#[case::zz_vertex_ai_embedding_models("zz-vertex_ai-embedding-models", Some("vertex_ai"))]
|
||||
#[case::zz_vertex_ai_image_models("zz-vertex_ai-image-models", Some("vertex_ai"))]
|
||||
#[case::zz_vertex_ai_language_models("zz-vertex_ai-language-models", Some("vertex_ai"))]
|
||||
#[case::zz_vertex_ai_llama_models("zz-vertex_ai-llama_models", None)]
|
||||
#[case::zz_vertex_ai_mistral_models("zz-vertex_ai-mistral_models", None)]
|
||||
#[case::zz_vertex_ai_text_models("zz-vertex_ai-text-models", Some("vertex_ai"))]
|
||||
#[case::zz_vertex_ai_video_models("zz-vertex_ai-video-models", Some("vertex_ai"))]
|
||||
#[case::zz_vertex_ai_vision_models("zz-vertex_ai-vision-models", Some("vertex_ai"))]
|
||||
#[case::zz_watsonx("zz-watsonx", Some("watsonx"))]
|
||||
#[case::zz_xai("zz-xai", None)]
|
||||
#[case::zz_vimg("zz-vimg", Some("vertex_ai"))]
|
||||
#[case::bytez_x("bytez/x", Some("bytez"))]
|
||||
#[case::amazon_nova_x("amazon_nova-x", Some("amazon_nova"))]
|
||||
#[case::sap_x("sap/x", Some("sap"))]
|
||||
#[case::gpt_image_9("gpt-image-9", Some("openai"))]
|
||||
#[case::sora_2("sora-2", Some("openai"))]
|
||||
#[case::ft_gpt_4o_o_1("ft:gpt-4o:o::1", Some("openai"))]
|
||||
#[case::ft_gpt_3_5_turbo_o_1("ft:gpt-3.5-turbo:o::1", Some("openai"))]
|
||||
#[case::replicate_64_char_version_id(&*format!("owner/m:{}b", "a".repeat(63)), Some("replicate"))]
|
||||
#[case::long_id_that_is_not_a_replicate_version(&*format!("owner/m:{}", "a".repeat(70)), None)]
|
||||
fn get_llm_provider_routes_every_model_list_like_python(
|
||||
#[case] model: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(
|
||||
provider_list_catalog()
|
||||
.get_llm_provider(model)
|
||||
.map(|resolved| resolved.custom_llm_provider)
|
||||
.as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue