mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
feat(model-catalog): expose registration row-name metadata
This commit is contained in:
parent
36383c91fd
commit
430d3be2c7
4 changed files with 318 additions and 31 deletions
|
|
@ -37,6 +37,7 @@ pub enum Resolved {
|
|||
pub struct RegistrationTarget {
|
||||
pub write: String,
|
||||
pub inherit_from: Option<String>,
|
||||
pub names_the_row: bool,
|
||||
}
|
||||
|
||||
pub fn resolve(keys: &impl Keys, query: &Query<'_>) -> Resolved {
|
||||
|
|
@ -74,7 +75,7 @@ pub fn registration_target_with_provider_names(
|
|||
via: Via::Exact | Via::ProviderPrefix,
|
||||
} => Some(key),
|
||||
resolved => {
|
||||
registration_alias(keys, caller_key, provider, is_provider).or(match resolved {
|
||||
registration_alias(keys, caller_key, provider, &is_provider).or(match resolved {
|
||||
Resolved::Found { key, .. } => Some(key),
|
||||
Resolved::NotFound { .. } => None,
|
||||
})
|
||||
|
|
@ -82,24 +83,83 @@ pub fn registration_target_with_provider_names(
|
|||
};
|
||||
RegistrationTarget {
|
||||
write: caller_key.to_owned(),
|
||||
names_the_row: inherit_from.as_deref().is_some_and(|inherited| {
|
||||
registration_names_the_row(caller_key, provider, &is_provider, inherited)
|
||||
}),
|
||||
inherit_from,
|
||||
}
|
||||
}
|
||||
|
||||
fn registration_provider<'a>(
|
||||
model: &'a str,
|
||||
provider: Option<&'a str>,
|
||||
is_provider: &impl Fn(&str) -> bool,
|
||||
) -> Option<&'a str> {
|
||||
provider
|
||||
.filter(|provider| !provider.is_empty())
|
||||
.or_else(|| {
|
||||
model
|
||||
.split_once('/')
|
||||
.map(|(prefix, _)| prefix)
|
||||
.filter(|prefix| is_provider(prefix))
|
||||
})
|
||||
}
|
||||
|
||||
fn registration_names_the_row(
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
is_provider: &impl Fn(&str) -> bool,
|
||||
inherited: &str,
|
||||
) -> bool {
|
||||
if inherited == model {
|
||||
return true;
|
||||
}
|
||||
let provider = registration_provider(model, provider, is_provider);
|
||||
let mut base = model;
|
||||
while let Some((prefix, tail)) = base.split_once('/') {
|
||||
if !is_provider(prefix) {
|
||||
break;
|
||||
}
|
||||
base = tail;
|
||||
if inherited == base {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if matches!(provider, Some("bedrock" | "bedrock_converse")) {
|
||||
base = strip_bedrock_nova_route(strip_bedrock_route(base));
|
||||
}
|
||||
inherited == base || provider.is_some() && inherited == registration_stripped(base, provider)
|
||||
}
|
||||
|
||||
fn registration_stripped<'a>(model: &'a str, provider: Option<&str>) -> &'a str {
|
||||
if !matches!(provider, Some("bedrock" | "bedrock_converse")) {
|
||||
return strip_suffix(model, provider);
|
||||
}
|
||||
let custom = model
|
||||
.strip_prefix("bedrock/converse/")
|
||||
.or_else(|| model.strip_prefix("bedrock/"))
|
||||
.or_else(|| model.strip_prefix("converse/"))
|
||||
.unwrap_or(model);
|
||||
if custom.starts_with("nova-2/") || custom.starts_with("nova/") {
|
||||
return strip_suffix(custom, provider);
|
||||
}
|
||||
let model = bedrock_arn_model(strip_bedrock_nova_route(strip_bedrock_route(model)));
|
||||
let model = strip_bedrock_throughput(model);
|
||||
let geo_free = strip_bedrock_geo(model);
|
||||
if geo_free != model {
|
||||
geo_free
|
||||
} else {
|
||||
strip_region(model, provider, None)
|
||||
}
|
||||
}
|
||||
|
||||
fn registration_alias(
|
||||
keys: &impl Keys,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
is_provider: impl Fn(&str) -> bool,
|
||||
) -> Option<String> {
|
||||
let provider = provider
|
||||
.filter(|provider| !provider.is_empty())
|
||||
.or_else(|| {
|
||||
model
|
||||
.split_once('/')
|
||||
.map(|(prefix, _)| prefix)
|
||||
.filter(|prefix| is_provider(prefix))
|
||||
});
|
||||
let provider = registration_provider(model, provider, &is_provider);
|
||||
let mut lookup = Lookup {
|
||||
keys,
|
||||
provider,
|
||||
|
|
@ -119,9 +179,7 @@ fn registration_alias(
|
|||
}
|
||||
}
|
||||
if matches!(provider, Some("bedrock" | "bedrock_converse")) {
|
||||
base = bedrock_routing_model(base);
|
||||
base = base.strip_prefix("nova-2/").unwrap_or(base);
|
||||
base = base.strip_prefix("nova/").unwrap_or(base);
|
||||
base = strip_bedrock_nova_route(bedrock_routing_model(base));
|
||||
if let Some(key) = lookup.registration_candidate(base, Via::ProviderStripped) {
|
||||
return Some(key);
|
||||
}
|
||||
|
|
@ -337,7 +395,7 @@ fn is_aws_region(region: &str) -> bool {
|
|||
&& number.bytes().all(|c| c.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn bedrock_routing_model(mut model: &str) -> &str {
|
||||
fn strip_bedrock_route(mut model: &str) -> &str {
|
||||
for prefix in [
|
||||
"bedrock/",
|
||||
"chat_completions/",
|
||||
|
|
@ -348,6 +406,19 @@ fn bedrock_routing_model(mut model: &str) -> &str {
|
|||
] {
|
||||
model = model.strip_prefix(prefix).unwrap_or(model);
|
||||
}
|
||||
model
|
||||
}
|
||||
|
||||
fn bedrock_routing_model(model: &str) -> &str {
|
||||
bedrock_arn_model(strip_bedrock_route(model))
|
||||
}
|
||||
|
||||
fn strip_bedrock_nova_route(model: &str) -> &str {
|
||||
let model = model.strip_prefix("nova-2/").unwrap_or(model);
|
||||
model.strip_prefix("nova/").unwrap_or(model)
|
||||
}
|
||||
|
||||
fn bedrock_arn_model(model: &str) -> &str {
|
||||
let region_free = strip_region(model, Some("bedrock"), None);
|
||||
if region_free.starts_with("arn:") {
|
||||
return region_free.rsplit('/').next().unwrap_or(region_free);
|
||||
|
|
@ -621,11 +692,8 @@ mod tests {
|
|||
|prefix| ["bedrock", "openai", "azure_ai", "hosted_vllm"].contains(&prefix),
|
||||
);
|
||||
assert_eq!(
|
||||
target,
|
||||
RegistrationTarget {
|
||||
write: key.into(),
|
||||
inherit_from: inherited.map(str::to_owned)
|
||||
}
|
||||
(target.write.as_str(), target.inherit_from.as_deref()),
|
||||
(key, inherited)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -653,7 +721,8 @@ mod tests {
|
|||
),
|
||||
RegistrationTarget {
|
||||
write: key,
|
||||
inherit_from: Some("anthropic.model-2026-01-01".into())
|
||||
inherit_from: Some("anthropic.model-2026-01-01".into()),
|
||||
names_the_row: region != "ap-northeast-1",
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
@ -671,6 +740,168 @@ mod tests {
|
|||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::exact("anthropic.model", Some("bedrock"), "anthropic.model", true)]
|
||||
#[case::repeated(
|
||||
"bedrock/bedrock/bedrock/us.anthropic.model",
|
||||
Some("bedrock"),
|
||||
"us.anthropic.model",
|
||||
true
|
||||
)]
|
||||
#[case::inferred(
|
||||
"bedrock/bedrock/bedrock/us.anthropic.model",
|
||||
None,
|
||||
"us.anthropic.model",
|
||||
true
|
||||
)]
|
||||
#[case::empty_scope(
|
||||
"bedrock/bedrock/bedrock/us.anthropic.model",
|
||||
Some(""),
|
||||
"us.anthropic.model",
|
||||
true
|
||||
)]
|
||||
#[case::intermediate(
|
||||
"bedrock/bedrock/bedrock/us.anthropic.model",
|
||||
Some("bedrock"),
|
||||
"bedrock/us.anthropic.model",
|
||||
true
|
||||
)]
|
||||
#[case::routing_base(
|
||||
"bedrock/converse/ap-northeast-1.anthropic.model",
|
||||
Some("bedrock"),
|
||||
"ap-northeast-1.anthropic.model",
|
||||
true
|
||||
)]
|
||||
#[case::region_only(
|
||||
"bedrock/ap-northeast-1.anthropic.model",
|
||||
Some("bedrock"),
|
||||
"anthropic.model",
|
||||
false
|
||||
)]
|
||||
#[case::region_suffix_overlap(
|
||||
"bedrock/us.anthropic.model",
|
||||
Some("bedrock"),
|
||||
"anthropic.model",
|
||||
true
|
||||
)]
|
||||
#[case::dated_overlap(
|
||||
"bedrock/converse/us.anthropic.model-2026-01-01",
|
||||
Some("bedrock"),
|
||||
"anthropic.model-2026-01-01",
|
||||
true
|
||||
)]
|
||||
#[case::dated_region_only(
|
||||
"bedrock/converse/ap-northeast-1.anthropic.model-2026-01-01",
|
||||
Some("bedrock"),
|
||||
"anthropic.model-2026-01-01",
|
||||
false
|
||||
)]
|
||||
#[case::throughput(
|
||||
"bedrock/us.anthropic.model:0:51k",
|
||||
Some("bedrock"),
|
||||
"anthropic.model:0",
|
||||
true
|
||||
)]
|
||||
#[case::context_window(
|
||||
"bedrock/us.anthropic.model[1m]",
|
||||
Some("bedrock"),
|
||||
"anthropic.model",
|
||||
true
|
||||
)]
|
||||
#[case::slash_region(
|
||||
"bedrock/us-east-1/anthropic.model",
|
||||
Some("bedrock"),
|
||||
"anthropic.model",
|
||||
true
|
||||
)]
|
||||
#[case::resource(
|
||||
"bedrock/arn:aws:bedrock:us-east-1:000000000000:inference-profile/us.anthropic.model",
|
||||
Some("bedrock"),
|
||||
"anthropic.model",
|
||||
true
|
||||
)]
|
||||
#[case::native_date_removal(
|
||||
"bedrock/anthropic.model-20260101",
|
||||
Some("bedrock"),
|
||||
"anthropic.model",
|
||||
false
|
||||
)]
|
||||
#[case::selected_qualified_row(
|
||||
"anthropic.model",
|
||||
Some("bedrock"),
|
||||
"bedrock/anthropic.model",
|
||||
false
|
||||
)]
|
||||
fn registration_classifies_selected_row_names(
|
||||
#[case] key: &str,
|
||||
#[case] provider: Option<&str>,
|
||||
#[case] row: &str,
|
||||
#[case] names_the_row: bool,
|
||||
) {
|
||||
let models = catalog(json!({row: {
|
||||
"litellm_provider": "bedrock",
|
||||
"input_cost_per_token": 0.001,
|
||||
"input_cost_per_token_above_128k_tokens": 0.002
|
||||
}}));
|
||||
assert_eq!(
|
||||
registration_target_with_provider_names(&models, key, provider, |prefix| prefix
|
||||
== "bedrock"),
|
||||
RegistrationTarget {
|
||||
write: key.into(),
|
||||
inherit_from: Some(row.into()),
|
||||
names_the_row,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing("bedrock/missing", Some("bedrock"))]
|
||||
#[case::wrong_scope("bedrock/ap-northeast-1.anthropic.model", Some("openai"))]
|
||||
#[case::case_variant("bedrock/ap-northeast-1.anthropic.MODEL", Some("bedrock"))]
|
||||
#[case::unknown_prefix("bedrock/unknown/anthropic.model", Some("bedrock"))]
|
||||
fn registration_without_inheritance_does_not_name_a_row(
|
||||
registration_models: Catalog,
|
||||
#[case] key: &str,
|
||||
#[case] provider: Option<&str>,
|
||||
) {
|
||||
assert_eq!(
|
||||
registration_target_with_provider_names(
|
||||
®istration_models,
|
||||
key,
|
||||
provider,
|
||||
|prefix| prefix == "bedrock"
|
||||
),
|
||||
RegistrationTarget {
|
||||
write: key.into(),
|
||||
inherit_from: None,
|
||||
names_the_row: false,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::dated("openai/model-2026-01-01", "openai", "model")]
|
||||
#[case::finetuned("openai/ft:model:org:custom:id", "openai", "ft:model")]
|
||||
#[case::vertex("vertex_ai/model-123", "vertex_ai", "model")]
|
||||
#[case::gemini("gemini/model-123", "gemini", "model")]
|
||||
#[case::databricks("databricks/model-123", "databricks", "model")]
|
||||
fn registration_suffixes_name_the_selected_row(
|
||||
#[case] key: &str,
|
||||
#[case] provider: &str,
|
||||
#[case] row: &str,
|
||||
) {
|
||||
let models = catalog(json!({row: {"litellm_provider": provider}}));
|
||||
assert_eq!(
|
||||
registration_target_with_provider_names(&models, key, Some(provider), |prefix| prefix
|
||||
== provider),
|
||||
RegistrationTarget {
|
||||
write: key.into(),
|
||||
inherit_from: Some(row.into()),
|
||||
names_the_row: true,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn models() -> Catalog {
|
||||
catalog(json!({
|
||||
|
|
@ -812,12 +1043,10 @@ mod tests {
|
|||
"bedrock/anthropic.claude-sonnet-4-6": {"litellm_provider":"bedrock"},
|
||||
"hosted_vllm/Acme-Model": {"litellm_provider":"hosted_vllm"}
|
||||
}));
|
||||
let target = registration_target(&models, caller);
|
||||
assert_eq!(
|
||||
registration_target(&models, caller),
|
||||
RegistrationTarget {
|
||||
write: caller.into(),
|
||||
inherit_from: inherit_from.map(str::to_owned),
|
||||
}
|
||||
(target.write.as_str(), target.inherit_from.as_deref()),
|
||||
(caller, inherit_from)
|
||||
);
|
||||
assert_eq!(models.model_count(), 5);
|
||||
}
|
||||
|
|
@ -829,6 +1058,7 @@ mod tests {
|
|||
RegistrationTarget {
|
||||
write: "openai/o3-pro".into(),
|
||||
inherit_from: Some("openai/o3-pro".into()),
|
||||
names_the_row: true,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
@ -1209,7 +1439,8 @@ mod tests {
|
|||
registration_target_with_provider(&counted, "MODEL", provider),
|
||||
RegistrationTarget {
|
||||
write: "MODEL".into(),
|
||||
inherit_from: None
|
||||
inherit_from: None,
|
||||
names_the_row: false,
|
||||
}
|
||||
);
|
||||
assert_eq!(counted.lowercase_calls.get(), 0);
|
||||
|
|
@ -1247,12 +1478,10 @@ mod tests {
|
|||
"case": {"litellm_provider": "openai"},
|
||||
"databricks/version": {"litellm_provider": "databricks"}
|
||||
}));
|
||||
let target = registration_target_with_provider(&models, key, provider);
|
||||
assert_eq!(
|
||||
registration_target_with_provider(&models, key, provider),
|
||||
RegistrationTarget {
|
||||
write: key.into(),
|
||||
inherit_from: inherited.map(str::to_owned),
|
||||
}
|
||||
(target.write.as_str(), target.inherit_from.as_deref()),
|
||||
(key, inherited)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -164,6 +164,7 @@ pub(crate) struct ModelNotFound {
|
|||
pub(crate) struct RegistrationTarget {
|
||||
write: String,
|
||||
inherit_from: Option<String>,
|
||||
names_the_row: bool,
|
||||
}
|
||||
|
||||
#[derive(IntoPyObject)]
|
||||
|
|
@ -232,6 +233,7 @@ pub(crate) fn registration_target(
|
|||
Ok(RegistrationTarget {
|
||||
write: target.write,
|
||||
inherit_from: target.inherit_from,
|
||||
names_the_row: target.names_the_row,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from pydantic import JsonValue
|
|||
|
||||
import litellm
|
||||
from litellm.rust_bridge import _native as native
|
||||
from litellm.utils import _resolve_builtin_model_cost_entry
|
||||
from litellm.utils import _fallback_registration, _resolve_builtin_model_cost_entry, _without_overridden_price_variants
|
||||
|
||||
|
||||
class RegistrationTargetTests(unittest.TestCase):
|
||||
|
|
@ -80,6 +80,58 @@ class RegistrationTargetTests(unittest.TestCase):
|
|||
self.assertEqual(None if legacy is None else legacy.key, expected)
|
||||
target: Final = native.registration_target(rows, key, provider=provider)
|
||||
self.assertEqual((target.write, target.inherit_from), (key, expected))
|
||||
self.assertIs(target.names_the_row, False if legacy is None else legacy.names_the_row)
|
||||
|
||||
def test_repeated_prefix_names_row_and_preserves_fallback_price_variants(self) -> None:
|
||||
self._assert_fallback_price_variants("us.anthropic.model", "bedrock/bedrock/bedrock/us.anthropic.model", True)
|
||||
|
||||
def test_region_only_alias_does_not_name_row_and_suppresses_fallback_price_variants(self) -> None:
|
||||
self._assert_fallback_price_variants("anthropic.model", "bedrock/ap-northeast-1.anthropic.model", False)
|
||||
|
||||
def test_region_candidate_also_in_suffix_names_preserves_fallback_price_variants(self) -> None:
|
||||
self._assert_fallback_price_variants(
|
||||
"anthropic.model-2026-01-01", "bedrock/converse/us.anthropic.model-2026-01-01", True
|
||||
)
|
||||
|
||||
def _assert_fallback_price_variants(self, source: str, key: str, names_the_row: bool) -> None:
|
||||
row: Final = {
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
"input_cost_per_token_above_128k_tokens": 0.01,
|
||||
"tiered_pricing": [
|
||||
{"input_cost_per_token": 0.003, "output_cost_per_token": 0.004, "range": [0, 256_000]},
|
||||
{"input_cost_per_token": 0.005, "output_cost_per_token": 0.006, "range": [256_000, 1_000_000]},
|
||||
],
|
||||
}
|
||||
rows: Final[dict[str, object]] = {source: row}
|
||||
override: Final = {"input_cost_per_token": 0.007}
|
||||
before: Final = json.dumps(rows)
|
||||
with patch.object(litellm, "model_cost", rows):
|
||||
legacy: Final = _resolve_builtin_model_cost_entry(key, "bedrock")
|
||||
self.assertIsNotNone(legacy)
|
||||
if legacy is None:
|
||||
self.fail("synthetic source must match the M2 reference")
|
||||
self.assertEqual((legacy.key, legacy.names_the_row), (source, names_the_row))
|
||||
fallback: Final = _fallback_registration(key, "bedrock", override)
|
||||
self.assertTrue(fallback.matched_builtin)
|
||||
self.assertEqual("input_cost_per_token_above_128k_tokens" in fallback.entry, names_the_row)
|
||||
target: Final = native.registration_target(rows, key, provider="bedrock")
|
||||
self.assertEqual((target.write, target.inherit_from, target.names_the_row), (key, source, names_the_row))
|
||||
inherited: Final = row if target.names_the_row else _without_overridden_price_variants(row, override)
|
||||
self.assertEqual(
|
||||
fallback.entry, {name: value for name, value in inherited.items() if name != "litellm_provider"}
|
||||
)
|
||||
self.assertEqual(json.dumps(rows), before)
|
||||
|
||||
def test_row_name_metadata_is_read_only_and_false_without_inheritance(self) -> None:
|
||||
rows: Final[dict[str, object]] = {"same": {"litellm_provider": "openai"}}
|
||||
target: Final = native.registration_target(rows, "same")
|
||||
self.assertIs(target.names_the_row, True)
|
||||
self.assertRaises(AttributeError, setattr, target, "names_the_row", False)
|
||||
self.assertIs(target.names_the_row, True)
|
||||
self.assertIs(native.registration_target(rows, "absent").names_the_row, False)
|
||||
self.assertIs(native.registration_target(rows, "same", provider="azure").names_the_row, False)
|
||||
|
||||
def test_repeated_prefix_inherits_prices_without_expanding_read_resolution(self) -> None:
|
||||
self._assert_inherits_live_prices("us.anthropic.model", "bedrock/bedrock/bedrock/us.anthropic.model")
|
||||
|
|
@ -120,6 +172,8 @@ class RegistrationTargetTests(unittest.TestCase):
|
|||
explicit: Final = native.registration_target(rows, "same", provider=None)
|
||||
self.assertEqual((old.write, old.inherit_from), ("same", "same"))
|
||||
self.assertEqual((explicit.write, explicit.inherit_from), (old.write, old.inherit_from))
|
||||
self.assertIs(old.names_the_row, True)
|
||||
self.assertIs(explicit.names_the_row, True)
|
||||
provider: Final = inspect.signature(native.registration_target).parameters["provider"]
|
||||
self.assertEqual(provider.kind, inspect.Parameter.KEYWORD_ONLY)
|
||||
self.assertIsNone(provider.default)
|
||||
|
|
|
|||
|
|
@ -154,6 +154,8 @@ class RegistrationTarget:
|
|||
def write(self) -> str: ...
|
||||
@property
|
||||
def inherit_from(self) -> str | None: ...
|
||||
@property
|
||||
def names_the_row(self) -> bool: ...
|
||||
|
||||
@final
|
||||
class RowErrors:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue