feat(catalog): separate inferred model-info resolution intent

This commit is contained in:
Devin AI 2026-10-26 06:44:17 +00:00
parent 69aef480f4
commit 272da08170
4 changed files with 458 additions and 21 deletions

View file

@ -44,6 +44,78 @@ pub fn resolve(keys: &impl Keys, query: &Query<'_>) -> Resolved {
resolve_inner(keys, query, true)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct InferredProvider<'a> {
pub model: &'a str,
pub provider: Option<&'a str>,
}
pub fn resolve_inferred(
keys: &impl Keys,
query: &Query<'_>,
inferred: InferredProvider<'_>,
) -> Resolved {
if query.provider.is_some_and(|provider| !provider.is_empty()) {
return resolve(keys, query);
}
let model = query.base_model.unwrap_or(query.model);
let provider = inferred.provider.filter(|provider| !provider.is_empty());
let mut lookup = Lookup {
keys,
provider,
case_insensitive: true,
tried: Vec::new(),
};
let exact_via = if query.base_model.is_some() {
Via::BaseModel
} else {
Via::Exact
};
if let Some(found) = lookup.candidate(model, exact_via) {
return found;
}
let split = if matches!(provider, Some("bedrock" | "bedrock_converse")) {
strip_bedrock_nova_route(bedrock_routing_model(inferred.model))
} else {
inferred.model
};
let split = if provider == Some("bedrock_mantle") {
strip_region(split, provider, query.region)
} else {
split
};
if provider == Some("bedrock_mantle")
&& let Some(found) =
lookup.candidate(&format!("bedrock_mantle/{split}"), Via::RegionStripped)
{
return found;
}
if let Some(found) = lookup.candidate(split, Via::ProviderStripped) {
return found;
}
if provider == Some("bedrock_mantle")
&& let Some(found) = lookup.candidate(
&format!("bedrock_mantle/{}", strip_suffix(split, provider)),
Via::ProviderSuffixStripped,
)
{
return found;
}
if let Some(found) =
lookup.candidate(registration_stripped(model, provider), Via::SuffixStripped)
{
return found;
}
if provider == Some("fireworks_ai")
&& let Some(found) = lookup.candidate(&fireworks_resource(split), Via::ProviderResource)
{
return found;
}
Resolved::NotFound {
tried: lookup.tried,
}
}
pub fn registration_target(keys: &impl Keys, caller_key: &str) -> RegistrationTarget {
registration_target_with_provider(keys, caller_key, None)
}
@ -546,6 +618,144 @@ mod tests {
}
}
#[rstest]
#[case::bare("model", "model", "openai", json!({"model": {}, "openai/model": {}}), "model")]
#[case::rejected_exact("azure/model", "model", "azure", json!({"azure/model": {"litellm_provider": "openai"}, "model": {"litellm_provider": "azure"}}), "model")]
#[case::dated_bare("model-2026-01-01", "model-2026-01-01", "openai", json!({"model": {}, "openai/model": {}}), "model")]
#[case::dated_prefixed("openai/model-2026-01-01", "model-2026-01-01", "openai", json!({"model-2026-01-01": {}, "openai/model": {}}), "model-2026-01-01")]
#[case::dated_original("openai/model-2026-01-01", "model-2026-01-01", "openai", json!({"model": {}, "openai/model": {}}), "openai/model")]
#[case::numeric_suffix("model-001", "model-001", "vertex_ai", json!({"model": {}, "vertex_ai/model": {}}), "model")]
#[case::regional("bedrock_mantle/us-east-1/model", "us-east-1/model", "bedrock_mantle", json!({"bedrock_mantle/model": {}, "model": {}}), "bedrock_mantle/model")]
#[case::regional_suffix("bedrock_mantle/us-east-1/model-2026-01-01", "us-east-1/model-2026-01-01", "bedrock_mantle", json!({"bedrock_mantle/model": {}, "model": {}}), "bedrock_mantle/model")]
#[case::route("bedrock/converse/anthropic.model", "converse/anthropic.model", "bedrock", json!({"anthropic.model": {}, "bedrock/anthropic.model": {}}), "anthropic.model")]
#[case::bedrock_region("bedrock/us-east-1/anthropic.model", "us-east-1/anthropic.model", "bedrock", json!({"anthropic.model": {}, "bedrock/anthropic.model": {}}), "anthropic.model")]
#[case::arn("bedrock/arn:aws:bedrock:us-east-1:123:inference-profile/us.anthropic.model", "arn:aws:bedrock:us-east-1:123:inference-profile/us.anthropic.model", "bedrock", json!({"us.anthropic.model": {}, "anthropic.model": {}}), "us.anthropic.model")]
#[case::bedrock_geo("bedrock/us.anthropic.model", "us.anthropic.model", "bedrock", json!({"anthropic.model": {}, "bedrock/anthropic.model": {}}), "anthropic.model")]
#[case::nova("bedrock/nova-2/arn:aws:bedrock:us-east-1:123:custom-model/a", "nova-2/arn:aws:bedrock:us-east-1:123:custom-model/a", "bedrock", json!({"amazon.nova-2-custom": {}}), "amazon.nova-2-custom")]
#[case::lowercase("MODEL", "MODEL", "openai", json!({"Model": {}, "openai/MODEL": {}}), "Model")]
#[case::lowercase_mismatch("azure/MODEL", "MODEL", "azure", json!({"azure/Model": {"litellm_provider": "openai"}, "Model": {"litellm_provider": "azure"}}), "Model")]
#[case::wildcard("model", "model", "azure", json!({"model": {"litellm_provider": null}}), "model")]
#[case::azure_ai("model", "model", "azure_ai", json!({"model": {"litellm_provider": "openai"}}), "model")]
#[case::family("model", "model", "vertex_ai", json!({"model": {"litellm_provider": "vertex_ai-language-models"}}), "model")]
#[case::resource("model", "model", "fireworks_ai", json!({"fireworks_ai/accounts/fireworks/models/model": {}}), "fireworks_ai/accounts/fireworks/models/model")]
fn inferred_provider_preserves_model_info_priority(
#[case] model: &str,
#[case] normalized: &str,
#[case] provider: &str,
#[case] rows: Value,
#[case] expected: &str,
) {
let result = resolve_inferred(
&catalog(rows),
&query(model, None),
InferredProvider {
model: normalized,
provider: Some(provider),
},
);
let Resolved::Found { key, .. } = result else {
panic!("{result:?}")
};
assert_eq!(key, expected);
}
#[rstest]
#[case::mismatch("j2-light", "j2-light", Some("ai21_chat"), json!({"j2-light": {"litellm_provider": "ai21"}}), vec!["j2-light"])]
#[case::normalized_retains_prefix("azure/command-r-plus", "azure/command-r-plus", Some("openai"), json!({"command-r-plus": {"litellm_provider": "openai"}}), vec!["azure/command-r-plus"])]
#[case::inference_failed("unknown/model", "unknown/model", None, json!({"model": {}}), vec!["unknown/model"])]
#[case::inference_empty("unknown/model", "unknown/model", Some(""), json!({"model": {}}), vec!["unknown/model"])]
#[case::suffix_not_normalized("openai/model-2026-01-01", "model-2026-01-01", Some("openai"), json!({"model": {}}), vec!["openai/model-2026-01-01", "model-2026-01-01", "openai/model"])]
#[case::unqualified_suffix("model-2026-01-01", "model-2026-01-01", Some("openai"), json!({"openai/model": {}}), vec!["model-2026-01-01", "model"])]
fn inferred_misses_preserve_candidate_order(
#[case] model: &str,
#[case] normalized: &str,
#[case] provider: Option<&str>,
#[case] rows: Value,
#[case] tried: Vec<&str>,
) {
assert_eq!(
resolve_inferred(
&catalog(rows),
&query(model, None),
InferredProvider {
model: normalized,
provider
}
),
Resolved::NotFound {
tried: tried.into_iter().map(str::to_owned).collect()
},
);
}
#[rstest]
#[case::explicit(Some("openai"), "openai/model")]
#[case::empty(Some(""), "model")]
#[case::absent(None, "model")]
fn explicit_nonempty_provider_wins(#[case] provider: Option<&str>, #[case] expected: &str) {
let models = catalog(
json!({"model": {"litellm_provider": "azure"}, "openai/model": {"litellm_provider": "openai"}}),
);
let result = resolve_inferred(
&models,
&query("model", provider),
InferredProvider {
model: "model",
provider: Some("azure"),
},
);
let Resolved::Found { key, .. } = result else {
panic!("{result:?}")
};
assert_eq!(key, expected);
}
#[rstest]
#[case::base(None, "base", Some("openai"), "base", Via::BaseModel)]
#[case::region(
None,
"custom-region/base",
Some("bedrock_mantle"),
"bedrock_mantle/base",
Via::RegionStripped
)]
#[case::explicit_region(
Some("openai"),
"ignored",
Some("azure"),
"openai/custom-region/base",
Via::Region
)]
fn inferred_base_and_region(
#[case] explicit: Option<&str>,
#[case] normalized: &str,
#[case] provider: Option<&str>,
#[case] key: &str,
#[case] via: Via,
) {
let models = catalog(json!({key: {}}));
let query = Query {
model: "deployment",
provider: explicit,
base_model: Some("base"),
region: Some("custom-region"),
};
assert_eq!(
resolve_inferred(
&models,
&query,
InferredProvider {
model: normalized,
provider
}
),
Resolved::Found {
key: key.into(),
via
}
);
}
#[fixture]
fn registration_models() -> Catalog {
catalog(json!({

View file

@ -173,7 +173,7 @@ pub(crate) enum ResolutionResult {
NotFound(ModelNotFound),
}
#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None))]
#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None, inferred=None))]
pub(crate) fn resolve_model(
module: &Bound<'_, PyModule>,
model_cost: &Bound<'_, PyDict>,
@ -181,6 +181,7 @@ pub(crate) fn resolve_model(
provider: Option<&str>,
base_model: Option<&str>,
region: Option<&str>,
inferred: Option<(&str, Option<&str>)>,
) -> PyResult<ResolutionResult> {
let index = module.getattr("_model_key_index")?;
let index = index.extract::<PyRef<'_, ModelKeyIndex>>()?;
@ -189,15 +190,20 @@ pub(crate) fn resolve_model(
index: &index,
error: RefCell::default(),
};
let result = resolution::resolve(
&view,
&Query {
model,
provider,
base_model,
region,
},
);
let query = Query {
model,
provider,
base_model,
region,
};
let result = match inferred {
Some((model, provider)) => resolution::resolve_inferred(
&view,
&query,
resolution::InferredProvider { model, provider },
),
None => resolution::resolve(&view, &query),
};
view.finish()?;
Ok(match result {
Resolved::Found { key, via } => ResolutionResult::Found(ResolvedModel {
@ -785,7 +791,7 @@ mod tests {
}),
);
let ResolutionResult::Found(found) =
resolve_model(&module(py), &rows, model, provider, base, region).unwrap()
resolve_model(&module(py), &rows, model, provider, base, region, None).unwrap()
else {
panic!("not found")
};
@ -879,7 +885,7 @@ mod tests {
Python::attach(|py| {
let rows = dict(py, json!({key: {"litellm_provider": provider}}));
let ResolutionResult::Found(found) =
resolve_model(&module(py), &rows, model, Some(provider), None, None).unwrap()
resolve_model(&module(py), &rows, model, Some(provider), None, None, None).unwrap()
else {
panic!("not found")
};
@ -908,7 +914,16 @@ mod tests {
let rows = dict(py, json!({"model": {"litellm_provider": actual}}));
assert_eq!(
matches!(
resolve_model(&module(py), &rows, "model", Some(provider), None, None).unwrap(),
resolve_model(
&module(py),
&rows,
"model",
Some(provider),
None,
None,
None
)
.unwrap(),
ResolutionResult::Found(_)
),
found
@ -968,7 +983,7 @@ mod tests {
Python::attach(|py| {
let rows = dict(py, json!({"model": {}}));
let ResolutionResult::NotFound(missing) =
resolve_model(&module(py), &rows, model, None, None, None).unwrap()
resolve_model(&module(py), &rows, model, None, None, None, None).unwrap()
else {
panic!("unexpected match")
};
@ -1019,7 +1034,7 @@ assert n.price(row, usage, when).total == 21.0
let module = module(py);
let rows = dict(py, json!({"Model": {}}));
let resolve = |rows: &Bound<'_, PyDict>, key| match resolve_model(
&module, rows, key, None, None, None,
&module, rows, key, None, None, None, None,
)
.unwrap()
{
@ -1180,7 +1195,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
json!({"custom/Model": {}, "custom/model": {}, "MODEL": {}}),
);
let ResolutionResult::Found(found) =
resolve_model(&module(py), &rows, model, provider, None, None).unwrap()
resolve_model(&module(py), &rows, model, provider, None, None, None).unwrap()
else {
panic!("not found")
};
@ -1188,6 +1203,47 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
});
}
#[rstest]
#[case::bare("read-model", "read-model", "openai")]
#[case::mismatch("azure/read-model", "read-model", "azure")]
fn inferred_intent_is_bound_without_mutating_rows(
#[from(interpreter)] _interpreter: (),
#[case] model: &str,
#[case] normalized: &str,
#[case] provider: &str,
) {
Python::attach(|py| {
let module = crate::native_module(py);
let rows = dict(
py,
json!({
"read-model": {"litellm_provider": provider, "supports_vision": true},
"openai/read-model": {"litellm_provider": "openai", "supports_vision": false},
"azure/read-model": {"litellm_provider": "openai", "supports_vision": false}
}),
);
let locals = PyDict::new(py);
locals.set_item("n", &module).unwrap();
locals.set_item("rows", &rows).unwrap();
locals.set_item("model", model).unwrap();
locals.set_item("inferred", (normalized, provider)).unwrap();
py.run(
cr#"
before = tuple((key, id(row), dict(row)) for key, row in rows.items())
result = n.resolve_model(rows, model, inferred=inferred)
assert result.key == "read-model"
assert n.model_info(rows, result.key).capabilities["supports_vision"] is True
assert tuple((key, id(row), dict(row)) for key, row in rows.items()) == before
rows["read-model"]["litellm_provider"] = "wrong-scope"
assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFound)
"#,
Some(&locals),
None,
)
.unwrap();
});
}
#[rstest]
fn base_misses_and_provider_fallbacks_preserve_order(#[from(interpreter)] _interpreter: ()) {
Python::attach(|py| {
@ -1196,14 +1252,21 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
py,
json!({"openai/model": {"litellm_provider":"anthropic"}, "model": {"litellm_provider":"openai"}}),
);
let ResolutionResult::NotFound(missing) =
resolve_model(&module, &rows, "model", None, Some("missing-base"), None).unwrap()
else {
let ResolutionResult::NotFound(missing) = resolve_model(
&module,
&rows,
"model",
None,
Some("missing-base"),
None,
None,
)
.unwrap() else {
panic!("unexpected base fallback")
};
assert_eq!(missing.tried, ["missing-base"]);
let ResolutionResult::Found(found) =
resolve_model(&module, &rows, "model", Some("openai"), None, None).unwrap()
resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap()
else {
panic!("not found")
};
@ -1214,7 +1277,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
.set_item("litellm_provider", "openai")
.unwrap();
let ResolutionResult::Found(found) =
resolve_model(&module, &rows, "model", Some("openai"), None, None).unwrap()
resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap()
else {
panic!("not found")
};
@ -1233,6 +1296,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
Some("databricks"),
None,
None,
None,
)
.unwrap() else {
panic!("unexpected match")

View file

@ -0,0 +1,162 @@
from __future__ import annotations
import inspect
import unittest
from typing import Final
from unittest.mock import patch
import litellm
from litellm import utils
from litellm.rust_bridge import _native as native
class ModelInfoResolutionTests(unittest.TestCase):
def test_inferred_reads_match_m3_selection(self) -> None:
cases: Final = (
("read-model", "read-model", "openai", "read-model", "openai/read-model"),
("read-model-2026-01-01", "read-model-2026-01-01", "openai", "read-model", "openai/read-model"),
("azure/read-model", "read-model", "azure", "read-model", "azure/read-model"),
("azure/READ-MODEL", "READ-MODEL", "azure", "read-model", "azure/READ-MODEL"),
("openai/read-model-2026-01-01", "read-model-2026-01-01", "openai", "openai/read-model", "read-model"),
(
"bedrock/converse/anthropic.read-model",
"converse/anthropic.read-model",
"bedrock",
"anthropic.read-model",
"bedrock/anthropic.read-model",
),
(
"bedrock/us.anthropic.read-model",
"us.anthropic.read-model",
"bedrock",
"anthropic.read-model",
"bedrock/anthropic.read-model",
),
(
"bedrock_mantle/us-east-1/read-model",
"read-model",
"bedrock_mantle",
"bedrock_mantle/read-model",
"read-model",
),
("vertex_ai/read-model-001", "read-model-001", "vertex_ai", "vertex_ai/read-model", "read-model"),
)
for model, normalized, provider, expected, other in cases:
with self.subTest(model=model):
self.assert_m3_selection(model, normalized, provider, expected, other)
def assert_m3_selection(self, model: str, normalized: str, provider: str, expected: str, other: str) -> None:
rows: Final[dict[str, object]] = {
expected: {"litellm_provider": provider, "mode": "chat", "supports_vision": True},
other: {
"litellm_provider": "openai" if provider == "azure" else provider,
"mode": "chat",
"supports_vision": False,
},
}
with (
patch.object(litellm, "model_cost", rows),
patch.object(litellm, "open_ai_chat_completion_models", [model]),
patch.object(utils, "_model_cost_lowercase_map", None),
):
native.invalidate_model_cost_index()
routed_model, routed_provider, _, _ = litellm.get_llm_provider(model)
self.assertEqual((routed_model, routed_provider), (normalized, provider))
legacy: Final = utils.get_model_info_helper(model)
selected: Final = native.resolve_model(rows, model, inferred=(routed_model, routed_provider))
self.assertIsInstance(selected, native.ResolvedModel)
if not isinstance(selected, native.ResolvedModel):
self.fail(str(selected.tried))
self.assertEqual((legacy["key"], selected.key), (expected, expected))
lowered: Final = native.model_info(rows, selected.key)
self.assertIsInstance(lowered, native.ModelInfoView)
if not isinstance(lowered, native.ModelInfoView):
self.fail(str(lowered.errors))
self.assertEqual((legacy.get("supports_vision"), lowered.capabilities["supports_vision"]), (True, True))
def test_host_normalization_can_preserve_azure_prefix(self) -> None:
model: Final = "azure/command-r-plus"
rows: Final[dict[str, object]] = {"command-r-plus": {"litellm_provider": "openai", "mode": "chat"}}
with (
patch.object(litellm, "model_cost", rows),
patch.object(litellm, "cohere_chat_models", ["command-r-plus"]),
patch.object(utils, "_model_cost_lowercase_map", None),
):
normalized, provider, _, _ = litellm.get_llm_provider(model)
self.assertEqual((normalized, provider), (model, "openai"))
self.assertEqual(utils.get_potential_model_names(model, None)["split_model"], model)
with self.assertRaisesRegex(Exception, "isn't mapped yet"):
utils.get_model_info_helper(model)
selected: Final = native.resolve_model(rows, model, inferred=(normalized, provider))
self.assertIsInstance(selected, native.ModelNotFound)
if not isinstance(selected, native.ModelNotFound):
self.fail(selected.key)
self.assertEqual(selected.tried, [model])
def test_provider_mismatch_is_not_accepted(self) -> None:
rows: Final[dict[str, object]] = {"j2-light": {"litellm_provider": "ai21", "mode": "chat"}}
with (
patch.object(litellm, "model_cost", rows),
patch.object(litellm, "ai21_chat_models", ["j2-light"]),
patch.object(utils, "_model_cost_lowercase_map", None),
):
normalized, provider, _, _ = litellm.get_llm_provider("j2-light")
self.assertEqual(provider, "ai21_chat")
with self.assertRaisesRegex(Exception, "isn't mapped yet"):
utils.get_model_info_helper("j2-light")
self.assertIsInstance(
native.resolve_model(rows, "j2-light", inferred=(normalized, provider)), native.ModelNotFound
)
self.assertIsInstance(native.resolve_model(rows, "j2-light"), native.ResolvedModel)
def test_legacy_calls_and_registration_do_not_inherit_read_intent(self) -> None:
rows: Final[dict[str, object]] = {
"read-model": {"litellm_provider": "azure"},
"azure/read-model": {"litellm_provider": "openai"},
"openai/read-model": {"litellm_provider": "openai"},
}
cases: Final = (
("read-model", None, None, "read-model"),
("azure/read-model", None, None, "azure/read-model"),
("read-model", "openai", ("ignored", "azure"), "openai/read-model"),
("read-model", "", ("read-model", "azure"), "read-model"),
("azure/read-model", None, ("read-model", "azure"), "read-model"),
)
before: Final = native.registration_target(rows, "azure/read-model", provider="azure")
for model, provider, inferred, expected in cases:
with self.subTest(model=model, provider=provider, inferred=inferred):
if not isinstance(
selected := native.resolve_model(rows, model, provider=provider, inferred=inferred),
native.ResolvedModel,
):
self.fail(str(selected.tried))
self.assertEqual(selected.key, expected)
after: Final = native.registration_target(rows, "azure/read-model", provider="azure")
self.assertEqual(
(before.write, before.inherit_from, before.names_the_row),
(after.write, after.inherit_from, after.names_the_row),
)
self.assertEqual(
(after.write, after.inherit_from, after.names_the_row), ("azure/read-model", "read-model", True)
)
def test_failed_inference_is_distinct_from_omitting_inference(self) -> None:
rows: Final[dict[str, object]] = {"read-model": {}}
self.assertIsInstance(native.resolve_model(rows, "unknown/read-model"), native.ResolvedModel)
selected: Final = native.resolve_model(rows, "unknown/read-model", inferred=("unknown/read-model", None))
self.assertIsInstance(selected, native.ModelNotFound)
if not isinstance(selected, native.ModelNotFound):
self.fail(selected.key)
self.assertEqual(selected.tried, ["unknown/read-model"])
def test_inferred_binding_shape(self) -> None:
inferred: Final = inspect.signature(native.resolve_model).parameters["inferred"]
self.assertEqual(inferred.kind, inspect.Parameter.KEYWORD_ONLY)
self.assertIn("inferred=None", str(inspect.signature(native.resolve_model)))
for invalid in ("openai", ("model",), ("model", "openai", None), (None, "openai"), ("model", 123)):
with self.subTest(invalid=invalid):
self.assertRaises((TypeError, ValueError), native.resolve_model, {}, "model", inferred=invalid)
if __name__ == "__main__":
unittest.main()

View file

@ -248,6 +248,7 @@ def resolve_model(
provider: str | None = None,
base_model: str | None = None,
region: str | None = None,
inferred: tuple[str, str | None] | None = None,
) -> ResolvedModel | ModelNotFound: ...
def registration_target(
model_cost: dict[str, object], key: str, *, provider: str | None = None