diff --git a/litellm-rust/crates/model-catalog/src/resolution.rs b/litellm-rust/crates/model-catalog/src/resolution.rs index 17585758cea..d3952840ce6 100644 --- a/litellm-rust/crates/model-catalog/src/resolution.rs +++ b/litellm-rust/crates/model-catalog/src/resolution.rs @@ -44,6 +44,78 @@ pub fn resolve(keys: &impl Keys, query: &Query<'_>) -> Resolved { resolve_inner(keys, query, true) } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InferredProvider<'a> { + pub model: &'a str, + pub provider: Option<&'a str>, +} + +pub fn resolve_inferred( + keys: &impl Keys, + query: &Query<'_>, + inferred: InferredProvider<'_>, +) -> Resolved { + if query.provider.is_some_and(|provider| !provider.is_empty()) { + return resolve(keys, query); + } + let model = query.base_model.unwrap_or(query.model); + let provider = inferred.provider.filter(|provider| !provider.is_empty()); + let mut lookup = Lookup { + keys, + provider, + case_insensitive: true, + tried: Vec::new(), + }; + let exact_via = if query.base_model.is_some() { + Via::BaseModel + } else { + Via::Exact + }; + if let Some(found) = lookup.candidate(model, exact_via) { + return found; + } + let split = if matches!(provider, Some("bedrock" | "bedrock_converse")) { + strip_bedrock_nova_route(bedrock_routing_model(inferred.model)) + } else { + inferred.model + }; + let split = if provider == Some("bedrock_mantle") { + strip_region(split, provider, query.region) + } else { + split + }; + if provider == Some("bedrock_mantle") + && let Some(found) = + lookup.candidate(&format!("bedrock_mantle/{split}"), Via::RegionStripped) + { + return found; + } + if let Some(found) = lookup.candidate(split, Via::ProviderStripped) { + return found; + } + if provider == Some("bedrock_mantle") + && let Some(found) = lookup.candidate( + &format!("bedrock_mantle/{}", strip_suffix(split, provider)), + Via::ProviderSuffixStripped, + ) + { + return found; + } + if let Some(found) = + lookup.candidate(registration_stripped(model, provider), Via::SuffixStripped) + { + return found; + } + if provider == Some("fireworks_ai") + && let Some(found) = lookup.candidate(&fireworks_resource(split), Via::ProviderResource) + { + return found; + } + Resolved::NotFound { + tried: lookup.tried, + } +} + pub fn registration_target(keys: &impl Keys, caller_key: &str) -> RegistrationTarget { registration_target_with_provider(keys, caller_key, None) } @@ -546,6 +618,144 @@ mod tests { } } + #[rstest] + #[case::bare("model", "model", "openai", json!({"model": {}, "openai/model": {}}), "model")] + #[case::rejected_exact("azure/model", "model", "azure", json!({"azure/model": {"litellm_provider": "openai"}, "model": {"litellm_provider": "azure"}}), "model")] + #[case::dated_bare("model-2026-01-01", "model-2026-01-01", "openai", json!({"model": {}, "openai/model": {}}), "model")] + #[case::dated_prefixed("openai/model-2026-01-01", "model-2026-01-01", "openai", json!({"model-2026-01-01": {}, "openai/model": {}}), "model-2026-01-01")] + #[case::dated_original("openai/model-2026-01-01", "model-2026-01-01", "openai", json!({"model": {}, "openai/model": {}}), "openai/model")] + #[case::numeric_suffix("model-001", "model-001", "vertex_ai", json!({"model": {}, "vertex_ai/model": {}}), "model")] + #[case::regional("bedrock_mantle/us-east-1/model", "us-east-1/model", "bedrock_mantle", json!({"bedrock_mantle/model": {}, "model": {}}), "bedrock_mantle/model")] + #[case::regional_suffix("bedrock_mantle/us-east-1/model-2026-01-01", "us-east-1/model-2026-01-01", "bedrock_mantle", json!({"bedrock_mantle/model": {}, "model": {}}), "bedrock_mantle/model")] + #[case::route("bedrock/converse/anthropic.model", "converse/anthropic.model", "bedrock", json!({"anthropic.model": {}, "bedrock/anthropic.model": {}}), "anthropic.model")] + #[case::bedrock_region("bedrock/us-east-1/anthropic.model", "us-east-1/anthropic.model", "bedrock", json!({"anthropic.model": {}, "bedrock/anthropic.model": {}}), "anthropic.model")] + #[case::arn("bedrock/arn:aws:bedrock:us-east-1:123:inference-profile/us.anthropic.model", "arn:aws:bedrock:us-east-1:123:inference-profile/us.anthropic.model", "bedrock", json!({"us.anthropic.model": {}, "anthropic.model": {}}), "us.anthropic.model")] + #[case::bedrock_geo("bedrock/us.anthropic.model", "us.anthropic.model", "bedrock", json!({"anthropic.model": {}, "bedrock/anthropic.model": {}}), "anthropic.model")] + #[case::nova("bedrock/nova-2/arn:aws:bedrock:us-east-1:123:custom-model/a", "nova-2/arn:aws:bedrock:us-east-1:123:custom-model/a", "bedrock", json!({"amazon.nova-2-custom": {}}), "amazon.nova-2-custom")] + #[case::lowercase("MODEL", "MODEL", "openai", json!({"Model": {}, "openai/MODEL": {}}), "Model")] + #[case::lowercase_mismatch("azure/MODEL", "MODEL", "azure", json!({"azure/Model": {"litellm_provider": "openai"}, "Model": {"litellm_provider": "azure"}}), "Model")] + #[case::wildcard("model", "model", "azure", json!({"model": {"litellm_provider": null}}), "model")] + #[case::azure_ai("model", "model", "azure_ai", json!({"model": {"litellm_provider": "openai"}}), "model")] + #[case::family("model", "model", "vertex_ai", json!({"model": {"litellm_provider": "vertex_ai-language-models"}}), "model")] + #[case::resource("model", "model", "fireworks_ai", json!({"fireworks_ai/accounts/fireworks/models/model": {}}), "fireworks_ai/accounts/fireworks/models/model")] + fn inferred_provider_preserves_model_info_priority( + #[case] model: &str, + #[case] normalized: &str, + #[case] provider: &str, + #[case] rows: Value, + #[case] expected: &str, + ) { + let result = resolve_inferred( + &catalog(rows), + &query(model, None), + InferredProvider { + model: normalized, + provider: Some(provider), + }, + ); + let Resolved::Found { key, .. } = result else { + panic!("{result:?}") + }; + assert_eq!(key, expected); + } + + #[rstest] + #[case::mismatch("j2-light", "j2-light", Some("ai21_chat"), json!({"j2-light": {"litellm_provider": "ai21"}}), vec!["j2-light"])] + #[case::normalized_retains_prefix("azure/command-r-plus", "azure/command-r-plus", Some("openai"), json!({"command-r-plus": {"litellm_provider": "openai"}}), vec!["azure/command-r-plus"])] + #[case::inference_failed("unknown/model", "unknown/model", None, json!({"model": {}}), vec!["unknown/model"])] + #[case::inference_empty("unknown/model", "unknown/model", Some(""), json!({"model": {}}), vec!["unknown/model"])] + #[case::suffix_not_normalized("openai/model-2026-01-01", "model-2026-01-01", Some("openai"), json!({"model": {}}), vec!["openai/model-2026-01-01", "model-2026-01-01", "openai/model"])] + #[case::unqualified_suffix("model-2026-01-01", "model-2026-01-01", Some("openai"), json!({"openai/model": {}}), vec!["model-2026-01-01", "model"])] + fn inferred_misses_preserve_candidate_order( + #[case] model: &str, + #[case] normalized: &str, + #[case] provider: Option<&str>, + #[case] rows: Value, + #[case] tried: Vec<&str>, + ) { + assert_eq!( + resolve_inferred( + &catalog(rows), + &query(model, None), + InferredProvider { + model: normalized, + provider + } + ), + Resolved::NotFound { + tried: tried.into_iter().map(str::to_owned).collect() + }, + ); + } + + #[rstest] + #[case::explicit(Some("openai"), "openai/model")] + #[case::empty(Some(""), "model")] + #[case::absent(None, "model")] + fn explicit_nonempty_provider_wins(#[case] provider: Option<&str>, #[case] expected: &str) { + let models = catalog( + json!({"model": {"litellm_provider": "azure"}, "openai/model": {"litellm_provider": "openai"}}), + ); + let result = resolve_inferred( + &models, + &query("model", provider), + InferredProvider { + model: "model", + provider: Some("azure"), + }, + ); + let Resolved::Found { key, .. } = result else { + panic!("{result:?}") + }; + assert_eq!(key, expected); + } + + #[rstest] + #[case::base(None, "base", Some("openai"), "base", Via::BaseModel)] + #[case::region( + None, + "custom-region/base", + Some("bedrock_mantle"), + "bedrock_mantle/base", + Via::RegionStripped + )] + #[case::explicit_region( + Some("openai"), + "ignored", + Some("azure"), + "openai/custom-region/base", + Via::Region + )] + fn inferred_base_and_region( + #[case] explicit: Option<&str>, + #[case] normalized: &str, + #[case] provider: Option<&str>, + #[case] key: &str, + #[case] via: Via, + ) { + let models = catalog(json!({key: {}})); + let query = Query { + model: "deployment", + provider: explicit, + base_model: Some("base"), + region: Some("custom-region"), + }; + assert_eq!( + resolve_inferred( + &models, + &query, + InferredProvider { + model: normalized, + provider + } + ), + Resolved::Found { + key: key.into(), + via + } + ); + } + #[fixture] fn registration_models() -> Catalog { catalog(json!({ diff --git a/litellm-rust/crates/python-bridge/src/model_registry/live.rs b/litellm-rust/crates/python-bridge/src/model_registry/live.rs index df4541df39d..4ec668147a6 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry/live.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry/live.rs @@ -173,7 +173,7 @@ pub(crate) enum ResolutionResult { NotFound(ModelNotFound), } -#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None))] +#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None, inferred=None))] pub(crate) fn resolve_model( module: &Bound<'_, PyModule>, model_cost: &Bound<'_, PyDict>, @@ -181,6 +181,7 @@ pub(crate) fn resolve_model( provider: Option<&str>, base_model: Option<&str>, region: Option<&str>, + inferred: Option<(&str, Option<&str>)>, ) -> PyResult { let index = module.getattr("_model_key_index")?; let index = index.extract::>()?; @@ -189,15 +190,20 @@ pub(crate) fn resolve_model( index: &index, error: RefCell::default(), }; - let result = resolution::resolve( - &view, - &Query { - model, - provider, - base_model, - region, - }, - ); + let query = Query { + model, + provider, + base_model, + region, + }; + let result = match inferred { + Some((model, provider)) => resolution::resolve_inferred( + &view, + &query, + resolution::InferredProvider { model, provider }, + ), + None => resolution::resolve(&view, &query), + }; view.finish()?; Ok(match result { Resolved::Found { key, via } => ResolutionResult::Found(ResolvedModel { @@ -785,7 +791,7 @@ mod tests { }), ); let ResolutionResult::Found(found) = - resolve_model(&module(py), &rows, model, provider, base, region).unwrap() + resolve_model(&module(py), &rows, model, provider, base, region, None).unwrap() else { panic!("not found") }; @@ -879,7 +885,7 @@ mod tests { Python::attach(|py| { let rows = dict(py, json!({key: {"litellm_provider": provider}})); let ResolutionResult::Found(found) = - resolve_model(&module(py), &rows, model, Some(provider), None, None).unwrap() + resolve_model(&module(py), &rows, model, Some(provider), None, None, None).unwrap() else { panic!("not found") }; @@ -908,7 +914,16 @@ mod tests { let rows = dict(py, json!({"model": {"litellm_provider": actual}})); assert_eq!( matches!( - resolve_model(&module(py), &rows, "model", Some(provider), None, None).unwrap(), + resolve_model( + &module(py), + &rows, + "model", + Some(provider), + None, + None, + None + ) + .unwrap(), ResolutionResult::Found(_) ), found @@ -968,7 +983,7 @@ mod tests { Python::attach(|py| { let rows = dict(py, json!({"model": {}})); let ResolutionResult::NotFound(missing) = - resolve_model(&module(py), &rows, model, None, None, None).unwrap() + resolve_model(&module(py), &rows, model, None, None, None, None).unwrap() else { panic!("unexpected match") }; @@ -1019,7 +1034,7 @@ assert n.price(row, usage, when).total == 21.0 let module = module(py); let rows = dict(py, json!({"Model": {}})); let resolve = |rows: &Bound<'_, PyDict>, key| match resolve_model( - &module, rows, key, None, None, None, + &module, rows, key, None, None, None, None, ) .unwrap() { @@ -1180,7 +1195,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2 json!({"custom/Model": {}, "custom/model": {}, "MODEL": {}}), ); let ResolutionResult::Found(found) = - resolve_model(&module(py), &rows, model, provider, None, None).unwrap() + resolve_model(&module(py), &rows, model, provider, None, None, None).unwrap() else { panic!("not found") }; @@ -1188,6 +1203,47 @@ assert n.model_info(models, "Model").input_cost_per_token == 2 }); } + #[rstest] + #[case::bare("read-model", "read-model", "openai")] + #[case::mismatch("azure/read-model", "read-model", "azure")] + fn inferred_intent_is_bound_without_mutating_rows( + #[from(interpreter)] _interpreter: (), + #[case] model: &str, + #[case] normalized: &str, + #[case] provider: &str, + ) { + Python::attach(|py| { + let module = crate::native_module(py); + let rows = dict( + py, + json!({ + "read-model": {"litellm_provider": provider, "supports_vision": true}, + "openai/read-model": {"litellm_provider": "openai", "supports_vision": false}, + "azure/read-model": {"litellm_provider": "openai", "supports_vision": false} + }), + ); + let locals = PyDict::new(py); + locals.set_item("n", &module).unwrap(); + locals.set_item("rows", &rows).unwrap(); + locals.set_item("model", model).unwrap(); + locals.set_item("inferred", (normalized, provider)).unwrap(); + py.run( + cr#" +before = tuple((key, id(row), dict(row)) for key, row in rows.items()) +result = n.resolve_model(rows, model, inferred=inferred) +assert result.key == "read-model" +assert n.model_info(rows, result.key).capabilities["supports_vision"] is True +assert tuple((key, id(row), dict(row)) for key, row in rows.items()) == before +rows["read-model"]["litellm_provider"] = "wrong-scope" +assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFound) +"#, + Some(&locals), + None, + ) + .unwrap(); + }); + } + #[rstest] fn base_misses_and_provider_fallbacks_preserve_order(#[from(interpreter)] _interpreter: ()) { Python::attach(|py| { @@ -1196,14 +1252,21 @@ assert n.model_info(models, "Model").input_cost_per_token == 2 py, json!({"openai/model": {"litellm_provider":"anthropic"}, "model": {"litellm_provider":"openai"}}), ); - let ResolutionResult::NotFound(missing) = - resolve_model(&module, &rows, "model", None, Some("missing-base"), None).unwrap() - else { + let ResolutionResult::NotFound(missing) = resolve_model( + &module, + &rows, + "model", + None, + Some("missing-base"), + None, + None, + ) + .unwrap() else { panic!("unexpected base fallback") }; assert_eq!(missing.tried, ["missing-base"]); let ResolutionResult::Found(found) = - resolve_model(&module, &rows, "model", Some("openai"), None, None).unwrap() + resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap() else { panic!("not found") }; @@ -1214,7 +1277,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2 .set_item("litellm_provider", "openai") .unwrap(); let ResolutionResult::Found(found) = - resolve_model(&module, &rows, "model", Some("openai"), None, None).unwrap() + resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap() else { panic!("not found") }; @@ -1233,6 +1296,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2 Some("databricks"), None, None, + None, ) .unwrap() else { panic!("unexpected match") diff --git a/litellm-rust/crates/python-bridge/tests/catalog_resolution.py b/litellm-rust/crates/python-bridge/tests/catalog_resolution.py new file mode 100644 index 00000000000..4057d67aa1d --- /dev/null +++ b/litellm-rust/crates/python-bridge/tests/catalog_resolution.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import inspect +import unittest +from typing import Final +from unittest.mock import patch + +import litellm +from litellm import utils +from litellm.rust_bridge import _native as native + + +class ModelInfoResolutionTests(unittest.TestCase): + def test_inferred_reads_match_m3_selection(self) -> None: + cases: Final = ( + ("read-model", "read-model", "openai", "read-model", "openai/read-model"), + ("read-model-2026-01-01", "read-model-2026-01-01", "openai", "read-model", "openai/read-model"), + ("azure/read-model", "read-model", "azure", "read-model", "azure/read-model"), + ("azure/READ-MODEL", "READ-MODEL", "azure", "read-model", "azure/READ-MODEL"), + ("openai/read-model-2026-01-01", "read-model-2026-01-01", "openai", "openai/read-model", "read-model"), + ( + "bedrock/converse/anthropic.read-model", + "converse/anthropic.read-model", + "bedrock", + "anthropic.read-model", + "bedrock/anthropic.read-model", + ), + ( + "bedrock/us.anthropic.read-model", + "us.anthropic.read-model", + "bedrock", + "anthropic.read-model", + "bedrock/anthropic.read-model", + ), + ( + "bedrock_mantle/us-east-1/read-model", + "read-model", + "bedrock_mantle", + "bedrock_mantle/read-model", + "read-model", + ), + ("vertex_ai/read-model-001", "read-model-001", "vertex_ai", "vertex_ai/read-model", "read-model"), + ) + for model, normalized, provider, expected, other in cases: + with self.subTest(model=model): + self.assert_m3_selection(model, normalized, provider, expected, other) + + def assert_m3_selection(self, model: str, normalized: str, provider: str, expected: str, other: str) -> None: + rows: Final[dict[str, object]] = { + expected: {"litellm_provider": provider, "mode": "chat", "supports_vision": True}, + other: { + "litellm_provider": "openai" if provider == "azure" else provider, + "mode": "chat", + "supports_vision": False, + }, + } + with ( + patch.object(litellm, "model_cost", rows), + patch.object(litellm, "open_ai_chat_completion_models", [model]), + patch.object(utils, "_model_cost_lowercase_map", None), + ): + native.invalidate_model_cost_index() + routed_model, routed_provider, _, _ = litellm.get_llm_provider(model) + self.assertEqual((routed_model, routed_provider), (normalized, provider)) + legacy: Final = utils.get_model_info_helper(model) + selected: Final = native.resolve_model(rows, model, inferred=(routed_model, routed_provider)) + self.assertIsInstance(selected, native.ResolvedModel) + if not isinstance(selected, native.ResolvedModel): + self.fail(str(selected.tried)) + self.assertEqual((legacy["key"], selected.key), (expected, expected)) + lowered: Final = native.model_info(rows, selected.key) + self.assertIsInstance(lowered, native.ModelInfoView) + if not isinstance(lowered, native.ModelInfoView): + self.fail(str(lowered.errors)) + self.assertEqual((legacy.get("supports_vision"), lowered.capabilities["supports_vision"]), (True, True)) + + def test_host_normalization_can_preserve_azure_prefix(self) -> None: + model: Final = "azure/command-r-plus" + rows: Final[dict[str, object]] = {"command-r-plus": {"litellm_provider": "openai", "mode": "chat"}} + with ( + patch.object(litellm, "model_cost", rows), + patch.object(litellm, "cohere_chat_models", ["command-r-plus"]), + patch.object(utils, "_model_cost_lowercase_map", None), + ): + normalized, provider, _, _ = litellm.get_llm_provider(model) + self.assertEqual((normalized, provider), (model, "openai")) + self.assertEqual(utils.get_potential_model_names(model, None)["split_model"], model) + with self.assertRaisesRegex(Exception, "isn't mapped yet"): + utils.get_model_info_helper(model) + selected: Final = native.resolve_model(rows, model, inferred=(normalized, provider)) + self.assertIsInstance(selected, native.ModelNotFound) + if not isinstance(selected, native.ModelNotFound): + self.fail(selected.key) + self.assertEqual(selected.tried, [model]) + + def test_provider_mismatch_is_not_accepted(self) -> None: + rows: Final[dict[str, object]] = {"j2-light": {"litellm_provider": "ai21", "mode": "chat"}} + with ( + patch.object(litellm, "model_cost", rows), + patch.object(litellm, "ai21_chat_models", ["j2-light"]), + patch.object(utils, "_model_cost_lowercase_map", None), + ): + normalized, provider, _, _ = litellm.get_llm_provider("j2-light") + self.assertEqual(provider, "ai21_chat") + with self.assertRaisesRegex(Exception, "isn't mapped yet"): + utils.get_model_info_helper("j2-light") + self.assertIsInstance( + native.resolve_model(rows, "j2-light", inferred=(normalized, provider)), native.ModelNotFound + ) + self.assertIsInstance(native.resolve_model(rows, "j2-light"), native.ResolvedModel) + + def test_legacy_calls_and_registration_do_not_inherit_read_intent(self) -> None: + rows: Final[dict[str, object]] = { + "read-model": {"litellm_provider": "azure"}, + "azure/read-model": {"litellm_provider": "openai"}, + "openai/read-model": {"litellm_provider": "openai"}, + } + cases: Final = ( + ("read-model", None, None, "read-model"), + ("azure/read-model", None, None, "azure/read-model"), + ("read-model", "openai", ("ignored", "azure"), "openai/read-model"), + ("read-model", "", ("read-model", "azure"), "read-model"), + ("azure/read-model", None, ("read-model", "azure"), "read-model"), + ) + before: Final = native.registration_target(rows, "azure/read-model", provider="azure") + for model, provider, inferred, expected in cases: + with self.subTest(model=model, provider=provider, inferred=inferred): + if not isinstance( + selected := native.resolve_model(rows, model, provider=provider, inferred=inferred), + native.ResolvedModel, + ): + self.fail(str(selected.tried)) + self.assertEqual(selected.key, expected) + after: Final = native.registration_target(rows, "azure/read-model", provider="azure") + self.assertEqual( + (before.write, before.inherit_from, before.names_the_row), + (after.write, after.inherit_from, after.names_the_row), + ) + self.assertEqual( + (after.write, after.inherit_from, after.names_the_row), ("azure/read-model", "read-model", True) + ) + + def test_failed_inference_is_distinct_from_omitting_inference(self) -> None: + rows: Final[dict[str, object]] = {"read-model": {}} + self.assertIsInstance(native.resolve_model(rows, "unknown/read-model"), native.ResolvedModel) + selected: Final = native.resolve_model(rows, "unknown/read-model", inferred=("unknown/read-model", None)) + self.assertIsInstance(selected, native.ModelNotFound) + if not isinstance(selected, native.ModelNotFound): + self.fail(selected.key) + self.assertEqual(selected.tried, ["unknown/read-model"]) + + def test_inferred_binding_shape(self) -> None: + inferred: Final = inspect.signature(native.resolve_model).parameters["inferred"] + self.assertEqual(inferred.kind, inspect.Parameter.KEYWORD_ONLY) + self.assertIn("inferred=None", str(inspect.signature(native.resolve_model))) + for invalid in ("openai", ("model",), ("model", "openai", None), (None, "openai"), ("model", 123)): + with self.subTest(invalid=invalid): + self.assertRaises((TypeError, ValueError), native.resolve_model, {}, "model", inferred=invalid) + + +if __name__ == "__main__": + unittest.main() diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index c04ffcc6dbe..953d6403390 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -248,6 +248,7 @@ def resolve_model( provider: str | None = None, base_model: str | None = None, region: str | None = None, + inferred: tuple[str, str | None] | None = None, ) -> ResolvedModel | ModelNotFound: ... def registration_target( model_cost: dict[str, object], key: str, *, provider: str | None = None