mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
feat(catalog): separate inferred model-info resolution intent
This commit is contained in:
parent
69aef480f4
commit
272da08170
4 changed files with 458 additions and 21 deletions
|
|
@ -44,6 +44,78 @@ pub fn resolve(keys: &impl Keys, query: &Query<'_>) -> Resolved {
|
|||
resolve_inner(keys, query, true)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct InferredProvider<'a> {
|
||||
pub model: &'a str,
|
||||
pub provider: Option<&'a str>,
|
||||
}
|
||||
|
||||
pub fn resolve_inferred(
|
||||
keys: &impl Keys,
|
||||
query: &Query<'_>,
|
||||
inferred: InferredProvider<'_>,
|
||||
) -> Resolved {
|
||||
if query.provider.is_some_and(|provider| !provider.is_empty()) {
|
||||
return resolve(keys, query);
|
||||
}
|
||||
let model = query.base_model.unwrap_or(query.model);
|
||||
let provider = inferred.provider.filter(|provider| !provider.is_empty());
|
||||
let mut lookup = Lookup {
|
||||
keys,
|
||||
provider,
|
||||
case_insensitive: true,
|
||||
tried: Vec::new(),
|
||||
};
|
||||
let exact_via = if query.base_model.is_some() {
|
||||
Via::BaseModel
|
||||
} else {
|
||||
Via::Exact
|
||||
};
|
||||
if let Some(found) = lookup.candidate(model, exact_via) {
|
||||
return found;
|
||||
}
|
||||
let split = if matches!(provider, Some("bedrock" | "bedrock_converse")) {
|
||||
strip_bedrock_nova_route(bedrock_routing_model(inferred.model))
|
||||
} else {
|
||||
inferred.model
|
||||
};
|
||||
let split = if provider == Some("bedrock_mantle") {
|
||||
strip_region(split, provider, query.region)
|
||||
} else {
|
||||
split
|
||||
};
|
||||
if provider == Some("bedrock_mantle")
|
||||
&& let Some(found) =
|
||||
lookup.candidate(&format!("bedrock_mantle/{split}"), Via::RegionStripped)
|
||||
{
|
||||
return found;
|
||||
}
|
||||
if let Some(found) = lookup.candidate(split, Via::ProviderStripped) {
|
||||
return found;
|
||||
}
|
||||
if provider == Some("bedrock_mantle")
|
||||
&& let Some(found) = lookup.candidate(
|
||||
&format!("bedrock_mantle/{}", strip_suffix(split, provider)),
|
||||
Via::ProviderSuffixStripped,
|
||||
)
|
||||
{
|
||||
return found;
|
||||
}
|
||||
if let Some(found) =
|
||||
lookup.candidate(registration_stripped(model, provider), Via::SuffixStripped)
|
||||
{
|
||||
return found;
|
||||
}
|
||||
if provider == Some("fireworks_ai")
|
||||
&& let Some(found) = lookup.candidate(&fireworks_resource(split), Via::ProviderResource)
|
||||
{
|
||||
return found;
|
||||
}
|
||||
Resolved::NotFound {
|
||||
tried: lookup.tried,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn registration_target(keys: &impl Keys, caller_key: &str) -> RegistrationTarget {
|
||||
registration_target_with_provider(keys, caller_key, None)
|
||||
}
|
||||
|
|
@ -546,6 +618,144 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bare("model", "model", "openai", json!({"model": {}, "openai/model": {}}), "model")]
|
||||
#[case::rejected_exact("azure/model", "model", "azure", json!({"azure/model": {"litellm_provider": "openai"}, "model": {"litellm_provider": "azure"}}), "model")]
|
||||
#[case::dated_bare("model-2026-01-01", "model-2026-01-01", "openai", json!({"model": {}, "openai/model": {}}), "model")]
|
||||
#[case::dated_prefixed("openai/model-2026-01-01", "model-2026-01-01", "openai", json!({"model-2026-01-01": {}, "openai/model": {}}), "model-2026-01-01")]
|
||||
#[case::dated_original("openai/model-2026-01-01", "model-2026-01-01", "openai", json!({"model": {}, "openai/model": {}}), "openai/model")]
|
||||
#[case::numeric_suffix("model-001", "model-001", "vertex_ai", json!({"model": {}, "vertex_ai/model": {}}), "model")]
|
||||
#[case::regional("bedrock_mantle/us-east-1/model", "us-east-1/model", "bedrock_mantle", json!({"bedrock_mantle/model": {}, "model": {}}), "bedrock_mantle/model")]
|
||||
#[case::regional_suffix("bedrock_mantle/us-east-1/model-2026-01-01", "us-east-1/model-2026-01-01", "bedrock_mantle", json!({"bedrock_mantle/model": {}, "model": {}}), "bedrock_mantle/model")]
|
||||
#[case::route("bedrock/converse/anthropic.model", "converse/anthropic.model", "bedrock", json!({"anthropic.model": {}, "bedrock/anthropic.model": {}}), "anthropic.model")]
|
||||
#[case::bedrock_region("bedrock/us-east-1/anthropic.model", "us-east-1/anthropic.model", "bedrock", json!({"anthropic.model": {}, "bedrock/anthropic.model": {}}), "anthropic.model")]
|
||||
#[case::arn("bedrock/arn:aws:bedrock:us-east-1:123:inference-profile/us.anthropic.model", "arn:aws:bedrock:us-east-1:123:inference-profile/us.anthropic.model", "bedrock", json!({"us.anthropic.model": {}, "anthropic.model": {}}), "us.anthropic.model")]
|
||||
#[case::bedrock_geo("bedrock/us.anthropic.model", "us.anthropic.model", "bedrock", json!({"anthropic.model": {}, "bedrock/anthropic.model": {}}), "anthropic.model")]
|
||||
#[case::nova("bedrock/nova-2/arn:aws:bedrock:us-east-1:123:custom-model/a", "nova-2/arn:aws:bedrock:us-east-1:123:custom-model/a", "bedrock", json!({"amazon.nova-2-custom": {}}), "amazon.nova-2-custom")]
|
||||
#[case::lowercase("MODEL", "MODEL", "openai", json!({"Model": {}, "openai/MODEL": {}}), "Model")]
|
||||
#[case::lowercase_mismatch("azure/MODEL", "MODEL", "azure", json!({"azure/Model": {"litellm_provider": "openai"}, "Model": {"litellm_provider": "azure"}}), "Model")]
|
||||
#[case::wildcard("model", "model", "azure", json!({"model": {"litellm_provider": null}}), "model")]
|
||||
#[case::azure_ai("model", "model", "azure_ai", json!({"model": {"litellm_provider": "openai"}}), "model")]
|
||||
#[case::family("model", "model", "vertex_ai", json!({"model": {"litellm_provider": "vertex_ai-language-models"}}), "model")]
|
||||
#[case::resource("model", "model", "fireworks_ai", json!({"fireworks_ai/accounts/fireworks/models/model": {}}), "fireworks_ai/accounts/fireworks/models/model")]
|
||||
fn inferred_provider_preserves_model_info_priority(
|
||||
#[case] model: &str,
|
||||
#[case] normalized: &str,
|
||||
#[case] provider: &str,
|
||||
#[case] rows: Value,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let result = resolve_inferred(
|
||||
&catalog(rows),
|
||||
&query(model, None),
|
||||
InferredProvider {
|
||||
model: normalized,
|
||||
provider: Some(provider),
|
||||
},
|
||||
);
|
||||
let Resolved::Found { key, .. } = result else {
|
||||
panic!("{result:?}")
|
||||
};
|
||||
assert_eq!(key, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::mismatch("j2-light", "j2-light", Some("ai21_chat"), json!({"j2-light": {"litellm_provider": "ai21"}}), vec!["j2-light"])]
|
||||
#[case::normalized_retains_prefix("azure/command-r-plus", "azure/command-r-plus", Some("openai"), json!({"command-r-plus": {"litellm_provider": "openai"}}), vec!["azure/command-r-plus"])]
|
||||
#[case::inference_failed("unknown/model", "unknown/model", None, json!({"model": {}}), vec!["unknown/model"])]
|
||||
#[case::inference_empty("unknown/model", "unknown/model", Some(""), json!({"model": {}}), vec!["unknown/model"])]
|
||||
#[case::suffix_not_normalized("openai/model-2026-01-01", "model-2026-01-01", Some("openai"), json!({"model": {}}), vec!["openai/model-2026-01-01", "model-2026-01-01", "openai/model"])]
|
||||
#[case::unqualified_suffix("model-2026-01-01", "model-2026-01-01", Some("openai"), json!({"openai/model": {}}), vec!["model-2026-01-01", "model"])]
|
||||
fn inferred_misses_preserve_candidate_order(
|
||||
#[case] model: &str,
|
||||
#[case] normalized: &str,
|
||||
#[case] provider: Option<&str>,
|
||||
#[case] rows: Value,
|
||||
#[case] tried: Vec<&str>,
|
||||
) {
|
||||
assert_eq!(
|
||||
resolve_inferred(
|
||||
&catalog(rows),
|
||||
&query(model, None),
|
||||
InferredProvider {
|
||||
model: normalized,
|
||||
provider
|
||||
}
|
||||
),
|
||||
Resolved::NotFound {
|
||||
tried: tried.into_iter().map(str::to_owned).collect()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::explicit(Some("openai"), "openai/model")]
|
||||
#[case::empty(Some(""), "model")]
|
||||
#[case::absent(None, "model")]
|
||||
fn explicit_nonempty_provider_wins(#[case] provider: Option<&str>, #[case] expected: &str) {
|
||||
let models = catalog(
|
||||
json!({"model": {"litellm_provider": "azure"}, "openai/model": {"litellm_provider": "openai"}}),
|
||||
);
|
||||
let result = resolve_inferred(
|
||||
&models,
|
||||
&query("model", provider),
|
||||
InferredProvider {
|
||||
model: "model",
|
||||
provider: Some("azure"),
|
||||
},
|
||||
);
|
||||
let Resolved::Found { key, .. } = result else {
|
||||
panic!("{result:?}")
|
||||
};
|
||||
assert_eq!(key, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::base(None, "base", Some("openai"), "base", Via::BaseModel)]
|
||||
#[case::region(
|
||||
None,
|
||||
"custom-region/base",
|
||||
Some("bedrock_mantle"),
|
||||
"bedrock_mantle/base",
|
||||
Via::RegionStripped
|
||||
)]
|
||||
#[case::explicit_region(
|
||||
Some("openai"),
|
||||
"ignored",
|
||||
Some("azure"),
|
||||
"openai/custom-region/base",
|
||||
Via::Region
|
||||
)]
|
||||
fn inferred_base_and_region(
|
||||
#[case] explicit: Option<&str>,
|
||||
#[case] normalized: &str,
|
||||
#[case] provider: Option<&str>,
|
||||
#[case] key: &str,
|
||||
#[case] via: Via,
|
||||
) {
|
||||
let models = catalog(json!({key: {}}));
|
||||
let query = Query {
|
||||
model: "deployment",
|
||||
provider: explicit,
|
||||
base_model: Some("base"),
|
||||
region: Some("custom-region"),
|
||||
};
|
||||
assert_eq!(
|
||||
resolve_inferred(
|
||||
&models,
|
||||
&query,
|
||||
InferredProvider {
|
||||
model: normalized,
|
||||
provider
|
||||
}
|
||||
),
|
||||
Resolved::Found {
|
||||
key: key.into(),
|
||||
via
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn registration_models() -> Catalog {
|
||||
catalog(json!({
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ pub(crate) enum ResolutionResult {
|
|||
NotFound(ModelNotFound),
|
||||
}
|
||||
|
||||
#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None))]
|
||||
#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None, inferred=None))]
|
||||
pub(crate) fn resolve_model(
|
||||
module: &Bound<'_, PyModule>,
|
||||
model_cost: &Bound<'_, PyDict>,
|
||||
|
|
@ -181,6 +181,7 @@ pub(crate) fn resolve_model(
|
|||
provider: Option<&str>,
|
||||
base_model: Option<&str>,
|
||||
region: Option<&str>,
|
||||
inferred: Option<(&str, Option<&str>)>,
|
||||
) -> PyResult<ResolutionResult> {
|
||||
let index = module.getattr("_model_key_index")?;
|
||||
let index = index.extract::<PyRef<'_, ModelKeyIndex>>()?;
|
||||
|
|
@ -189,15 +190,20 @@ pub(crate) fn resolve_model(
|
|||
index: &index,
|
||||
error: RefCell::default(),
|
||||
};
|
||||
let result = resolution::resolve(
|
||||
&view,
|
||||
&Query {
|
||||
model,
|
||||
provider,
|
||||
base_model,
|
||||
region,
|
||||
},
|
||||
);
|
||||
let query = Query {
|
||||
model,
|
||||
provider,
|
||||
base_model,
|
||||
region,
|
||||
};
|
||||
let result = match inferred {
|
||||
Some((model, provider)) => resolution::resolve_inferred(
|
||||
&view,
|
||||
&query,
|
||||
resolution::InferredProvider { model, provider },
|
||||
),
|
||||
None => resolution::resolve(&view, &query),
|
||||
};
|
||||
view.finish()?;
|
||||
Ok(match result {
|
||||
Resolved::Found { key, via } => ResolutionResult::Found(ResolvedModel {
|
||||
|
|
@ -785,7 +791,7 @@ mod tests {
|
|||
}),
|
||||
);
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module(py), &rows, model, provider, base, region).unwrap()
|
||||
resolve_model(&module(py), &rows, model, provider, base, region, None).unwrap()
|
||||
else {
|
||||
panic!("not found")
|
||||
};
|
||||
|
|
@ -879,7 +885,7 @@ mod tests {
|
|||
Python::attach(|py| {
|
||||
let rows = dict(py, json!({key: {"litellm_provider": provider}}));
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module(py), &rows, model, Some(provider), None, None).unwrap()
|
||||
resolve_model(&module(py), &rows, model, Some(provider), None, None, None).unwrap()
|
||||
else {
|
||||
panic!("not found")
|
||||
};
|
||||
|
|
@ -908,7 +914,16 @@ mod tests {
|
|||
let rows = dict(py, json!({"model": {"litellm_provider": actual}}));
|
||||
assert_eq!(
|
||||
matches!(
|
||||
resolve_model(&module(py), &rows, "model", Some(provider), None, None).unwrap(),
|
||||
resolve_model(
|
||||
&module(py),
|
||||
&rows,
|
||||
"model",
|
||||
Some(provider),
|
||||
None,
|
||||
None,
|
||||
None
|
||||
)
|
||||
.unwrap(),
|
||||
ResolutionResult::Found(_)
|
||||
),
|
||||
found
|
||||
|
|
@ -968,7 +983,7 @@ mod tests {
|
|||
Python::attach(|py| {
|
||||
let rows = dict(py, json!({"model": {}}));
|
||||
let ResolutionResult::NotFound(missing) =
|
||||
resolve_model(&module(py), &rows, model, None, None, None).unwrap()
|
||||
resolve_model(&module(py), &rows, model, None, None, None, None).unwrap()
|
||||
else {
|
||||
panic!("unexpected match")
|
||||
};
|
||||
|
|
@ -1019,7 +1034,7 @@ assert n.price(row, usage, when).total == 21.0
|
|||
let module = module(py);
|
||||
let rows = dict(py, json!({"Model": {}}));
|
||||
let resolve = |rows: &Bound<'_, PyDict>, key| match resolve_model(
|
||||
&module, rows, key, None, None, None,
|
||||
&module, rows, key, None, None, None, None,
|
||||
)
|
||||
.unwrap()
|
||||
{
|
||||
|
|
@ -1180,7 +1195,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
|
|||
json!({"custom/Model": {}, "custom/model": {}, "MODEL": {}}),
|
||||
);
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module(py), &rows, model, provider, None, None).unwrap()
|
||||
resolve_model(&module(py), &rows, model, provider, None, None, None).unwrap()
|
||||
else {
|
||||
panic!("not found")
|
||||
};
|
||||
|
|
@ -1188,6 +1203,47 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
|
|||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::bare("read-model", "read-model", "openai")]
|
||||
#[case::mismatch("azure/read-model", "read-model", "azure")]
|
||||
fn inferred_intent_is_bound_without_mutating_rows(
|
||||
#[from(interpreter)] _interpreter: (),
|
||||
#[case] model: &str,
|
||||
#[case] normalized: &str,
|
||||
#[case] provider: &str,
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let module = crate::native_module(py);
|
||||
let rows = dict(
|
||||
py,
|
||||
json!({
|
||||
"read-model": {"litellm_provider": provider, "supports_vision": true},
|
||||
"openai/read-model": {"litellm_provider": "openai", "supports_vision": false},
|
||||
"azure/read-model": {"litellm_provider": "openai", "supports_vision": false}
|
||||
}),
|
||||
);
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("n", &module).unwrap();
|
||||
locals.set_item("rows", &rows).unwrap();
|
||||
locals.set_item("model", model).unwrap();
|
||||
locals.set_item("inferred", (normalized, provider)).unwrap();
|
||||
py.run(
|
||||
cr#"
|
||||
before = tuple((key, id(row), dict(row)) for key, row in rows.items())
|
||||
result = n.resolve_model(rows, model, inferred=inferred)
|
||||
assert result.key == "read-model"
|
||||
assert n.model_info(rows, result.key).capabilities["supports_vision"] is True
|
||||
assert tuple((key, id(row), dict(row)) for key, row in rows.items()) == before
|
||||
rows["read-model"]["litellm_provider"] = "wrong-scope"
|
||||
assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFound)
|
||||
"#,
|
||||
Some(&locals),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn base_misses_and_provider_fallbacks_preserve_order(#[from(interpreter)] _interpreter: ()) {
|
||||
Python::attach(|py| {
|
||||
|
|
@ -1196,14 +1252,21 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
|
|||
py,
|
||||
json!({"openai/model": {"litellm_provider":"anthropic"}, "model": {"litellm_provider":"openai"}}),
|
||||
);
|
||||
let ResolutionResult::NotFound(missing) =
|
||||
resolve_model(&module, &rows, "model", None, Some("missing-base"), None).unwrap()
|
||||
else {
|
||||
let ResolutionResult::NotFound(missing) = resolve_model(
|
||||
&module,
|
||||
&rows,
|
||||
"model",
|
||||
None,
|
||||
Some("missing-base"),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("unexpected base fallback")
|
||||
};
|
||||
assert_eq!(missing.tried, ["missing-base"]);
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module, &rows, "model", Some("openai"), None, None).unwrap()
|
||||
resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap()
|
||||
else {
|
||||
panic!("not found")
|
||||
};
|
||||
|
|
@ -1214,7 +1277,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
|
|||
.set_item("litellm_provider", "openai")
|
||||
.unwrap();
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module, &rows, "model", Some("openai"), None, None).unwrap()
|
||||
resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap()
|
||||
else {
|
||||
panic!("not found")
|
||||
};
|
||||
|
|
@ -1233,6 +1296,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
|
|||
Some("databricks"),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("unexpected match")
|
||||
|
|
|
|||
162
litellm-rust/crates/python-bridge/tests/catalog_resolution.py
Normal file
162
litellm-rust/crates/python-bridge/tests/catalog_resolution.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import unittest
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import litellm
|
||||
from litellm import utils
|
||||
from litellm.rust_bridge import _native as native
|
||||
|
||||
|
||||
class ModelInfoResolutionTests(unittest.TestCase):
|
||||
def test_inferred_reads_match_m3_selection(self) -> None:
|
||||
cases: Final = (
|
||||
("read-model", "read-model", "openai", "read-model", "openai/read-model"),
|
||||
("read-model-2026-01-01", "read-model-2026-01-01", "openai", "read-model", "openai/read-model"),
|
||||
("azure/read-model", "read-model", "azure", "read-model", "azure/read-model"),
|
||||
("azure/READ-MODEL", "READ-MODEL", "azure", "read-model", "azure/READ-MODEL"),
|
||||
("openai/read-model-2026-01-01", "read-model-2026-01-01", "openai", "openai/read-model", "read-model"),
|
||||
(
|
||||
"bedrock/converse/anthropic.read-model",
|
||||
"converse/anthropic.read-model",
|
||||
"bedrock",
|
||||
"anthropic.read-model",
|
||||
"bedrock/anthropic.read-model",
|
||||
),
|
||||
(
|
||||
"bedrock/us.anthropic.read-model",
|
||||
"us.anthropic.read-model",
|
||||
"bedrock",
|
||||
"anthropic.read-model",
|
||||
"bedrock/anthropic.read-model",
|
||||
),
|
||||
(
|
||||
"bedrock_mantle/us-east-1/read-model",
|
||||
"read-model",
|
||||
"bedrock_mantle",
|
||||
"bedrock_mantle/read-model",
|
||||
"read-model",
|
||||
),
|
||||
("vertex_ai/read-model-001", "read-model-001", "vertex_ai", "vertex_ai/read-model", "read-model"),
|
||||
)
|
||||
for model, normalized, provider, expected, other in cases:
|
||||
with self.subTest(model=model):
|
||||
self.assert_m3_selection(model, normalized, provider, expected, other)
|
||||
|
||||
def assert_m3_selection(self, model: str, normalized: str, provider: str, expected: str, other: str) -> None:
|
||||
rows: Final[dict[str, object]] = {
|
||||
expected: {"litellm_provider": provider, "mode": "chat", "supports_vision": True},
|
||||
other: {
|
||||
"litellm_provider": "openai" if provider == "azure" else provider,
|
||||
"mode": "chat",
|
||||
"supports_vision": False,
|
||||
},
|
||||
}
|
||||
with (
|
||||
patch.object(litellm, "model_cost", rows),
|
||||
patch.object(litellm, "open_ai_chat_completion_models", [model]),
|
||||
patch.object(utils, "_model_cost_lowercase_map", None),
|
||||
):
|
||||
native.invalidate_model_cost_index()
|
||||
routed_model, routed_provider, _, _ = litellm.get_llm_provider(model)
|
||||
self.assertEqual((routed_model, routed_provider), (normalized, provider))
|
||||
legacy: Final = utils.get_model_info_helper(model)
|
||||
selected: Final = native.resolve_model(rows, model, inferred=(routed_model, routed_provider))
|
||||
self.assertIsInstance(selected, native.ResolvedModel)
|
||||
if not isinstance(selected, native.ResolvedModel):
|
||||
self.fail(str(selected.tried))
|
||||
self.assertEqual((legacy["key"], selected.key), (expected, expected))
|
||||
lowered: Final = native.model_info(rows, selected.key)
|
||||
self.assertIsInstance(lowered, native.ModelInfoView)
|
||||
if not isinstance(lowered, native.ModelInfoView):
|
||||
self.fail(str(lowered.errors))
|
||||
self.assertEqual((legacy.get("supports_vision"), lowered.capabilities["supports_vision"]), (True, True))
|
||||
|
||||
def test_host_normalization_can_preserve_azure_prefix(self) -> None:
|
||||
model: Final = "azure/command-r-plus"
|
||||
rows: Final[dict[str, object]] = {"command-r-plus": {"litellm_provider": "openai", "mode": "chat"}}
|
||||
with (
|
||||
patch.object(litellm, "model_cost", rows),
|
||||
patch.object(litellm, "cohere_chat_models", ["command-r-plus"]),
|
||||
patch.object(utils, "_model_cost_lowercase_map", None),
|
||||
):
|
||||
normalized, provider, _, _ = litellm.get_llm_provider(model)
|
||||
self.assertEqual((normalized, provider), (model, "openai"))
|
||||
self.assertEqual(utils.get_potential_model_names(model, None)["split_model"], model)
|
||||
with self.assertRaisesRegex(Exception, "isn't mapped yet"):
|
||||
utils.get_model_info_helper(model)
|
||||
selected: Final = native.resolve_model(rows, model, inferred=(normalized, provider))
|
||||
self.assertIsInstance(selected, native.ModelNotFound)
|
||||
if not isinstance(selected, native.ModelNotFound):
|
||||
self.fail(selected.key)
|
||||
self.assertEqual(selected.tried, [model])
|
||||
|
||||
def test_provider_mismatch_is_not_accepted(self) -> None:
|
||||
rows: Final[dict[str, object]] = {"j2-light": {"litellm_provider": "ai21", "mode": "chat"}}
|
||||
with (
|
||||
patch.object(litellm, "model_cost", rows),
|
||||
patch.object(litellm, "ai21_chat_models", ["j2-light"]),
|
||||
patch.object(utils, "_model_cost_lowercase_map", None),
|
||||
):
|
||||
normalized, provider, _, _ = litellm.get_llm_provider("j2-light")
|
||||
self.assertEqual(provider, "ai21_chat")
|
||||
with self.assertRaisesRegex(Exception, "isn't mapped yet"):
|
||||
utils.get_model_info_helper("j2-light")
|
||||
self.assertIsInstance(
|
||||
native.resolve_model(rows, "j2-light", inferred=(normalized, provider)), native.ModelNotFound
|
||||
)
|
||||
self.assertIsInstance(native.resolve_model(rows, "j2-light"), native.ResolvedModel)
|
||||
|
||||
def test_legacy_calls_and_registration_do_not_inherit_read_intent(self) -> None:
|
||||
rows: Final[dict[str, object]] = {
|
||||
"read-model": {"litellm_provider": "azure"},
|
||||
"azure/read-model": {"litellm_provider": "openai"},
|
||||
"openai/read-model": {"litellm_provider": "openai"},
|
||||
}
|
||||
cases: Final = (
|
||||
("read-model", None, None, "read-model"),
|
||||
("azure/read-model", None, None, "azure/read-model"),
|
||||
("read-model", "openai", ("ignored", "azure"), "openai/read-model"),
|
||||
("read-model", "", ("read-model", "azure"), "read-model"),
|
||||
("azure/read-model", None, ("read-model", "azure"), "read-model"),
|
||||
)
|
||||
before: Final = native.registration_target(rows, "azure/read-model", provider="azure")
|
||||
for model, provider, inferred, expected in cases:
|
||||
with self.subTest(model=model, provider=provider, inferred=inferred):
|
||||
if not isinstance(
|
||||
selected := native.resolve_model(rows, model, provider=provider, inferred=inferred),
|
||||
native.ResolvedModel,
|
||||
):
|
||||
self.fail(str(selected.tried))
|
||||
self.assertEqual(selected.key, expected)
|
||||
after: Final = native.registration_target(rows, "azure/read-model", provider="azure")
|
||||
self.assertEqual(
|
||||
(before.write, before.inherit_from, before.names_the_row),
|
||||
(after.write, after.inherit_from, after.names_the_row),
|
||||
)
|
||||
self.assertEqual(
|
||||
(after.write, after.inherit_from, after.names_the_row), ("azure/read-model", "read-model", True)
|
||||
)
|
||||
|
||||
def test_failed_inference_is_distinct_from_omitting_inference(self) -> None:
|
||||
rows: Final[dict[str, object]] = {"read-model": {}}
|
||||
self.assertIsInstance(native.resolve_model(rows, "unknown/read-model"), native.ResolvedModel)
|
||||
selected: Final = native.resolve_model(rows, "unknown/read-model", inferred=("unknown/read-model", None))
|
||||
self.assertIsInstance(selected, native.ModelNotFound)
|
||||
if not isinstance(selected, native.ModelNotFound):
|
||||
self.fail(selected.key)
|
||||
self.assertEqual(selected.tried, ["unknown/read-model"])
|
||||
|
||||
def test_inferred_binding_shape(self) -> None:
|
||||
inferred: Final = inspect.signature(native.resolve_model).parameters["inferred"]
|
||||
self.assertEqual(inferred.kind, inspect.Parameter.KEYWORD_ONLY)
|
||||
self.assertIn("inferred=None", str(inspect.signature(native.resolve_model)))
|
||||
for invalid in ("openai", ("model",), ("model", "openai", None), (None, "openai"), ("model", 123)):
|
||||
with self.subTest(invalid=invalid):
|
||||
self.assertRaises((TypeError, ValueError), native.resolve_model, {}, "model", inferred=invalid)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -248,6 +248,7 @@ def resolve_model(
|
|||
provider: str | None = None,
|
||||
base_model: str | None = None,
|
||||
region: str | None = None,
|
||||
inferred: tuple[str, str | None] | None = None,
|
||||
) -> ResolvedModel | ModelNotFound: ...
|
||||
def registration_target(
|
||||
model_cost: dict[str, object], key: str, *, provider: str | None = None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue