mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-19 09:53:33 +00:00
fix(catalog): preserve SDK info provider and suffix semantics
This commit is contained in:
parent
272da08170
commit
e622016f55
6 changed files with 714 additions and 32 deletions
|
|
@ -2,6 +2,13 @@ use std::fmt;
|
|||
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error, PartialEq, Eq)]
|
||||
#[error("provider metadata comparison failed for {key} with {provider}")]
|
||||
pub struct ProviderReadError {
|
||||
pub key: String,
|
||||
pub provider: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error("invalid JSON: {0}")]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,95 @@
|
|||
use std::cell::RefCell;
|
||||
|
||||
use crate::ProviderReadError;
|
||||
|
||||
pub trait Keys {
|
||||
fn contains(&self, key: &str) -> bool;
|
||||
fn lowercase(&self, key: &str) -> Option<String>;
|
||||
fn provider_of(&self, key: &str) -> Option<String>;
|
||||
|
||||
fn matches_provider(&self, key: &str, provider: &str) -> bool {
|
||||
provider_matches(provider, self.provider_of(key).as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
pub trait InfoKeys: Keys {
|
||||
fn info_provider_match(&self, key: &str, provider: &str) -> ProviderMatch;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::VariantArray)]
|
||||
pub enum ProviderMatch {
|
||||
Match,
|
||||
Mismatch,
|
||||
Invalid,
|
||||
}
|
||||
|
||||
pub trait ProviderValue {
|
||||
fn differs(&self, provider: &str) -> Option<bool>;
|
||||
fn equals(&self, provider: &str) -> Option<bool>;
|
||||
fn starts_with(&self, prefix: &str) -> Option<bool>;
|
||||
}
|
||||
|
||||
pub fn info_provider_match(provider: &str, value: &impl ProviderValue) -> ProviderMatch {
|
||||
match compare_info_provider(provider, value) {
|
||||
Some(true) => ProviderMatch::Match,
|
||||
Some(false) => ProviderMatch::Mismatch,
|
||||
None => ProviderMatch::Invalid,
|
||||
}
|
||||
}
|
||||
|
||||
fn compare_info_provider(provider: &str, value: &impl ProviderValue) -> Option<bool> {
|
||||
if !value.differs(provider)? {
|
||||
return Some(true);
|
||||
}
|
||||
if matches!(provider, "vertex_ai" | "fireworks_ai") && value.starts_with(provider)?
|
||||
|| provider.starts_with("bedrock") && value.starts_with("bedrock")?
|
||||
|| matches!(provider, "litellm_proxy" | "github" | "nadir")
|
||||
{
|
||||
return Some(true);
|
||||
}
|
||||
Some(provider == "azure_ai" && (value.equals("azure")? || value.equals("openai")?))
|
||||
}
|
||||
|
||||
struct InfoView<'a, K> {
|
||||
keys: &'a K,
|
||||
error: RefCell<Option<ProviderReadError>>,
|
||||
}
|
||||
|
||||
impl<K: InfoKeys> Keys for InfoView<'_, K> {
|
||||
fn contains(&self, key: &str) -> bool {
|
||||
self.error.borrow().is_none() && self.keys.contains(key)
|
||||
}
|
||||
|
||||
fn lowercase(&self, key: &str) -> Option<String> {
|
||||
if self.error.borrow().is_some() {
|
||||
return None;
|
||||
}
|
||||
self.keys.lowercase(key)
|
||||
}
|
||||
|
||||
fn provider_of(&self, key: &str) -> Option<String> {
|
||||
if self.error.borrow().is_some() {
|
||||
return None;
|
||||
}
|
||||
self.keys.provider_of(key)
|
||||
}
|
||||
|
||||
fn matches_provider(&self, key: &str, provider: &str) -> bool {
|
||||
if self.error.borrow().is_some() {
|
||||
return false;
|
||||
}
|
||||
match self.keys.info_provider_match(key, provider) {
|
||||
ProviderMatch::Match => true,
|
||||
ProviderMatch::Mismatch => false,
|
||||
ProviderMatch::Invalid => {
|
||||
self.error.replace(Some(ProviderReadError {
|
||||
key: key.to_owned(),
|
||||
provider: provider.to_owned(),
|
||||
}));
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
|
|
@ -50,6 +138,116 @@ pub struct InferredProvider<'a> {
|
|||
pub provider: Option<&'a str>,
|
||||
}
|
||||
|
||||
pub fn resolve_info(
|
||||
keys: &impl InfoKeys,
|
||||
query: &Query<'_>,
|
||||
inferred: Option<InferredProvider<'_>>,
|
||||
) -> Result<Resolved, ProviderReadError> {
|
||||
let view = InfoView {
|
||||
keys,
|
||||
error: RefCell::default(),
|
||||
};
|
||||
let resolved = match query.provider.filter(|provider| !provider.is_empty()) {
|
||||
Some(provider) => resolve_info_explicit(&view, query, provider),
|
||||
None => match inferred {
|
||||
Some(inferred) => resolve_inferred(&view, query, inferred),
|
||||
None => resolve(&view, query),
|
||||
},
|
||||
};
|
||||
match view.error.into_inner() {
|
||||
Some(error) => Err(error),
|
||||
None => Ok(resolved),
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_info_explicit(keys: &impl Keys, query: &Query<'_>, provider: &str) -> Resolved {
|
||||
let model = query.base_model.unwrap_or(query.model);
|
||||
let split = model
|
||||
.strip_prefix(provider)
|
||||
.and_then(|model| model.strip_prefix('/'))
|
||||
.unwrap_or(model);
|
||||
let combined = if split == model {
|
||||
format!("{provider}/{model}")
|
||||
} else {
|
||||
model.to_owned()
|
||||
};
|
||||
let stripped = registration_stripped(split, Some(provider));
|
||||
let split = if matches!(provider, "bedrock" | "bedrock_converse") {
|
||||
strip_bedrock_nova_route(bedrock_routing_model(split))
|
||||
} else {
|
||||
split
|
||||
};
|
||||
let split = if provider == "bedrock_mantle" {
|
||||
strip_region(split, Some(provider), query.region)
|
||||
} else {
|
||||
split
|
||||
};
|
||||
let mut lookup = Lookup {
|
||||
keys,
|
||||
provider: Some(provider),
|
||||
case_insensitive: true,
|
||||
tried: Vec::new(),
|
||||
};
|
||||
if let Some(region) = query.region {
|
||||
let regional = if split.starts_with(&format!("{region}/")) {
|
||||
format!("{provider}/{split}")
|
||||
} else {
|
||||
format!("{provider}/{region}/{split}")
|
||||
};
|
||||
if let Some(found) = lookup.candidate(®ional, Via::Region) {
|
||||
return found;
|
||||
}
|
||||
}
|
||||
let exact_via = if query.base_model.is_some() {
|
||||
Via::BaseModel
|
||||
} else {
|
||||
Via::Exact
|
||||
};
|
||||
if let Some(found) = lookup.candidate(
|
||||
&combined,
|
||||
if combined == model {
|
||||
exact_via
|
||||
} else {
|
||||
Via::ProviderPrefix
|
||||
},
|
||||
) {
|
||||
return found;
|
||||
}
|
||||
if let Some(found) = lookup.candidate(model, exact_via) {
|
||||
return found;
|
||||
}
|
||||
if provider == "bedrock_mantle"
|
||||
&& let Some(found) = lookup.candidate(&format!("{provider}/{split}"), Via::RegionStripped)
|
||||
{
|
||||
return found;
|
||||
}
|
||||
if let Some(found) = lookup.candidate(split, Via::ProviderStripped) {
|
||||
return found;
|
||||
}
|
||||
let combined_stripped = if provider == "bedrock_mantle" {
|
||||
format!("{provider}/{}", strip_suffix(split, Some(provider)))
|
||||
} else {
|
||||
format!("{provider}/{stripped}")
|
||||
};
|
||||
if let Some(found) = lookup.candidate(&combined_stripped, Via::ProviderSuffixStripped) {
|
||||
return found;
|
||||
}
|
||||
if let Some(found) = lookup.candidate(stripped, Via::SuffixStripped) {
|
||||
return found;
|
||||
}
|
||||
let prefixed = if provider == "fireworks_ai" {
|
||||
fireworks_resource(split)
|
||||
} else {
|
||||
format!("{provider}/{model}")
|
||||
};
|
||||
if let Some(found) = lookup.candidate(&prefixed, Via::RepeatedProvider) {
|
||||
return found;
|
||||
}
|
||||
Resolved::NotFound {
|
||||
tried: lookup.tried,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resolve_inferred(
|
||||
keys: &impl Keys,
|
||||
query: &Query<'_>,
|
||||
|
|
@ -306,7 +504,7 @@ impl<K: Keys> Lookup<'_, K> {
|
|||
return None;
|
||||
};
|
||||
if let Some(provider) = self.provider
|
||||
&& !provider_matches(provider, self.keys.provider_of(&key).as_deref())
|
||||
&& !self.keys.matches_provider(&key, provider)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
|
@ -609,6 +807,129 @@ mod tests {
|
|||
Catalog::parse(rows.to_string().as_bytes()).unwrap().value
|
||||
}
|
||||
|
||||
impl InfoKeys for Catalog {
|
||||
fn info_provider_match(&self, key: &str, provider: &str) -> ProviderMatch {
|
||||
if self.matches_provider(key, provider) {
|
||||
ProviderMatch::Match
|
||||
} else {
|
||||
ProviderMatch::Mismatch
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TestProvider(Option<&'static str>);
|
||||
|
||||
impl ProviderValue for TestProvider {
|
||||
fn differs(&self, provider: &str) -> Option<bool> {
|
||||
Some(self.0 != Some(provider))
|
||||
}
|
||||
|
||||
fn equals(&self, provider: &str) -> Option<bool> {
|
||||
Some(self.0 == Some(provider))
|
||||
}
|
||||
|
||||
fn starts_with(&self, prefix: &str) -> Option<bool> {
|
||||
self.0.map(|value| value.starts_with(prefix))
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::ordinary("openai", None, ProviderMatch::Mismatch)]
|
||||
#[case::azure_ai("azure_ai", None, ProviderMatch::Mismatch)]
|
||||
#[case::bedrock("bedrock", None, ProviderMatch::Invalid)]
|
||||
#[case::converse("bedrock_converse", None, ProviderMatch::Invalid)]
|
||||
#[case::vertex("vertex_ai", None, ProviderMatch::Invalid)]
|
||||
#[case::fireworks("fireworks_ai", None, ProviderMatch::Invalid)]
|
||||
#[case::github("github", None, ProviderMatch::Match)]
|
||||
#[case::proxy("litellm_proxy", None, ProviderMatch::Match)]
|
||||
#[case::nadir("nadir", None, ProviderMatch::Match)]
|
||||
#[case::exact("openai", Some("openai"), ProviderMatch::Match)]
|
||||
#[case::family("bedrock", Some("bedrock_converse"), ProviderMatch::Match)]
|
||||
#[case::valid_mismatch("bedrock", Some("openai"), ProviderMatch::Mismatch)]
|
||||
#[case::azure_alias("azure_ai", Some("openai"), ProviderMatch::Match)]
|
||||
fn info_provider_protocol(
|
||||
#[case] requested: &str,
|
||||
#[case] actual: Option<&'static str>,
|
||||
#[case] expected: ProviderMatch,
|
||||
) {
|
||||
assert_eq!(
|
||||
info_provider_match(requested, &TestProvider(actual)),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
struct AbortingKeys(Cell<bool>);
|
||||
|
||||
impl Keys for AbortingKeys {
|
||||
fn contains(&self, _: &str) -> bool {
|
||||
assert!(!self.0.get(), "lookup continued after invalid provider");
|
||||
true
|
||||
}
|
||||
|
||||
fn lowercase(&self, _: &str) -> Option<String> {
|
||||
panic!("lookup continued after invalid provider")
|
||||
}
|
||||
|
||||
fn provider_of(&self, _: &str) -> Option<String> {
|
||||
panic!("info reads must preserve provider state")
|
||||
}
|
||||
}
|
||||
|
||||
impl InfoKeys for AbortingKeys {
|
||||
fn info_provider_match(&self, _: &str, _: &str) -> ProviderMatch {
|
||||
self.0.set(true);
|
||||
ProviderMatch::Invalid
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::explicit(Some("bedrock"))]
|
||||
#[case::inferred(None)]
|
||||
#[case::empty(Some(""))]
|
||||
fn info_provider_failure_stops_live_access(#[case] explicit: Option<&str>) {
|
||||
let keys = AbortingKeys(Cell::new(false));
|
||||
let error = resolve_info(
|
||||
&keys,
|
||||
&query("bedrock/model", explicit),
|
||||
Some(InferredProvider {
|
||||
model: "model",
|
||||
provider: Some("bedrock"),
|
||||
}),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(error.key, "bedrock/model");
|
||||
assert_eq!(error.provider, "bedrock");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::dated("probe-model-20260101", false)]
|
||||
#[case::qualified_priority("probe-model", true)]
|
||||
fn info_explicit_bedrock_is_not_generic_suffix_resolution(
|
||||
#[case] model: &str,
|
||||
#[case] found: bool,
|
||||
) {
|
||||
let rows = catalog(json!({
|
||||
"probe-model": {"litellm_provider": "bedrock"},
|
||||
"bedrock/probe-model": {"litellm_provider": "bedrock"}
|
||||
}));
|
||||
let result = resolve_info(&rows, &query(model, Some("bedrock")), None).unwrap();
|
||||
let expected = if found {
|
||||
Resolved::Found {
|
||||
key: "bedrock/probe-model".into(),
|
||||
via: Via::ProviderPrefix,
|
||||
}
|
||||
} else {
|
||||
Resolved::NotFound {
|
||||
tried: vec![format!("bedrock/{model}"), model.into()],
|
||||
}
|
||||
};
|
||||
assert_eq!(result, expected);
|
||||
assert!(matches!(
|
||||
resolve(&rows, &query(model, Some("bedrock"))),
|
||||
Resolved::Found { .. }
|
||||
));
|
||||
}
|
||||
|
||||
fn query<'a>(model: &'a str, provider: Option<&'a str>) -> Query<'a> {
|
||||
Query {
|
||||
model,
|
||||
|
|
|
|||
|
|
@ -32,8 +32,8 @@ mod _native {
|
|||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::model_registry::{
|
||||
CatalogRefresher, FetchedRegistry, ModelCostLayers, ModelInfoView, ModelNotFound,
|
||||
ModelRegistry, PriceLine, Priced, RegistrationTarget, RegistryNotModified,
|
||||
CatalogRefresher, FetchedRegistry, InvalidProvider, ModelCostLayers, ModelInfoView,
|
||||
ModelNotFound, ModelRegistry, PriceLine, Priced, RegistrationTarget, RegistryNotModified,
|
||||
RegistryUnavailable, ResolvedModel, RowError, RowErrors, Unpriced, UsageInput, WhenInput,
|
||||
fetch_once, fetch_with_retry, invalidate_model_cost_index, model_info, model_row_errors,
|
||||
price, registration_target, resolve_model, write_rows,
|
||||
|
|
@ -138,6 +138,7 @@ assert live == {"model": {"litellm_provider": "openai", "input_cost_per_token":
|
|||
"write_rows",
|
||||
"ResolvedModel",
|
||||
"ModelNotFound",
|
||||
"InvalidProvider",
|
||||
"RegistrationTarget",
|
||||
"RowErrors",
|
||||
"ModelInfoView",
|
||||
|
|
|
|||
|
|
@ -9,9 +9,9 @@ use litellm_model_catalog::wire::{Fields, FlatKey};
|
|||
use litellm_model_catalog::{self as resolution, Keys, Query, Resolved};
|
||||
use litellm_model_catalog::{Error, Located, Model};
|
||||
use pyo3::class::gc::{PyTraverseError, PyVisit};
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::exceptions::{PyException, PyValueError};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyAny, PyDict, PyModule, PySet};
|
||||
use pyo3::types::{PyAny, PyDict, PyModule, PySet, PyString};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::{RowError, escape_pointer_segment, fields_to_dict, nonfinite_errors, row_error};
|
||||
|
|
@ -149,6 +149,66 @@ impl Keys for PyView<'_, '_> {
|
|||
}
|
||||
}
|
||||
|
||||
struct PyProvider<'a, 'py> {
|
||||
value: Bound<'py, PyAny>,
|
||||
error: &'a RefCell<Option<PyErr>>,
|
||||
}
|
||||
|
||||
impl PyProvider<'_, '_> {
|
||||
fn comparison(&self, result: PyResult<bool>) -> Option<bool> {
|
||||
match result {
|
||||
Ok(value) => Some(value),
|
||||
Err(error) => {
|
||||
if !error.is_instance_of::<PyException>(self.value.py()) {
|
||||
self.error.borrow_mut().get_or_insert(error);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl resolution::ProviderValue for PyProvider<'_, '_> {
|
||||
fn differs(&self, provider: &str) -> Option<bool> {
|
||||
self.comparison(self.value.ne(provider))
|
||||
}
|
||||
|
||||
fn equals(&self, provider: &str) -> Option<bool> {
|
||||
self.comparison(
|
||||
PyString::new(self.value.py(), provider)
|
||||
.as_any()
|
||||
.eq(&self.value),
|
||||
)
|
||||
}
|
||||
|
||||
fn starts_with(&self, prefix: &str) -> Option<bool> {
|
||||
self.comparison(
|
||||
self.value
|
||||
.call_method1("startswith", (prefix,))
|
||||
.and_then(|value| value.is_truthy()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl resolution::InfoKeys for PyView<'_, '_> {
|
||||
fn info_provider_match(&self, key: &str, provider: &str) -> resolution::ProviderMatch {
|
||||
let value = self
|
||||
.read(self.dict.get_item(key))
|
||||
.and_then(|row| row.cast::<PyDict>().ok().cloned())
|
||||
.and_then(|row| self.read(row.get_item("litellm_provider")));
|
||||
match value {
|
||||
Some(value) if !value.is_none() => resolution::info_provider_match(
|
||||
provider,
|
||||
&PyProvider {
|
||||
value,
|
||||
error: &self.error,
|
||||
},
|
||||
),
|
||||
_ => resolution::ProviderMatch::Match,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct ResolvedModel {
|
||||
key: String,
|
||||
|
|
@ -160,6 +220,12 @@ pub(crate) struct ModelNotFound {
|
|||
tried: Vec<String>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct InvalidProvider {
|
||||
key: String,
|
||||
provider: String,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct RegistrationTarget {
|
||||
write: String,
|
||||
|
|
@ -171,9 +237,14 @@ pub(crate) struct RegistrationTarget {
|
|||
pub(crate) enum ResolutionResult {
|
||||
Found(ResolvedModel),
|
||||
NotFound(ModelNotFound),
|
||||
Invalid(InvalidProvider),
|
||||
}
|
||||
|
||||
#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None, inferred=None))]
|
||||
#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None, inferred=None, info_read=false))]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "preserve the keyword-only binding contract"
|
||||
)]
|
||||
pub(crate) fn resolve_model(
|
||||
module: &Bound<'_, PyModule>,
|
||||
model_cost: &Bound<'_, PyDict>,
|
||||
|
|
@ -182,6 +253,7 @@ pub(crate) fn resolve_model(
|
|||
base_model: Option<&str>,
|
||||
region: Option<&str>,
|
||||
inferred: Option<(&str, Option<&str>)>,
|
||||
info_read: bool,
|
||||
) -> PyResult<ResolutionResult> {
|
||||
let index = module.getattr("_model_key_index")?;
|
||||
let index = index.extract::<PyRef<'_, ModelKeyIndex>>()?;
|
||||
|
|
@ -196,21 +268,27 @@ pub(crate) fn resolve_model(
|
|||
base_model,
|
||||
region,
|
||||
};
|
||||
let result = match inferred {
|
||||
Some((model, provider)) => resolution::resolve_inferred(
|
||||
&view,
|
||||
&query,
|
||||
resolution::InferredProvider { model, provider },
|
||||
),
|
||||
None => resolution::resolve(&view, &query),
|
||||
let inferred =
|
||||
inferred.map(|(model, provider)| resolution::InferredProvider { model, provider });
|
||||
let result = if info_read {
|
||||
resolution::resolve_info(&view, &query, inferred)
|
||||
} else {
|
||||
Ok(match inferred {
|
||||
Some(inferred) => resolution::resolve_inferred(&view, &query, inferred),
|
||||
None => resolution::resolve(&view, &query),
|
||||
})
|
||||
};
|
||||
view.finish()?;
|
||||
Ok(match result {
|
||||
Resolved::Found { key, via } => ResolutionResult::Found(ResolvedModel {
|
||||
Ok(Resolved::Found { key, via }) => ResolutionResult::Found(ResolvedModel {
|
||||
key,
|
||||
via: format!("{via:?}"),
|
||||
}),
|
||||
Resolved::NotFound { tried } => ResolutionResult::NotFound(ModelNotFound { tried }),
|
||||
Ok(Resolved::NotFound { tried }) => ResolutionResult::NotFound(ModelNotFound { tried }),
|
||||
Err(error) => ResolutionResult::Invalid(InvalidProvider {
|
||||
key: error.key,
|
||||
provider: error.provider,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -790,9 +868,17 @@ mod tests {
|
|||
"部署": {"litellm_provider": null}
|
||||
}),
|
||||
);
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module(py), &rows, model, provider, base, region, None).unwrap()
|
||||
else {
|
||||
let ResolutionResult::Found(found) = resolve_model(
|
||||
&module(py),
|
||||
&rows,
|
||||
model,
|
||||
provider,
|
||||
base,
|
||||
region,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("not found")
|
||||
};
|
||||
assert_eq!((found.key.as_str(), found.via.as_str()), (key, via));
|
||||
|
|
@ -884,9 +970,17 @@ mod tests {
|
|||
) {
|
||||
Python::attach(|py| {
|
||||
let rows = dict(py, json!({key: {"litellm_provider": provider}}));
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module(py), &rows, model, Some(provider), None, None, None).unwrap()
|
||||
else {
|
||||
let ResolutionResult::Found(found) = resolve_model(
|
||||
&module(py),
|
||||
&rows,
|
||||
model,
|
||||
Some(provider),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("not found")
|
||||
};
|
||||
assert_eq!((found.key.as_str(), found.via.as_str()), (key, via));
|
||||
|
|
@ -921,7 +1015,8 @@ mod tests {
|
|||
Some(provider),
|
||||
None,
|
||||
None,
|
||||
None
|
||||
None,
|
||||
false
|
||||
)
|
||||
.unwrap(),
|
||||
ResolutionResult::Found(_)
|
||||
|
|
@ -931,6 +1026,59 @@ mod tests {
|
|||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn info_reads_distinguish_provider_mismatch_from_invalid(
|
||||
#[from(interpreter)] _interpreter: (),
|
||||
#[values("openai", "bedrock", "github")] provider: &str,
|
||||
#[values(json!(1), json!(true), json!([]), json!({}))] value: Value,
|
||||
#[values(None, Some(""), Some("explicit"))] explicit: Option<&str>,
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let model = format!("{provider}/model");
|
||||
let rows = dict(
|
||||
py,
|
||||
json!({
|
||||
&model: {"litellm_provider": value},
|
||||
"model": {"litellm_provider": provider}
|
||||
}),
|
||||
);
|
||||
let result = resolve_model(
|
||||
&module(py),
|
||||
&rows,
|
||||
&model,
|
||||
explicit.map(|explicit| if explicit.is_empty() { "" } else { provider }),
|
||||
None,
|
||||
None,
|
||||
Some(("model", Some(provider))),
|
||||
true,
|
||||
)
|
||||
.unwrap();
|
||||
match result {
|
||||
ResolutionResult::Found(found) => {
|
||||
assert_eq!(
|
||||
found.key,
|
||||
if provider == "openai" {
|
||||
"model"
|
||||
} else {
|
||||
&model
|
||||
}
|
||||
);
|
||||
assert_ne!(provider, "bedrock");
|
||||
}
|
||||
ResolutionResult::Invalid(invalid) => {
|
||||
assert_eq!(provider, "bedrock");
|
||||
assert_eq!(
|
||||
(invalid.key.as_str(), invalid.provider.as_str()),
|
||||
(model.as_str(), provider)
|
||||
);
|
||||
}
|
||||
ResolutionResult::NotFound(_) => {
|
||||
panic!("mismatch must continue, invalid must abort")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::prefixed("openai/o3-pro", Some("o3-pro"))]
|
||||
#[case::versioned(
|
||||
|
|
@ -983,7 +1131,7 @@ mod tests {
|
|||
Python::attach(|py| {
|
||||
let rows = dict(py, json!({"model": {}}));
|
||||
let ResolutionResult::NotFound(missing) =
|
||||
resolve_model(&module(py), &rows, model, None, None, None, None).unwrap()
|
||||
resolve_model(&module(py), &rows, model, None, None, None, None, false).unwrap()
|
||||
else {
|
||||
panic!("unexpected match")
|
||||
};
|
||||
|
|
@ -1034,12 +1182,15 @@ assert n.price(row, usage, when).total == 21.0
|
|||
let module = module(py);
|
||||
let rows = dict(py, json!({"Model": {}}));
|
||||
let resolve = |rows: &Bound<'_, PyDict>, key| match resolve_model(
|
||||
&module, rows, key, None, None, None, None,
|
||||
&module, rows, key, None, None, None, None, false,
|
||||
)
|
||||
.unwrap()
|
||||
{
|
||||
ResolutionResult::Found(found) => Some(found.key),
|
||||
ResolutionResult::NotFound(_) => None,
|
||||
ResolutionResult::Invalid(_) => {
|
||||
panic!("default lookup must not validate provider values")
|
||||
}
|
||||
};
|
||||
assert_eq!(resolve(&rows, "MODEL").as_deref(), Some("Model"));
|
||||
rows.del_item("Model").unwrap();
|
||||
|
|
@ -1195,7 +1346,8 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
|
|||
json!({"custom/Model": {}, "custom/model": {}, "MODEL": {}}),
|
||||
);
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module(py), &rows, model, provider, None, None, None).unwrap()
|
||||
resolve_model(&module(py), &rows, model, provider, None, None, None, false)
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("not found")
|
||||
};
|
||||
|
|
@ -1260,14 +1412,23 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou
|
|||
Some("missing-base"),
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("unexpected base fallback")
|
||||
};
|
||||
assert_eq!(missing.tried, ["missing-base"]);
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap()
|
||||
else {
|
||||
let ResolutionResult::Found(found) = resolve_model(
|
||||
&module,
|
||||
&rows,
|
||||
"model",
|
||||
Some("openai"),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("not found")
|
||||
};
|
||||
assert_eq!((found.key.as_str(), found.via.as_str()), ("model", "Exact"));
|
||||
|
|
@ -1276,9 +1437,17 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou
|
|||
.unwrap()
|
||||
.set_item("litellm_provider", "openai")
|
||||
.unwrap();
|
||||
let ResolutionResult::Found(found) =
|
||||
resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap()
|
||||
else {
|
||||
let ResolutionResult::Found(found) = resolve_model(
|
||||
&module,
|
||||
&rows,
|
||||
"model",
|
||||
Some("openai"),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("not found")
|
||||
};
|
||||
assert_eq!(found.key, "openai/model");
|
||||
|
|
@ -1297,6 +1466,7 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou
|
|||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("unexpected match")
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import inspect
|
||||
import unittest
|
||||
from collections import UserString
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
|
|
@ -11,6 +12,161 @@ from litellm.rust_bridge import _native as native
|
|||
|
||||
|
||||
class ModelInfoResolutionTests(unittest.TestCase):
|
||||
def test_provider_callback_control_flow_is_not_a_lookup_failure(self) -> None:
|
||||
class InterruptedProvider:
|
||||
def __ne__(self, other: object) -> bool:
|
||||
raise KeyboardInterrupt
|
||||
|
||||
rows: Final[dict[str, object]] = {"model": {"litellm_provider": InterruptedProvider()}}
|
||||
with self.assertRaises(KeyboardInterrupt):
|
||||
native.resolve_model(rows, "model", provider="openai", info_read=True)
|
||||
|
||||
def test_info_read_preserves_m3_provider_protocol(self) -> None:
|
||||
values: Final[tuple[object, ...]] = (
|
||||
1,
|
||||
True,
|
||||
False,
|
||||
[],
|
||||
{},
|
||||
(),
|
||||
1.25,
|
||||
b"openai",
|
||||
bytearray(b"openai"),
|
||||
object(),
|
||||
UserString("openai"),
|
||||
UserString("bedrock_converse"),
|
||||
None,
|
||||
"",
|
||||
"openai",
|
||||
"bedrock_converse",
|
||||
)
|
||||
for provider in (
|
||||
"openai",
|
||||
"azure_ai",
|
||||
"bedrock",
|
||||
"bedrock_converse",
|
||||
"vertex_ai",
|
||||
"fireworks_ai",
|
||||
"github",
|
||||
"litellm_proxy",
|
||||
"nadir",
|
||||
):
|
||||
for value in values:
|
||||
for explicit in (None, "", provider):
|
||||
with self.subTest(provider=provider, value=value, explicit=explicit):
|
||||
self.assert_provider_protocol(provider, value, explicit)
|
||||
|
||||
def assert_provider_protocol(self, provider: str, value: object, explicit: str | None) -> None:
|
||||
model: Final = f"{provider}/invalid-model"
|
||||
first: Final = {"litellm_provider": value, "mode": "chat", "supports_vision": False}
|
||||
rows: Final[dict[str, object]] = {
|
||||
model: first,
|
||||
"invalid-model": {"litellm_provider": provider, "mode": "chat", "supports_vision": True},
|
||||
}
|
||||
result: Final = native.resolve_model(
|
||||
rows, model, provider=explicit, inferred=("invalid-model", provider), info_read=True
|
||||
)
|
||||
try:
|
||||
matches: Final = utils._check_provider_match(first, provider) # pyright: ignore[reportUnknownMemberType, reportPrivateUsage] # Exact M3 oracle has an untyped private boundary
|
||||
except Exception:
|
||||
self.assertIsInstance(result, native.InvalidProvider)
|
||||
if not isinstance(result, native.InvalidProvider):
|
||||
self.fail("provider comparison error became a lookup result")
|
||||
self.assertEqual((result.key, result.provider), (model, provider))
|
||||
else:
|
||||
self.assertIsInstance(result, native.ResolvedModel)
|
||||
if not isinstance(result, native.ResolvedModel):
|
||||
self.fail("matching candidate was lost")
|
||||
self.assertEqual(result.key, model if matches else "invalid-model")
|
||||
self.assertIs(rows[model], first)
|
||||
self.assertIs(first["litellm_provider"], value)
|
||||
|
||||
def test_info_read_matches_public_m3_invalid_provider_results(self) -> None:
|
||||
for provider in ("openai", "bedrock", "vertex_ai", "fireworks_ai", "github"):
|
||||
for explicit in (None, "", provider):
|
||||
with self.subTest(provider=provider, explicit=explicit):
|
||||
self.assert_public_m3_invalid_provider(provider, explicit)
|
||||
|
||||
def assert_public_m3_invalid_provider(self, provider: str, explicit: str | None) -> None:
|
||||
model: Final = f"{provider}/invalid-model"
|
||||
rows: Final[dict[str, object]] = {
|
||||
model: {"litellm_provider": 1, "mode": "chat"},
|
||||
"invalid-model": {"litellm_provider": provider, "mode": "chat"},
|
||||
}
|
||||
with patch.object(litellm, "model_cost", rows), patch.object(utils, "_model_cost_lowercase_map", None):
|
||||
result: Final = native.resolve_model(
|
||||
rows, model, provider=explicit, inferred=("invalid-model", provider), info_read=True
|
||||
)
|
||||
if provider in ("bedrock", "vertex_ai", "fireworks_ai"):
|
||||
with self.assertRaisesRegex(Exception, "isn't mapped yet"):
|
||||
utils.get_model_info_helper(model, custom_llm_provider=explicit)
|
||||
self.assertIsInstance(result, native.InvalidProvider)
|
||||
else:
|
||||
legacy: Final = utils.get_model_info_helper(model, custom_llm_provider=explicit)
|
||||
if not isinstance(result, native.ResolvedModel):
|
||||
self.fail("public M3 selected a row")
|
||||
self.assertEqual(result.key, legacy["key"])
|
||||
|
||||
def test_info_read_sees_mutations_without_changing_registration(self) -> None:
|
||||
row: Final[dict[str, object]] = {"litellm_provider": 1, "opaque": object()}
|
||||
rows: Final[dict[str, object]] = {"bedrock/model": row, "model": {"litellm_provider": "bedrock"}}
|
||||
before: Final = native.registration_target(rows, "bedrock/model", provider="bedrock")
|
||||
self.assertIsInstance(native.resolve_model(rows, "bedrock/model", provider="bedrock"), native.ResolvedModel)
|
||||
self.assertIsInstance(
|
||||
native.resolve_model(rows, "bedrock/model", provider="bedrock", info_read=True), native.InvalidProvider
|
||||
)
|
||||
row["litellm_provider"] = "openai"
|
||||
selected: Final = native.resolve_model(rows, "bedrock/model", provider="bedrock", info_read=True)
|
||||
if not isinstance(selected, native.ResolvedModel):
|
||||
self.fail("later valid row lost after provider mutation")
|
||||
self.assertEqual(selected.key, "model")
|
||||
row["litellm_provider"] = 1
|
||||
after: Final = native.registration_target(rows, "bedrock/model", provider="bedrock")
|
||||
self.assertEqual(
|
||||
(before.write, before.inherit_from, before.names_the_row),
|
||||
(after.write, after.inherit_from, after.names_the_row),
|
||||
)
|
||||
self.assertIsInstance(native.model_info(rows, "bedrock/model"), native.RowErrors)
|
||||
values: Final[tuple[object, ...]] = (None, 1, True, [], {}, object())
|
||||
for value in values:
|
||||
row["litellm_provider"] = value
|
||||
self.assertIsInstance(
|
||||
native.resolve_model(rows, "bedrock/model", inferred=("bedrock/model", None), info_read=True),
|
||||
native.ResolvedModel,
|
||||
)
|
||||
del row["litellm_provider"]
|
||||
self.assertIsInstance(
|
||||
native.resolve_model(rows, "bedrock/model", provider="bedrock", info_read=True), native.ResolvedModel
|
||||
)
|
||||
|
||||
def test_sdk_explicit_bedrock_suffix_and_priority_match_m3(self) -> None:
|
||||
for model in ("probe-model-20260101", "probe-model"):
|
||||
with self.subTest(model=model):
|
||||
self.assert_sdk_bedrock_selection(model)
|
||||
|
||||
def assert_sdk_bedrock_selection(self, model: str) -> None:
|
||||
rows: Final[dict[str, object]] = {
|
||||
"probe-model": {"litellm_provider": "bedrock", "mode": "chat", "supports_function_calling": True},
|
||||
"bedrock/probe-model": {"litellm_provider": "bedrock", "mode": "chat", "supports_function_calling": False},
|
||||
}
|
||||
with patch.object(litellm, "model_cost", rows), patch.object(utils, "_model_cost_lowercase_map", None):
|
||||
result: Final = native.resolve_model(
|
||||
rows, model, provider="bedrock", inferred=("ignored", "openai"), info_read=True
|
||||
)
|
||||
self.assertIsInstance(native.resolve_model(rows, model, provider="bedrock"), native.ResolvedModel)
|
||||
if model.endswith("-20260101"):
|
||||
with self.assertRaisesRegex(Exception, "isn't mapped yet"):
|
||||
utils.get_model_info_helper(model, custom_llm_provider="bedrock")
|
||||
self.assertIsInstance(result, native.ModelNotFound)
|
||||
if not isinstance(result, native.ModelNotFound):
|
||||
self.fail("SDK dated miss became a match")
|
||||
self.assertEqual(result.tried, [f"bedrock/{model}", model])
|
||||
else:
|
||||
legacy: Final = utils.get_model_info_helper(model, custom_llm_provider="bedrock")
|
||||
if not isinstance(result, native.ResolvedModel):
|
||||
self.fail("qualified candidate was lost")
|
||||
self.assertEqual((result.key, legacy["key"]), ("bedrock/probe-model", "bedrock/probe-model"))
|
||||
|
||||
def test_inferred_reads_match_m3_selection(self) -> None:
|
||||
cases: Final = (
|
||||
("read-model", "read-model", "openai", "read-model", "openai/read-model"),
|
||||
|
|
@ -68,6 +224,12 @@ class ModelInfoResolutionTests(unittest.TestCase):
|
|||
if not isinstance(selected, native.ResolvedModel):
|
||||
self.fail(str(selected.tried))
|
||||
self.assertEqual((legacy["key"], selected.key), (expected, expected))
|
||||
info_selected: Final = native.resolve_model(
|
||||
rows, model, inferred=(routed_model, routed_provider), info_read=True
|
||||
)
|
||||
if not isinstance(info_selected, native.ResolvedModel):
|
||||
self.fail("info-read intent changed an inferred selection")
|
||||
self.assertEqual(info_selected.key, expected)
|
||||
lowered: Final = native.model_info(rows, selected.key)
|
||||
self.assertIsInstance(lowered, native.ModelInfoView)
|
||||
if not isinstance(lowered, native.ModelInfoView):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from asyncio import Future
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Literal, Never, final
|
||||
from typing import Literal, Never, final, overload
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue
|
||||
|
|
@ -148,6 +148,13 @@ class ModelNotFound:
|
|||
@property
|
||||
def tried(self) -> list[str]: ...
|
||||
|
||||
@final
|
||||
class InvalidProvider:
|
||||
@property
|
||||
def key(self) -> str: ...
|
||||
@property
|
||||
def provider(self) -> str: ...
|
||||
|
||||
@final
|
||||
class RegistrationTarget:
|
||||
@property
|
||||
|
|
@ -241,6 +248,7 @@ class Unpriced:
|
|||
@property
|
||||
def partial(self) -> Priced: ...
|
||||
|
||||
@overload
|
||||
def resolve_model(
|
||||
model_cost: dict[str, object],
|
||||
model: str,
|
||||
|
|
@ -249,7 +257,19 @@ def resolve_model(
|
|||
base_model: str | None = None,
|
||||
region: str | None = None,
|
||||
inferred: tuple[str, str | None] | None = None,
|
||||
info_read: Literal[False] = False,
|
||||
) -> ResolvedModel | ModelNotFound: ...
|
||||
@overload
|
||||
def resolve_model(
|
||||
model_cost: dict[str, object],
|
||||
model: str,
|
||||
*,
|
||||
provider: str | None = None,
|
||||
base_model: str | None = None,
|
||||
region: str | None = None,
|
||||
inferred: tuple[str, str | None] | None = None,
|
||||
info_read: bool,
|
||||
) -> ResolvedModel | ModelNotFound | InvalidProvider: ...
|
||||
def registration_target(
|
||||
model_cost: dict[str, object], key: str, *, provider: str | None = None
|
||||
) -> RegistrationTarget: ...
|
||||
|
|
@ -450,6 +470,7 @@ __all__ = [
|
|||
"FetchedRegistry",
|
||||
"ForkedAfterNativeRuntimeStarted",
|
||||
"HuggingFaceEncoding",
|
||||
"InvalidProvider",
|
||||
"ModelCostLayers",
|
||||
"ModelInfoView",
|
||||
"ModelNotFound",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue