From e622016f55f9d95fd7856a949d3a8a69f3cccb87 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:13:09 +0000 Subject: [PATCH] fix(catalog): preserve SDK info provider and suffix semantics --- .../crates/model-catalog/src/error.rs | 7 + .../crates/model-catalog/src/resolution.rs | 323 +++++++++++++++++- litellm-rust/crates/python-bridge/src/lib.rs | 5 +- .../python-bridge/src/model_registry/live.rs | 226 ++++++++++-- .../python-bridge/tests/catalog_resolution.py | 162 +++++++++ litellm/rust_bridge/_native.pyi | 23 +- 6 files changed, 714 insertions(+), 32 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs index e6c068242a0..9007ffa79b9 100644 --- a/litellm-rust/crates/model-catalog/src/error.rs +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -2,6 +2,13 @@ use std::fmt; use thiserror::Error; +#[derive(Debug, Error, PartialEq, Eq)] +#[error("provider metadata comparison failed for {key} with {provider}")] +pub struct ProviderReadError { + pub key: String, + pub provider: String, +} + #[derive(Debug, Error)] pub enum Error { #[error("invalid JSON: {0}")] diff --git a/litellm-rust/crates/model-catalog/src/resolution.rs b/litellm-rust/crates/model-catalog/src/resolution.rs index d3952840ce6..9c0b6359780 100644 --- a/litellm-rust/crates/model-catalog/src/resolution.rs +++ b/litellm-rust/crates/model-catalog/src/resolution.rs @@ -1,7 +1,95 @@ +use std::cell::RefCell; + +use crate::ProviderReadError; + pub trait Keys { fn contains(&self, key: &str) -> bool; fn lowercase(&self, key: &str) -> Option; fn provider_of(&self, key: &str) -> Option; + + fn matches_provider(&self, key: &str, provider: &str) -> bool { + provider_matches(provider, self.provider_of(key).as_deref()) + } +} + +pub trait InfoKeys: Keys { + fn info_provider_match(&self, key: &str, provider: &str) -> ProviderMatch; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::VariantArray)] +pub enum ProviderMatch { + Match, + Mismatch, + Invalid, +} + +pub trait ProviderValue { + fn differs(&self, provider: &str) -> Option; + fn equals(&self, provider: &str) -> Option; + fn starts_with(&self, prefix: &str) -> Option; +} + +pub fn info_provider_match(provider: &str, value: &impl ProviderValue) -> ProviderMatch { + match compare_info_provider(provider, value) { + Some(true) => ProviderMatch::Match, + Some(false) => ProviderMatch::Mismatch, + None => ProviderMatch::Invalid, + } +} + +fn compare_info_provider(provider: &str, value: &impl ProviderValue) -> Option { + if !value.differs(provider)? { + return Some(true); + } + if matches!(provider, "vertex_ai" | "fireworks_ai") && value.starts_with(provider)? + || provider.starts_with("bedrock") && value.starts_with("bedrock")? + || matches!(provider, "litellm_proxy" | "github" | "nadir") + { + return Some(true); + } + Some(provider == "azure_ai" && (value.equals("azure")? || value.equals("openai")?)) +} + +struct InfoView<'a, K> { + keys: &'a K, + error: RefCell>, +} + +impl Keys for InfoView<'_, K> { + fn contains(&self, key: &str) -> bool { + self.error.borrow().is_none() && self.keys.contains(key) + } + + fn lowercase(&self, key: &str) -> Option { + if self.error.borrow().is_some() { + return None; + } + self.keys.lowercase(key) + } + + fn provider_of(&self, key: &str) -> Option { + if self.error.borrow().is_some() { + return None; + } + self.keys.provider_of(key) + } + + fn matches_provider(&self, key: &str, provider: &str) -> bool { + if self.error.borrow().is_some() { + return false; + } + match self.keys.info_provider_match(key, provider) { + ProviderMatch::Match => true, + ProviderMatch::Mismatch => false, + ProviderMatch::Invalid => { + self.error.replace(Some(ProviderReadError { + key: key.to_owned(), + provider: provider.to_owned(), + })); + false + } + } + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -50,6 +138,116 @@ pub struct InferredProvider<'a> { pub provider: Option<&'a str>, } +pub fn resolve_info( + keys: &impl InfoKeys, + query: &Query<'_>, + inferred: Option>, +) -> Result { + let view = InfoView { + keys, + error: RefCell::default(), + }; + let resolved = match query.provider.filter(|provider| !provider.is_empty()) { + Some(provider) => resolve_info_explicit(&view, query, provider), + None => match inferred { + Some(inferred) => resolve_inferred(&view, query, inferred), + None => resolve(&view, query), + }, + }; + match view.error.into_inner() { + Some(error) => Err(error), + None => Ok(resolved), + } +} + +fn resolve_info_explicit(keys: &impl Keys, query: &Query<'_>, provider: &str) -> Resolved { + let model = query.base_model.unwrap_or(query.model); + let split = model + .strip_prefix(provider) + .and_then(|model| model.strip_prefix('/')) + .unwrap_or(model); + let combined = if split == model { + format!("{provider}/{model}") + } else { + model.to_owned() + }; + let stripped = registration_stripped(split, Some(provider)); + let split = if matches!(provider, "bedrock" | "bedrock_converse") { + strip_bedrock_nova_route(bedrock_routing_model(split)) + } else { + split + }; + let split = if provider == "bedrock_mantle" { + strip_region(split, Some(provider), query.region) + } else { + split + }; + let mut lookup = Lookup { + keys, + provider: Some(provider), + case_insensitive: true, + tried: Vec::new(), + }; + if let Some(region) = query.region { + let regional = if split.starts_with(&format!("{region}/")) { + format!("{provider}/{split}") + } else { + format!("{provider}/{region}/{split}") + }; + if let Some(found) = lookup.candidate(®ional, Via::Region) { + return found; + } + } + let exact_via = if query.base_model.is_some() { + Via::BaseModel + } else { + Via::Exact + }; + if let Some(found) = lookup.candidate( + &combined, + if combined == model { + exact_via + } else { + Via::ProviderPrefix + }, + ) { + return found; + } + if let Some(found) = lookup.candidate(model, exact_via) { + return found; + } + if provider == "bedrock_mantle" + && let Some(found) = lookup.candidate(&format!("{provider}/{split}"), Via::RegionStripped) + { + return found; + } + if let Some(found) = lookup.candidate(split, Via::ProviderStripped) { + return found; + } + let combined_stripped = if provider == "bedrock_mantle" { + format!("{provider}/{}", strip_suffix(split, Some(provider))) + } else { + format!("{provider}/{stripped}") + }; + if let Some(found) = lookup.candidate(&combined_stripped, Via::ProviderSuffixStripped) { + return found; + } + if let Some(found) = lookup.candidate(stripped, Via::SuffixStripped) { + return found; + } + let prefixed = if provider == "fireworks_ai" { + fireworks_resource(split) + } else { + format!("{provider}/{model}") + }; + if let Some(found) = lookup.candidate(&prefixed, Via::RepeatedProvider) { + return found; + } + Resolved::NotFound { + tried: lookup.tried, + } +} + pub fn resolve_inferred( keys: &impl Keys, query: &Query<'_>, @@ -306,7 +504,7 @@ impl Lookup<'_, K> { return None; }; if let Some(provider) = self.provider - && !provider_matches(provider, self.keys.provider_of(&key).as_deref()) + && !self.keys.matches_provider(&key, provider) { return None; } @@ -609,6 +807,129 @@ mod tests { Catalog::parse(rows.to_string().as_bytes()).unwrap().value } + impl InfoKeys for Catalog { + fn info_provider_match(&self, key: &str, provider: &str) -> ProviderMatch { + if self.matches_provider(key, provider) { + ProviderMatch::Match + } else { + ProviderMatch::Mismatch + } + } + } + + struct TestProvider(Option<&'static str>); + + impl ProviderValue for TestProvider { + fn differs(&self, provider: &str) -> Option { + Some(self.0 != Some(provider)) + } + + fn equals(&self, provider: &str) -> Option { + Some(self.0 == Some(provider)) + } + + fn starts_with(&self, prefix: &str) -> Option { + self.0.map(|value| value.starts_with(prefix)) + } + } + + #[rstest] + #[case::ordinary("openai", None, ProviderMatch::Mismatch)] + #[case::azure_ai("azure_ai", None, ProviderMatch::Mismatch)] + #[case::bedrock("bedrock", None, ProviderMatch::Invalid)] + #[case::converse("bedrock_converse", None, ProviderMatch::Invalid)] + #[case::vertex("vertex_ai", None, ProviderMatch::Invalid)] + #[case::fireworks("fireworks_ai", None, ProviderMatch::Invalid)] + #[case::github("github", None, ProviderMatch::Match)] + #[case::proxy("litellm_proxy", None, ProviderMatch::Match)] + #[case::nadir("nadir", None, ProviderMatch::Match)] + #[case::exact("openai", Some("openai"), ProviderMatch::Match)] + #[case::family("bedrock", Some("bedrock_converse"), ProviderMatch::Match)] + #[case::valid_mismatch("bedrock", Some("openai"), ProviderMatch::Mismatch)] + #[case::azure_alias("azure_ai", Some("openai"), ProviderMatch::Match)] + fn info_provider_protocol( + #[case] requested: &str, + #[case] actual: Option<&'static str>, + #[case] expected: ProviderMatch, + ) { + assert_eq!( + info_provider_match(requested, &TestProvider(actual)), + expected + ); + } + + struct AbortingKeys(Cell); + + impl Keys for AbortingKeys { + fn contains(&self, _: &str) -> bool { + assert!(!self.0.get(), "lookup continued after invalid provider"); + true + } + + fn lowercase(&self, _: &str) -> Option { + panic!("lookup continued after invalid provider") + } + + fn provider_of(&self, _: &str) -> Option { + panic!("info reads must preserve provider state") + } + } + + impl InfoKeys for AbortingKeys { + fn info_provider_match(&self, _: &str, _: &str) -> ProviderMatch { + self.0.set(true); + ProviderMatch::Invalid + } + } + + #[rstest] + #[case::explicit(Some("bedrock"))] + #[case::inferred(None)] + #[case::empty(Some(""))] + fn info_provider_failure_stops_live_access(#[case] explicit: Option<&str>) { + let keys = AbortingKeys(Cell::new(false)); + let error = resolve_info( + &keys, + &query("bedrock/model", explicit), + Some(InferredProvider { + model: "model", + provider: Some("bedrock"), + }), + ) + .unwrap_err(); + assert_eq!(error.key, "bedrock/model"); + assert_eq!(error.provider, "bedrock"); + } + + #[rstest] + #[case::dated("probe-model-20260101", false)] + #[case::qualified_priority("probe-model", true)] + fn info_explicit_bedrock_is_not_generic_suffix_resolution( + #[case] model: &str, + #[case] found: bool, + ) { + let rows = catalog(json!({ + "probe-model": {"litellm_provider": "bedrock"}, + "bedrock/probe-model": {"litellm_provider": "bedrock"} + })); + let result = resolve_info(&rows, &query(model, Some("bedrock")), None).unwrap(); + let expected = if found { + Resolved::Found { + key: "bedrock/probe-model".into(), + via: Via::ProviderPrefix, + } + } else { + Resolved::NotFound { + tried: vec![format!("bedrock/{model}"), model.into()], + } + }; + assert_eq!(result, expected); + assert!(matches!( + resolve(&rows, &query(model, Some("bedrock"))), + Resolved::Found { .. } + )); + } + fn query<'a>(model: &'a str, provider: Option<&'a str>) -> Query<'a> { Query { model, diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index ce53fe798ec..28c2b4f5863 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -32,8 +32,8 @@ mod _native { use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] use crate::model_registry::{ - CatalogRefresher, FetchedRegistry, ModelCostLayers, ModelInfoView, ModelNotFound, - ModelRegistry, PriceLine, Priced, RegistrationTarget, RegistryNotModified, + CatalogRefresher, FetchedRegistry, InvalidProvider, ModelCostLayers, ModelInfoView, + ModelNotFound, ModelRegistry, PriceLine, Priced, RegistrationTarget, RegistryNotModified, RegistryUnavailable, ResolvedModel, RowError, RowErrors, Unpriced, UsageInput, WhenInput, fetch_once, fetch_with_retry, invalidate_model_cost_index, model_info, model_row_errors, price, registration_target, resolve_model, write_rows, @@ -138,6 +138,7 @@ assert live == {"model": {"litellm_provider": "openai", "input_cost_per_token": "write_rows", "ResolvedModel", "ModelNotFound", + "InvalidProvider", "RegistrationTarget", "RowErrors", "ModelInfoView", diff --git a/litellm-rust/crates/python-bridge/src/model_registry/live.rs b/litellm-rust/crates/python-bridge/src/model_registry/live.rs index 4ec668147a6..8feb77bac5b 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry/live.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry/live.rs @@ -9,9 +9,9 @@ use litellm_model_catalog::wire::{Fields, FlatKey}; use litellm_model_catalog::{self as resolution, Keys, Query, Resolved}; use litellm_model_catalog::{Error, Located, Model}; use pyo3::class::gc::{PyTraverseError, PyVisit}; -use pyo3::exceptions::PyValueError; +use pyo3::exceptions::{PyException, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyAny, PyDict, PyModule, PySet}; +use pyo3::types::{PyAny, PyDict, PyModule, PySet, PyString}; use time::OffsetDateTime; use super::{RowError, escape_pointer_segment, fields_to_dict, nonfinite_errors, row_error}; @@ -149,6 +149,66 @@ impl Keys for PyView<'_, '_> { } } +struct PyProvider<'a, 'py> { + value: Bound<'py, PyAny>, + error: &'a RefCell>, +} + +impl PyProvider<'_, '_> { + fn comparison(&self, result: PyResult) -> Option { + match result { + Ok(value) => Some(value), + Err(error) => { + if !error.is_instance_of::(self.value.py()) { + self.error.borrow_mut().get_or_insert(error); + } + None + } + } + } +} + +impl resolution::ProviderValue for PyProvider<'_, '_> { + fn differs(&self, provider: &str) -> Option { + self.comparison(self.value.ne(provider)) + } + + fn equals(&self, provider: &str) -> Option { + self.comparison( + PyString::new(self.value.py(), provider) + .as_any() + .eq(&self.value), + ) + } + + fn starts_with(&self, prefix: &str) -> Option { + self.comparison( + self.value + .call_method1("startswith", (prefix,)) + .and_then(|value| value.is_truthy()), + ) + } +} + +impl resolution::InfoKeys for PyView<'_, '_> { + fn info_provider_match(&self, key: &str, provider: &str) -> resolution::ProviderMatch { + let value = self + .read(self.dict.get_item(key)) + .and_then(|row| row.cast::().ok().cloned()) + .and_then(|row| self.read(row.get_item("litellm_provider"))); + match value { + Some(value) if !value.is_none() => resolution::info_provider_match( + provider, + &PyProvider { + value, + error: &self.error, + }, + ), + _ => resolution::ProviderMatch::Match, + } + } +} + #[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")] pub(crate) struct ResolvedModel { key: String, @@ -160,6 +220,12 @@ pub(crate) struct ModelNotFound { tried: Vec, } +#[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")] +pub(crate) struct InvalidProvider { + key: String, + provider: String, +} + #[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")] pub(crate) struct RegistrationTarget { write: String, @@ -171,9 +237,14 @@ pub(crate) struct RegistrationTarget { pub(crate) enum ResolutionResult { Found(ResolvedModel), NotFound(ModelNotFound), + Invalid(InvalidProvider), } -#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None, inferred=None))] +#[pyfunction(pass_module, signature = (model_cost, model, *, provider=None, base_model=None, region=None, inferred=None, info_read=false))] +#[expect( + clippy::too_many_arguments, + reason = "preserve the keyword-only binding contract" +)] pub(crate) fn resolve_model( module: &Bound<'_, PyModule>, model_cost: &Bound<'_, PyDict>, @@ -182,6 +253,7 @@ pub(crate) fn resolve_model( base_model: Option<&str>, region: Option<&str>, inferred: Option<(&str, Option<&str>)>, + info_read: bool, ) -> PyResult { let index = module.getattr("_model_key_index")?; let index = index.extract::>()?; @@ -196,21 +268,27 @@ pub(crate) fn resolve_model( base_model, region, }; - let result = match inferred { - Some((model, provider)) => resolution::resolve_inferred( - &view, - &query, - resolution::InferredProvider { model, provider }, - ), - None => resolution::resolve(&view, &query), + let inferred = + inferred.map(|(model, provider)| resolution::InferredProvider { model, provider }); + let result = if info_read { + resolution::resolve_info(&view, &query, inferred) + } else { + Ok(match inferred { + Some(inferred) => resolution::resolve_inferred(&view, &query, inferred), + None => resolution::resolve(&view, &query), + }) }; view.finish()?; Ok(match result { - Resolved::Found { key, via } => ResolutionResult::Found(ResolvedModel { + Ok(Resolved::Found { key, via }) => ResolutionResult::Found(ResolvedModel { key, via: format!("{via:?}"), }), - Resolved::NotFound { tried } => ResolutionResult::NotFound(ModelNotFound { tried }), + Ok(Resolved::NotFound { tried }) => ResolutionResult::NotFound(ModelNotFound { tried }), + Err(error) => ResolutionResult::Invalid(InvalidProvider { + key: error.key, + provider: error.provider, + }), }) } @@ -790,9 +868,17 @@ mod tests { "部署": {"litellm_provider": null} }), ); - let ResolutionResult::Found(found) = - resolve_model(&module(py), &rows, model, provider, base, region, None).unwrap() - else { + let ResolutionResult::Found(found) = resolve_model( + &module(py), + &rows, + model, + provider, + base, + region, + None, + false, + ) + .unwrap() else { panic!("not found") }; assert_eq!((found.key.as_str(), found.via.as_str()), (key, via)); @@ -884,9 +970,17 @@ mod tests { ) { Python::attach(|py| { let rows = dict(py, json!({key: {"litellm_provider": provider}})); - let ResolutionResult::Found(found) = - resolve_model(&module(py), &rows, model, Some(provider), None, None, None).unwrap() - else { + let ResolutionResult::Found(found) = resolve_model( + &module(py), + &rows, + model, + Some(provider), + None, + None, + None, + false, + ) + .unwrap() else { panic!("not found") }; assert_eq!((found.key.as_str(), found.via.as_str()), (key, via)); @@ -921,7 +1015,8 @@ mod tests { Some(provider), None, None, - None + None, + false ) .unwrap(), ResolutionResult::Found(_) @@ -931,6 +1026,59 @@ mod tests { }); } + #[rstest] + fn info_reads_distinguish_provider_mismatch_from_invalid( + #[from(interpreter)] _interpreter: (), + #[values("openai", "bedrock", "github")] provider: &str, + #[values(json!(1), json!(true), json!([]), json!({}))] value: Value, + #[values(None, Some(""), Some("explicit"))] explicit: Option<&str>, + ) { + Python::attach(|py| { + let model = format!("{provider}/model"); + let rows = dict( + py, + json!({ + &model: {"litellm_provider": value}, + "model": {"litellm_provider": provider} + }), + ); + let result = resolve_model( + &module(py), + &rows, + &model, + explicit.map(|explicit| if explicit.is_empty() { "" } else { provider }), + None, + None, + Some(("model", Some(provider))), + true, + ) + .unwrap(); + match result { + ResolutionResult::Found(found) => { + assert_eq!( + found.key, + if provider == "openai" { + "model" + } else { + &model + } + ); + assert_ne!(provider, "bedrock"); + } + ResolutionResult::Invalid(invalid) => { + assert_eq!(provider, "bedrock"); + assert_eq!( + (invalid.key.as_str(), invalid.provider.as_str()), + (model.as_str(), provider) + ); + } + ResolutionResult::NotFound(_) => { + panic!("mismatch must continue, invalid must abort") + } + } + }); + } + #[rstest] #[case::prefixed("openai/o3-pro", Some("o3-pro"))] #[case::versioned( @@ -983,7 +1131,7 @@ mod tests { Python::attach(|py| { let rows = dict(py, json!({"model": {}})); let ResolutionResult::NotFound(missing) = - resolve_model(&module(py), &rows, model, None, None, None, None).unwrap() + resolve_model(&module(py), &rows, model, None, None, None, None, false).unwrap() else { panic!("unexpected match") }; @@ -1034,12 +1182,15 @@ assert n.price(row, usage, when).total == 21.0 let module = module(py); let rows = dict(py, json!({"Model": {}})); let resolve = |rows: &Bound<'_, PyDict>, key| match resolve_model( - &module, rows, key, None, None, None, None, + &module, rows, key, None, None, None, None, false, ) .unwrap() { ResolutionResult::Found(found) => Some(found.key), ResolutionResult::NotFound(_) => None, + ResolutionResult::Invalid(_) => { + panic!("default lookup must not validate provider values") + } }; assert_eq!(resolve(&rows, "MODEL").as_deref(), Some("Model")); rows.del_item("Model").unwrap(); @@ -1195,7 +1346,8 @@ assert n.model_info(models, "Model").input_cost_per_token == 2 json!({"custom/Model": {}, "custom/model": {}, "MODEL": {}}), ); let ResolutionResult::Found(found) = - resolve_model(&module(py), &rows, model, provider, None, None, None).unwrap() + resolve_model(&module(py), &rows, model, provider, None, None, None, false) + .unwrap() else { panic!("not found") }; @@ -1260,14 +1412,23 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou Some("missing-base"), None, None, + false, ) .unwrap() else { panic!("unexpected base fallback") }; assert_eq!(missing.tried, ["missing-base"]); - let ResolutionResult::Found(found) = - resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap() - else { + let ResolutionResult::Found(found) = resolve_model( + &module, + &rows, + "model", + Some("openai"), + None, + None, + None, + false, + ) + .unwrap() else { panic!("not found") }; assert_eq!((found.key.as_str(), found.via.as_str()), ("model", "Exact")); @@ -1276,9 +1437,17 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou .unwrap() .set_item("litellm_provider", "openai") .unwrap(); - let ResolutionResult::Found(found) = - resolve_model(&module, &rows, "model", Some("openai"), None, None, None).unwrap() - else { + let ResolutionResult::Found(found) = resolve_model( + &module, + &rows, + "model", + Some("openai"), + None, + None, + None, + false, + ) + .unwrap() else { panic!("not found") }; assert_eq!(found.key, "openai/model"); @@ -1297,6 +1466,7 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou None, None, None, + false, ) .unwrap() else { panic!("unexpected match") diff --git a/litellm-rust/crates/python-bridge/tests/catalog_resolution.py b/litellm-rust/crates/python-bridge/tests/catalog_resolution.py index 4057d67aa1d..b0b982b3dfd 100644 --- a/litellm-rust/crates/python-bridge/tests/catalog_resolution.py +++ b/litellm-rust/crates/python-bridge/tests/catalog_resolution.py @@ -2,6 +2,7 @@ from __future__ import annotations import inspect import unittest +from collections import UserString from typing import Final from unittest.mock import patch @@ -11,6 +12,161 @@ from litellm.rust_bridge import _native as native class ModelInfoResolutionTests(unittest.TestCase): + def test_provider_callback_control_flow_is_not_a_lookup_failure(self) -> None: + class InterruptedProvider: + def __ne__(self, other: object) -> bool: + raise KeyboardInterrupt + + rows: Final[dict[str, object]] = {"model": {"litellm_provider": InterruptedProvider()}} + with self.assertRaises(KeyboardInterrupt): + native.resolve_model(rows, "model", provider="openai", info_read=True) + + def test_info_read_preserves_m3_provider_protocol(self) -> None: + values: Final[tuple[object, ...]] = ( + 1, + True, + False, + [], + {}, + (), + 1.25, + b"openai", + bytearray(b"openai"), + object(), + UserString("openai"), + UserString("bedrock_converse"), + None, + "", + "openai", + "bedrock_converse", + ) + for provider in ( + "openai", + "azure_ai", + "bedrock", + "bedrock_converse", + "vertex_ai", + "fireworks_ai", + "github", + "litellm_proxy", + "nadir", + ): + for value in values: + for explicit in (None, "", provider): + with self.subTest(provider=provider, value=value, explicit=explicit): + self.assert_provider_protocol(provider, value, explicit) + + def assert_provider_protocol(self, provider: str, value: object, explicit: str | None) -> None: + model: Final = f"{provider}/invalid-model" + first: Final = {"litellm_provider": value, "mode": "chat", "supports_vision": False} + rows: Final[dict[str, object]] = { + model: first, + "invalid-model": {"litellm_provider": provider, "mode": "chat", "supports_vision": True}, + } + result: Final = native.resolve_model( + rows, model, provider=explicit, inferred=("invalid-model", provider), info_read=True + ) + try: + matches: Final = utils._check_provider_match(first, provider) # pyright: ignore[reportUnknownMemberType, reportPrivateUsage] # Exact M3 oracle has an untyped private boundary + except Exception: + self.assertIsInstance(result, native.InvalidProvider) + if not isinstance(result, native.InvalidProvider): + self.fail("provider comparison error became a lookup result") + self.assertEqual((result.key, result.provider), (model, provider)) + else: + self.assertIsInstance(result, native.ResolvedModel) + if not isinstance(result, native.ResolvedModel): + self.fail("matching candidate was lost") + self.assertEqual(result.key, model if matches else "invalid-model") + self.assertIs(rows[model], first) + self.assertIs(first["litellm_provider"], value) + + def test_info_read_matches_public_m3_invalid_provider_results(self) -> None: + for provider in ("openai", "bedrock", "vertex_ai", "fireworks_ai", "github"): + for explicit in (None, "", provider): + with self.subTest(provider=provider, explicit=explicit): + self.assert_public_m3_invalid_provider(provider, explicit) + + def assert_public_m3_invalid_provider(self, provider: str, explicit: str | None) -> None: + model: Final = f"{provider}/invalid-model" + rows: Final[dict[str, object]] = { + model: {"litellm_provider": 1, "mode": "chat"}, + "invalid-model": {"litellm_provider": provider, "mode": "chat"}, + } + with patch.object(litellm, "model_cost", rows), patch.object(utils, "_model_cost_lowercase_map", None): + result: Final = native.resolve_model( + rows, model, provider=explicit, inferred=("invalid-model", provider), info_read=True + ) + if provider in ("bedrock", "vertex_ai", "fireworks_ai"): + with self.assertRaisesRegex(Exception, "isn't mapped yet"): + utils.get_model_info_helper(model, custom_llm_provider=explicit) + self.assertIsInstance(result, native.InvalidProvider) + else: + legacy: Final = utils.get_model_info_helper(model, custom_llm_provider=explicit) + if not isinstance(result, native.ResolvedModel): + self.fail("public M3 selected a row") + self.assertEqual(result.key, legacy["key"]) + + def test_info_read_sees_mutations_without_changing_registration(self) -> None: + row: Final[dict[str, object]] = {"litellm_provider": 1, "opaque": object()} + rows: Final[dict[str, object]] = {"bedrock/model": row, "model": {"litellm_provider": "bedrock"}} + before: Final = native.registration_target(rows, "bedrock/model", provider="bedrock") + self.assertIsInstance(native.resolve_model(rows, "bedrock/model", provider="bedrock"), native.ResolvedModel) + self.assertIsInstance( + native.resolve_model(rows, "bedrock/model", provider="bedrock", info_read=True), native.InvalidProvider + ) + row["litellm_provider"] = "openai" + selected: Final = native.resolve_model(rows, "bedrock/model", provider="bedrock", info_read=True) + if not isinstance(selected, native.ResolvedModel): + self.fail("later valid row lost after provider mutation") + self.assertEqual(selected.key, "model") + row["litellm_provider"] = 1 + after: Final = native.registration_target(rows, "bedrock/model", provider="bedrock") + self.assertEqual( + (before.write, before.inherit_from, before.names_the_row), + (after.write, after.inherit_from, after.names_the_row), + ) + self.assertIsInstance(native.model_info(rows, "bedrock/model"), native.RowErrors) + values: Final[tuple[object, ...]] = (None, 1, True, [], {}, object()) + for value in values: + row["litellm_provider"] = value + self.assertIsInstance( + native.resolve_model(rows, "bedrock/model", inferred=("bedrock/model", None), info_read=True), + native.ResolvedModel, + ) + del row["litellm_provider"] + self.assertIsInstance( + native.resolve_model(rows, "bedrock/model", provider="bedrock", info_read=True), native.ResolvedModel + ) + + def test_sdk_explicit_bedrock_suffix_and_priority_match_m3(self) -> None: + for model in ("probe-model-20260101", "probe-model"): + with self.subTest(model=model): + self.assert_sdk_bedrock_selection(model) + + def assert_sdk_bedrock_selection(self, model: str) -> None: + rows: Final[dict[str, object]] = { + "probe-model": {"litellm_provider": "bedrock", "mode": "chat", "supports_function_calling": True}, + "bedrock/probe-model": {"litellm_provider": "bedrock", "mode": "chat", "supports_function_calling": False}, + } + with patch.object(litellm, "model_cost", rows), patch.object(utils, "_model_cost_lowercase_map", None): + result: Final = native.resolve_model( + rows, model, provider="bedrock", inferred=("ignored", "openai"), info_read=True + ) + self.assertIsInstance(native.resolve_model(rows, model, provider="bedrock"), native.ResolvedModel) + if model.endswith("-20260101"): + with self.assertRaisesRegex(Exception, "isn't mapped yet"): + utils.get_model_info_helper(model, custom_llm_provider="bedrock") + self.assertIsInstance(result, native.ModelNotFound) + if not isinstance(result, native.ModelNotFound): + self.fail("SDK dated miss became a match") + self.assertEqual(result.tried, [f"bedrock/{model}", model]) + else: + legacy: Final = utils.get_model_info_helper(model, custom_llm_provider="bedrock") + if not isinstance(result, native.ResolvedModel): + self.fail("qualified candidate was lost") + self.assertEqual((result.key, legacy["key"]), ("bedrock/probe-model", "bedrock/probe-model")) + def test_inferred_reads_match_m3_selection(self) -> None: cases: Final = ( ("read-model", "read-model", "openai", "read-model", "openai/read-model"), @@ -68,6 +224,12 @@ class ModelInfoResolutionTests(unittest.TestCase): if not isinstance(selected, native.ResolvedModel): self.fail(str(selected.tried)) self.assertEqual((legacy["key"], selected.key), (expected, expected)) + info_selected: Final = native.resolve_model( + rows, model, inferred=(routed_model, routed_provider), info_read=True + ) + if not isinstance(info_selected, native.ResolvedModel): + self.fail("info-read intent changed an inferred selection") + self.assertEqual(info_selected.key, expected) lowered: Final = native.model_info(rows, selected.key) self.assertIsInstance(lowered, native.ModelInfoView) if not isinstance(lowered, native.ModelInfoView): diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 953d6403390..405fbf4512d 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -1,6 +1,6 @@ from asyncio import Future from collections.abc import Callable, Mapping, Sequence -from typing import Literal, Never, final +from typing import Literal, Never, final, overload import httpx from pydantic import JsonValue @@ -148,6 +148,13 @@ class ModelNotFound: @property def tried(self) -> list[str]: ... +@final +class InvalidProvider: + @property + def key(self) -> str: ... + @property + def provider(self) -> str: ... + @final class RegistrationTarget: @property @@ -241,6 +248,7 @@ class Unpriced: @property def partial(self) -> Priced: ... +@overload def resolve_model( model_cost: dict[str, object], model: str, @@ -249,7 +257,19 @@ def resolve_model( base_model: str | None = None, region: str | None = None, inferred: tuple[str, str | None] | None = None, + info_read: Literal[False] = False, ) -> ResolvedModel | ModelNotFound: ... +@overload +def resolve_model( + model_cost: dict[str, object], + model: str, + *, + provider: str | None = None, + base_model: str | None = None, + region: str | None = None, + inferred: tuple[str, str | None] | None = None, + info_read: bool, +) -> ResolvedModel | ModelNotFound | InvalidProvider: ... def registration_target( model_cost: dict[str, object], key: str, *, provider: str | None = None ) -> RegistrationTarget: ... @@ -450,6 +470,7 @@ __all__ = [ "FetchedRegistry", "ForkedAfterNativeRuntimeStarted", "HuggingFaceEncoding", + "InvalidProvider", "ModelCostLayers", "ModelInfoView", "ModelNotFound",