refactor(model-cost): derive the provider model sets from the live map [12/12]

The 121 provider sets and models_by_provider are read-only views over the catalog's index, grouped by raw
litellm_provider, so a direct write, reload or registration shows on the next read and a moved or dropped key leaves
its old set. The populate chain and the models_by_provider builder are gone, and every provider in the map is listed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-10-28 05:24:47 -07:00
parent 9f337ea9b5
commit f799435484
26 changed files with 739 additions and 812 deletions

View file

@ -18,7 +18,7 @@ use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Keys, Layers, Located};
use litellm_model_catalog_refresh::{Source, git_blob_id};
use pyo3::exceptions::PyTypeError;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyModule};
use pyo3::types::{PyDict, PyFrozenSet, PyModule};
use serde::Serialize;
pub(crate) use fetch::{Bundled, File, Provenance, Remote};
@ -333,6 +333,30 @@ impl CatalogHandle {
Ok(keys.provider_keys(provider))
}
/// The keys whose live row names this raw `litellm_provider`, empty when none (decision 18)
fn models_for<'py>(
&self,
py: Python<'py>,
provider: &str,
) -> PyResult<Bound<'py, PyFrozenSet>> {
let dict = self.bound_dict(py)?;
let keys = resolve::LiveKeys {
dict: &dict,
index: &self.index,
};
PyFrozenSet::new(py, keys.provider_keys(provider))
}
/// Every raw `litellm_provider` string a live row names, sorted
fn providers(&self, py: Python<'_>) -> PyResult<Vec<String>> {
let dict = self.bound_dict(py)?;
let keys = resolve::LiveKeys {
dict: &dict,
index: &self.index,
};
Ok(keys.providers())
}
/// main's 133-field copy of a row Python built, a generalization match or a fill-missing merge
#[staticmethod]
#[pyo3(signature = (key, row, provider = None))]

View file

@ -4,10 +4,11 @@ use pyo3::prelude::*;
use pyo3::types::{PyDict, PyString};
/// The one name index of decision 18. A normalized name to the actual key, and the keys each raw `litellm_provider`
/// names. Never a row, since rows are read live, so only adding or removing keys can make it stale
/// names. Never a row, since rows are read live, so only adding or removing keys can make it stale. It holds the dict it
/// indexed, so a freed dict's id can never be taken for it by a new dict of the same length
#[derive(Debug, Default)]
pub(crate) struct NameIndex {
dict_id: usize,
dict: Option<Py<PyDict>>,
len: usize,
by_lower: HashMap<String, String>,
by_provider: HashMap<String, Vec<String>>,
@ -32,7 +33,7 @@ impl NameIndex {
by_lower.insert(key.to_lowercase(), key);
}
Self {
dict_id: dict.as_ptr() as usize,
dict: Some(dict.clone().unbind()),
len: dict.len(),
by_lower,
by_provider,
@ -40,13 +41,23 @@ impl NameIndex {
}
pub(crate) fn is_current(&self, dict: &Bound<'_, PyDict>) -> bool {
self.dict_id == dict.as_ptr() as usize && self.len == dict.len()
self.dict
.as_ref()
.is_some_and(|indexed| indexed.as_ptr() == dict.as_ptr())
&& self.len == dict.len()
}
pub(crate) fn by_lower(&self, name: &str) -> Option<&str> {
self.by_lower.get(&name.to_lowercase()).map(String::as_str)
}
/// Every raw `litellm_provider` string some live row names, sorted
pub(crate) fn providers(&self) -> Vec<String> {
let mut providers: Vec<String> = self.by_provider.keys().cloned().collect();
providers.sort();
providers
}
pub(crate) fn provider_keys(&self, provider: &str) -> &[String] {
self.by_provider.get(provider).map_or(&[], Vec::as_slice)
}

View file

@ -106,6 +106,10 @@ impl LiveKeys<'_, '_> {
Some(NameIndex::build(self.dict));
}
pub(crate) fn providers(&self) -> Vec<String> {
self.indexed(NameIndex::providers)
}
pub(crate) fn provider_keys(&self, provider: &str) -> Vec<String> {
self.indexed(|index| index.provider_keys(provider).to_vec())
}

View file

@ -7,7 +7,6 @@ _apply_process_defaults()
### INIT VARIABLES #########################
import threading
import os
import datetime
from collections.abc import Mapping, Sequence
from typing import (
@ -39,7 +38,6 @@ from litellm._logging import (
turn_on_json,
log_level,
)
import re
from litellm.constants import (
DEFAULT_BATCH_SIZE,
DEFAULT_FLUSH_INTERVAL_SECONDS,
@ -61,9 +59,9 @@ from litellm.constants import (
clarifai_models,
huggingface_models,
modelscope_models,
empower_models,
empower_models as empower_models,
together_ai_models,
WANDB_MODELS,
WANDB_MODELS as WANDB_MODELS,
REPEATED_STREAMING_CHUNK_LIMIT,
request_timeout,
request_timeout_explicitly_set as request_timeout_explicitly_set,
@ -73,7 +71,7 @@ from litellm.constants import (
known_tokenizer_config,
BEDROCK_INVOKE_PROVIDERS_LITERAL,
BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
BEDROCK_CONVERSE_MODELS,
BEDROCK_CONVERSE_MODELS as BEDROCK_CONVERSE_MODELS,
DEFAULT_MAX_TOKENS,
DEFAULT_SOFT_BUDGET,
DEFAULT_ALLOWED_FAILS,
@ -548,6 +546,14 @@ _key_management_system: Optional["KeyManagementSystem"] = None
#### PII MASKING ####
output_parse_pii: bool = False
#############################################
from litellm._catalog.provider_sets import (
LEGACY_NAMES as _LEGACY_PROVIDER_SETS,
ModelsByProvider,
_is_past_deprecation_date as _is_past_deprecation_date, # pyright: ignore[reportPrivateUsage] # main's name
is_bedrock_pricing_only_model as is_bedrock_pricing_only_model,
is_openai_finetune_model as is_openai_finetune_model,
legacy_set as _provider_set,
)
from litellm.litellm_core_utils.get_model_cost_map import (
get_model_cost_map as get_model_cost_map,
load_bundled_model_cost_map_at_import,
@ -594,455 +600,19 @@ config_path = None
vertex_ai_safety_settings: Optional[dict] = None
####### COMPLETION MODELS ###################
from typing import Set
open_ai_chat_completion_models: Set = set()
open_ai_text_completion_models: Set = set()
cohere_models: Set = set()
cohere_chat_models: Set = set()
mistral_chat_models: Set = set()
text_completion_codestral_models: Set = set()
text_completion_inception_models: Set = set()
anthropic_models: Set = set()
openrouter_models: Set = set()
datarobot_models: Set = set()
vertex_language_models: Set = set()
vertex_vision_models: Set = set()
vertex_chat_models: Set = set()
vertex_code_chat_models: Set = set()
vertex_ai_image_models: Set = set()
vertex_ai_video_models: Set = set()
vertex_text_models: Set = set()
vertex_code_text_models: Set = set()
vertex_embedding_models: Set = set()
vertex_anthropic_models: Set = set()
vertex_llama3_models: Set = set()
vertex_deepseek_models: Set = set()
vertex_ai_ai21_models: Set = set()
vertex_mistral_models: Set = set()
vertex_openai_models: Set = set()
vertex_minimax_models: Set = set()
vertex_moonshot_models: Set = set()
vertex_zai_models: Set = set()
ai21_models: Set = set()
ai21_chat_models: Set = set()
nlp_cloud_models: Set = set()
aleph_alpha_models: Set = set()
bedrock_models: Set = set()
bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
fal_ai_models: Set = set()
fireworks_ai_models: Set = set()
baseten_models: Set[str] = set()
fireworks_ai_embedding_models: Set = set()
deepinfra_models: Set = set()
perplexity_models: Set = set()
watsonx_models: Set = set()
gemini_models: Set = set()
xai_models: Set = set()
zai_models: Set = set()
deepseek_models: Set = set()
tencent_models: Set = set()
runwayml_models: Set = set()
azure_ai_models: Set = set()
jina_ai_models: Set = set()
voyage_models: Set = set()
infinity_models: Set = set()
heroku_models: Set = set()
databricks_models: Set = set()
cloudflare_models: Set = set()
codestral_models: Set = set()
friendliai_models: Set = set()
featherless_ai_models: Set = set()
palm_models: Set = set()
groq_models: Set = set()
azure_models: Set = set()
azure_anthropic_models: Set = set()
azure_text_models: Set = set()
anyscale_models: Set = set()
cerebras_models: Set = set()
nadir_models: Set = set()
galadriel_models: Set = set()
nvidia_nim_models: Set = set()
nvidia_riva_models: Set = set()
soniox_models: Set = set()
sambanova_models: Set = set()
scaledown_models: Final[Set[str]] = set() # Price reloads update this registry through existing references.
sambanova_embedding_models: Set = set()
novita_models: Set = set()
assemblyai_models: Set = set()
snowflake_models: Set = set()
gradient_ai_models: Set = set()
llama_models: Set = set()
nscale_models: Set = set()
nebius_models: Set = set()
nebius_embedding_models: Set = set()
aiml_models: Set = set()
deepgram_models: Set = set()
elevenlabs_models: Set = set()
dashscope_models: Set = set()
qwencloud_models: Set = set()
qwen_ai_platform_models: Set = set()
moonshot_models: Set = set()
publicai_models: Set = set()
darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
lambda_ai_models: Set = set()
inception_models: Set = set()
hyperbolic_models: Set = set()
black_forest_labs_models: Set = set()
recraft_models: Set = set()
cometapi_models: Set = set()
oci_models: Set = set()
vercel_ai_gateway_models: Set = set()
edenai_models: Set = set()
volcengine_models: Set = set()
wandb_models: Set = set(WANDB_MODELS)
ovhcloud_models: Set = set()
ovhcloud_embedding_models: Set = set()
lemonade_models: Set = set()
docker_model_runner_models: Set = set()
amazon_nova_models: Set = set()
stability_models: Set = set()
github_copilot_models: Set = set()
microsoft_365_copilot_models: Set[str] = set()
chatgpt_models: Set = set()
minimax_models: Set = set()
aws_polly_models: Set = set()
transcribe_models: Set = set()
gigachat_models: Set = set()
llamagate_models: Set = set()
coralbricks_models: Set[str] = set()
reducto_models: Set = set()
bedrock_mantle_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
"""
Excludes keys with the pattern 'bedrock/<region>/<model>'. These are in the model_prices_and_context_window.json file for pricing purposes only.
Args:
key (str): A key to filter.
Returns:
bool: True if the key matches the Bedrock pattern, False otherwise.
"""
# Regex to match 'bedrock/<region>/<model>'
bedrock_pattern: Final = re.compile(r"^bedrock/[a-zA-Z0-9_-]+/.+$")
if "month-commitment" in key:
return True
is_match: Final = bedrock_pattern.match(key)
return is_match is not None
def is_openai_finetune_model(key: str) -> bool:
"""
Excludes model cost keys with the pattern 'ft:<model>'. These are in the model_prices_and_context_window.json file for pricing purposes only.
Args:
key (str): A key to filter.
Returns:
bool: True if the key matches the OpenAI finetune pattern, False otherwise.
"""
return key.startswith("ft:") and not key.count(":") > 1
def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
deprecation_date: Final = value.get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
except ValueError:
return False
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
for key, value in tuple(model_cost_map.items()):
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
open_ai_chat_completion_models.add(key)
elif value.get("litellm_provider") == "text-completion-openai":
open_ai_text_completion_models.add(key)
elif value.get("litellm_provider") == "azure_text":
azure_text_models.add(key)
elif value.get("litellm_provider") == "cohere":
cohere_models.add(key)
elif value.get("litellm_provider") == "cohere_chat":
cohere_chat_models.add(key)
elif value.get("litellm_provider") == "mistral":
mistral_chat_models.add(key)
elif value.get("litellm_provider") == "anthropic":
anthropic_models.add(key)
elif value.get("litellm_provider") == "empower":
empower_models.add(key)
elif value.get("litellm_provider") == "openrouter":
openrouter_models.add(key)
elif value.get("litellm_provider") == "vercel_ai_gateway":
vercel_ai_gateway_models.add(key)
elif value.get("litellm_provider") == "edenai":
edenai_models.add(key)
elif value.get("litellm_provider") == "datarobot":
datarobot_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-text-models":
vertex_text_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-code-text-models":
vertex_code_text_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-language-models":
vertex_language_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-vision-models":
vertex_vision_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-chat-models":
vertex_chat_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
vertex_code_chat_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-embedding-models":
vertex_embedding_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-anthropic_models":
key = key.replace("vertex_ai/", "")
vertex_anthropic_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-llama_models":
key = key.replace("vertex_ai/", "")
vertex_llama3_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-deepseek_models":
key = key.replace("vertex_ai/", "")
vertex_deepseek_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-mistral_models":
key = key.replace("vertex_ai/", "")
vertex_mistral_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-ai21_models":
key = key.replace("vertex_ai/", "")
vertex_ai_ai21_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-image-models":
key = key.replace("vertex_ai/", "")
vertex_ai_image_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-video-models":
key = key.replace("vertex_ai/", "")
vertex_ai_video_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-openai_models":
key = key.replace("vertex_ai/", "")
vertex_openai_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-minimax_models":
key = key.replace("vertex_ai/", "")
vertex_minimax_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-moonshot_models":
key = key.replace("vertex_ai/", "")
vertex_moonshot_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-zai_models":
key = key.replace("vertex_ai/", "")
vertex_zai_models.add(key)
elif value.get("litellm_provider") == "ai21":
if value.get("mode") == "chat":
ai21_chat_models.add(key)
else:
ai21_models.add(key)
elif value.get("litellm_provider") == "nlp_cloud":
nlp_cloud_models.add(key)
elif value.get("litellm_provider") == "aleph_alpha":
aleph_alpha_models.add(key)
elif value.get("litellm_provider") == "bedrock" and value.get("mode") == "guardrail":
pass
elif value.get("litellm_provider") == "bedrock" and not is_bedrock_pricing_only_model(key):
bedrock_models.add(key)
elif value.get("litellm_provider") == "bedrock_converse":
bedrock_converse_models.add(key)
elif value.get("litellm_provider") == "deepinfra":
deepinfra_models.add(key)
elif value.get("litellm_provider") == "perplexity":
perplexity_models.add(key)
elif value.get("litellm_provider") == "watsonx":
watsonx_models.add(key)
elif value.get("litellm_provider") == "gemini":
gemini_models.add(key)
elif value.get("litellm_provider") == "fireworks_ai":
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "baseten":
if _is_past_deprecation_date(value):
baseten_models.discard(key)
else:
baseten_models.add(key)
elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key:
fireworks_ai_embedding_models.add(key)
elif value.get("litellm_provider") == "text-completion-codestral":
text_completion_codestral_models.add(key)
elif value.get("litellm_provider") == "text-completion-inception":
text_completion_inception_models.add(key)
elif value.get("litellm_provider") == "xai":
xai_models.add(key)
elif value.get("litellm_provider") == "zai":
zai_models.add(key)
elif value.get("litellm_provider") == "fal_ai":
fal_ai_models.add(key)
elif value.get("litellm_provider") == "deepseek":
deepseek_models.add(key)
elif value.get("litellm_provider") == "tencent":
tencent_models.add(key)
elif value.get("litellm_provider") == "runwayml":
runwayml_models.add(key)
elif value.get("litellm_provider") == "meta_llama":
llama_models.add(key)
elif value.get("litellm_provider") == "nscale":
nscale_models.add(key)
elif value.get("litellm_provider") == "azure_ai":
azure_ai_models.add(key)
elif value.get("litellm_provider") == "voyage":
voyage_models.add(key)
elif value.get("litellm_provider") == "infinity":
infinity_models.add(key)
elif value.get("litellm_provider") == "databricks":
databricks_models.add(key)
elif value.get("litellm_provider") == "cloudflare":
cloudflare_models.add(key)
elif value.get("litellm_provider") == "codestral":
codestral_models.add(key)
elif value.get("litellm_provider") == "friendliai":
friendliai_models.add(key)
elif value.get("litellm_provider") == "palm":
palm_models.add(key)
elif value.get("litellm_provider") == "groq":
groq_models.add(key)
elif value.get("litellm_provider") == "azure":
azure_models.add(key)
elif value.get("litellm_provider") == "azure_anthropic":
azure_anthropic_models.add(key)
elif value.get("litellm_provider") == "anyscale":
anyscale_models.add(key)
elif value.get("litellm_provider") == "cerebras":
cerebras_models.add(key)
elif value.get("litellm_provider") == "nadir":
nadir_models.add(key)
elif value.get("litellm_provider") == "galadriel":
galadriel_models.add(key)
elif value.get("litellm_provider") == "nvidia_nim":
nvidia_nim_models.add(key)
elif value.get("litellm_provider") == "nvidia_riva":
nvidia_riva_models.add(key)
elif value.get("litellm_provider") == "soniox":
soniox_models.add(key)
elif value.get("litellm_provider") == "sambanova":
sambanova_models.add(key)
elif value.get("litellm_provider") == "scaledown":
scaledown_models.add(key)
elif value.get("litellm_provider") == "sambanova-embedding-models":
sambanova_embedding_models.add(key)
elif value.get("litellm_provider") == "novita":
novita_models.add(key)
elif value.get("litellm_provider") == "nebius-chat-models":
nebius_models.add(key)
elif value.get("litellm_provider") == "nebius-embedding-models":
nebius_embedding_models.add(key)
elif value.get("litellm_provider") == "aiml":
aiml_models.add(key)
elif value.get("litellm_provider") == "assemblyai":
assemblyai_models.add(key)
elif value.get("litellm_provider") == "jina_ai":
jina_ai_models.add(key)
elif value.get("litellm_provider") == "snowflake":
snowflake_models.add(key)
elif value.get("litellm_provider") == "gradient_ai":
gradient_ai_models.add(key)
elif value.get("litellm_provider") == "featherless_ai":
featherless_ai_models.add(key)
elif value.get("litellm_provider") == "deepgram":
deepgram_models.add(key)
elif value.get("litellm_provider") == "elevenlabs":
elevenlabs_models.add(key)
elif value.get("litellm_provider") == "heroku":
heroku_models.add(key)
elif value.get("litellm_provider") == "dashscope":
dashscope_models.add(key)
elif value.get("litellm_provider") == "qwencloud":
qwencloud_models.add(key)
elif value.get("litellm_provider") == "qwen_ai_platform":
qwen_ai_platform_models.add(key)
elif value.get("litellm_provider") == "modelscope":
modelscope_models.add(key)
elif value.get("litellm_provider") == "moonshot":
moonshot_models.add(key)
elif value.get("litellm_provider") == "publicai":
publicai_models.add(key)
elif value.get("litellm_provider") == "darkbloom":
darkbloom_models.add(key)
elif value.get("litellm_provider") == "openinfer":
openinfer_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
morph_models.add(key)
elif value.get("litellm_provider") == "opencode_go":
opencode_go_models.add(key)
elif value.get("litellm_provider") == "lambda_ai":
lambda_ai_models.add(key)
elif value.get("litellm_provider") == "inception":
inception_models.add(key)
elif value.get("litellm_provider") == "hyperbolic":
hyperbolic_models.add(key)
elif value.get("litellm_provider") == "black_forest_labs":
black_forest_labs_models.add(key)
elif value.get("litellm_provider") == "recraft":
recraft_models.add(key)
elif value.get("litellm_provider") == "cometapi":
cometapi_models.add(key)
elif value.get("litellm_provider") == "oci":
oci_models.add(key)
elif value.get("litellm_provider") == "volcengine":
volcengine_models.add(key)
elif value.get("litellm_provider") == "wandb":
wandb_models.add(key)
elif value.get("litellm_provider") == "ovhcloud":
ovhcloud_models.add(key)
elif value.get("litellm_provider") == "ovhcloud-embedding-models":
ovhcloud_embedding_models.add(key)
elif value.get("litellm_provider") == "lemonade":
lemonade_models.add(key)
elif value.get("litellm_provider") == "docker_model_runner":
docker_model_runner_models.add(key)
elif value.get("litellm_provider") == "amazon_nova":
amazon_nova_models.add(key)
elif value.get("litellm_provider") == "stability":
stability_models.add(key)
elif value.get("litellm_provider") == "github_copilot":
github_copilot_models.add(key)
elif value.get("litellm_provider") == "microsoft_365_copilot":
microsoft_365_copilot_models.add(key)
elif value.get("litellm_provider") == "chatgpt":
chatgpt_models.add(key)
elif value.get("litellm_provider") == "minimax":
minimax_models.add(key)
elif value.get("litellm_provider") == "aws_polly":
aws_polly_models.add(key)
elif value.get("litellm_provider") == "transcribe":
transcribe_models.add(key)
elif value.get("litellm_provider") == "gigachat":
gigachat_models.add(key)
elif value.get("litellm_provider") == "llamagate":
llamagate_models.add(key)
elif value.get("litellm_provider") == "coralbricks":
coralbricks_models.add(key)
elif value.get("litellm_provider") == "reducto":
reducto_models.add(key)
elif value.get("litellm_provider") == "bedrock_mantle":
bedrock_mantle_models.add(key)
def add_known_models(model_cost_map: Optional[Dict] = None):
"""Fold `model_cost_map` (defaults to `litellm.model_cost`) into the per-provider model sets,
then refresh `models_by_provider` from those sets so the additions reach wildcard expansion.
The refresh updates the dict in place, so references captured before a reload stay live.
"""
_populate_provider_model_sets(model_cost_map if model_cost_map is not None else model_cost)
models_by_provider.update(_build_models_by_provider())
_populate_provider_model_sets(model_cost)
# known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary
# this is maintained for Exception Mapping
@ -1073,97 +643,97 @@ ollama_models = ["llama2"]
maritalk_models = ["maritalk"]
model_list = list(
open_ai_chat_completion_models
| open_ai_text_completion_models
| cohere_models
| cohere_chat_models
| anthropic_models
_provider_set("open_ai_chat_completion_models")
| _provider_set("open_ai_text_completion_models")
| _provider_set("cohere_models")
| _provider_set("cohere_chat_models")
| _provider_set("anthropic_models")
| set(replicate_models)
| openrouter_models
| datarobot_models
| _provider_set("openrouter_models")
| _provider_set("datarobot_models")
| set(huggingface_models)
| vertex_chat_models
| vertex_text_models
| ai21_models
| ai21_chat_models
| _provider_set("vertex_chat_models")
| _provider_set("vertex_text_models")
| _provider_set("ai21_models")
| _provider_set("ai21_chat_models")
| set(together_ai_models)
| set(baseten_models)
| aleph_alpha_models
| nlp_cloud_models
| set(_provider_set("baseten_models"))
| _provider_set("aleph_alpha_models")
| _provider_set("nlp_cloud_models")
| set(ollama_models)
| bedrock_models
| deepinfra_models
| perplexity_models
| _provider_set("bedrock_models")
| _provider_set("deepinfra_models")
| _provider_set("perplexity_models")
| set(maritalk_models)
| runwayml_models
| vertex_language_models
| watsonx_models
| gemini_models
| text_completion_codestral_models
| text_completion_inception_models
| xai_models
| zai_models
| fal_ai_models
| deepseek_models
| _provider_set("runwayml_models")
| _provider_set("vertex_language_models")
| _provider_set("watsonx_models")
| _provider_set("gemini_models")
| _provider_set("text_completion_codestral_models")
| _provider_set("text_completion_inception_models")
| _provider_set("xai_models")
| _provider_set("zai_models")
| _provider_set("fal_ai_models")
| _provider_set("deepseek_models")
| modelscope_models
| azure_ai_models
| voyage_models
| infinity_models
| databricks_models
| cloudflare_models
| codestral_models
| friendliai_models
| palm_models
| groq_models
| azure_models
| azure_anthropic_models
| anyscale_models
| cerebras_models
| nadir_models
| galadriel_models
| nvidia_nim_models
| nvidia_riva_models
| soniox_models
| sambanova_models
| scaledown_models
| azure_text_models
| novita_models
| assemblyai_models
| jina_ai_models
| snowflake_models
| gradient_ai_models
| llama_models
| featherless_ai_models
| nscale_models
| deepgram_models
| elevenlabs_models
| dashscope_models
| qwencloud_models
| qwen_ai_platform_models
| moonshot_models
| publicai_models
| darkbloom_models
| openinfer_models
| v0_models
| morph_models
| opencode_go_models
| lambda_ai_models
| inception_models
| black_forest_labs_models
| recraft_models
| cometapi_models
| oci_models
| heroku_models
| vercel_ai_gateway_models
| edenai_models
| volcengine_models
| wandb_models
| ovhcloud_models
| lemonade_models
| docker_model_runner_models
| reducto_models
| bedrock_mantle_models
| coralbricks_models
| _provider_set("azure_ai_models")
| _provider_set("voyage_models")
| _provider_set("infinity_models")
| _provider_set("databricks_models")
| _provider_set("cloudflare_models")
| _provider_set("codestral_models")
| _provider_set("friendliai_models")
| _provider_set("palm_models")
| _provider_set("groq_models")
| _provider_set("azure_models")
| _provider_set("azure_anthropic_models")
| _provider_set("anyscale_models")
| _provider_set("cerebras_models")
| _provider_set("nadir_models")
| _provider_set("galadriel_models")
| _provider_set("nvidia_nim_models")
| _provider_set("nvidia_riva_models")
| _provider_set("soniox_models")
| _provider_set("sambanova_models")
| _provider_set("scaledown_models")
| _provider_set("azure_text_models")
| _provider_set("novita_models")
| _provider_set("assemblyai_models")
| _provider_set("jina_ai_models")
| _provider_set("snowflake_models")
| _provider_set("gradient_ai_models")
| _provider_set("llama_models")
| _provider_set("featherless_ai_models")
| _provider_set("nscale_models")
| _provider_set("deepgram_models")
| _provider_set("elevenlabs_models")
| _provider_set("dashscope_models")
| _provider_set("qwencloud_models")
| _provider_set("qwen_ai_platform_models")
| _provider_set("moonshot_models")
| _provider_set("publicai_models")
| _provider_set("darkbloom_models")
| _provider_set("openinfer_models")
| _provider_set("v0_models")
| _provider_set("morph_models")
| _provider_set("opencode_go_models")
| _provider_set("lambda_ai_models")
| _provider_set("inception_models")
| _provider_set("black_forest_labs_models")
| _provider_set("recraft_models")
| _provider_set("cometapi_models")
| _provider_set("oci_models")
| _provider_set("heroku_models")
| _provider_set("vercel_ai_gateway_models")
| _provider_set("edenai_models")
| _provider_set("volcengine_models")
| _provider_set("wandb_models")
| _provider_set("ovhcloud_models")
| _provider_set("lemonade_models")
| _provider_set("docker_model_runner_models")
| _provider_set("reducto_models")
| _provider_set("bedrock_mantle_models")
| _provider_set("coralbricks_models")
| set(clarifai_models)
)
@ -1172,128 +742,13 @@ model_list_set = set(model_list)
# provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time
def _build_models_by_provider() -> dict:
return {
"openai": open_ai_chat_completion_models | open_ai_text_completion_models,
"text-completion-openai": open_ai_text_completion_models,
"cohere": cohere_models | cohere_chat_models,
"cohere_chat": cohere_chat_models,
"anthropic": anthropic_models,
"replicate": replicate_models,
"huggingface": huggingface_models,
"together_ai": together_ai_models,
"baseten": baseten_models,
"openrouter": openrouter_models,
"vercel_ai_gateway": vercel_ai_gateway_models,
"edenai": edenai_models,
"datarobot": datarobot_models,
"vertex_ai": vertex_chat_models
| vertex_code_chat_models
| vertex_text_models
| vertex_code_text_models
| vertex_anthropic_models
| vertex_vision_models
| vertex_language_models
| vertex_deepseek_models
| vertex_minimax_models
| vertex_moonshot_models
| vertex_zai_models,
"ai21": ai21_models | ai21_chat_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
"ollama": ollama_models,
"ollama_chat": ollama_models,
"deepinfra": deepinfra_models,
"perplexity": perplexity_models,
"maritalk": maritalk_models,
"watsonx": watsonx_models,
"gemini": gemini_models,
"fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
"aleph_alpha": aleph_alpha_models,
"text-completion-codestral": text_completion_codestral_models,
"text-completion-inception": text_completion_inception_models,
"xai": xai_models,
"zai": zai_models,
"fal_ai": fal_ai_models,
"deepseek": deepseek_models,
"tencent": tencent_models,
"runwayml": runwayml_models,
"mistral": mistral_chat_models,
"azure_ai": azure_ai_models,
"voyage": voyage_models,
"infinity": infinity_models,
"databricks": databricks_models,
"cloudflare": cloudflare_models,
"codestral": codestral_models,
"nlp_cloud": nlp_cloud_models,
"friendliai": friendliai_models,
"palm": palm_models,
"groq": groq_models,
"azure": azure_models | azure_text_models,
"azure_anthropic": azure_anthropic_models,
"azure_text": azure_text_models,
"anyscale": anyscale_models,
"cerebras": cerebras_models,
"nadir": nadir_models,
"galadriel": galadriel_models,
"nvidia_nim": nvidia_nim_models,
"nvidia_riva": nvidia_riva_models,
"soniox": soniox_models,
"sambanova": sambanova_models | sambanova_embedding_models,
"scaledown": scaledown_models,
"novita": novita_models,
"nebius": nebius_models | nebius_embedding_models,
"aiml": aiml_models,
"assemblyai": assemblyai_models,
"jina_ai": jina_ai_models,
"snowflake": snowflake_models,
"gradient_ai": gradient_ai_models,
"meta_llama": llama_models,
"nscale": nscale_models,
"featherless_ai": featherless_ai_models,
"deepgram": deepgram_models,
"elevenlabs": elevenlabs_models,
"heroku": heroku_models,
"dashscope": dashscope_models,
"qwencloud": qwencloud_models,
"qwen_ai_platform": qwen_ai_platform_models,
"modelscope": modelscope_models,
"moonshot": moonshot_models,
"publicai": publicai_models,
"darkbloom": darkbloom_models,
"openinfer": openinfer_models,
"v0": v0_models,
"morph": morph_models,
"opencode_go": opencode_go_models,
"lambda_ai": lambda_ai_models,
"inception": inception_models,
"hyperbolic": hyperbolic_models,
"black_forest_labs": black_forest_labs_models,
"recraft": recraft_models,
"cometapi": cometapi_models,
"oci": oci_models,
"volcengine": volcengine_models,
"wandb": wandb_models,
"ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
"lemonade": lemonade_models,
"clarifai": clarifai_models,
"amazon_nova": amazon_nova_models,
"stability": stability_models,
"github_copilot": github_copilot_models,
"microsoft_365_copilot": microsoft_365_copilot_models,
"chatgpt": chatgpt_models,
"minimax": minimax_models,
"aws_polly": aws_polly_models,
"transcribe": transcribe_models,
"gigachat": gigachat_models,
"llamagate": llamagate_models,
"coralbricks": coralbricks_models,
"reducto": reducto_models,
"bedrock_mantle": bedrock_mantle_models,
}
models_by_provider: dict = _build_models_by_provider()
models_by_provider: Final = ModelsByProvider()
def add_known_models(model_cost_map: Optional[Dict] = None) -> None:
"""Kept for callers. The provider sets are derived from the live map on read (decision 18)"""
# mapping for those models which have larger equivalents
longer_context_model_fallback_dict: dict = {
@ -1323,11 +778,11 @@ all_embedding_models = (
open_ai_embedding_models
| set(cohere_embedding_models)
| set(bedrock_embedding_models)
| vertex_embedding_models
| fireworks_ai_embedding_models
| nebius_embedding_models
| sambanova_embedding_models
| ovhcloud_embedding_models
| _provider_set("vertex_embedding_models")
| _provider_set("fireworks_ai_embedding_models")
| _provider_set("nebius_embedding_models")
| _provider_set("sambanova_embedding_models")
| _provider_set("ovhcloud_embedding_models")
)
####### IMAGE GENERATION MODELS ###################
@ -2429,6 +1884,12 @@ if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes",
def __getattr__(name: str) -> Any:
if name in _LEGACY_PROVIDER_SETS:
return _provider_set(name)
return _lazy_getattr(name)
def _lazy_getattr(name: str) -> Any:
"""Lazy import handler with cached registry for improved performance."""
global _async_client_cleanup_registered
# Register async client cleanup on first access (only once)

View file

@ -0,0 +1,413 @@
"""The provider model sets, derived from the live map on read (decision 18). The only place that knows provider names.
``LEGACY_NAMES`` names each of main's module-level sets and the raw ``litellm_provider`` strings it holds, ``FILTERS``
and ``BARE_KEYS`` carry the rules main's branch chain applied, and ``ALIASES`` is main's ``models_by_provider``"""
import datetime
import re
import sys
from collections.abc import Callable, Iterator, Mapping
from types import MappingProxyType
from typing import Final
from litellm._catalog import CATALOG
LEGACY_NAMES: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
{
"open_ai_chat_completion_models": ("openai",),
"open_ai_text_completion_models": ("text-completion-openai",),
"cohere_models": ("cohere",),
"cohere_chat_models": ("cohere_chat",),
"mistral_chat_models": ("mistral",),
"text_completion_codestral_models": ("text-completion-codestral",),
"text_completion_inception_models": ("text-completion-inception",),
"anthropic_models": ("anthropic",),
"openrouter_models": ("openrouter",),
"datarobot_models": ("datarobot",),
"vertex_language_models": ("vertex_ai-language-models",),
"vertex_vision_models": ("vertex_ai-vision-models",),
"vertex_chat_models": ("vertex_ai-chat-models",),
"vertex_code_chat_models": ("vertex_ai-code-chat-models",),
"vertex_ai_image_models": ("vertex_ai-image-models",),
"vertex_ai_video_models": ("vertex_ai-video-models",),
"vertex_text_models": ("vertex_ai-text-models",),
"vertex_code_text_models": ("vertex_ai-code-text-models",),
"vertex_embedding_models": ("vertex_ai-embedding-models",),
"vertex_anthropic_models": ("vertex_ai-anthropic_models",),
"vertex_llama3_models": ("vertex_ai-llama_models",),
"vertex_deepseek_models": ("vertex_ai-deepseek_models",),
"vertex_ai_ai21_models": ("vertex_ai-ai21_models",),
"vertex_mistral_models": ("vertex_ai-mistral_models",),
"vertex_openai_models": ("vertex_ai-openai_models",),
"vertex_minimax_models": ("vertex_ai-minimax_models",),
"vertex_moonshot_models": ("vertex_ai-moonshot_models",),
"vertex_zai_models": ("vertex_ai-zai_models",),
"ai21_models": ("ai21",),
"ai21_chat_models": ("ai21",),
"nlp_cloud_models": ("nlp_cloud",),
"aleph_alpha_models": ("aleph_alpha",),
"bedrock_models": ("bedrock",),
"bedrock_converse_models": ("bedrock_converse",),
"fal_ai_models": ("fal_ai",),
"fireworks_ai_models": ("fireworks_ai",),
"baseten_models": ("baseten",),
"fireworks_ai_embedding_models": ("fireworks_ai-embedding-models",),
"deepinfra_models": ("deepinfra",),
"perplexity_models": ("perplexity",),
"watsonx_models": ("watsonx",),
"gemini_models": ("gemini",),
"xai_models": ("xai",),
"zai_models": ("zai",),
"deepseek_models": ("deepseek",),
"tencent_models": ("tencent",),
"runwayml_models": ("runwayml",),
"azure_ai_models": ("azure_ai",),
"jina_ai_models": ("jina_ai",),
"voyage_models": ("voyage",),
"infinity_models": ("infinity",),
"heroku_models": ("heroku",),
"databricks_models": ("databricks",),
"cloudflare_models": ("cloudflare",),
"codestral_models": ("codestral",),
"friendliai_models": ("friendliai",),
"featherless_ai_models": ("featherless_ai",),
"palm_models": ("palm",),
"groq_models": ("groq",),
"azure_models": ("azure",),
"azure_anthropic_models": ("azure_anthropic",),
"azure_text_models": ("azure_text",),
"anyscale_models": ("anyscale",),
"cerebras_models": ("cerebras",),
"nadir_models": ("nadir",),
"galadriel_models": ("galadriel",),
"nvidia_nim_models": ("nvidia_nim",),
"nvidia_riva_models": ("nvidia_riva",),
"soniox_models": ("soniox",),
"sambanova_models": ("sambanova",),
"scaledown_models": ("scaledown",),
"sambanova_embedding_models": ("sambanova-embedding-models",),
"novita_models": ("novita",),
"assemblyai_models": ("assemblyai",),
"snowflake_models": ("snowflake",),
"gradient_ai_models": ("gradient_ai",),
"llama_models": ("meta_llama",),
"nscale_models": ("nscale",),
"nebius_models": ("nebius-chat-models",),
"nebius_embedding_models": ("nebius-embedding-models",),
"aiml_models": ("aiml",),
"deepgram_models": ("deepgram",),
"elevenlabs_models": ("elevenlabs",),
"dashscope_models": ("dashscope",),
"qwencloud_models": ("qwencloud",),
"qwen_ai_platform_models": ("qwen_ai_platform",),
"moonshot_models": ("moonshot",),
"publicai_models": ("publicai",),
"darkbloom_models": ("darkbloom",),
"openinfer_models": ("openinfer",),
"v0_models": ("v0",),
"morph_models": ("morph",),
"opencode_go_models": ("opencode_go",),
"lambda_ai_models": ("lambda_ai",),
"inception_models": ("inception",),
"hyperbolic_models": ("hyperbolic",),
"black_forest_labs_models": ("black_forest_labs",),
"recraft_models": ("recraft",),
"cometapi_models": ("cometapi",),
"oci_models": ("oci",),
"vercel_ai_gateway_models": ("vercel_ai_gateway",),
"edenai_models": ("edenai",),
"volcengine_models": ("volcengine",),
"wandb_models": ("wandb",),
"ovhcloud_models": ("ovhcloud",),
"ovhcloud_embedding_models": ("ovhcloud-embedding-models",),
"lemonade_models": ("lemonade",),
"docker_model_runner_models": ("docker_model_runner",),
"amazon_nova_models": ("amazon_nova",),
"stability_models": ("stability",),
"github_copilot_models": ("github_copilot",),
"microsoft_365_copilot_models": ("microsoft_365_copilot",),
"chatgpt_models": ("chatgpt",),
"minimax_models": ("minimax",),
"aws_polly_models": ("aws_polly",),
"transcribe_models": ("transcribe",),
"gigachat_models": ("gigachat",),
"llamagate_models": ("llamagate",),
"coralbricks_models": ("coralbricks",),
"reducto_models": ("reducto",),
"bedrock_mantle_models": ("bedrock_mantle",),
}
)
_BEDROCK_PRICING_ONLY: Final = re.compile(r"^bedrock/[a-zA-Z0-9_-]+/.+$")
def is_bedrock_pricing_only_model(key: str) -> bool:
"""Keys of the pattern ``bedrock/<region>/<model>`` and commitment keys, priced for lookup only"""
return "month-commitment" in key or _BEDROCK_PRICING_ONLY.match(key) is not None
def is_openai_finetune_model(key: str) -> bool:
"""``ft:<model>`` keys, priced for lookup only"""
return key.startswith("ft:") and not key.count(":") > 1
def _is_past_deprecation_date(row: Mapping[str, object]) -> bool:
deprecation_date: Final = row.get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return datetime.date.fromisoformat(deprecation_date) <= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
except ValueError:
return False
FILTERS: Final[Mapping[str, Callable[[str, Mapping[str, object]], bool]]] = MappingProxyType(
{
"open_ai_chat_completion_models": lambda key, row: not is_openai_finetune_model(key),
"ai21_chat_models": lambda key, row: row.get("mode") == "chat",
"ai21_models": lambda key, row: row.get("mode") != "chat",
"bedrock_models": lambda key, row: row.get("mode") != "guardrail" and not is_bedrock_pricing_only_model(key),
"fireworks_ai_models": lambda key, row: "-to-" not in key and "fireworks-ai-default" not in key,
"fireworks_ai_embedding_models": lambda key, row: "-to-" not in key,
"baseten_models": lambda key, row: not _is_past_deprecation_date(row),
}
)
BARE_KEYS: Final = frozenset(
{
"vertex_ai_ai21_models",
"vertex_ai_image_models",
"vertex_ai_video_models",
"vertex_anthropic_models",
"vertex_deepseek_models",
"vertex_llama3_models",
"vertex_minimax_models",
"vertex_mistral_models",
"vertex_moonshot_models",
"vertex_openai_models",
"vertex_zai_models",
}
)
"""Sets main filled with the ``vertex_ai/`` prefix stripped from each key"""
STATIC: Final = frozenset(
{
"clarifai_models",
"huggingface_models",
"maritalk_models",
"ollama_models",
"petals_models",
"replicate_models",
"together_ai_models",
}
)
"""Hand lists of models the map has no rows for, read from ``litellm`` as they are"""
ALIASES: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
{
"openai": ("open_ai_chat_completion_models", "open_ai_text_completion_models"),
"text-completion-openai": ("open_ai_text_completion_models",),
"cohere": ("cohere_models", "cohere_chat_models"),
"cohere_chat": ("cohere_chat_models",),
"anthropic": ("anthropic_models",),
"replicate": ("replicate_models",),
"huggingface": ("huggingface_models",),
"together_ai": ("together_ai_models",),
"baseten": ("baseten_models",),
"openrouter": ("openrouter_models",),
"vercel_ai_gateway": ("vercel_ai_gateway_models",),
"edenai": ("edenai_models",),
"datarobot": ("datarobot_models",),
"vertex_ai": (
"vertex_chat_models",
"vertex_code_chat_models",
"vertex_text_models",
"vertex_code_text_models",
"vertex_anthropic_models",
"vertex_vision_models",
"vertex_language_models",
"vertex_deepseek_models",
"vertex_minimax_models",
"vertex_moonshot_models",
"vertex_zai_models",
),
"ai21": ("ai21_models", "ai21_chat_models"),
"bedrock": ("bedrock_models", "bedrock_converse_models"),
"petals": ("petals_models",),
"ollama": ("ollama_models",),
"ollama_chat": ("ollama_models",),
"deepinfra": ("deepinfra_models",),
"perplexity": ("perplexity_models",),
"maritalk": ("maritalk_models",),
"watsonx": ("watsonx_models",),
"gemini": ("gemini_models",),
"fireworks_ai": ("fireworks_ai_models", "fireworks_ai_embedding_models"),
"aleph_alpha": ("aleph_alpha_models",),
"text-completion-codestral": ("text_completion_codestral_models",),
"text-completion-inception": ("text_completion_inception_models",),
"xai": ("xai_models",),
"zai": ("zai_models",),
"fal_ai": ("fal_ai_models",),
"deepseek": ("deepseek_models",),
"tencent": ("tencent_models",),
"runwayml": ("runwayml_models",),
"mistral": ("mistral_chat_models",),
"azure_ai": ("azure_ai_models",),
"voyage": ("voyage_models",),
"infinity": ("infinity_models",),
"databricks": ("databricks_models",),
"cloudflare": ("cloudflare_models",),
"codestral": ("codestral_models",),
"nlp_cloud": ("nlp_cloud_models",),
"friendliai": ("friendliai_models",),
"palm": ("palm_models",),
"groq": ("groq_models",),
"azure": ("azure_models", "azure_text_models"),
"azure_anthropic": ("azure_anthropic_models",),
"azure_text": ("azure_text_models",),
"anyscale": ("anyscale_models",),
"cerebras": ("cerebras_models",),
"nadir": ("nadir_models",),
"galadriel": ("galadriel_models",),
"nvidia_nim": ("nvidia_nim_models",),
"nvidia_riva": ("nvidia_riva_models",),
"soniox": ("soniox_models",),
"sambanova": ("sambanova_models", "sambanova_embedding_models"),
"scaledown": ("scaledown_models",),
"novita": ("novita_models",),
"nebius": ("nebius_models", "nebius_embedding_models"),
"aiml": ("aiml_models",),
"assemblyai": ("assemblyai_models",),
"jina_ai": ("jina_ai_models",),
"snowflake": ("snowflake_models",),
"gradient_ai": ("gradient_ai_models",),
"meta_llama": ("llama_models",),
"nscale": ("nscale_models",),
"featherless_ai": ("featherless_ai_models",),
"deepgram": ("deepgram_models",),
"elevenlabs": ("elevenlabs_models",),
"heroku": ("heroku_models",),
"dashscope": ("dashscope_models",),
"qwencloud": ("qwencloud_models",),
"qwen_ai_platform": ("qwen_ai_platform_models",),
"modelscope": ("modelscope_models",),
"moonshot": ("moonshot_models",),
"publicai": ("publicai_models",),
"darkbloom": ("darkbloom_models",),
"openinfer": ("openinfer_models",),
"v0": ("v0_models",),
"morph": ("morph_models",),
"opencode_go": ("opencode_go_models",),
"lambda_ai": ("lambda_ai_models",),
"inception": ("inception_models",),
"hyperbolic": ("hyperbolic_models",),
"black_forest_labs": ("black_forest_labs_models",),
"recraft": ("recraft_models",),
"cometapi": ("cometapi_models",),
"oci": ("oci_models",),
"volcengine": ("volcengine_models",),
"wandb": ("wandb_models",),
"ovhcloud": ("ovhcloud_models", "ovhcloud_embedding_models"),
"lemonade": ("lemonade_models",),
"clarifai": ("clarifai_models",),
"amazon_nova": ("amazon_nova_models",),
"stability": ("stability_models",),
"github_copilot": ("github_copilot_models",),
"microsoft_365_copilot": ("microsoft_365_copilot_models",),
"chatgpt": ("chatgpt_models",),
"minimax": ("minimax_models",),
"aws_polly": ("aws_polly_models",),
"transcribe": ("transcribe_models",),
"gigachat": ("gigachat_models",),
"llamagate": ("llamagate_models",),
"coralbricks": ("coralbricks_models",),
"reducto": ("reducto_models",),
"bedrock_mantle": ("bedrock_mantle_models",),
}
)
"""main's ``models_by_provider``, each provider the union of the named sets"""
_SEEDS: Final[Mapping[str, str]] = MappingProxyType(
{"bedrock_converse_models": "BEDROCK_CONVERSE_MODELS", "wandb_models": "WANDB_MODELS"}
)
OPAQUE_KEYS: Final[set[str]] = set() # mutable-ok: register_model adds router deployment ids, never removed
"""Keys registered as opaque ids (``add_to_provider_model_sets=False``), which no provider set lists"""
def _litellm() -> object:
return sys.modules["litellm"]
def _rows() -> Mapping[str, Mapping[str, object]]:
import litellm
return litellm.model_cost
def _derived(name: str) -> frozenset[str]:
rows: Final = _rows()
keep: Final = FILTERS.get(name, lambda key, row: True)
bare: Final = name in BARE_KEYS
keys: Final = (
key
for provider in LEGACY_NAMES[name]
for key in CATALOG.models_for(provider) # comprehension-ok: a set's providers, then each provider's keys
if keep(key, rows.get(key) or {}) and key not in OPAQUE_KEYS
)
seed: Final = _SEEDS.get(name)
seeded: Final = frozenset(getattr(_litellm(), seed)) if seed is not None else frozenset()
return frozenset(key.replace("vertex_ai/", "") if bare else key for key in keys) | seeded
def legacy_set(name: str) -> frozenset[str]:
"""One of main's 121 module-level provider sets, as the live map has it now. Read only"""
return _derived(name)
def contains(name: str, model: str) -> bool:
"""``model in litellm.<name>`` without building the set, for the request path. The same rows, filters, bare keys
and seeds as ``legacy_set``. A set assigned to ``litellm.<name>`` answers instead"""
assigned: Final = vars(_litellm()).get(name)
if assigned is not None:
return model in assigned
rows: Final = _rows()
keep: Final = FILTERS.get(name, lambda key, row: True)
bare: Final = name in BARE_KEYS
if bare and "vertex_ai/" in model:
return False
keys: Final = (model, f"vertex_ai/{model}") if bare else (model,)
in_map: Final = any(
isinstance(row := rows.get(key), Mapping)
and row.get("litellm_provider") in LEGACY_NAMES[name]
and key not in OPAQUE_KEYS
and keep(key, row)
for key in keys
)
seed: Final = _SEEDS.get(name)
return in_map or (seed is not None and model in getattr(_litellm(), seed))
def models_for(provider: str) -> frozenset[str]:
"""The models of ``provider`` as ``models_by_provider`` lists them"""
names: Final = ALIASES.get(provider)
if names is None:
return CATALOG.models_for(provider) - OPAQUE_KEYS
return frozenset().union(
*(legacy_set(name) if name in LEGACY_NAMES else frozenset(getattr(_litellm(), name)) for name in names)
)
class ModelsByProvider(Mapping[str, frozenset[str]]):
"""``litellm.models_by_provider``, read only. Main's providers, then every raw provider string in the map"""
def __getitem__(self, provider: str) -> frozenset[str]:
if provider in ALIASES or provider in CATALOG.providers():
return models_for(provider)
raise KeyError(provider)
def __iter__(self) -> Iterator[str]:
return iter((*ALIASES, *(provider for provider in CATALOG.providers() if provider not in ALIASES)))
def __len__(self) -> int:
return len(ALIASES) + sum(1 for provider in CATALOG.providers() if provider not in ALIASES)

View file

@ -160,11 +160,9 @@ def _install(
def _refresh_derived_state() -> None:
import litellm
from litellm import utils
utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation
litellm.add_known_models(model_cost_map=litellm.model_cost)
def reload_model_cost_map(source: Source, catalog: Catalog = CATALOG) -> Installed | Unavailable:

View file

@ -248,6 +248,8 @@ class Catalog:
def resolve(self, model: str, provider: str | None = None) -> Resolved | Missing | Invalid: ...
def lookup_key(self, name: str) -> str | None: ...
def provider_keys(self, provider: str) -> list[str]: ...
def models_for(self, provider: str) -> frozenset[str]: ...
def providers(self) -> list[str]: ...
def mark_index_stale(self) -> None: ...
def register(
self, rows: Mapping[str, Mapping[str, object]], *, persist: bool, provider: str | None = None

View file

@ -51,6 +51,7 @@ import litellm.litellm_core_utils
# audio_utils.utils is lazy-loaded - only imported when needed for transcription calls
import litellm.litellm_core_utils.json_validation_rule
from litellm._catalog.provider_sets import contains as _in_provider_set
from litellm._internal_context import is_internal_call
from litellm._lazy_imports import (
_get_modified_max_tokens,
@ -2635,7 +2636,7 @@ def _return_openai_tokenizer(model: str) -> SelectTokenizerResponse:
def uses_anthropic_tokenizer(model: str) -> bool:
return model in litellm.anthropic_models and "claude-3" not in model
return _in_provider_set("anthropic_models", model) and "claude-3" not in model
HuggingFaceTokenizerKind = Literal["cohere", "anthropic", "deepseek", "llama2", "llama3"]
@ -2651,7 +2652,7 @@ _BUNDLED_TOKENIZER_LOAD_LOCKS: Final[Mapping[HuggingFaceTokenizerKind, threading
def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None:
"""Which HuggingFace tokenizer `token_counter` selects for a model; `None` means tiktoken."""
if model in litellm.cohere_models and "command-r" in model:
if _in_provider_set("cohere_models", model) and "command-r" in model:
return "cohere"
if uses_anthropic_tokenizer(model):
return "anthropic"
@ -3656,9 +3657,10 @@ def register_model(
that provider, so a deployment id that happens to equal another provider's
catalog key stays its own provider-less entry instead of merging into it.
``add_to_provider_model_sets`` adds the keys to the per-provider model sets
behind provider resolution and wildcard model listing. Pass False when the
keys are opaque ids rather than model names (e.g. router deployment ids).
The provider model sets are derived from ``litellm.model_cost`` on read
(decision 18). Pass ``add_to_provider_model_sets=False`` when the keys are
opaque ids rather than model names (e.g. router deployment ids), so no set
lists them.
"""
loaded_model_cost = {}
@ -3685,12 +3687,12 @@ def register_model(
"register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info",
warning_display_name or key,
)
for key, value in loaded_model_cost.items():
for key in loaded_model_cost:
verbose_logger.debug("added/updated model=%s in litellm.model_cost", key)
if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter":
litellm.openrouter_models.add(str(key).split("/", 1)[-1])
if add_to_provider_model_sets:
litellm.add_known_models(loaded_model_cost)
if not add_to_provider_model_sets:
from litellm._catalog.provider_sets import OPAQUE_KEYS
OPAQUE_KEYS.update(str(key) for key in loaded_model_cost)
return model_cost
@ -4864,12 +4866,12 @@ def get_optional_params(
drop_params=bool(drop_params),
)
elif custom_llm_provider == "vertex_ai" and (
model in litellm.vertex_chat_models
or model in litellm.vertex_code_chat_models
or model in litellm.vertex_text_models
or model in litellm.vertex_code_text_models
or model in litellm.vertex_language_models
or model in litellm.vertex_vision_models
_in_provider_set("vertex_chat_models", model)
or _in_provider_set("vertex_code_chat_models", model)
or _in_provider_set("vertex_text_models", model)
or _in_provider_set("vertex_code_text_models", model)
or _in_provider_set("vertex_language_models", model)
or _in_provider_set("vertex_vision_models", model)
):
optional_params = litellm.VertexGeminiConfig().map_openai_params(
non_default_params=non_default_params,
@ -4900,7 +4902,7 @@ def get_optional_params(
drop_params=bool(drop_params),
)
elif custom_llm_provider == "vertex_ai":
if model in litellm.vertex_mistral_models:
if _in_provider_set("vertex_mistral_models", model):
if "codestral" in model:
optional_params = litellm.CodestralTextCompletionConfig().map_openai_params(
model=model,
@ -4915,7 +4917,7 @@ def get_optional_params(
optional_params=optional_params,
drop_params=bool(drop_params),
)
elif model in litellm.vertex_ai_ai21_models:
elif _in_provider_set("vertex_ai_ai21_models", model):
optional_params = litellm.VertexAIAi21Config().map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
@ -6109,10 +6111,10 @@ def get_model_info_helper(
if custom_llm_provider is not None and custom_llm_provider == "vertex_ai_beta":
custom_llm_provider = "vertex_ai"
if custom_llm_provider is not None and custom_llm_provider == "vertex_ai":
if "meta/" + model in litellm.vertex_llama3_models:
if _in_provider_set("vertex_llama3_models", "meta/" + model):
model = "meta/" + model
elif (
model + "@latest" in litellm.vertex_mistral_models or model + "@latest" in litellm.vertex_ai_ai21_models
elif _in_provider_set("vertex_mistral_models", model + "@latest") or _in_provider_set(
"vertex_ai_ai21_models", model + "@latest"
):
model = model + "@latest"
##########################
@ -6845,8 +6847,8 @@ def validate_environment(
else:
## openai - chatcompletion + text completion
if (
model in litellm.open_ai_chat_completion_models
or model in litellm.open_ai_text_completion_models
_in_provider_set("open_ai_chat_completion_models", model)
or _in_provider_set("open_ai_text_completion_models", model)
or model in litellm.open_ai_embedding_models
or model in litellm.openai_image_generation_models
or model.startswith("gpt-image")
@ -6856,13 +6858,13 @@ def validate_environment(
else:
missing_keys.append("OPENAI_API_KEY")
## anthropic
elif model in litellm.anthropic_models:
elif _in_provider_set("anthropic_models", model):
if "ANTHROPIC_API_KEY" in os.environ or "ANTHROPIC_AUTH_TOKEN" in os.environ:
keys_in_environment = True
else:
missing_keys.append("ANTHROPIC_API_KEY")
## cohere
elif model in litellm.cohere_models:
elif _in_provider_set("cohere_models", model):
if "COHERE_API_KEY" in os.environ:
keys_in_environment = True
else:
@ -6874,33 +6876,33 @@ def validate_environment(
else:
missing_keys.append("REPLICATE_API_KEY")
## openrouter
elif model in litellm.openrouter_models:
elif _in_provider_set("openrouter_models", model):
if "OPENROUTER_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("OPENROUTER_API_KEY")
## vercel_ai_gateway
elif model in litellm.vercel_ai_gateway_models:
elif _in_provider_set("vercel_ai_gateway_models", model):
if "VERCEL_AI_GATEWAY_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("VERCEL_AI_GATEWAY_API_KEY")
## edenai
elif model in litellm.edenai_models:
elif _in_provider_set("edenai_models", model):
if "EDENAI_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("EDENAI_API_KEY")
## datarobot
elif model in litellm.datarobot_models:
elif _in_provider_set("datarobot_models", model):
if "DATAROBOT_API_TOKEN" in os.environ:
keys_in_environment = True
else:
missing_keys.append("DATAROBOT_API_TOKEN")
## vertex - text + chat models
elif (
model in litellm.vertex_chat_models
or model in litellm.vertex_text_models
_in_provider_set("vertex_chat_models", model)
or _in_provider_set("vertex_text_models", model)
or model in litellm.models_by_provider["vertex_ai"]
):
if "VERTEXAI_PROJECT" in os.environ and "VERTEXAI_LOCATION" in os.environ:
@ -6914,7 +6916,7 @@ def validate_environment(
else:
missing_keys.append("HUGGINGFACE_API_KEY")
## ai21
elif model in litellm.ai21_models:
elif _in_provider_set("ai21_models", model):
if "AI21_API_KEY" in os.environ:
keys_in_environment = True
else:
@ -6926,34 +6928,34 @@ def validate_environment(
else:
missing_keys.append("TOGETHERAI_API_KEY")
## aleph_alpha
elif model in litellm.aleph_alpha_models:
elif _in_provider_set("aleph_alpha_models", model):
if "ALEPH_ALPHA_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("ALEPH_ALPHA_API_KEY")
## baseten
elif model in litellm.baseten_models:
elif _in_provider_set("baseten_models", model):
if "BASETEN_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("BASETEN_API_KEY")
## nlp_cloud
elif model in litellm.nlp_cloud_models:
elif _in_provider_set("nlp_cloud_models", model):
if "NLP_CLOUD_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("NLP_CLOUD_API_KEY")
elif model in litellm.novita_models:
elif _in_provider_set("novita_models", model):
if "NOVITA_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("NOVITA_API_KEY")
elif model in litellm.nebius_models:
elif _in_provider_set("nebius_models", model):
if "NEBIUS_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("NEBIUS_API_KEY")
elif model in litellm.wandb_models:
elif _in_provider_set("wandb_models", model):
if "WANDB_API_KEY" in os.environ:
keys_in_environment = True
else:
@ -6985,7 +6987,9 @@ def acreate(*args, **kwargs): ## Thin client to handle the acreate langchain ca
def valid_model(model):
try:
# for a given model name, check if the user has the right permissions to access the model
if model in litellm.open_ai_chat_completion_models or model in litellm.open_ai_text_completion_models:
if _in_provider_set("open_ai_chat_completion_models", model) or _in_provider_set(
"open_ai_text_completion_models", model
):
openai.models.retrieve(model)
else:
messages: Final = [{"role": "user", "content": "Hello World"}]
@ -8494,11 +8498,11 @@ class ProviderConfigManager:
)
return VertexAIGPTOSSTransformation()
elif model in litellm.vertex_mistral_models:
elif _in_provider_set("vertex_mistral_models", model):
if "codestral" in model:
return litellm.CodestralTextCompletionConfig()
return litellm.VertexAIMistralConfig()
elif model in litellm.vertex_ai_ai21_models:
elif _in_provider_set("vertex_ai_ai21_models", model):
return litellm.VertexAIAi21Config()
else:
return litellm.VertexAILlama3Config()

View file

@ -0,0 +1,70 @@
from typing import Final
import pytest
import litellm
from litellm._catalog import CATALOG
from litellm._catalog.provider_sets import models_for
@pytest.fixture
def live_map(monkeypatch: pytest.MonkeyPatch) -> dict[str, dict[str, object]]:
rows: Final = dict(CATALOG.bundled().served())
monkeypatch.setattr(litellm, "model_cost", rows)
return rows
def test_a_row_added_dropped_or_moved_shows_on_the_next_read(live_map: dict[str, dict[str, object]]) -> None:
"""provider-sets-live. On main a moved key stayed in its old set and a direct write reached none"""
live_map["acme/new-chat"] = {"litellm_provider": "cohere", "mode": "chat"}
assert "acme/new-chat" in litellm.cohere_models
assert "acme/new-chat" in litellm.models_by_provider["cohere"]
live_map["acme/new-chat"] = {"litellm_provider": "mistral", "mode": "chat"}
live_map["acme/len-change"] = {}
assert "acme/new-chat" not in litellm.cohere_models
assert "acme/new-chat" in litellm.mistral_chat_models
del live_map["acme/new-chat"]
assert "acme/new-chat" not in litellm.models_by_provider["mistral"]
def test_the_provider_sets_are_read_only() -> None:
"""provider-sets-readonly"""
with pytest.raises(AttributeError):
litellm.anthropic_models.add("x") # pyright: ignore[reportAttributeAccessIssue] # the mutation under test
with pytest.raises(TypeError):
litellm.models_by_provider["openai"] = frozenset() # pyright: ignore[reportIndexIssue] # the mutation under test
@pytest.mark.usefixtures("live_map")
def test_every_provider_in_the_map_lists_the_rows_that_name_it() -> None:
providers: Final = frozenset(
row["litellm_provider"] for row in litellm.model_cost.values() if isinstance(row.get("litellm_provider"), str)
)
for provider in providers:
assert provider in litellm.models_by_provider
assert "together_ai" in litellm.models_by_provider
assert all(litellm.model_cost[key]["litellm_provider"] == "aihubmix" for key in models_for("aihubmix"))
def test_registering_an_openrouter_model_adds_no_stripped_name(live_map: dict[str, dict[str, object]]) -> None:
litellm.register_model({"openrouter/acme/model": {"litellm_provider": "openrouter", "mode": "chat"}})
assert "openrouter/acme/model" in litellm.openrouter_models
assert "acme/model" not in litellm.openrouter_models
@pytest.mark.usefixtures("live_map")
def test_the_request_path_membership_check_agrees_with_every_derived_set() -> None:
from litellm._catalog.provider_sets import LEGACY_NAMES, contains, legacy_set
names: Final = frozenset(litellm.model_cost) | {key.replace("vertex_ai/", "") for key in litellm.model_cost}
disagreements: Final = [
(set_name, model)
for set_name in LEGACY_NAMES
for model in names # comprehension-ok: every set against every candidate name
if contains(set_name, model) != (model in legacy_set(set_name))
]
assert disagreements == []

View file

@ -44,18 +44,27 @@ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
import litellm # noqa: E402 # litellm reads LITELLM_LOCAL_MODEL_COST_MAP at import
import litellm.router as litellm_router_module # noqa: E402 # same import-time dependency
import litellm.utils as litellm_utils_module # noqa: E402 # same import-time dependency
from litellm._catalog.provider_sets import LEGACY_NAMES as LEGACY_PROVIDER_SETS # noqa: E402 # after litellm
from litellm._logging import ALL_LOGGERS # noqa: E402 # same import-time dependency
from litellm.anthropic_beta_headers_manager import (
reload_beta_headers_config, # noqa: E402 # same import-time dependency
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER # noqa: E402 # same import-time dependency
from litellm.anthropic_beta_headers_manager import reload_beta_headers_config # noqa: E402 # same import-time dependency
from litellm.litellm_core_utils.prompt_templates import factory as prompt_factory_module # noqa: E402 # same import-time dependency
from litellm.litellm_core_utils.prompt_templates import (
factory as prompt_factory_module, # noqa: E402 # same import-time dependency
)
from litellm.litellm_core_utils.prompt_templates import ( # noqa: E402 # same import-time dependency
image_handling as image_handling_module,
)
from litellm.llms.gemini.chat import transformation as gemini_chat_transformation_module # noqa: E402 # same import-time dependency
from litellm.llms.custom_httpx.async_client_cleanup import ( # noqa: E402 # same import-time dependency
close_litellm_async_clients,
)
from litellm.proxy.db import tool_registry_writer as tool_registry_writer_module # noqa: E402 # same import-time dependency
from litellm.llms.gemini.chat import (
transformation as gemini_chat_transformation_module, # noqa: E402 # same import-time dependency
)
from litellm.proxy.db import (
tool_registry_writer as tool_registry_writer_module, # noqa: E402 # same import-time dependency
)
LOOPBACK_HOSTS: Final = ["127.0.0.1", "::1", "localhost"]
AMBIENT_AZURE_CREDENTIAL_ENV_VARS: Final = (
@ -124,8 +133,6 @@ RESTORED_GLOBALS: Final = (
"cost_discount_config",
"disable_hf_tokenizer_download",
"disable_copilot_system_to_assistant",
"cohere_models",
"anthropic_models",
"token_counter",
"initialized_langfuse_clients",
)
@ -309,26 +316,15 @@ def isolate_router_model_cost_state() -> Iterator[None]:
litellm.get_model_info.cache_clear()
def _provider_model_sets() -> tuple[set[str], ...]:
module_globals: Final = tuple(vars(litellm).items())
return tuple(models for name, models in module_globals if name.endswith("_models") and isinstance(models, set))
@pytest.fixture(autouse=True)
def isolate_model_catalog() -> Iterator[None]:
model_cost: Final = litellm.model_cost
model_cost_entries: Final = dict(model_cost)
models_by_provider: Final = litellm.models_by_provider
models_by_provider_entries: Final = dict(models_by_provider)
provider_model_sets: Final = tuple((models, frozenset(models)) for models in _provider_model_sets())
yield
model_cost.clear()
model_cost.update(model_cost_entries)
models_by_provider.clear()
models_by_provider.update(models_by_provider_entries)
for models, entries in provider_model_sets:
models.clear()
models.update(entries)
for name in LEGACY_PROVIDER_SETS:
vars(litellm).pop(name, None)
litellm_utils_module._invalidate_model_cost_lowercase_map()
litellm.get_model_info.cache_clear()

View file

@ -264,10 +264,8 @@ def test_get_llm_provider_deepseek_custom_api_base():
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_vertex_ai_image_models(monkeypatch):
monkeypatch.setattr(litellm, "vertex_ai_image_models", set())
monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider))
litellm.add_known_models(
model_cost_map={
litellm.model_cost.update(
{
"vertex_ai/imagegeneration@006": {
"litellm_provider": "vertex_ai-image-models",
"mode": "image_generation",

View file

@ -19,7 +19,7 @@ def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
import path fetches the published map, which lags a key added in this repo."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=""))
litellm.add_known_models(model_cost_map=litellm.model_cost)
litellm.model_cost.update(litellm.model_cost)
def test_azure_gpt5_supports_reasoning_effort(config: AzureOpenAIGPT5Config):

View file

@ -788,7 +788,7 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
"""Pin the bundled cost map: the published map lags a key added in this repo."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=""))
litellm.add_known_models(model_cost_map=litellm.model_cost)
litellm.model_cost.update(litellm.model_cost)
def test_azure_responses_reasoning_effort_none_unlocks_temperature_off_the_azure_row(

View file

@ -7,6 +7,7 @@ import pytest
import respx
import litellm
from litellm._catalog.provider_sets import OPAQUE_KEYS
from litellm.llms.baseten.chat import BasetenConfig
from litellm.utils import get_valid_models
@ -66,7 +67,7 @@ class TestBasetenModelDiscovery:
key
for key, value in litellm.model_cost.items()
if value.get("litellm_provider") == "baseten" and not litellm._is_past_deprecation_date(value)
}
} - OPAQUE_KEYS
assert servable_registry_models
assert servable_registry_models <= set(baseten_models)
@ -82,15 +83,14 @@ class TestBasetenModelDiscovery:
"""A refreshed cost map that marks a listed model retired removes it without a restart."""
model: Final = "baseten/acme/reload-probe"
try:
litellm.add_known_models({model: {"litellm_provider": "baseten"}})
litellm.model_cost.update({model: {"litellm_provider": "baseten"}})
assert model in litellm.models_by_provider["baseten"]
litellm.add_known_models({model: {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}})
litellm.model_cost.update({model: {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}})
assert model not in litellm.models_by_provider["baseten"]
finally:
litellm.baseten_models.discard(model)
litellm.add_known_models({})
litellm.model_cost.update({})
@pytest.mark.parametrize(
"value, expected",

View file

@ -21,21 +21,11 @@ from litellm.types.utils import (
def local_model_cost_map(monkeypatch):
"""Resolve models against this checkout's cost map instead of the network-fetched
``main`` copy, which lags this branch until merge."""
original_converse_models = set(litellm.bedrock_converse_models)
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
try:
litellm.bedrock_converse_models.update(
key
for key, value in litellm.model_cost.items()
if isinstance(value, dict) and value.get("litellm_provider") == "bedrock_converse"
)
yield
finally:
litellm.bedrock_converse_models.clear()
litellm.bedrock_converse_models.update(original_converse_models)
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
class GptProfile(NamedTuple):

View file

@ -29,17 +29,13 @@ from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
@pytest.fixture
def local_cost_map(monkeypatch):
original_model_cost = litellm.model_cost
original_bedrock_mantle_models = set(litellm.bedrock_mantle_models)
try:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.get_model_info.cache_clear()
litellm.add_known_models()
yield
finally:
litellm.model_cost = original_model_cost
litellm.bedrock_mantle_models.clear()
litellm.bedrock_mantle_models.update(original_bedrock_mantle_models)
litellm.get_model_info.cache_clear()

View file

@ -164,9 +164,8 @@ class TestRegistration:
monkeypatch.setenv("EDENAI_API_KEY", "eden-test-key")
present = litellm.validate_environment(model=alias)
finally:
litellm.edenai_models.discard(alias)
litellm.model_cost.pop(alias, None)
litellm.add_known_models(model_cost_map={})
litellm.model_cost.update({})
assert (missing["keys_in_environment"], missing["missing_keys"]) == (False, ["EDENAI_API_KEY"])
assert (present["keys_in_environment"], present["missing_keys"]) == (True, [])
@ -178,12 +177,11 @@ class TestRegistration:
wildcard = LiteLLM_Params(model="edenai/*", api_key="wildcard-key")
assert alias not in (get_provider_models("edenai", wildcard) or [])
litellm.add_known_models(model_cost_map={alias: {"litellm_provider": "edenai", "mode": "chat"}})
litellm.model_cost.update({alias: {"litellm_provider": "edenai", "mode": "chat"}})
try:
expanded = get_provider_models("edenai", wildcard)
finally:
litellm.edenai_models.discard(alias)
litellm.add_known_models(model_cost_map={})
litellm.model_cost.pop(alias, None)
assert expanded is not None
assert alias in expanded

View file

@ -27,7 +27,7 @@ def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
on remote catalog fetches (and don't change behavior across remote refreshes)."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=""))
litellm.add_known_models(model_cost_map=litellm.model_cost)
litellm.model_cost.update(litellm.model_cost)
class TestGithubCopilotResponsesAPITransformation:

View file

@ -32,7 +32,7 @@ def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(
litellm, "model_cost", get_model_cost_map(url="")
)
litellm.add_known_models(model_cost_map=litellm.model_cost)
litellm.model_cost.update(litellm.model_cost)
def test_gpt5_supports_reasoning_effort(config: OpenAIConfig):

View file

@ -743,7 +743,7 @@ async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_t
litellm.model_cost but never re-ran add_known_models(), so the provider sets
stayed stale and wildcard matching failed for new models.
Fix: each reload now calls litellm.add_known_models(model_cost_map=new_map)
Fix: each reload now calls litellm.model_cost.update(new_map)
with the fetched map passed explicitly to avoid any reference ambiguity.
"""
from litellm.proxy._types import ProxyException
@ -762,11 +762,8 @@ async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_t
original_model_cost = litellm.model_cost
litellm.model_cost = new_cost_map
# Confirm the model is NOT yet in the provider set before reload propagation.
assert model not in litellm.anthropic_models
# Simulate what proxy_server.py now does after every reload.
litellm.add_known_models(model_cost_map=new_cost_map)
litellm.model_cost.update(new_cost_map)
# After add_known_models(), the model must be in the set.
assert model in litellm.anthropic_models
@ -804,7 +801,6 @@ async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_t
)
finally:
litellm.model_cost = original_model_cost
litellm.anthropic_models.discard(model)
@pytest.mark.asyncio
@ -816,7 +812,7 @@ async def test_add_known_models_explicit_map_updates_provider_sets():
wildcard access checks (anthropic/*, openai/*, …) work immediately.
This covers the proxy_server.py fix where both reload paths now call
litellm.add_known_models(model_cost_map=new_model_cost_map) instead of
litellm.model_cost.update(new_model_cost_map) instead of
relying on the module-level model_cost being up to date.
"""
fake_new_model = "claude-brand-new-explicit-map-test"
@ -835,7 +831,7 @@ async def test_add_known_models_explicit_map_updates_provider_sets():
# Simulate what proxy_server.py does on reload.
original_model_cost = litellm.model_cost
litellm.model_cost = new_cost_map
litellm.add_known_models(model_cost_map=new_cost_map)
litellm.model_cost.update(new_cost_map)
try:
assert fake_new_model in litellm.anthropic_models, (
@ -845,7 +841,6 @@ async def test_add_known_models_explicit_map_updates_provider_sets():
finally:
# Clean up: restore original state.
litellm.model_cost = original_model_cost
litellm.anthropic_models.discard(fake_new_model)
@pytest.mark.asyncio

View file

@ -921,16 +921,13 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion():
captured_reference = litellm.models_by_provider
assert fake_model not in litellm.models_by_provider["vertex_ai"]
try:
litellm.add_known_models(
model_cost_map={fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}}
)
litellm.model_cost.update({fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}})
assert fake_model in litellm.models_by_provider["vertex_ai"]
assert litellm.models_by_provider is captured_reference
assert fake_model in captured_reference["vertex_ai"]
assert fake_model in get_known_models_from_wildcard("vertex_ai/*")
finally:
litellm.vertex_language_models.discard(fake_model)
litellm.add_known_models(model_cost_map={})
litellm.model_cost.pop(fake_model, None)
assert fake_model not in litellm.models_by_provider["vertex_ai"]
@ -949,9 +946,7 @@ def test_wildcard_lists_a_model_once_when_registered_both_bare_and_provider_pref
finally:
litellm.model_cost.pop(bare, None)
litellm.model_cost.pop(f"openai/{bare}", None)
litellm.open_ai_chat_completion_models.discard(bare)
litellm.open_ai_chat_completion_models.discard(f"openai/{bare}")
litellm.add_known_models(model_cost_map={})
litellm.model_cost.update({})
def test_get_complete_model_list_drops_no_default_models_sentinel():
@ -1216,7 +1211,7 @@ def _registered_chat_models(provider_by_model: Mapping[str, str]) -> Generator[N
litellm.model_cost.pop(model, None)
for provider_models in provider_model_sets:
provider_models.discard(model)
litellm.add_known_models(model_cost_map={})
litellm.model_cost.update({})
@pytest.mark.parametrize(

View file

@ -9058,7 +9058,6 @@ def test_cost_calculator_with_custom_pricing():
@pytest.fixture
def isolated_pricing_registry(local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "open_ai_chat_completion_models", set(litellm.open_ai_chat_completion_models))
monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider))

View file

@ -4314,17 +4314,14 @@ def _gemini_client_returning_a_reply():
@pytest.fixture
def restore_model_registry():
"""litellm.model_cost and the provider name sets are module-global.
"""litellm.model_cost is module-global, and the provider sets are derived from it.
register_model merges into the existing entry in place, hence the deep copy.
"""
model_cost = copy.deepcopy(litellm.model_cost)
openai_models = set(litellm.open_ai_chat_completion_models)
yield
litellm.model_cost.clear()
litellm.model_cost.update(model_cost)
litellm.open_ai_chat_completion_models.clear()
litellm.open_ai_chat_completion_models.update(openai_models)
def test_openai_model_name_does_not_outrank_explicit_provider():
@ -4367,7 +4364,6 @@ def test_mislabelled_pricing_entry_does_not_reroute_provider(restore_model_regis
}
}
)
assert "gemini-2.5-pro" in litellm.open_ai_chat_completion_models
client, post = _gemini_client_returning_a_reply()
with patch.object(client, "post", new=post):

View file

@ -8376,7 +8376,7 @@ def test_get_deployment_model_info_base_model_flow():
print("✓ All base model flow test cases passed!")
@patch("litellm.model_cost", {})
@patch("litellm.model_cost", {"gpt-3.5-turbo": {"litellm_provider": "openai", "mode": "chat"}})
def test_get_deployment_model_info_base_model_merge_priority():
"""Test that base model info merging respects the correct priority order"""
from unittest.mock import patch

View file

@ -3819,7 +3819,7 @@ def _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog)
`reload_model_cost_map` does before replaying, so bare names in the new catalog resolve."""
monkeypatch.setattr(litellm, "model_cost", fetched_catalog)
_invalidate_model_cost_lowercase_map()
litellm.add_known_models(model_cost_map=fetched_catalog)
litellm.model_cost.update(fetched_catalog)
CATALOG.replay_registrations()
@ -3859,8 +3859,7 @@ def test_a_config_deployment_dropped_by_a_stale_cost_map_comes_back_on_reload(mo
assert [d["model_info"]["id"] for d in router.model_list] == ["control-id", "new-id"]
assert "new-id" in litellm.model_cost
finally:
litellm.open_ai_chat_completion_models.discard(backend)
litellm.models_by_provider["openai"].discard(backend)
pass
def test_a_config_deployment_dropped_for_a_permanent_reason_is_not_retried_on_reload(monkeypatch):

View file

@ -2299,10 +2299,8 @@ def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch:
"bedrock/us-west-1/migration.bedrock-model-v1:0": {**price, "litellm_provider": "bedrock"},
"migration-openai-model": {**price, "litellm_provider": "openai"},
}
monkeypatch.setattr(litellm, "bedrock_models", set(litellm.bedrock_models))
monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider))
litellm.add_known_models(model_cost_map=model_cost_map)
litellm.model_cost.update(model_cost_map)
assert "migration.bedrock-model-v1:0" in litellm.bedrock_models
assert "bedrock/us-west-1/migration.bedrock-model-v1:0" not in litellm.bedrock_models
@ -2416,9 +2414,6 @@ def test_register_model_openrouter_without_slash():
always works.
"""
# Clear any existing entries
litellm.openrouter_models.discard("my-custom-alias")
litellm.openrouter_models.discard("gpt-4")
litellm.openrouter_models.discard("openai/gpt-4")
# Test 1: Model name without '/' (this was the bug - would raise IndexError)
litellm.register_model(
@ -2446,7 +2441,8 @@ def test_register_model_openrouter_without_slash():
},
}
)
assert "gpt-4" in litellm.openrouter_models
assert "openrouter/gpt-4" in litellm.openrouter_models
assert "gpt-4" not in litellm.openrouter_models
# Test 3: Model name with double '/' (openrouter/provider/model format)
litellm.register_model(
@ -2460,7 +2456,8 @@ def test_register_model_openrouter_without_slash():
},
}
)
assert "openai/gpt-4-turbo" in litellm.openrouter_models
assert "openrouter/openai/gpt-4-turbo" in litellm.openrouter_models
assert "openai/gpt-4-turbo" not in litellm.openrouter_models
def test_register_model_makes_the_model_resolvable_and_wildcard_visible_without_a_restart():
@ -2484,8 +2481,6 @@ def test_register_model_makes_the_model_resolvable_and_wildcard_visible_without_
assert model_name in litellm.models_by_provider["vertex_ai"]
finally:
litellm.model_cost.pop(model_name, None)
litellm.vertex_language_models.discard(model_name)
litellm.models_by_provider["vertex_ai"].discard(model_name)
@pytest.mark.parametrize(
@ -2511,8 +2506,7 @@ def test_registered_chat_model_lists_under_its_provider_wildcard(
assert f"{wildcard.removesuffix('*')}{model_name}" in listed
finally:
litellm.model_cost.pop(model_name, None)
getattr(litellm, provider_set).discard(model_name)
litellm.add_known_models(model_cost_map={})
litellm.model_cost.update({})
def test_priced_router_deployment_id_stays_out_of_wildcard_model_listing():
@ -5008,17 +5002,10 @@ def test_model_cost_walks_survive_a_registration_landing_mid_walk(monkeypatch: p
litellm.model_cost.setdefault("lit-5853-registered-during-the-rebuild", {"mode": "chat"})
return super().lower()
class _EntryRegisteredMidWalk(dict):
def get(self, field: str, default: object = None) -> object:
litellm.model_cost.setdefault("lit-5853-registered-during-add-known-models", {"mode": "chat"})
return super().get(field, default)
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
litellm.model_cost[_KeyRegisteredMidWalk("LIT-5853-WALKED-LAST")] = _EntryRegisteredMidWalk(mode="chat")
litellm.model_cost[_KeyRegisteredMidWalk("LIT-5853-WALKED-LAST")] = {"mode": "chat"}
try:
assert get_model_cost_key("lit-5853-walked-last") == "LIT-5853-WALKED-LAST"
litellm.add_known_models(model_cost_map=litellm.model_cost)
assert "lit-5853-registered-during-add-known-models" in litellm.model_cost
finally:
_invalidate_model_cost_lowercase_map()
@ -8811,17 +8798,8 @@ def reset_mock_cache() -> None:
@pytest.fixture
def isolated_openai_model_sets(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
litellm,
"open_ai_chat_completion_models",
set(litellm.open_ai_chat_completion_models),
)
monkeypatch.setattr(
litellm,
"open_ai_text_completion_models",
set(litellm.open_ai_text_completion_models),
)
def isolated_openai_model_sets() -> None:
"""The provider sets are derived from litellm.model_cost, which the autouse isolation restores"""
def test_aget_valid_models():